You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by cp...@apache.org on 2017/05/25 17:55:10 UTC
[01/44] lucene-solr:jira/solr-8668: SOLR-10725: Remove trailing
spaces in Lucene's NOTICE to make smoke tester happy
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-8668 ab9aad20a -> 7524bd020
SOLR-10725: Remove trailing spaces in Lucene's NOTICE to make smoke tester happy
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/08d24570
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/08d24570
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/08d24570
Branch: refs/heads/jira/solr-8668
Commit: 08d24570a7e67c81325ed43baa809073a4f02725
Parents: 2fc41d5
Author: Jan Høydahl <ja...@apache.org>
Authored: Tue May 23 09:13:25 2017 +0200
Committer: Jan Høydahl <ja...@apache.org>
Committed: Tue May 23 09:13:25 2017 +0200
----------------------------------------------------------------------
lucene/NOTICE.txt | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/08d24570/lucene/NOTICE.txt
----------------------------------------------------------------------
diff --git a/lucene/NOTICE.txt b/lucene/NOTICE.txt
index ecf0820..6260f3b 100644
--- a/lucene/NOTICE.txt
+++ b/lucene/NOTICE.txt
@@ -18,13 +18,13 @@ Some data files (under analysis/icu/src/data) are derived from Unicode data such
as the Unicode Character Database. See http://unicode.org/copyright.html for more
details.
-Brics Automaton (under core/src/java/org/apache/lucene/util/automaton) is
+Brics Automaton (under core/src/java/org/apache/lucene/util/automaton) is
BSD-licensed, created by Anders Møller. See http://www.brics.dk/automaton/
The levenshtein automata tables (under core/src/java/org/apache/lucene/util/automaton) were
automatically generated with the moman/finenight FSA library, created by
Jean-Philippe Barrette-LaPierre. This library is available under an MIT license,
-see http://sites.google.com/site/rrettesite/moman and
+see http://sites.google.com/site/rrettesite/moman and
http://bitbucket.org/jpbarrette/moman/overview/
The class org.apache.lucene.util.WeakIdentityMap was derived from
@@ -78,7 +78,7 @@ analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseLightStemmer.ja
analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLightStemmer.java
analysis/common/src/java/org/apache/lucene/analysis/sv/SwedishLightStemmer.java
-The Stempel analyzer (stempel) includes BSD-licensed software developed
+The Stempel analyzer (stempel) includes BSD-licensed software developed
by the Egothor project http://egothor.sf.net/, created by Leo Galambos, Martin Kvapil,
and Edmond Nolan.
@@ -90,8 +90,8 @@ See http://project.carrot2.org/license.html.
The SmartChineseAnalyzer source code (smartcn) was
provided by Xiaoping Gao and copyright 2009 by www.imdict.net.
-WordBreakTestUnicode_*.java (under modules/analysis/common/src/test/)
-is derived from Unicode data such as the Unicode Character Database.
+WordBreakTestUnicode_*.java (under modules/analysis/common/src/test/)
+is derived from Unicode data such as the Unicode Character Database.
See http://unicode.org/copyright.html for more details.
The Morfologik analyzer (morfologik) includes BSD-licensed software
[14/44] lucene-solr:jira/solr-8668: SOLR-10721: Provide a way to know
when Core Discovery is finished and when all async cores are done loading
Posted by cp...@apache.org.
SOLR-10721: Provide a way to know when Core Discovery is finished and when all async cores are done loading
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/28b8696d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/28b8696d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/28b8696d
Branch: refs/heads/jira/solr-8668
Commit: 28b8696d771d6cccdbab673918cbf04d34b998ad
Parents: 6a82b42
Author: Erick <er...@apache.org>
Authored: Tue May 23 12:20:58 2017 -0700
Committer: Erick <er...@apache.org>
Committed: Tue May 23 12:20:58 2017 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +++
.../org/apache/solr/core/CoreContainer.java | 17 ++++++++++++++++-
.../org/apache/solr/core/TestCoreDiscovery.java | 20 ++++++++++++++++----
3 files changed, 35 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/28b8696d/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 9889945..6b91c4d 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -222,6 +222,9 @@ New Features
* SOLR-10307: Allow Passing SSL passwords through environment variables. (Mano Kovacs via Mark Miller)
+* SOLR-10721: Provide a way to know when Core Discovery is finished and when all async cores are done loading
+ (Erick Erickson)
+
Bug Fixes
----------------------
* SOLR-10723 JSON Facet API: resize() implemented incorrectly for CountSlotAcc, HllAgg.NumericAcc
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/28b8696d/solr/core/src/java/org/apache/solr/core/CoreContainer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 7471c08..505a235 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -186,6 +186,13 @@ public class CoreContainer {
protected MetricsCollectorHandler metricsCollectorHandler;
+
+ // Bits for the state variable.
+ public final static long LOAD_COMPLETE = 0x1L;
+ public final static long CORE_DISCOVERY_COMPLETE = 0x2L;
+ public final static long INITIAL_CORE_LOAD_COMPLETE = 0x4L;
+ private volatile long status = 0L;
+
private enum CoreInitFailedAction { fromleader, none }
/**
@@ -579,7 +586,8 @@ public class CoreContainer {
Collections.sort(cds, coreComparator::compare);
}
checkForDuplicateCoreNames(cds);
-
+ status |= CORE_DISCOVERY_COMPLETE;
+
for (final CoreDescriptor cd : cds) {
if (cd.isTransient() || !cd.isLoadOnStartup()) {
getTransientCacheHandler().addTransientDescriptor(cd.getName(), cd);
@@ -641,8 +649,11 @@ public class CoreContainer {
if (isZooKeeperAware()) {
zkSys.getZkController().checkOverseerDesignate();
}
+ // This is a bit redundant but these are two distinct concepts for all they're accomplished at the same time.
+ status |= LOAD_COMPLETE | INITIAL_CORE_LOAD_COMPLETE;
}
+
public TransientSolrCoreCache getTransientCacheHandler() {
if (transientCoreCache == null) {
@@ -1533,6 +1544,10 @@ public class CoreContainer {
return cfg;
}
+ public long getStatus() {
+ return status;
+ }
+
}
class CloserThread extends Thread {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/28b8696d/solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java b/solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java
index 0c05d83..95c8cb9 100644
--- a/solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java
+++ b/solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java
@@ -33,6 +33,9 @@ import org.junit.After;
import org.junit.BeforeClass;
import org.junit.Test;
+import static org.apache.solr.core.CoreContainer.CORE_DISCOVERY_COMPLETE;
+import static org.apache.solr.core.CoreContainer.INITIAL_CORE_LOAD_COMPLETE;
+import static org.apache.solr.core.CoreContainer.LOAD_COMPLETE;
import static org.hamcrest.CoreMatchers.not;
import static org.junit.internal.matchers.StringContains.containsString;
@@ -107,14 +110,23 @@ public class TestCoreDiscovery extends SolrTestCaseJ4 {
}
private CoreContainer init() throws Exception {
- final CoreContainer cores = new CoreContainer();
+ final CoreContainer container = new CoreContainer();
try {
- cores.load();
+ container.load();
} catch (Exception e) {
- cores.shutdown();
+ container.shutdown();
throw e;
}
- return cores;
+
+ long status = container.getStatus();
+
+ assertTrue("Load complete flag should be set",
+ (status & LOAD_COMPLETE) == LOAD_COMPLETE);
+ assertTrue("Core discovery should be complete",
+ (status & CORE_DISCOVERY_COMPLETE) == CORE_DISCOVERY_COMPLETE);
+ assertTrue("Initial core loading should be complete",
+ (status & INITIAL_CORE_LOAD_COMPLETE) == INITIAL_CORE_LOAD_COMPLETE);
+ return container;
}
@After
[32/44] lucene-solr:jira/solr-8668: SOLR-10731: Add k parameter
Posted by cp...@apache.org.
SOLR-10731: Add k parameter
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/bdde9a16
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/bdde9a16
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/bdde9a16
Branch: refs/heads/jira/solr-8668
Commit: bdde9a163f6760c9cc7ef98c5a64fffcde1e2211
Parents: 94731aa
Author: Joel Bernstein <jb...@apache.org>
Authored: Wed May 24 20:04:20 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Wed May 24 20:04:43 2017 -0400
----------------------------------------------------------------------
.../org/apache/solr/client/solrj/io/stream/KnnStream.java | 9 +++++++++
.../solr/client/solrj/io/stream/StreamExpressionTest.java | 9 ++++++++-
2 files changed, 17 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bdde9a16/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/KnnStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/KnnStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/KnnStream.java
index 1d5f187..f2ba0b9 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/KnnStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/KnnStream.java
@@ -47,6 +47,8 @@ import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.params.ModifiableSolrParams;
import static org.apache.solr.common.params.CommonParams.Q;
+import static org.apache.solr.common.params.CommonParams.ROWS;
+
public class KnnStream extends TupleStream implements Expressible {
@@ -195,6 +197,13 @@ public class KnnStream extends TupleStream implements Expressible {
}
}
+ String k = params.get("k");
+
+ if(k != null) {
+ params.add(ROWS, k);
+ params.remove(k);
+ }
+
params.add(Q, "{!mlt"+builder.toString()+"}"+id);
QueryRequest request = new QueryRequest(params);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bdde9a16/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index 04af631..47b1d31 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -943,6 +943,13 @@ public class StreamExpressionTest extends SolrCloudTestCase {
assertOrder(tuples, 2, 3, 4);
sParams = new ModifiableSolrParams(StreamingTest.mapParams(CommonParams.QT, "/stream"));
+ sParams.add("expr", "knn(" + COLLECTIONORALIAS + ", id=\"1\", qf=\"a_t\", k=\"2\", fl=\"id, score\", mintf=\"1\")");
+ solrStream = new SolrStream(jetty.getBaseUrl().toString() + "/collection1", sParams);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 2);
+ assertOrder(tuples, 2, 3);
+
+ sParams = new ModifiableSolrParams(StreamingTest.mapParams(CommonParams.QT, "/stream"));
sParams.add("expr", "knn(" + COLLECTIONORALIAS + ", id=\"1\", qf=\"a_t\", rows=\"4\", fl=\"id, score\", mintf=\"1\", maxdf=\"0\")");
solrStream = new SolrStream(jetty.getBaseUrl().toString() + "/collection1", sParams);
tuples = getTuples(solrStream);
@@ -955,7 +962,7 @@ public class StreamExpressionTest extends SolrCloudTestCase {
assertTrue(tuples.size() == 0);
sParams = new ModifiableSolrParams(StreamingTest.mapParams(CommonParams.QT, "/stream"));
- sParams.add("expr", "knn(" + COLLECTIONORALIAS + ", id=\"1\", qf=\"a_t\", rows=\"2\", fl=\"id, score\", mintf=\"1\", minwl=\"20\")");
+ sParams.add("expr", "knn(" + COLLECTIONORALIAS + ", id=\"1\", qf=\"a_t\", r=\"2\", fl=\"id, score\", mintf=\"1\", minwl=\"20\")");
solrStream = new SolrStream(jetty.getBaseUrl().toString() + "/collection1", sParams);
tuples = getTuples(solrStream);
assertTrue(tuples.size() == 0);
[36/44] lucene-solr:jira/solr-8668: SOLR-10004: Adding descriptions
for a few classes to assess impact on smoke tester warnings
Posted by cp...@apache.org.
SOLR-10004: Adding descriptions for a few classes to assess impact on smoke tester warnings
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6d80320a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6d80320a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6d80320a
Branch: refs/heads/jira/solr-8668
Commit: 6d80320aef9b1bdcbe5a9bba153af5adc45f84aa
Parents: effd376
Author: Ishan Chattopadhyaya <is...@apache.org>
Authored: Thu May 25 17:43:45 2017 +0530
Committer: Ishan Chattopadhyaya <is...@apache.org>
Committed: Thu May 25 17:47:21 2017 +0530
----------------------------------------------------------------------
solr/core/src/java/org/apache/solr/update/PeerSync.java | 6 +++++-
.../java/org/apache/solr/update/SolrCmdDistributor.java | 4 +++-
solr/core/src/java/org/apache/solr/update/UpdateLog.java | 11 +++++++++--
3 files changed, 17 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6d80320a/solr/core/src/java/org/apache/solr/update/PeerSync.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/PeerSync.java b/solr/core/src/java/org/apache/solr/update/PeerSync.java
index dfadb0c..c6bfbbf 100644
--- a/solr/core/src/java/org/apache/solr/update/PeerSync.java
+++ b/solr/core/src/java/org/apache/solr/update/PeerSync.java
@@ -65,7 +65,11 @@ import static org.apache.solr.common.params.CommonParams.ID;
import static org.apache.solr.update.processor.DistributedUpdateProcessor.DistribPhase.FROMLEADER;
import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
-/** @lucene.experimental */
+/**
+ * @lucene.experimental
+ * This class is useful for performing peer to peer synchronization of recently indexed update commands during
+ * recovery process.
+ */
public class PeerSync implements SolrMetricProducer {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private boolean debug = log.isDebugEnabled();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6d80320a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
index dac4000..5772b2e 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
@@ -51,7 +51,9 @@ import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
-
+/**
+ * Used for distributing commands from a shard leader to its replicas.
+ */
public class SolrCmdDistributor implements Closeable {
private static final int MAX_RETRIES_ON_FORWARD = 25;
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6d80320a/solr/core/src/java/org/apache/solr/update/UpdateLog.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLog.java b/solr/core/src/java/org/apache/solr/update/UpdateLog.java
index 87b93f4..4928aa1 100644
--- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java
+++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java
@@ -80,7 +80,11 @@ import static org.apache.solr.update.processor.DistributedUpdateProcessor.Distri
import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
-/** @lucene.experimental */
+/**
+ * @lucene.experimental
+ * This holds references to the transaction logs and pointers for the document IDs to their
+ * exact positions in the transaction logs.
+ */
public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
private static final long STATUS_TIME = TimeUnit.NANOSECONDS.convert(60, TimeUnit.SECONDS);
public static String LOG_FILENAME_PATTERN = "%s.%019d";
@@ -143,7 +147,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
/**
* The index of the _version_ value in an entry from the transaction log.
*/
-public static final int VERSION_IDX = 1;
+ public static final int VERSION_IDX = 1;
/**
* The index of the previous pointer in an entry from the transaction log.
@@ -204,6 +208,9 @@ public static final int VERSION_IDX = 1;
}
};
+ /**
+ * Holds the query and the version for a DeleteByQuery command
+ */
public static class DBQ {
public String q; // the query string
public long version; // positive version of the DBQ
[07/44] lucene-solr:jira/solr-8668: Ref Guide: fix note for atomic
updates after SOLR-9530
Posted by cp...@apache.org.
Ref Guide: fix note for atomic updates after SOLR-9530
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/31e02e93
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/31e02e93
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/31e02e93
Branch: refs/heads/jira/solr-8668
Commit: 31e02e93a5712d8fa2cafff3b13ce329563c6579
Parents: 73aa53b
Author: Cassandra Targett <ct...@apache.org>
Authored: Tue May 23 11:05:18 2017 -0500
Committer: Cassandra Targett <ct...@apache.org>
Committed: Tue May 23 11:05:53 2017 -0500
----------------------------------------------------------------------
.../src/update-request-processors.adoc | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/31e02e93/solr/solr-ref-guide/src/update-request-processors.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/update-request-processors.adoc b/solr/solr-ref-guide/src/update-request-processors.adoc
index 22d14ed..eb69d3b 100644
--- a/solr/solr-ref-guide/src/update-request-processors.adoc
+++ b/solr/solr-ref-guide/src/update-request-processors.adoc
@@ -90,7 +90,7 @@ Do not forget to add `RunUpdateProcessorFactory` at the end of any chains you de
Update request processors can also be configured independent of a chain in `solrconfig.xml`.
-.updateProcessor
+.updateProcessor Configuration
[source,xml]
----
<updateProcessor class="solr.processor.SignatureUpdateProcessorFactory" name="signature">
@@ -105,7 +105,7 @@ Update request processors can also be configured independent of a chain in `solr
In this case, an instance of `SignatureUpdateProcessorFactory` is configured with the name "signature" and a `RemoveBlankFieldUpdateProcessorFactory` is defined with the name "remove_blanks". Once the above has been specified in `solrconfig.xml`, we can be refer to them in update request processor chains in `solrconfig.xml` as follows:
-.updateRequestProcessorChains and updateProcessors
+.updateRequestProcessorChain Configuration
[source,xml]
----
<updateProcessorChain name="custom" processor="remove_blanks,signature">
@@ -135,7 +135,7 @@ In summary:
In the previous section, we saw that the `updateRequestProcessorChain` was configured with `processor="remove_blanks, signature"`. This means that such processors are of the #1 kind and are run only on the forwarding nodes. Similarly, we can configure them as the #2 kind by specifying with the attribute "post-processor" as follows:
-.post-processors
+.post-processor Configuration
[source,xml]
----
<updateProcessorChain name="custom" processor="signature" post-processor="remove_blanks">
@@ -145,19 +145,19 @@ In the previous section, we saw that the `updateRequestProcessorChain` was confi
However executing a processor only on the forwarding nodes is a great way of distributing an expensive computation such as de-duplication across a SolrCloud cluster by sending requests randomly via a load balancer. Otherwise the expensive computation is repeated on both the leader and replica nodes.
-// TODO 6.6 I think this can be removed after SOLR-9530 -CT
-.Pre-processors and Atomic Updates
-[WARNING]
-====
-Because `DistributedUpdateProcessor` is responsible for processing <<updating-parts-of-documents.adoc#updating-parts-of-documents,Atomic Updates>> into full documents on the leader node, this means that pre-processors which are executed only on the forwarding nodes can only operate on the partial document. If you have a processor which must process a full document then the only choice is to specify it as a post-processor.
-====
-
.Custom update chain post-processors may never be invoked on a recovering replica
[WARNING]
====
While a replica is in <<read-and-write-side-fault-tolerance.adoc#ReadandWriteSideFaultTolerance-WriteSideFaultTolerance,recovery>>, inbound update requests are buffered to the transaction log. After recovery has completed successfully, those buffered update requests are replayed. As of this writing, however, custom update chain post-processors are never invoked for buffered update requests. See https://issues.apache.org/jira/browse/SOLR-8030[SOLR-8030]. To work around this problem until SOLR-8030 has been fixed, *avoid specifying post-processors in custom update chains*.
====
+=== Atomic Updates
+
+If the `AtomicUpdateProcessorFactory` is in the update chain before the `DistributedUpdateProcessor`, the incoming document to the chain will be a partial document.
+
+Because `DistributedUpdateProcessor` is responsible for processing <<updating-parts-of-documents.adoc#updating-parts-of-documents,Atomic Updates>> into full documents on the leader node, this means that pre-processors which are executed only on the forwarding nodes can only operate on the partial document. If you have a processor which must process a full document then the only choice is to specify it as a post-processor.
+
+
[[UpdateRequestProcessors-UsingCustomChains]]
== Using Custom Chains
[20/44] lucene-solr:jira/solr-8668: SOLR-10700: Remove
PostingsHighlighter references from docs
Posted by cp...@apache.org.
SOLR-10700: Remove PostingsHighlighter references from docs
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/872ed81c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/872ed81c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/872ed81c
Branch: refs/heads/jira/solr-8668
Commit: 872ed81cc971646c10fdb27b23ffe6ca7decd91a
Parents: d60c72f
Author: David Smiley <ds...@apache.org>
Authored: Tue May 23 23:24:22 2017 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Tue May 23 23:24:22 2017 -0400
----------------------------------------------------------------------
.../apispec/core.SchemaEdit.addField.json | 2 +-
solr/solr-ref-guide/src/highlighting.adoc | 9 +-----
.../solr/common/params/HighlightParams.java | 31 ++++++++++----------
3 files changed, 17 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/872ed81c/solr/core/src/resources/apispec/core.SchemaEdit.addField.json
----------------------------------------------------------------------
diff --git a/solr/core/src/resources/apispec/core.SchemaEdit.addField.json b/solr/core/src/resources/apispec/core.SchemaEdit.addField.json
index b8cb40b..19265ab 100644
--- a/solr/core/src/resources/apispec/core.SchemaEdit.addField.json
+++ b/solr/core/src/resources/apispec/core.SchemaEdit.addField.json
@@ -82,7 +82,7 @@
},
"termPayloads": {
"type": "boolean",
- "description": "If true, term payloads will be stored for use with highlighting. If this is not defined, it will inherit the value from the fieldType. If the fieldType does not define a value, it will default to false. Do not enable this if using the PostingsHighlighter.",
+ "description": "If true, term vectors will include payloads. If this is not defined, it will inherit the value from the fieldType. If the fieldType does not define a value, it will default to false.",
"default": "false"
},
"useDocValuesAsStored": {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/872ed81c/solr/solr-ref-guide/src/highlighting.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/highlighting.adoc b/solr/solr-ref-guide/src/highlighting.adoc
index 7207ce9..77fae07 100644
--- a/solr/solr-ref-guide/src/highlighting.adoc
+++ b/solr/solr-ref-guide/src/highlighting.adoc
@@ -35,7 +35,7 @@ You only need to set the `hl` and often `hl.fl` parameters to get results. The f
|===
|Parameter |Default |Description
|hl |false |Use this parameter to enable or disable highlighting.
-|hl.method |original |The highlighting implementation to use. Acceptable values are: `unified`, `original`, `fastVector`, and `postings`. See the <<Highlighting-ChoosingaHighlighter,Choosing a Highlighter>> section below for more details on the differences between the available highlighters.
+|hl.method |original |The highlighting implementation to use. Acceptable values are: `unified`, `original`, `fastVector`. See the <<Highlighting-ChoosingaHighlighter,Choosing a Highlighter>> section below for more details on the differences between the available highlighters.
|hl.fl |_(df=)_ |Specifies a list of fields to highlight. Accepts a comma- or space-delimited list of fields for which Solr should generate highlighted snippets. A wildcard of `\*` (asterisk) can be used to match field globs, such as `text_*` or even `\*` to highlight on all fields where highlighting is possible. When using `\*`, consider adding `hl.requireFieldMatch=true`.
|hl.q |_(q=)_ |A query to use for highlighting. This parameter allows you to highlight different terms than those being used to retrieve documents.
|hl.qparser |_(defType=)_ |The query parser to use for the `hl.q` query.
@@ -137,13 +137,6 @@ This highlighter's query-representation is less advanced than the Original or Un
+
Note that both the FastVector and Original Highlighters can be used in conjunction in a search request to highlight some fields with one and some the other. In contrast, the other highlighters can only be chosen exclusively.
-The Postings Highlighter:: (`hl.method=postings`)
-+
-The Postings Highlighter is the ancestor of the Unified Highlighter, supporting a subset of its options and none of its index configuration flexibility - it _requires_ `storeOffsetsWithPositions` on all fields to highlight.
-This option is here for backwards compatibility; it's deprecated.
-In 7.0, it is internally implemented as a reconfiguration of the Unified Highlighter.
-See older reference guide editions for its options.
-
The Unified Highlighter is exclusively configured via search parameters. In contrast, some settings for the Original and FastVector Highlighters are set in `solrconfig.xml`. There's a robust example of the latter in the "```techproducts```" configset.
In addition to further information below, more information can be found in the {solr-javadocs}/solr-core/org/apache/solr/highlight/package-summary.html[Solr javadocs].
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/872ed81c/solr/solrj/src/java/org/apache/solr/common/params/HighlightParams.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/HighlightParams.java b/solr/solrj/src/java/org/apache/solr/common/params/HighlightParams.java
index 3741214..ef254cc 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/HighlightParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/HighlightParams.java
@@ -32,8 +32,7 @@ public interface HighlightParams {
// KEY:
// OH = (original) Highlighter (AKA the standard Highlighter)
// FVH = FastVectorHighlighter
- // PH = PostingsHighlighter
- // UH = UnifiedHighlighter
+ // UH = UnifiedHighlighter (evolved from PostingsHighlighter)
// query interpretation
public static final String Q = HIGHLIGHT+".q"; // all
@@ -43,7 +42,7 @@ public interface HighlightParams {
public static final String HIGHLIGHT_MULTI_TERM = HIGHLIGHT+".highlightMultiTerm"; // all
// if no snippets...
- public static final String DEFAULT_SUMMARY = HIGHLIGHT + ".defaultSummary"; // UH, PH
+ public static final String DEFAULT_SUMMARY = HIGHLIGHT + ".defaultSummary"; // UH
public static final String ALTERNATE_FIELD = HIGHLIGHT+".alternateField"; // OH, FVH
public static final String ALTERNATE_FIELD_LENGTH = HIGHLIGHT+".maxAlternateFieldLength"; // OH, FVH
public static final String HIGHLIGHT_ALTERNATE = HIGHLIGHT+".highlightAlternate"; // OH, FVH
@@ -59,11 +58,11 @@ public interface HighlightParams {
public static final String BOUNDARY_SCANNER = HIGHLIGHT+".boundaryScanner"; // FVH
public static final String BS_MAX_SCAN = HIGHLIGHT+".bs.maxScan"; // FVH
public static final String BS_CHARS = HIGHLIGHT+".bs.chars"; // FVH
- public static final String BS_TYPE = HIGHLIGHT+".bs.type"; // FVH, UH, PH
- public static final String BS_LANGUAGE = HIGHLIGHT+".bs.language"; // FVH, UH, PH
- public static final String BS_COUNTRY = HIGHLIGHT+".bs.country"; // FVH, UH, PH
- public static final String BS_VARIANT = HIGHLIGHT+".bs.variant"; // FVH, UH, PH
- public static final String BS_SEP = HIGHLIGHT+".bs.separator"; // UH, PH
+ public static final String BS_TYPE = HIGHLIGHT+".bs.type"; // FVH, UH
+ public static final String BS_LANGUAGE = HIGHLIGHT+".bs.language"; // FVH, UH
+ public static final String BS_COUNTRY = HIGHLIGHT+".bs.country"; // FVH, UH
+ public static final String BS_VARIANT = HIGHLIGHT+".bs.variant"; // FVH, UH
+ public static final String BS_SEP = HIGHLIGHT+".bs.separator"; // UH
// formatting
public static final String FORMATTER = HIGHLIGHT+".formatter"; // OH
@@ -73,18 +72,18 @@ public interface HighlightParams {
public static final String SIMPLE_PRE = HIGHLIGHT+"."+SIMPLE+".pre"; // OH
public static final String SIMPLE_POST = HIGHLIGHT+"."+SIMPLE+".post"; // OH
public static final String FRAGMENTS_BUILDER = HIGHLIGHT+".fragmentsBuilder"; // FVH
- public static final String TAG_PRE = HIGHLIGHT + ".tag.pre"; // FVH, UH, PH
- public static final String TAG_POST = HIGHLIGHT + ".tag.post"; // FVH, UH, PH
- public static final String TAG_ELLIPSIS= HIGHLIGHT + ".tag.ellipsis"; // FVH, UH, PH
- public static final String MULTI_VALUED_SEPARATOR = HIGHLIGHT + ".multiValuedSeparatorChar"; // FVH, PH
+ public static final String TAG_PRE = HIGHLIGHT + ".tag.pre"; // FVH, UH
+ public static final String TAG_POST = HIGHLIGHT + ".tag.post"; // FVH, UH
+ public static final String TAG_ELLIPSIS= HIGHLIGHT + ".tag.ellipsis"; // FVH, UH
+ public static final String MULTI_VALUED_SEPARATOR = HIGHLIGHT + ".multiValuedSeparatorChar"; // FVH
// ordering
public static final String PRESERVE_MULTI = HIGHLIGHT+".preserveMulti"; // OH
public static final String FRAG_LIST_BUILDER = HIGHLIGHT+".fragListBuilder"; // FVH
- public static final String SCORE = "score"; // UH, PH
- public static final String SCORE_K1 = HIGHLIGHT +"."+SCORE+".k1"; // UH, PH
- public static final String SCORE_B = HIGHLIGHT +"."+SCORE+".b"; // UH, PH
- public static final String SCORE_PIVOT = HIGHLIGHT +"."+SCORE+".pivot"; // UH, PH
+ public static final String SCORE = "score"; // UH
+ public static final String SCORE_K1 = HIGHLIGHT +"."+SCORE+".k1"; // UH
+ public static final String SCORE_B = HIGHLIGHT +"."+SCORE+".b"; // UH
+ public static final String SCORE_PIVOT = HIGHLIGHT +"."+SCORE+".pivot"; // UH
// misc
public static final String MAX_CHARS = HIGHLIGHT+".maxAnalyzedChars"; // all
[06/44] lucene-solr:jira/solr-8668: SOLR-10307: Add upgrade notes for
back compat change.
Posted by cp...@apache.org.
SOLR-10307: Add upgrade notes for back compat change.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/73aa53b0
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/73aa53b0
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/73aa53b0
Branch: refs/heads/jira/solr-8668
Commit: 73aa53b0630bfe69e5798747da57bbe114b19959
Parents: 92ed8b4
Author: Mark Miller <ma...@apache.org>
Authored: Tue May 23 12:50:07 2017 -0300
Committer: Mark Miller <ma...@apache.org>
Committed: Tue May 23 12:50:07 2017 -0300
----------------------------------------------------------------------
solr/CHANGES.txt | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/73aa53b0/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 951b9d5..9889945 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -210,6 +210,13 @@ Jetty 9.3.14.v20161028
Detailed Change List
----------------------
+Upgrade Notes
+----------------------
+
+* SOLR-10307: If starting Jetty without the Solr start script, you must now pass keystore and truststore
+ passwords via the env variables SOLR_SSL_KEY_STORE_PASSWORD and SOLR_SSL_TRUST_STORE_PASSWORD rather
+ than system properties.
+
New Features
----------------------
[16/44] lucene-solr:jira/solr-8668: Ref Guide: update DIH docs for
SOLR-7383; SOLR-9601; plus major surgery on page layout
Posted by cp...@apache.org.
Ref Guide: update DIH docs for SOLR-7383; SOLR-9601; plus major surgery on page layout
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2319d69f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2319d69f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2319d69f
Branch: refs/heads/jira/solr-8668
Commit: 2319d69fd3d5b67729f31b5796cc1eb68220b664
Parents: 4f16beb
Author: Cassandra Targett <ct...@apache.org>
Authored: Tue May 23 15:53:53 2017 -0500
Committer: Cassandra Targett <ct...@apache.org>
Committed: Tue May 23 15:54:35 2017 -0500
----------------------------------------------------------------------
...store-data-with-the-data-import-handler.adoc | 950 ++++++++++---------
1 file changed, 521 insertions(+), 429 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2319d69f/solr/solr-ref-guide/src/uploading-structured-data-store-data-with-the-data-import-handler.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/uploading-structured-data-store-data-with-the-data-import-handler.adoc b/solr/solr-ref-guide/src/uploading-structured-data-store-data-with-the-data-import-handler.adoc
index af33ad9..9004789 100644
--- a/solr/solr-ref-guide/src/uploading-structured-data-store-data-with-the-data-import-handler.adoc
+++ b/solr/solr-ref-guide/src/uploading-structured-data-store-data-with-the-data-import-handler.adoc
@@ -1,7 +1,7 @@
= Uploading Structured Data Store Data with the Data Import Handler
:page-shortname: uploading-structured-data-store-data-with-the-data-import-handler
:page-permalink: uploading-structured-data-store-data-with-the-data-import-handler.html
-:toclevels: 1
+:page-toclevels: 1
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@@ -19,38 +19,42 @@
// specific language governing permissions and limitations
// under the License.
-Many search applications store the content to be indexed in a structured data store, such as a relational database. The Data Import Handler (DIH) provides a mechanism for importing content from a data store and indexing it. In addition to relational databases, DIH can index content from HTTP based data sources such as RSS and ATOM feeds, e-mail repositories, and structured XML where an XPath processor is used to generate fields.
+Many search applications store the content to be indexed in a structured data store, such as a relational database. The Data Import Handler (DIH) provides a mechanism for importing content from a data store and indexing it.
-The `example/example-DIH` directory contains several collections many of the features of the data import handler. To run this "```dih```" example:
+In addition to relational databases, DIH can index content from HTTP based data sources such as RSS and ATOM feeds, e-mail repositories, and structured XML where an XPath processor is used to generate fields.
+
+== DIH Concepts and Terminology
+
+Descriptions of the Data Import Handler use several familiar terms, such as entity and processor, in specific ways, as explained in the table below.
+
+Datasource::
+As its name suggests, a datasource defines the location of the data of interest. For a database, it's a DSN. For an HTTP datasource, it's the base URL.
+
+Entity::
+Conceptually, an entity is processed to generate a set of documents, containing multiple fields, which (after optionally being transformed in various ways) are sent to Solr for indexing. For a RDBMS data source, an entity is a view or table, which would be processed by one or more SQL statements to generate a set of rows (documents) with one or more columns (fields).
+
+Processor::
+An entity processor does the work of extracting content from a data source, transforming it, and adding it to the index. Custom entity processors can be written to extend or replace the ones supplied.
+
+Transformer::
+Each set of fields fetched by the entity may optionally be transformed. This process can modify the fields, create new fields, or generate multiple rows/documents form a single row. There are several built-in transformers in the DIH, which perform functions such as modifying dates and stripping HTML. It is possible to write custom transformers using the publicly available interface.
+
+== Solr's DIH Examples
+
+The `example/example-DIH` directory contains several collections to demonstrate many of the features of the data import handler. These are available with the `dih` example from the <<solr-control-script-reference.adoc#solr-control-script-reference,Solr Control Script>>:
[source,bash]
----
bin/solr -e dih
----
-For more information about the Data Import Handler, see https://wiki.apache.org/solr/DataImportHandler.
-
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-ConceptsandTerminology]]
-== Concepts and Terminology
+This launches a standalone Solr instance with several collections that correspond to detailed examples. The available examples are `atom`, `db`, `mail`, `solr`, and `tika`.
-Descriptions of the Data Import Handler use several familiar terms, such as entity and processor, in specific ways, as explained in the table below.
-
-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed
-
-[cols="30,70",options="header"]
-|===
-|Term |Definition
-|Datasource |As its name suggests, a datasource defines the location of the data of interest. For a database, it's a DSN. For an HTTP datasource, it's the base URL.
-|Entity |Conceptually, an entity is processed to generate a set of documents, containing multiple fields, which (after optionally being transformed in various ways) are sent to Solr for indexing. For a RDBMS data source, an entity is a view or table, which would be processed by one or more SQL statements to generate a set of rows (documents) with one or more columns (fields).
-|Processor |An entity processor does the work of extracting content from a data source, transforming it, and adding it to the index. Custom entity processors can be written to extend or replace the ones supplied.
-|Transformer |Each set of fields fetched by the entity may optionally be transformed. This process can modify the fields, create new fields, or generate multiple rows/documents form a single row. There are several built-in transformers in the DIH, which perform functions such as modifying dates and stripping HTML. It is possible to write custom transformers using the publicly available interface.
-|===
+All examples in this section assume you are running the DIH example server.
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-Configuration]]
-== Configuration
+== Configuring DIH
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-Configuringsolrconfig.xml]]
-=== Configuring `solrconfig.xml`
+=== Configuring solrconfig.xml for DIH
The Data Import Handler has to be registered in `solrconfig.xml`. For example:
@@ -67,55 +71,29 @@ The only required parameter is the `config` parameter, which specifies the locat
You can have multiple DIH configuration files. Each file would require a separate definition in the `solrconfig.xml` file, specifying a path to the file.
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-ConfiguringtheDIHConfigurationFile]]
=== Configuring the DIH Configuration File
-An annotated configuration file, based on the "```db```" collection in the `dih` example server, is shown below (`example/example-DIH/solr/db/conf/db-data-config.xml`). It extracts fields from the four tables defining a simple product database, with this schema. More information about the parameters and options shown here are described in the sections following.
+An annotated configuration file, based on the `db` collection in the `dih` example server, is shown below (this file is located in `example/example-DIH/solr/db/conf/db-data-config.xml`).
+
+This example shows how to extract fields from four tables defining a simple product database. More information about the parameters and options shown here will be described in the sections following.
[source,xml]
----
-<dataConfig>
-<!-- The first element is the dataSource, in this case an HSQLDB database.
- The path to the JDBC driver and the JDBC URL and login credentials are all specified here.
- Other permissible attributes include whether or not to autocommit to Solr, the batchsize
- used in the JDBC connection, a 'readOnly' flag.
- The password attribute is optional if there is no password set for the DB.
--->
- <dataSource driver="org.hsqldb.jdbcDriver" url="jdbc:hsqldb:./example-DIH/hsqldb/ex" user="sa" password="secret"/>
-<!--
-Alternately the password can be encrypted as follows. This is the value obtained as a result of the command
-openssl enc -aes-128-cbc -a -salt -in pwd.txt
-password="U2FsdGVkX18QMjY0yfCqlfBMvAB4d3XkwY96L7gfO2o="
-WHen the password is encrypted, you must provide an extra attribute
-encryptKeyFile="/location/of/encryptionkey"
-This file should a text file with a single line containing the encrypt/decrypt password
-
--->
-<!-- A 'document' element follows, containing multiple 'entity' elements.
- Note that 'entity' elements can be nested, and this allows the entity
- relationships in the sample database to be mirrored here, so that we can
- generate a denormalized Solr record which may include multiple features
- for one item, for instance -->
- <document>
+<dataConfig> --<1>
-<!-- The possible attributes for the entity element are described below.
- Entity elements may contain one or more 'field' elements, which map
- the data source field names to Solr fields, and optionally specify
- per-field transformations -->
-<!-- this entity is the 'root' entity. -->
+ <dataSource driver="org.hsqldb.jdbcDriver" url="jdbc:hsqldb:./example-DIH/hsqldb/ex" user="sa" password="secret"/> --<2>
+ <document> --<3>
<entity name="item" query="select * from item"
- deltaQuery="select id from item where last_modified > '${dataimporter.last_index_time}'">
+ deltaQuery="select id from item where last_modified > '${dataimporter.last_index_time}'"> --<4>
<field column="NAME" name="name" />
-<!-- This entity is nested and reflects the one-to-many relationship between an item and its multiple features.
- Note the use of variables; ${item.ID} is the value of the column 'ID' for the current item
- ('item' referring to the entity name) -->
<entity name="feature"
query="select DESCRIPTION from FEATURE where ITEM_ID='${item.ID}'"
deltaQuery="select ITEM_ID from FEATURE where last_modified > '${dataimporter.last_index_time}'"
- parentDeltaQuery="select ID from item where ID=${feature.ITEM_ID}">
+ parentDeltaQuery="select ID from item where ID=${feature.ITEM_ID}"> --<5>
<field name="features" column="DESCRIPTION" />
</entity>
+
<entity name="item_category"
query="select CATEGORY_ID from item_category where ITEM_ID='${item.ID}'"
deltaQuery="select ITEM_ID, CATEGORY_ID from item_category where last_modified > '${dataimporter.last_index_time}'"
@@ -131,6 +109,11 @@ This file should a text file with a single line containing the encrypt/decrypt p
</document>
</dataConfig>
----
+<1> The first element is the `dataSource`, in this case an HSQLDB database. The path to the JDBC driver and the JDBC URL and login credentials are all specified here. Other permissible attributes include whether or not to autocommit to Solr, the batchsize used in the JDBC connection, and a `readOnly` flag. The password attribute is optional if there is no password set for the DB.
+<2> Alternately the password can be encrypted as follows. This is the value obtained as a result of the command `openssl enc -aes-128-cbc -a -salt -in pwd.txt password="U2FsdGVkX18QMjY0yfCqlfBMvAB4d3XkwY96L7gfO2o="`. When the password is encrypted, you must provide an extra attribute `encryptKeyFile="/location/of/encryptionkey"`. This file should a text file with a single line containing the encrypt/decrypt password.
+<3> A `document` element follows, containing multiple `entity` elements. Note that `entity` elements can be nested, and this allows the entity relationships in the sample database to be mirrored here, so that we can generate a denormalized Solr record which may include multiple features for one item, for instance.
+<4> The possible attributes for the `entity` element are described in later sections. Entity elements may contain one or more `field` elements, which map the data source field names to Solr fields, and optionally specify per-field transformations. This entity is the `root` entity.
+<5> This entity is nested and reflects the one-to-many relationship between an item and its multiple features. Note the use of variables; `${item.ID}` is the value of the column 'ID' for the current item (`item` referring to the entity name).
Datasources can still be specified in `solrconfig.xml`. These must be specified in the defaults section of the handler in `solrconfig.xml`. However, these are not parsed until the main configuration is loaded.
@@ -140,15 +123,12 @@ A `reload-config` command is also supported, which is useful for validating a ne
[TIP]
====
-
-You can also view the DIH configuration in the Solr Admin UI and there is an interface to import content.
-
+You can also view the DIH configuration in the Solr Admin UI from the <<dataimport-screen.adoc#dataimport-screen,Dataimport Screen>>. It includes an interface to import content.
====
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-RequestParameters]]
-==== Request Parameters
+==== DIH Request Parameters
-Request parameters can be substituted in configuration with placeholder `${dataimporter.request.paramname}`.
+Request parameters can be substituted in configuration with placeholder `${dataimporter.request._paramname_}`, as in this example:
[source,xml]
----
@@ -158,51 +138,61 @@ Request parameters can be substituted in configuration with placeholder `${datai
password=${dataimporter.request.jdbcpassword} />
----
-Then, these parameters can be passed to the full-import command or defined in the `<defaults>` section in `solrconfig.xml`. This example shows the parameters with the full-import command:
+These parameters can then be passed to the `full-import` command or defined in the `<defaults>` section in `solrconfig.xml`. This example shows the parameters with the full-import command:
-`dataimport?command=full-import&jdbcurl=jdbc:hsqldb:./example-DIH/hsqldb/ex&jdbcuser=sa&jdbcpassword=secret`
+[source,bash]
+http://localhost:8983/solr/dih/dataimport?command=full-import&jdbcurl=jdbc:hsqldb:./example-DIH/hsqldb/ex&jdbcuser=sa&jdbcpassword=secret
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-DataImportHandlerCommands]]
-== Data Import Handler Commands
+== DataImportHandler Commands
DIH commands are sent to Solr via an HTTP request. The following operations are supported.
-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed
+abort::
+Aborts an ongoing operation. For example: `\http://localhost:8983/solr/dih/dataimport?command=abort`.
+
+delta-import::
+For incremental imports and change detection. Only the <<The SQL Entity Processor,SqlEntityProcessor>> supports delta imports.
++
+For example: `\http://localhost:8983/solr/dih/dataimport?command=delta-import`.
++
+This command supports the same `clean`, `commit`, `optimize` and `debug` parameters as `full-import` command described below.
+
+full-import::
+A Full Import operation can be started with a URL such as `\http://localhost:8983/solr/dih/dataimport?command=full-import`. The command returns immediately.
++
+The operation will be started in a new thread and the _status_ attribute in the response should be shown as _busy_. The operation may take some time depending on the size of dataset. Queries to Solr are not blocked during full-imports.
++
+When a `full-import` command is executed, it stores the start time of the operation in a file located at `conf/dataimport.properties`. This stored timestamp is used when a `delta-import` operation is executed.
++
+Commands available to `full-import` are:
++
+clean:::
+Default is true. Tells whether to clean up the index before the indexing is started.
+commit:::
+Default is true. Tells whether to commit after the operation.
+debug:::
+Default is false. Runs the command in debug mode and is used by the interactive development mode.
++
+Note that in debug mode, documents are never committed automatically. If you want to run debug mode and commit the results too, add `commit=true` as a request parameter.
+entity:::
+The name of an entity directly under the `<document>` tag in the configuration file. Use this to execute one or more entities selectively.
++
+Multiple "entity" parameters can be passed on to run multiple entities at once. If nothing is passed, all entities are executed.
+optimize:::
+Default is true. Tells Solr whether to optimize after the operation.
+synchronous:::
+Blocks request until import is completed. Default is false.
+
+reload-config::
+If the configuration file has been changed and you wish to reload it without restarting Solr, run the command `\http://localhost:8983/solr/dih/dataimport?command=reload-config`
+
+status::
+This command returns statistics on the number of documents created, deleted, queries run, rows fetched, status, and so on. For example: `\http://localhost:8983/solr/dih/dataimport?command=status`.
+
+show-config::
+This command responds with configuration: `\http://localhost:8983/solr/dih/dataimport?command=show-config`.
-[cols="30,70",options="header"]
-|===
-|Command |Description
-|`abort` |Aborts an ongoing operation. The URL is `\http://<host>:<port>/solr/<collection_name>/dataimport?command=abort`.
-|`delta-import` |For incremental imports and change detection. The command is of the form `\http://<host>:<port>/solr/<collection_name>/dataimport?command=delta-import`. It supports the same clean, commit, optimize and debug parameters as full-import command. Only the SqlEntityProcessor supports delta imports.
-|`full-import` |A Full Import operation can be started with a URL of the form `\http://<host>:<port>/solr/<collection_name>/dataimport?command=full-import`. The command returns immediately. The operation will be started in a new thread and the _status_ attribute in the response should be shown as _busy_. The operation may take some time depending on the size of dataset. Queries to Solr are not blocked during full-imports. When a full-import command is executed, it stores the start time of the operation in a file located at `conf/dataimport.properties`. This stored timestamp is used when a delta-import operation is executed. For a list of parameters that can be passed to this command, see below.
-|`reload-config` a|
-If the configuration file has been changed and you wish to reload it without restarting Solr, run the command
-`\http://<host>:<port>/solr/<collection_name>/command=reload-config`
-
-|`status` |The URL is `\http://<host>:<port>/solr/<collection_name>/dataimport?command=status`. It returns statistics on the number of documents created, deleted, queries run, rows fetched, status, and so on.
-|`show-config` |responds with configuration
-|===
-
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-Parametersforthefull-importCommand]]
-=== Parameters for the `full-import` Command
-
-The `full-import` command accepts the following parameters:
-
-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed
-
-[cols="30,70",options="header"]
-|===
-|Parameter |Description
-|clean |Default is true. Tells whether to clean up the index before the indexing is started.
-|commit |Default is true. Tells whether to commit after the operation.
-|debug |Default is false Runs the command in debug mode. It is used by the interactive development mode. Note that in debug mode, documents are never committed automatically. If you want to run debug mode and commit the results too, add `commit=true` as a request parameter.
-|entity |The name of an entity directly under the `<document>` tag in the configuration file. Use this to execute one or more entities selectively. Multiple "entity" parameters can be passed on to run multiple entities at once. If nothing is passed, all entities are executed.
-|optimize |Default is true. Tells Solr whether to optimize after the operation.
-|synchronous |Blocks request until import is completed. Default is `false`.
-|===
-
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-PropertyWriter]]
== Property Writer
The `propertyWriter` element defines the date format and locale for use with delta queries. It is an optional configuration. Add the element to the DIH configuration file, directly under the `dataConfig` element.
@@ -215,19 +205,25 @@ The `propertyWriter` element defines the date format and locale for use with del
The parameters available are:
-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed
+dateFormat::
+A `java.text.SimpleDateFormat` to use when converting the date to text. The default is `yyyy-MM-dd HH:mm:ss`.
+
+type::
+The implementation class. Use `SimplePropertiesWriter` for non-SolrCloud installations. If using SolrCloud, use `ZKPropertiesWriter`.
++
+If this is not specified, it will default to the appropriate class depending on if SolrCloud mode is enabled.
+
+directory::
+Used with the `SimplePropertiesWriter` only. The directory for the properties file. If not specified, the default is `conf`.
-[cols="30,70",options="header"]
-|===
-|Parameter |Description
-|dateFormat |A java.text.SimpleDateFormat to use when converting the date to text. The default is "yyyy-MM-dd HH:mm:ss".
-|type |The implementation class. Use `SimplePropertiesWriter` for non-SolrCloud installations. If using SolrCloud, use `ZKPropertiesWriter`. If this is not specified, it will default to the appropriate class depending on if SolrCloud mode is enabled.
-|directory |Used with the `SimplePropertiesWriter` only). The directory for the properties file. If not specified, the default is "conf".
-|filename |Used with the `SimplePropertiesWriter` only). The name of the properties file. If not specified, the default is the requestHandler name (as defined in `solrconfig.xml`, appended by ".properties" (i.e., "dataimport.properties").
-|locale |The locale. If not defined, the ROOT locale is used. It must be specified as language-country (https://tools.ietf.org/html/bcp47[BCP 47 language tag]). For example, `en-US`.
-|===
+filename::
+Used with the `SimplePropertiesWriter` only. The name of the properties file.
++
+If not specified, the default is the requestHandler name (as defined in `solrconfig.xml`, appended by ".properties" (such as, `dataimport.properties`).
+
+locale::
+The locale. If not defined, the ROOT locale is used. It must be specified as language-country (https://tools.ietf.org/html/bcp47[BCP 47 language tag]). For example, `en-US`.
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-DataSources]]
== Data Sources
A data source specifies the origin of data and its type. Somewhat confusingly, some data sources are configured within the associated entity processor. Data sources can also be specified in `solrconfig.xml`, which is useful when you have multiple environments (for example, development, QA, and production) differing only in their data sources.
@@ -238,12 +234,10 @@ The mandatory attributes for a data source definition are its name and type. The
The types of data sources available are described below.
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-ContentStreamDataSource]]
=== ContentStreamDataSource
This takes the POST data as the data source. This can be used with any EntityProcessor that uses a `DataSource<Reader>`.
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-FieldReaderDataSource]]
=== FieldReaderDataSource
This can be used where a database field contains XML which you wish to process using the XPathEntityProcessor. You would set up a configuration with both JDBC and FieldReader data sources, and two entities, as follows:
@@ -255,68 +249,56 @@ This can be used where a database field contains XML which you wish to process u
<document>
<!-- processor for database -->
-
<entity name ="e1" dataSource="a1" processor="SqlEntityProcessor" pk="docid"
query="select * from t1 ...">
<!-- nested XpathEntity; the field in the parent which is to be used for
- Xpath is set in the "datafield" attribute in place of the "url" attribute -->
-
+ XPath is set in the "datafield" attribute in place of the "url" attribute -->
<entity name="e2" dataSource="a2" processor="XPathEntityProcessor"
dataField="e1.fieldToUseForXPath">
- <!-- Xpath configuration follows -->
+ <!-- XPath configuration follows -->
...
</entity>
</entity>
</document>
----
-The FieldReaderDataSource can take an `encoding` parameter, which will default to "UTF-8" if not specified.It must be specified as language-country. For example, `en-US`.
+The `FieldReaderDataSource` can take an `encoding` parameter, which will default to "UTF-8" if not specified. It must be specified as language-country. For example, `en-US`.
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-FileDataSource]]
=== FileDataSource
-This can be used like an <<UploadingStructuredDataStoreDatawiththeDataImportHandler-URLDataSource,URLDataSource>>, but is used to fetch content from files on disk. The only difference from URLDataSource, when accessing disk files, is how a pathname is specified.
+This can be used like a <<URLDataSource>>, but is used to fetch content from files on disk. The only difference from `URLDataSource`, when accessing disk files, is how a pathname is specified.
This data source accepts these optional attributes.
-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed
+basePath::
+The base path relative to which the value is evaluated if it is not absolute.
-[cols="30,70",options="header"]
-|===
-|Optional Attribute |Description
-|basePath |The base path relative to which the value is evaluated if it is not absolute.
-|encoding |Defines the character encoding to use. If not defined, UTF-8 is used.
-|===
+encoding::
+Defines the character encoding to use. If not defined, UTF-8 is used.
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-JdbcDataSource]]
=== JdbcDataSource
-This is the default datasource. It's used with the <<UploadingStructuredDataStoreDatawiththeDataImportHandler-TheSQLEntityProcessor,SqlEntityProcessor>>. See the example in the <<UploadingStructuredDataStoreDatawiththeDataImportHandler-FieldReaderDataSource,FieldReaderDataSource>> section for details on configuration. `JdbcDatasource` supports at least the following attributes: .
+This is the default datasource. It's used with the <<The SQL Entity Processor,SqlEntityProcessor>>. See the example in the <<FieldReaderDataSource>> section for details on configuration. `JdbcDatasource` supports at least the following attributes:
-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed
+driver, url, user, password, encryptKeyFile::
+Usual JDBC connection properties.
-[cols="30,70",options="header"]
-|===
-|Attributes |Description
-|driver, url, user, password, encryptKeyFile |Usual jdbc connection properties
-|`batchSize`
- a|
-Passed to Statement#setFetchSize, default value 500.
+batchSize::
+Passed to `Statement#setFetchSize`, default value 500.
++
For MySQL driver, which doesn't honor fetchSize and pulls whole resultSet, which often lead to OutOfMemoryError.
-
++
In this case, set `batchSize=-1` that pass setFetchSize(Integer.MIN_VALUE), and switch result set to pull row by row
-|===
-All of them substitute properties via $\{placeholders}.
+All of them substitute properties via `${placeholders}`.
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-URLDataSource]]
=== URLDataSource
-This data source is often used with XPathEntityProcessor to fetch content from an underlying `file://` or `http://` location. Here's an example:
+This data source is often used with <<The XPathEntityProcessor,XPathEntityProcessor>> to fetch content from an underlying `file://` or `http://` location. Here's an example:
[source,xml]
----
@@ -330,49 +312,88 @@ This data source is often used with XPathEntityProcessor to fetch content from a
The URLDataSource type accepts these optional parameters:
-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed
+baseURL::
+Specifies a new baseURL for pathnames. You can use this to specify host/port changes between Dev/QA/Prod environments. Using this attribute isolates the changes to be made to the `solrconfig.xml`
+
+connectionTimeout::
+Specifies the length of time in milliseconds after which the connection should time out. The default value is 5000ms.
+
+encoding::
+By default the encoding in the response header is used. You can use this property to override the default encoding.
+
+readTimeout::
+Specifies the length of time in milliseconds after which a read operation should time out. The default value is 10000ms.
-[cols="30,70",options="header"]
-|===
-|Optional Parameter |Description
-|baseURL |Specifies a new baseURL for pathnames. You can use this to specify host/port changes between Dev/QA/Prod environments. Using this attribute isolates the changes to be made to the `solrconfig.xml`
-|connectionTimeout |Specifies the length of time in milliseconds after which the connection should time out. The default value is 5000ms.
-|encoding |By default the encoding in the response header is used. You can use this property to override the default encoding.
-|readTimeout |Specifies the length of time in milliseconds after which a read operation should time out. The default value is 10000ms.
-|===
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-EntityProcessors]]
== Entity Processors
Entity processors extract data, transform it, and add it to a Solr index. Examples of entities include views or tables in a data store.
-Each processor has its own set of attributes, described in its own section below. In addition, there are non-specific attributes common to all entities which may be specified.
-
-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed
-
-[cols="30,70",options="header"]
-|===
-|Attribute |Use
-|dataSource |The name of a data source. If there are multiple data sources defined, use this attribute with the name of the data source for this entity.
-|name |Required. The unique name used to identify an entity.
-|pk |The primary key for the entity. It is optional, and required only when using delta-imports. It has no relation to the uniqueKey defined in `schema.xml` but they can both be the same. It is mandatory if you do delta-imports and then refers to the column name in `${dataimporter.delta.<column-name>`} which is used as the primary key.
-|processor |Default is SqlEntityProcessor. Required only if the datasource is not RDBMS.
-|onError |Permissible values are (abort|skip|continue) . The default value is 'abort'. 'Skip' skips the current document. 'Continue' ignores the error and processing continues.
-|preImportDeleteQuery |Before a full-import command, use this query this to cleanup the index instead of using '*:*'. This is honored only on an entity that is an immediate sub-child of `<document>`.
-|postImportDeleteQuery |Similar to the above, but executed after the import has completed.
-|rootEntity |By default the entities immediately under the `<document>` are root entities. If this attribute is set to false, the entity directly falling under that entity will be treated as the root entity (and so on). For every row returned by the root entity, a document is created in Solr.
-|transformer |Optional. One or more transformers to be applied on this entity.
-|cacheImpl |Optional. A class (which must implement `DIHCache`) to use for caching this entity when doing lookups from an entity which wraps it. Provided implementation is "```SortedMapBackedCache```".
-|cacheKey |The name of a property of this entity to use as a cache key if `cacheImpl` is specified.
-|cacheLookup |An entity + property name that will be used to lookup cached instances of this entity if `cacheImpl` is specified.
-|where |an alternative way to specify `cacheKey` and `cacheLookup` concatenated with '='. eg `where="CODE=People.COUNTRY_CODE"` is equal to `cacheKey="CODE" cacheLookup="People.COUNTRY_CODE"`
-|child="true" |Enables indexing document blocks aka <<uploading-data-with-index-handlers.adoc#uploading-data-with-index-handlers,Nested Child Documents>> for searching with <<other-parsers.adoc#other-parsers,Block Join Query Parsers>>. It can be only specified on `<entity>` under another root entity. It switches from default behavior (merging field values) to nesting documents as children documents. Note: parent `<entity>` should add a field which is used as a parent filter in query time.
-|join="zipper" |Enables merge join aka "zipper" algorithm for joining parent and child entities without cache. It should be specified at child (nested) `<entity>`. It implies that parent and child queries return results ordered by keys, otherwise it throws an exception. Keys should be specified either with `where` attribute or with `cacheKey` and `cacheLookup`.
-|===
+Each processor has its own set of attributes, described in its own section below. In addition, there are several attributes common to all entities which may be specified:
+
+dataSource::
+The name of a data source. If there are multiple data sources defined, use this attribute with the name of the data source for this entity.
+
+name::
+Required. The unique name used to identify an entity.
+
+pk::
+The primary key for the entity. It is optional, and required only when using delta-imports. It has no relation to the uniqueKey defined in `schema.xml` but they can both be the same.
++
+This attribute is mandatory if you do delta-imports and then refer to the column name in `${dataimporter.delta.<column-name>}` which is used as the primary key.
+
+processor::
+Default is <<The SQL Entity Processor,SqlEntityProcessor>>. Required only if the datasource is not RDBMS.
+onError::
+Defines what to do if an error is encountered.
++
+Permissible values are:
+
+abort::: Stops the import.
+
+skip::: Skips the current document.
+
+continue::: Ignores the error and processing continues.
+
+preImportDeleteQuery::
+Before a `full-import` command, use this query this to cleanup the index instead of using `\*:*`. This is honored only on an entity that is an immediate sub-child of `<document>`.
+
+postImportDeleteQuery::
+Similar to `preImportDeleteQuery`, but it executes after the import has completed.
+
+rootEntity::
+By default the entities immediately under `<document>` are root entities. If this attribute is set to false, the entity directly falling under that entity will be treated as the root entity (and so on). For every row returned by the root entity, a document is created in Solr.
+
+transformer::
+Optional. One or more transformers to be applied on this entity.
+
+cacheImpl::
+Optional. A class (which must implement `DIHCache`) to use for caching this entity when doing lookups from an entity which wraps it. Provided implementation is `SortedMapBackedCache`.
+
+cacheKey::
+The name of a property of this entity to use as a cache key if `cacheImpl` is specified.
+
+cacheLookup::
+An entity + property name that will be used to lookup cached instances of this entity if `cacheImpl` is specified.
+
+where::
+An alternative way to specify `cacheKey` and `cacheLookup` concatenated with '='.
++
+For example, `where="CODE=People.COUNTRY_CODE"` is equivalent to `cacheKey="CODE" cacheLookup="People.COUNTRY_CODE"`
+
+child="true"::
+Enables indexing document blocks aka <<uploading-data-with-index-handlers.adoc#uploading-data-with-index-handlers,Nested Child Documents>> for searching with <<other-parsers.adoc#other-parsers,Block Join Query Parsers>>. It can be only specified on the `<entity>` element under another root entity. It switches from default behavior (merging field values) to nesting documents as children documents.
++
+Note: parent `<entity>` should add a field which is used as a parent filter in query time.
+
+join="zipper"::
+Enables merge join, aka "zipper" algorithm, for joining parent and child entities without cache. It should be specified at child (nested) `<entity>`. It implies that parent and child queries return results ordered by keys, otherwise it throws an exception. Keys should be specified either with `where` attribute or with `cacheKey` and `cacheLookup`.
+
+=== Entity Caching
Caching of entities in DIH is provided to avoid repeated lookups for same entities again and again. The default `SortedMapBackedCache` is a `HashMap` where a key is a field in the row and the value is a bunch of rows for that same key.
-In the example below, each `manufacturer` entity is cached using the '`id`' property as a cache key. Cache lookups will be performed for each `product` entity based on the product's "```manu```" property. When the cache has no data for a particular key, the query is run and the cache is populated
+In the example below, each `manufacturer` entity is cached using the `id` property as a cache key. Cache lookups will be performed for each `product` entity based on the product's `manu` property. When the cache has no data for a particular key, the query is run and the cache is populated
[source,xml]
----
@@ -382,101 +403,153 @@ In the example below, each `manufacturer` entity is cached using the '`id`' prop
</entity>
----
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-TheSQLEntityProcessor]]
=== The SQL Entity Processor
-The SqlEntityProcessor is the default processor. The associated <<UploadingStructuredDataStoreDatawiththeDataImportHandler-JdbcDataSource,data source>> should be a JDBC URL.
+The SqlEntityProcessor is the default processor. The associated <<JdbcDataSource>> should be a JDBC URL.
+
+The entity attributes specific to this processor are shown in the table below. These are in addition to the attributes common to all entity processors described above.
-The entity attributes specific to this processor are shown in the table below.
+query::
+Required. The SQL query used to select rows.
-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed
+deltaQuery::
+SQL query used if the operation is delta-import. This query selects the primary keys of the rows which will be parts of the delta-update. The pks will be available to the deltaImportQuery through the variable `${dataimporter.delta.<column-name>}`.
-[cols="30,70",options="header"]
-|===
-|Attribute |Use
-|query |Required. The SQL query used to select rows.
-|deltaQuery |SQL query used if the operation is delta-import. This query selects the primary keys of the rows which will be parts of the delta-update. The pks will be available to the deltaImportQuery through the variable `${dataimporter.delta.<column-name>`}.
-|parentDeltaQuery |SQL query used if the operation is delta-import.
-|deletedPkQuery |SQL query used if the operation is delta-import.
-|deltaImportQuery |SQL query used if the operation is delta-import. If this is not present, DIH tries to construct the import query by(after identifying the delta) modifying the 'query' (this is error prone). There is a namespace `${dataimporter.delta.<column-name>`} which can be used in this query. For example, `select * from tbl where id=${dataimporter.delta.id`}.
-|===
+parentDeltaQuery::
+SQL query used if the operation is `delta-import`.
+
+deletedPkQuery::
+SQL query used if the operation is `delta-import`.
+
+deltaImportQuery::
+SQL query used if the operation is `delta-import`. If this is not present, DIH tries to construct the import query by (after identifying the delta) modifying the 'query' (this is error prone).
++
+There is a namespace `${dataimporter.delta.<column-name>}` which can be used in this query. For example, `select * from tbl where id=${dataimporter.delta.id}`.
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-TheXPathEntityProcessor]]
=== The XPathEntityProcessor
-This processor is used when indexing XML formatted data. The data source is typically <<UploadingStructuredDataStoreDatawiththeDataImportHandler-URLDataSource,URLDataSource>> or <<UploadingStructuredDataStoreDatawiththeDataImportHandler-FileDataSource,FileDataSource>>. Xpath can also be used with the <<UploadingStructuredDataStoreDatawiththeDataImportHandler-TheFileListEntityProcessor,FileListEntityProcessor>> described below, to generate a document from each file.
+This processor is used when indexing XML formatted data. The data source is typically <<URLDataSource>> or <<FileDataSource>>. XPath can also be used with the <<The FileListEntityProcessor,FileListEntityProcessor>> described below, to generate a document from each file.
+
+The entity attributes unique to this processor are shown below. These are in addition to the attributes common to all entity processors described above.
+
+Processor::
+Required. Must be set to `XpathEntityProcessor`.
-The entity attributes unique to this processor are shown below.
+url::
+Required. The HTTP URL or file location.
-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed
+stream::
+Optional: Set to true for a large file or download.
-[cols="30,70",options="header"]
-|===
-|Attribute |Use
-|Processor |Required. Must be set to "XpathEntityProcessor".
-|url |Required. HTTP URL or file location.
-|stream |Optional: Set to true for a large file or download.
-|forEach |Required unless you define `useSolrAddSchema`. The Xpath expression which demarcates each record. This will be used to set up the processing loop.
-|xsl |Optional: Its value (a URL or filesystem path) is the name of a resource used as a preprocessor for applying the XSL transformation.
-|useSolrAddSchema |Set this to true if the content is in the form of the standard Solr update XML schema.
-|===
+forEach::
+Required unless you define `useSolrAddSchema`. The XPath expression which demarcates each record. This will be used to set up the processing loop.
-Each field element in the entity can have the following attributes as well as the default ones.
+xsl::
+Optional: Its value (a URL or filesystem path) is the name of a resource used as a preprocessor for applying the XSL transformation.
-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed
+useSolrAddSchema::
+Set this to true if the content is in the form of the standard Solr update XML schema.
-[cols="30,70",options="header"]
-|===
-|Attribute |Use
-|xpath |Required. The XPath expression which will extract the content from the record for this field. Only a subset of Xpath syntax is supported.
-|commonField |Optional. If true, then when this field is encountered in a record it will be copied to future records when creating a Solr document.
-|flatten a|
-Optional: If set to true, then any children text nodes are collected to form the value of a field. image::images/icons/emoticons/warning.png[(warning)]
- The default value is false, meaning that
+Each `<field>` element in the entity can have the following attributes as well as the default ones.
-if there are any sub-elements of the node pointed to by the XPath expression, they will be quietly omitted.
+xpath::
+Required. The XPath expression which will extract the content from the record for this field. Only a subset of XPath syntax is supported.
-|===
+commonField::
+Optional. If true, then when this field is encountered in a record it will be copied to future records when creating a Solr document.
-Here is an example from the "```rss```" collection in the `dih` example (`example/example-DIH/solr/rss/conf/rss-data-config.xml`):
+flatten::
+Optional. If set to true, then any children text nodes are collected to form the value of a field.
++
+[WARNING]
+The default value is false, meaning that if there are any sub-elements of the node pointed to by the XPath expression, they will be quietly omitted.
+
+Here is an example from the `atom` collection in the `dih` example (data-config file found at `example/example-DIH/solr/atom/conf/atom-data-config.xml`):
[source,xml]
----
-<!-- slashdot RSS Feed --->
<dataConfig>
- <dataSource type="HttpDataSource" />
- <document>
- <entity name="slashdot"
- pk="link"
- url="http://rss.slashdot.org/Slashdot/slashdot"
- processor="XPathEntityProcessor"
- transformer="DateFormatTransformer"
- forEach="/RDF/channel | /RDF/item" >
- <!-- NOTE: forEach sets up a processing loop ; here there are two expressions -->
- <field column="source" xpath="/RDF/channel/title" commonField="true" />
- <field column="source-link" xpath="/RDF/channel/link" commonField="true"/>
- <field column="subject" xpath="/RDF/channel/subject" commonField="true" />
- <field column="title" xpath="/RDF/item/title" />
- <field column="link" xpath="/RDF/item/link" />
- <field column="description" xpath="/RDF/item/description" />
- <field column="creator" xpath="/RDF/item/creator" />
- <field column="item-subject" xpath="/RDF/item/subject" />
- <field column="date" xpath="/RDF/item/date"
- dateTimeFormat="yyyy-MM-dd'T'hh:mm:ss" />
- <field column="slash-department" xpath="/RDF/item/department" />
- <field column="slash-section" xpath="/RDF/item/section" />
- <field column="slash-comments" xpath="/RDF/item/comments" />
+ <dataSource type="URLDataSource"/>
+ <document>
+
+ <entity name="stackoverflow"
+ url="http://stackoverflow.com/feeds/tag/solr"
+ processor="XPathEntityProcessor"
+ forEach="/feed|/feed/entry"
+ transformer="HTMLStripTransformer,RegexTransformer">
+
+ <!-- Pick this value up from the feed level and apply to all documents -->
+ <field column="lastchecked_dt" xpath="/feed/updated" commonField="true"/>
+
+ <!-- Keep only the final numeric part of the URL -->
+ <field column="id" xpath="/feed/entry/id" regex=".*/" replaceWith=""/>
+
+ <field column="title" xpath="/feed/entry/title"/>
+ <field column="author" xpath="/feed/entry/author/name"/>
+ <field column="category" xpath="/feed/entry/category/@term"/>
+ <field column="link" xpath="/feed/entry/link[@rel='alternate']/@href"/>
+
+ <!-- Use transformers to convert HTML into plain text.
+ There is also an UpdateRequestProcess to trim remaining spaces.
+ -->
+ <field column="summary" xpath="/feed/entry/summary" stripHTML="true" regex="( |\n)+" replaceWith=" "/>
+
+ <!-- Ignore namespaces when matching XPath -->
+ <field column="rank" xpath="/feed/entry/rank"/>
+
+ <field column="published_dt" xpath="/feed/entry/published"/>
+ <field column="updated_dt" xpath="/feed/entry/updated"/>
</entity>
+
</document>
</dataConfig>
----
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-TheMailEntityProcessor]]
=== The MailEntityProcessor
-The MailEntityProcessor uses the Java Mail API to index email messages using the IMAP protocol. The MailEntityProcessor works by connecting to a specified mailbox using a username and password, fetching the email headers for each message, and then fetching the full email contents to construct a document (one document for each mail message).
+The MailEntityProcessor uses the Java Mail API to index email messages using the IMAP protocol.
+
+The MailEntityProcessor works by connecting to a specified mailbox using a username and password, fetching the email headers for each message, and then fetching the full email contents to construct a document (one document for each mail message).
+
+The entity attributes unique to the MailEntityProcessor are shown below. These are in addition to the attributes common to all entity processors described above.
+
+processor::
+Required. Must be set to `MailEntityProcessor`.
+
+user::
+Required. Username for authenticating to the IMAP server; this is typically the email address of the mailbox owner.
+
+password::
+Required. Password for authenticating to the IMAP server.
+
+host::
+Required. The IMAP server to connect to.
+
+protocol::
+Required. The IMAP protocol to use, valid values are: imap, imaps, gimap, and gimaps.
+
+fetchMailsSince::
+Optional. Date/time used to set a filter to import messages that occur after the specified date; expected format is: `yyyy-MM-dd HH:mm:ss`.
+
+folders::
+Required. Comma-delimited list of folder names to pull messages from, such as "inbox".
-Here is an example from the "```mail```" collection of the `dih` example (`example/example-DIH/mail/conf/mail-data-config.xml`):
+recurse::
+Optional. Default is true. Flag to indicate if the processor should recurse all child folders when looking for messages to import.
+
+include::
+Optional. Comma-delimited list of folder patterns to include when processing folders (can be a literal value or regular expression).
+
+exclude::
+Optional. Comma-delimited list of folder patterns to exclude when processing folders (can be a literal value or regular expression). Excluded folder patterns take precedence over include folder patterns.
+
+processAttachement or processAttachments::
+Optional. Default is true. Use Tika to process message attachments.
+
+includeContent::
+Optional. Default is true. Include the message body when constructing Solr documents for indexing.
+
+Here is an example from the `mail` collection of the `dih` example (data-config file found at `example/example-DIH/mail/conf/mail-data-config.xml`):
[source,xml]
----
@@ -487,119 +560,128 @@ Here is an example from the "```mail```" collection of the `dih` example (`examp
password="password"
host="imap.gmail.com"
protocol="imaps"
- fetchMailsSince="2009-09-20 00:00:00"
+ fetchMailsSince="2014-06-30 00:00:00"
batchSize="20"
folders="inbox"
processAttachement="false"
- name="sample_entity"/>
+ name="mail_entity"/>
</document>
</dataConfig>
----
-The entity attributes unique to the MailEntityProcessor are shown below.
+==== Importing New Emails Only
-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed
+After running a full import, the MailEntityProcessor keeps track of the timestamp of the previous import so that subsequent imports can use the fetchMailsSince filter to only pull new messages from the mail server. This occurs automatically using the DataImportHandler `dataimport.properties` file (stored in `conf`).
-[cols="30,70",options="header"]
-|===
-|Attribute |Use
-|processor |Required. Must be set to "MailEntityProcessor".
-|user |Required. Username for authenticating to the IMAP server; this is typically the email address of the mailbox owner.
-|password |Required. Password for authenticating to the IMAP server.
-|host |Required. The IMAP server to connect to.
-|protocol |Required. The IMAP protocol to use, valid values are: imap, imaps, gimap, and gimaps.
-|fetchMailsSince |Optional. Date/time used to set a filter to import messages that occur after the specified date; expected format is: `yyyy-MM-dd HH:mm:ss`.
-|folders |Required. Comma-delimited list of folder names to pull messages from, such as "inbox".
-|recurse |Optional (default is true). Flag to indicate if the processor should recurse all child folders when looking for messages to import.
-|include |Optional. Comma-delimited list of folder patterns to include when processing folders (can be a literal value or regular expression).
-|exclude |Optional. Comma-delimited list of folder patterns to exclude when processing folders (can be a literal value or regular expression); excluded folder patterns take precedence over include folder patterns.
-a|
-processAttachement
+For instance, if you set `fetchMailsSince="2014-08-22 00:00:00"` in your `mail-data-config.xml`, then all mail messages that occur after this date will be imported on the first run of the importer. Subsequent imports will use the date of the previous import as the `fetchMailsSince` filter, so that only new emails since the last import are indexed each time.
-or
+==== GMail Extensions
-processAttachments
+When connecting to a GMail account, you can improve the efficiency of the MailEntityProcessor by setting the protocol to *gimap* or *gimaps*.
- |Optional (default is true). Use Tika to process message attachments.
-|includeContent |Optional (default is true). Include the message body when constructing Solr documents for indexing.
-|===
+This allows the processor to send the `fetchMailsSince` filter to the GMail server to have the date filter applied on the server, which means the processor only receives new messages from the server. However, GMail only supports date granularity, so the server-side filter may return previously seen messages if run more than once a day.
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-ImportingNewEmailsOnly]]
-==== Importing New Emails Only
+=== The TikaEntityProcessor
-After running a full import, the MailEntityProcessor keeps track of the timestamp of the previous import so that subsequent imports can use the fetchMailsSince filter to only pull new messages from the mail server. This occurs automatically using the Data Import Handler dataimport.properties file (stored in conf). For instance, if you set `fetchMailsSince="2014-08-22 00:00:00"` in your `mail-data-config.xml`, then all mail messages that occur after this date will be imported on the first run of the importer. Subsequent imports will use the date of the previous import as the fetchMailsSince filter, so that only new emails since the last import are indexed each time.
+The TikaEntityProcessor uses Apache Tika to process incoming documents. This is similar to <<uploading-data-with-solr-cell-using-apache-tika.adoc#uploading-data-with-solr-cell-using-apache-tika,Uploading Data with Solr Cell using Apache Tika>>, but using DataImportHandler options instead.
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-GMailExtensions]]
-==== GMail Extensions
+The parameters for this processor are described in the table below. These are in addition to the attributes common to all entity processors described above.
-When connecting to a GMail account, you can improve the efficiency of the MailEntityProcessor by setting the protocol to *gimap* or *gimaps*. This allows the processor to send the fetchMailsSince filter to the GMail server to have the date filter applied on the server, which means the processor only receives new messages from the server. However, GMail only supports date granularity, so the server-side filter may return previously seen messages if run more than once a day.
+dataSource::
+This parameter defines the data source and an optional name which can be referred to in later parts of the configuration if needed. This is the same `dataSource` explained in the description of general entity processor attributes above.
++
+The available data source types for this processor are:
++
+* BinURLDataSource: used for HTTP resources, but can also be used for files.
+* BinContentStreamDataSource: used for uploading content as a stream.
+* BinFileDataSource: used for content on the local filesystem.
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-TheTikaEntityProcessor]]
-=== The TikaEntityProcessor
+url::
+Required. The path to the source file(s), as a file path or a traditional internet URL.
+
+htmlMapper::
+Optional. Allows control of how Tika parses HTML. If this parameter is defined, it must be either *default* or *identity*; if it is absent, "default" is assumed.
++
+The "default" mapper strips much of the HTML from documents while the "identity" mapper passes all HTML as-is with no modifications.
+
+format::
+The output format. The options are *text*, *xml*, *html* or *none*. The default is "text" if not defined. The format "none" can be used if metadata only should be indexed and not the body of the documents.
+
+parser::
+Optional. The default parser is `org.apache.tika.parser.AutoDetectParser`. If a custom or other parser should be used, it should be entered as a fully-qualified name of the class and path.
+
+fields::
+The list of fields from the input documents and how they should be mapped to Solr fields. If the attribute `meta` is defined as "true", the field will be obtained from the metadata of the document and not parsed from the body of the main text.
+
+extractEmbedded::
+Instructs the TikaEntityProcessor to extract embedded documents or attachments when *true*. If false, embedded documents and attachments will be ignored.
-The TikaEntityProcessor uses Apache Tika to process incoming documents. This is similar to <<uploading-data-with-solr-cell-using-apache-tika.adoc#uploading-data-with-solr-cell-using-apache-tika,Uploading Data with Solr Cell using Apache Tika>>, but using the DataImportHandler options instead.
+onError::
+By default, the TikaEntityProcessor will stop processing documents if it finds one that generates an error. If you define `onError` to "skip", the TikaEntityProcessor will instead skip documents that fail processing and log a message that the document was skipped.
-Here is an example from the "```tika```" collection of the `dih` example (`example/example-DIH/tika/conf/tika-data-config.xml`):
+Here is an example from the `tika` collection of the `dih` example (data-config file found in `example/example-DIH/tika/conf/tika-data-config.xml`):
[source,xml]
----
<dataConfig>
- <dataSource type="BinFileDataSource" />
+ <dataSource type="BinFileDataSource"/>
<document>
- <entity name="tika-test" processor="TikaEntityProcessor"
- url="../contrib/extraction/src/test-files/extraction/solr-word.pdf" format="text">
- <field column="Author" name="author" meta="true"/>
- <field column="title" name="title" meta="true"/>
- <field column="text" name="text"/>
+ <entity name="file" processor="FileListEntityProcessor" dataSource="null"
+ baseDir="${solr.install.dir}/example/exampledocs" fileName=".*pdf"
+ rootEntity="false">
+
+ <field column="file" name="id"/>
+
+ <entity name="pdf" processor="TikaEntityProcessor"
+ url="${file.fileAbsolutePath}" format="text">
+
+ <field column="Author" name="author" meta="true"/>
+ <!-- in the original PDF, the Author meta-field name is upper-cased,
+ but in Solr schema it is lower-cased
+ -->
+
+ <field column="title" name="title" meta="true"/>
+ <field column="dc:format" name="format" meta="true"/>
+
+ <field column="text" name="text"/>
+
+ </entity>
</entity>
</document>
</dataConfig>
----
-The parameters for this processor are described in the table below:
+=== The FileListEntityProcessor
-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed
+This processor is basically a wrapper, and is designed to generate a set of files satisfying conditions specified in the attributes which can then be passed to another processor, such as the <<The XPathEntityProcessor,XPathEntityProcessor>>.
-[cols="30,70",options="header"]
-|===
-|Attribute |Use
-|dataSource a|
-This parameter defines the data source and an optional name which can be referred to in later parts of the configuration if needed. This is the same dataSource explained in the description of general entity processor attributes above. The available data source types for this processor are:
+The entity information for this processor would be nested within the FileListEntity entry. It generates five implicit fields: `fileAbsolutePath`, `fileDir`, `fileSize`, `fileLastModified`, and `file`, which can be used in the nested processor. This processor does not use a data source.
-* BinURLDataSource: used for HTTP resources, but can also be used for files.
-* BinContentStreamDataSource: used for uploading content as a stream.
-* BinFileDataSource: used for content on the local filesystem.
+The attributes specific to this processor are described in the table below:
-|url |The path to the source file(s), as a file path or a traditional internet URL. This parameter is required.
-|htmlMapper |Allows control of how Tika parses HTML. The "default" mapper strips much of the HTML from documents while the "identity" mapper passes all HTML as-is with no modifications. If this parameter is defined, it must be either *default* or *identity*; if it is absent, "default" is assumed.
-|format |The output format. The options are *text*, *xml*, *html* or *none*. The default is "text" if not defined. The format "none" can be used if metadata only should be indexed and not the body of the documents.
-|parser |The default parser is `org.apache.tika.parser.AutoDetectParser`. If a custom or other parser should be used, it should be entered as a fully-qualified name of the class and path.
-|fields |The list of fields from the input documents and how they should be mapped to Solr fields. If the attribute `meta` is defined as "true", the field will be obtained from the metadata of the document and not parsed from the body of the main text.
-|extractEmbedded |Instructs the TikaEntityProcessor to extract embedded documents or attachments when *true*. If false, embedded documents and attachments will be ignored.
-|onError |By default, the TikaEntityProcessor will stop processing documents if it finds one that generates an error. If you define `onError` to "skip", the TikaEntityProcessor will instead skip documents that fail processing and log a message that the document was skipped.
-|===
+fileName::
+Required. A regular expression pattern to identify files to be included.
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-TheFileListEntityProcessor]]
-=== The FileListEntityProcessor
+basedir::
+Required. The base directory (absolute path).
-This processor is basically a wrapper, and is designed to generate a set of files satisfying conditions specified in the attributes which can then be passed to another processor, such as the <<UploadingStructuredDataStoreDatawiththeDataImportHandler-TheXPathEntityProcessor,XPathEntityProcessor>>. The entity information for this processor would be nested within the FileListEnitity entry. It generates five implicit fields: `fileAbsolutePath, ``fileDir, fileSize, ``fileLastModified, ``file,` which can be used in the nested processor. This processor does not use a data source.
+recursive::
+Whether to search directories recursively. Default is 'false'.
-The attributes specific to this processor are described in the table below:
+excludes::
+A regular expression pattern to identify files which will be excluded.
+
+newerThan::
+A date in the format `yyyy-MM-ddHH:mm:ss` or a date math expression (`NOW - 2YEARS`).
-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed
-
-[cols="30,70",options="header"]
-|===
-|Attribute |Use
-|fileName |Required. A regular expression pattern to identify files to be included.
-|basedir |Required. The base directory (absolute path).
-|recursive |Whether to search directories recursively. Default is 'false'.
-|excludes |A regular expression pattern to identify files which will be excluded.
-|newerThan |A date in the format `yyyy-MM-ddHH:mm:ss` or a date math expression (`NOW - 2YEARS`).
-|olderThan |A date, using the same formats as newerThan.
-|rootEntity |This should be set to false. This ensures that each row (filepath) emitted by this processor is considered to be a document.
-|dataSource |Must be set to null.
-|===
+olderThan::
+A date, using the same formats as newerThan.
+
+rootEntity::
+This should be set to false. This ensures that each row (filepath) emitted by this processor is considered to be a document.
+
+dataSource::
+Must be set to null.
The example below shows the combination of the FileListEntityProcessor with another processor which will generate a set of fields from each file found.
@@ -618,7 +700,7 @@ The example below shows the combination of the FileListEntityProcessor with anot
dataSource="null"
baseDir="/my/document/directory">
- <!-- this processor extracts content using Xpath from each file found -->
+ <!-- this processor extracts content using XPath from each file found -->
<entity name="nested" processor="XPathEntityProcessor"
forEach="/rootelement" url="${f.fileAbsolutePath}" >
@@ -630,22 +712,22 @@ The example below shows the combination of the FileListEntityProcessor with anot
</dataConfig>
----
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-LineEntityProcessor]]
=== LineEntityProcessor
This EntityProcessor reads all content from the data source on a line by line basis and returns a field called `rawLine` for each line read. The content is not parsed in any way; however, you may add transformers to manipulate the data within the `rawLine` field, or to create other additional fields.
-The lines read can be filtered by two regular expressions specified with the `acceptLineRegex` and `omitLineRegex` attributes. The table below describes the LineEntityProcessor's attributes:
+The lines read can be filtered by two regular expressions specified with the `acceptLineRegex` and `omitLineRegex` attributes.
+
+The LineEntityProcessor has the following attributes:
+
+url::
+A required attribute that specifies the location of the input file in a way that is compatible with the configured data source. If this value is relative and you are using FileDataSource or URLDataSource, it assumed to be relative to baseLoc.
-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed
+acceptLineRegex::
+An optional attribute that if present discards any line which does not match the regular expression.
-[cols="30,70",options="header"]
-|===
-|Attribute |Description
-|url |A required attribute that specifies the location of the input file in a way that is compatible with the configured data source. If this value is relative and you are using FileDataSource or URLDataSource, it assumed to be relative to baseLoc.
-|acceptLineRegex |An optional attribute that if present discards any line which does not match the regExp.
-|omitLineRegex |An optional attribute that is applied after any acceptLineRegex and that discards any line which matches this regExp.
-|===
+omitLineRegex::
+An optional attribute that is applied after any `acceptLineRegex` and that discards any line which matches this regular expression.
For example:
@@ -659,15 +741,14 @@ For example:
rootEntity="false"
dataSource="myURIreader1"
transformer="RegexTransformer,DateFormatTransformer">
- ...
+</entity>
----
-While there are use cases where you might need to create a Solr document for each line read from a file, it is expected that in most cases that the lines read by this processor will consist of a pathname, which in turn will be consumed by another EntityProcessor, such as XPathEntityProcessor.
+While there are use cases where you might need to create a Solr document for each line read from a file, it is expected that in most cases that the lines read by this processor will consist of a pathname, which in turn will be consumed by another entity processor, such as the XPathEntityProcessor.
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-PlainTextEntityProcessor]]
=== PlainTextEntityProcessor
-This EntityProcessor reads all content from the data source into an single implicit field called `plainText`. The content is not parsed in any way, however you may add transformers to manipulate the data within the `plainText` as needed, or to create other additional fields.
+This EntityProcessor reads all content from the data source into an single implicit field called `plainText`. The content is not parsed in any way, however you may add <<Transformers,transformers>> to manipulate the data within the `plainText` as needed, or to create other additional fields.
For example:
@@ -681,20 +762,56 @@ For example:
Ensure that the dataSource is of type `DataSource<Reader>` (`FileDataSource`, `URLDataSource`).
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-SolrEntityProcessor]]
=== SolrEntityProcessor
-Uses Solr instance as a datasource, see https://wiki.apache.org/solr/DataImportHandler#SolrEntityProcessor. In addition to that, SolrEntityProcessor also supports the following parameters:
+This EntityProcessor imports data from different Solr instances and cores. The data is retrieved based on a specified filter query. This EntityProcessor is useful in cases you want to copy your Solr index and want to modify the data in the target index.
+
+The SolrEntityProcessor can only copy fields that are stored in the source index.
+
+The SolrEntityProcessor supports the following parameters:
+
+url::
+Required. The URL of the source Solr instance and/or core.
+
+query::
+Required. The main query to execute on the source index.
+
+fq::
+Any filter queries to execute on the source index. If more than one filter query is defined, they must be separated by a comma.
+
+rows::
+The number of rows to return for each iteration. The default is 50 rows.
+
+fl::
+A comma-separated list of fields to fetch from the source index. Note, these fields must be stored in the source Solr instance.
+
+qt::
+The search handler to use, if not the default.
-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed
+wt::
+The response format to use, either *javabin* or *xml*.
-[cols="30,70",options="header"]
-|===
-|cursorMark="true" |specify it to enable cursor for efficient result set scrolling
-|sort="id asc" |in this case it usually needs to specify sort param referencing uniqueKey field. see <<pagination-of-results.adoc#pagination-of-results,Pagination of Results>> for details.
-|===
+timeout::
+The query timeout in seconds. The default is 5 minutes (300 seconds).
+
+cursorMark="true"::
+Use this to enable cursor for efficient result set scrolling
+
+sort="id asc"::
+This should be used to specify a sort parameter referencing the uniqueKey field of the source Solr instance. See <<pagination-of-results.adoc#pagination-of-results,Pagination of Results>> for details.
+
+Here is a simple example of a SolrEntityProcessor:
+
+[source,xml]
+<dataConfig>
+ <document>
+ <entity name="sep" processor="SolrEntityProcessor"
+ url="http://127.0.0.1:8983/solr/db "
+ query="*:*"
+ fl="*,orig_version_l:_version_,ignored_price_c:price_c"/>
+ </document>
+</dataConfig>
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-Transformers]]
== Transformers
Transformers manipulate the fields in a document returned by an entity. A transformer can create new fields or modify existing ones. You must tell the entity which transformers your import operation will be using, by adding an attribute containing a comma separated list to the `<entity>` element.
@@ -706,52 +823,30 @@ Transformers manipulate the fields in a document returned by an entity. A transf
Specific transformation rules are then added to the attributes of a `<field>` element, as shown in the examples below. The transformers are applied in the order in which they are specified in the transformer attribute.
-The Data Import Handler contains several built-in transformers. You can also write your own custom transformers, as described in the Solr Wiki (see http://wiki.apache.org/solr/DIHCustomTransformer). The ScriptTransformer (described below) offers an alternative method for writing your own transformers.
+The DataImportHandler contains several built-in transformers. You can also write your own custom transformers, as described in the http://wiki.apache.org/solr/DIHCustomTransformer[DIHCustomTransformer] section of the Solr Wiki. The ScriptTransformer (described below) offers an alternative method for writing your own transformers.
-Solr includes the following built-in transformers:
+=== ClobTransformer
-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed
+You can use the ClobTransformer to create a string out of a CLOB in a database. A http://en.wikipedia.org/wiki/Character_large_object[CLOB] is a character large object: a collection of character data typically stored in a separate location that is referenced in the database.
-[cols="40,60",options="header"]
-|===
-|Transformer Name |Use
-|<<UploadingStructuredDataStoreDatawiththeDataImportHandler-ClobTransformer,ClobTransformer>> |Used to create a String out of a Clob type in database.
-|<<UploadingStructuredDataStoreDatawiththeDataImportHandler-TheDateFormatTransformer,DateFormatTransformer>> |Parse date/time instances.
-|<<UploadingStructuredDataStoreDatawiththeDataImportHandler-TheHTMLStripTransformer,HTMLStripTransformer>> |Strip HTML from a field.
-|<<UploadingStructuredDataStoreDatawiththeDataImportHandler-TheLogTransformer,LogTransformer>> |Used to log data to log files or a console.
-|<<UploadingStructuredDataStoreDatawiththeDataImportHandler-TheNumberFormatTransformer,NumberFormatTransformer>> |Uses the NumberFormat class in java to parse a string into a number.
-|<<UploadingStructuredDataStoreDatawiththeDataImportHandler-TheRegexTransformer,RegexTransformer>> |Use regular expressions to manipulate fields.
-|<<UploadingStructuredDataStoreDatawiththeDataImportHandler-TheScriptTransformer,ScriptTransformer>> |Write transformers in Javascript or any other scripting language supported by Java.
-|<<UploadingStructuredDataStoreDatawiththeDataImportHandler-TheTemplateTransformer,TemplateTransformer>> |Transform a field using a template.
-|===
+The ClobTransformer accepts these attributes:
-These transformers are described below.
+clob::
+Boolean value to signal if ClobTransformer should process this field or not. If this attribute is omitted, then the corresponding field is not transformed.
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-ClobTransformer]]
-=== ClobTransformer
+sourceColName::
+The source column to be used as input. If this is absent source and target are same
-You can use the ClobTransformer to create a string out of a CLOB in a database. A CLOB is a character large object: a collection of character data typically stored in a separate location that is referenced in the database. See http://en.wikipedia.org/wiki/Character_large_object. Here's an example of invoking the ClobTransformer.
+Here's an example of invoking the ClobTransformer.
[source,xml]
----
-<entity name="e" transformer="ClobTransformer" ...>
+<entity name="example" transformer="ClobTransformer" ...>
<field column="hugeTextField" clob="true" />
...
</entity>
----
-The ClobTransformer accepts these attributes:
-
-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed
-
-[cols="30,70",options="header"]
-|===
-|Attribute |Description
-|clob |Boolean value to signal if ClobTransformer should process this field or not. If this attribute is omitted, then the corresponding field is not transformed.
-|sourceColName |The source column to be used as input. If this is absent source and target are same
-|===
-
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-TheDateFormatTransformer]]
=== The DateFormatTransformer
This transformer converts dates from one format to another. This would be useful, for example, in a situation where you wanted to convert a field with a fully specified date/time into a less precise date format, for use in faceting.
@@ -760,15 +855,14 @@ DateFormatTransformer applies only on the fields with an attribute `dateTimeForm
This transformer recognizes the following attributes:
-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed
+dateTimeFormat::
+The format used for parsing this field. This must comply with the syntax of the http://docs.oracle.com/javase/8/docs/api/java/text/SimpleDateFormat.html[Java SimpleDateFormat] class.
+
+sourceColName::
+The column on which the dateFormat is to be applied. If this is absent source and target are same.
-[cols="30,70",options="header"]
-|===
-|Attribute |Description
-|dateTimeFormat |The format used for parsing this field. This must comply with the syntax of the http://docs.oracle.com/javase/8/docs/api/java/text/SimpleDateFormat.html[Java SimpleDateFormat] class.
-|sourceColName |The column on which the dateFormat is to be applied. If this is absent source and target are same.
-|locale |The locale to use for date transformations. If not defined, the ROOT locale is used. It must be specified as language-country (https://tools.ietf.org/html/bcp47[BCP 47 language tag]). For example, `en-US`.
-|===
+locale::
+The locale to use for date transformations. If not defined, the ROOT locale is used. It must be specified as language-country (https://tools.ietf.org/html/bcp47[BCP 47 language tag]). For example, `en-US`.
Here is example code that returns the date rounded up to the month "2007-JUL":
@@ -780,10 +874,13 @@ Here is example code that returns the date rounded up to the month "2007-JUL":
</entity>
----
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-TheHTMLStripTransformer]]
=== The HTMLStripTransformer
-You can use this transformer to strip HTML out of a field. For example:
+You can use this transformer to strip HTML out of a field.
+
+There is one attribute for this transformer, `stripHTML`, which is a boolean value (true or false) to signal if the HTMLStripTransformer should process the field or not.
+
+For example:
[source,xml]
----
@@ -793,9 +890,6 @@ You can use this transformer to strip HTML out of a field. For example:
</entity>
----
-There is one attribute for this transformer, `stripHTML`, which is a boolean value (true/false) to signal if the HTMLStripTransformer should process the field or not.
-
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-TheLogTransformer]]
=== The LogTransformer
You can use this transformer to log data to the console or log files. For example:
@@ -811,7 +905,6 @@ You can use this transformer to log data to the console or log files. For exampl
Unlike other transformers, the LogTransformer does not apply to any field, so the attributes are applied on the entity itself.
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-TheNumberFormatTransformer]]
=== The NumberFormatTransformer
Use this transformer to parse a number from a string, converting it into the specified format, and optionally using a different locale.
@@ -820,15 +913,14 @@ NumberFormatTransformer will be applied only to fields with an attribute `format
This transformer recognizes the following attributes:
-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed
+formatStyle::
+The format used for parsing this field. The value of the attribute must be one of `number`, `percent`, `integer`, or `currency`. This uses the semantics of the Java NumberFormat class.
+
+sourceColName::
+The column on which the NumberFormat is to be applied. This is attribute is absent. The source column and the target column are the same.
-[cols="30,70",options="header"]
-|===
-|Attribute |Description
-|formatStyle |The format used for parsing this field. The value of the attribute must be one of (`number\|percent\|integer\|currency`). This uses the semantics of the Java NumberFormat class.
-|sourceColName |The column on which the NumberFormat is to be applied. This is attribute is absent. The source column and the target column are the same.
-|locale |The locale to be used for parsing the strings. The locale. If not defined, the ROOT locale is used. It must be specified as language-country (https://tools.ietf.org/html/bcp47[BCP 47 language tag]). For example, `en-US`.
-|===
+locale::
+The locale to be used for parsing the strings. The locale. If not defined, the ROOT locale is used. It must be specified as language-country (https://tools.ietf.org/html/bcp47[BCP 47 language tag]). For example, `en-US`.
For example:
@@ -843,49 +935,50 @@ For example:
</entity>
----
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-TheRegexTransformer]]
=== The RegexTransformer
The regex transformer helps in extracting or manipulating values from fields (from the source) using Regular Expressions. The actual class name is `org.apache.solr.handler.dataimport.RegexTransformer`. But as it belongs to the default package the package-name can be omitted.
The table below describes the attributes recognized by the regex transformer.
-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed
+regex::
+The regular expression that is used to match against the column or sourceColName's value(s). If replaceWith is absent, each regex _group_ is taken as a value and a list of values is returned.
+
+sourceColName::
+The column on which the regex is to be applied. If not present, then the source and target are identical.
+
+splitBy::
+Used to split a string. It returns a list of values. Note, this is a regular expression so it may need to be escaped (e.g., via back-slashes).
-[cols="30,70",options="header"]
-|===
-|Attribute |Description
-|regex |The regular expression that is used to match against the column or sourceColName's value(s). If replaceWith is absent, each regex _group_ is taken as a value and a list of values is returned.
-|sourceColName |The column on which the regex is to be applied. If not present, then the source and target are identical.
-|splitBy |Used to split a string. It returns a list of values. note: this is a regular expression – it may need to be escaped (e.g. via back-slashes)
-|groupNames |A comma separated list of field column names, used where the regex contains groups and each group is to be saved to a different field. If some groups are not to be named leave a space between commas.
-|replaceWith |Used along with regex . It is equivalent to the method `new String(<sourceColVal>).replaceAll(<regex>, <replaceWith>)`.
-|===
+groupNames::
+A comma separated list of field column names, used where the regex contains groups and each group is to be saved to a different field. If some groups are not to be named leave a space between commas.
+
+replaceWith::
+Used along with regex . It is equivalent to the method `new String(<sourceColVal>).replaceAll(<regex>, <replaceWith>)`.
Here is an example of configuring the regex transformer:
[source,xml]
----
<entity name="foo" transformer="RegexTransformer"
- query="select full_name, emailids from foo">
- <field column="full_name"/>
+ query="select full_name, emailids from foo"> --<1>
+ <field column="full_name"/> --<2>
<field column="firstName" regex="Mr(\w*)\b.*" sourceColName="full_name"/>
<field column="lastName" regex="Mr.*?\b(\w*)" sourceColName="full_name"/>
<!-- another way of doing the same -->
<field column="fullName" regex="Mr(\w*)\b(.*)" groupNames="firstName,lastName"/>
- <field column="mailId" splitBy="," sourceColName="emailids"/>
+ <field column="mailId" splitBy="," sourceColName="emailids"/> --<3>
</entity>
----
-In this example, regex and sourceColName are custom attributes used by the transformer. The transformer reads the field `full_name` from the resultset and transforms it to two new target fields, `firstName` and `lastName`. Even though the query returned only one column, `full_name`, in the result set, the Solr document gets two extra fields `firstName` and `lastName` which are "derived" fields. These new fields are only created if the regexp matches.
-
-The emailids field in the table can be a comma-separated value. It ends up producing one or more email IDs, and we expect the `mailId` to be a multivalued field in Solr.
+<1> In this example, `regex` and `sourceColName` are custom attributes used by the transformer.
+<2> The transformer reads the field `full_name` from the result set and transforms it to two new target fields, `firstName` and `lastName`. Even though the query returned only one column, `full_name`, in the result set, the Solr document gets two extra fields `firstName` and `lastName` which are "derived" fields. These new fields are only created if the regexp matches.
+<3> The `emailids` field in the table can be a comma-separated value. It ends up producing one or more email IDs, and we expect the `mailId` to be a multivalued field in Solr.
-Note that this transformer can either be used to split a string into tokens based on a splitBy pattern, or to perform a string substitution as per replaceWith, or it can assign groups within a pattern to a list of groupNames. It decides what it is to do based upon the above attributes `splitBy`, `replaceWith` and `groupNames` which are looked for in order. This first one found is acted upon and other unrelated attributes are ignored.
+Note that this transformer can be used to either split a string into tokens based on a splitBy pattern, or to perform a string substitution as per `replaceWith`, or it can assign groups within a pattern to a list of `groupNames`. It decides what it is to do based upon the above attributes `splitBy`, `replaceWith` and `groupNames` which are looked for in order. This first one found is acted upon and other unrelated attributes are ignored.
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-TheScriptTransformer]]
=== The ScriptTransformer
The script transformer allows arbitrary transformer functions to be written in any scripting language supported by Java, such as Javascript, JRuby, Jython, Groovy, or BeanShell. Javascript is integrated into Java 8; you'll need to integrate other languages yourself.
@@ -925,7 +1018,6 @@ Here is a simple example.
</dataConfig>
----
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-TheTemplateTransformer]]
=== The TemplateTransformer
You can use the template transformer to construct or modify a field value, perhaps using the value of other fields. You can insert extra text into the template.
@@ -939,18 +1031,18 @@ You can use the template transformer to construct or modify a field value, perha
</entity>
----
-[[UploadingStructuredDataStoreDatawiththeDataImportHandler-SpecialCommandsfortheDataImportHandler]]
-== Special Commands for the Data Import Handler
+== Special Commands for DIH
You can pass special commands to the DIH by adding any of the variables listed below to any row returned by any component:
-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed
+$skipDoc::
+Skip the current document; that is, do not add it to Solr. The value can be the string `true` or `false`.
+
+$skipRow::
+Skip the current row. The document will be added with rows from other entities. The value can be the string `true` or `false`.
+
+$deleteDocById::
+Delete a document from Solr with this ID. The value has to be the `uniqueKey` value of the document.
-[cols="30,70",options="header"]
-|===
-|Variable |Description
-|$skipDoc |Skip the current document; that is, do not add it to Solr. The value can be the string `true\|false`.
-|$skipRow |Skip the current row. The document will be added with rows from other entities. The value can be the string `true\|false`
-|$deleteDocById |Delete a document from Solr with this ID. The value has to be the `uniqueKey` value of the document.
-|$deleteDocByQuery |Delete documents from Solr using this query. The value must be a Solr Query.
-|===
+$deleteDocByQuery::
+Delete documents from Solr using this query. The value must be a Solr Query.
[08/44] lucene-solr:jira/solr-8668: LUCENE-7847: Fix the
all-docs-match optimization of range queries on range fields.
Posted by cp...@apache.org.
LUCENE-7847: Fix the all-docs-match optimization of range queries on range fields.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/14320a58
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/14320a58
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/14320a58
Branch: refs/heads/jira/solr-8668
Commit: 14320a584c7771c63fba4de868c51ee9a5cf06de
Parents: 31e02e9
Author: Adrien Grand <jp...@gmail.com>
Authored: Tue May 23 18:37:01 2017 +0200
Committer: Adrien Grand <jp...@gmail.com>
Committed: Tue May 23 18:46:50 2017 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 ++
.../apache/lucene/document/RangeFieldQuery.java | 48 ++++++++++----------
.../search/TestDoubleRangeFieldQueries.java | 15 ++++--
.../search/TestFloatRangeFieldQueries.java | 15 ++++--
.../lucene/search/TestIntRangeFieldQueries.java | 23 ++++++++--
.../search/TestLongRangeFieldQueries.java | 23 ++++++++--
.../search/TestInetAddressRangeQueries.java | 18 ++++++--
.../search/BaseRangeFieldQueryTestCase.java | 4 +-
8 files changed, 104 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/14320a58/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 2138321..3951cea 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -155,6 +155,9 @@ Bug Fixes
* LUCENE-7833: ToParentBlockJoinQuery computed the min score instead of the max
score with ScoreMode.MAX. (Adrien Grand)
+* LUCENE-7847: Fixed all-docs-match optimization of range queries on range
+ fields. (Adrien Grand)
+
Improvements
* LUCENE-7782: OfflineSorter now passes the total number of items it
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/14320a58/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java b/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java
index 10f10fa..750189d 100644
--- a/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java
@@ -112,6 +112,7 @@ abstract class RangeFieldQuery extends Query {
public final Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException {
return new ConstantScoreWeight(this, boost) {
final RangeFieldComparator target = new RangeFieldComparator();
+
private DocIdSet buildMatchingDocIdSet(LeafReader reader, PointValues values) throws IOException {
DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values, field);
values.intersect(
@@ -133,25 +134,29 @@ abstract class RangeFieldQuery extends Query {
}
@Override
public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
- byte[] node = getInternalRange(minPackedValue, maxPackedValue);
- // compute range relation for BKD traversal
- if (target.intersects(node) == false) {
- return Relation.CELL_OUTSIDE_QUERY;
- } else if (target.within(node)) {
- // target within cell; continue traversing:
- return Relation.CELL_CROSSES_QUERY;
- } else if (target.contains(node)) {
- // target contains cell; add iff queryType is not a CONTAINS or CROSSES query:
- return (queryType == QueryType.CONTAINS || queryType == QueryType.CROSSES) ?
- Relation.CELL_OUTSIDE_QUERY : Relation.CELL_INSIDE_QUERY;
- }
- // target intersects cell; continue traversing:
- return Relation.CELL_CROSSES_QUERY;
+ return compareRange(minPackedValue, maxPackedValue);
}
});
return result.build();
}
+ private Relation compareRange(byte[] minPackedValue, byte[] maxPackedValue) {
+ byte[] node = getInternalRange(minPackedValue, maxPackedValue);
+ // compute range relation for BKD traversal
+ if (target.intersects(node) == false) {
+ return Relation.CELL_OUTSIDE_QUERY;
+ } else if (target.within(node)) {
+ // target within cell; continue traversing:
+ return Relation.CELL_CROSSES_QUERY;
+ } else if (target.contains(node)) {
+ // target contains cell; add iff queryType is not a CONTAINS or CROSSES query:
+ return (queryType == QueryType.CONTAINS || queryType == QueryType.CROSSES) ?
+ Relation.CELL_OUTSIDE_QUERY : Relation.CELL_INSIDE_QUERY;
+ }
+ // target intersects cell; continue traversing:
+ return Relation.CELL_CROSSES_QUERY;
+ }
+
@Override
public Scorer scorer(LeafReaderContext context) throws IOException {
LeafReader reader = context.reader();
@@ -166,17 +171,10 @@ abstract class RangeFieldQuery extends Query {
return null;
}
checkFieldInfo(fieldInfo);
- boolean allDocsMatch = true;
- if (values.getDocCount() == reader.maxDoc()) {
- // if query crosses, docs need to be further scrutinized
- byte[] range = getInternalRange(values.getMinPackedValue(), values.getMaxPackedValue());
- // if the internal node is not equal and not contained by the query, all docs do not match
- if (queryType == QueryType.CROSSES || (!Arrays.equals(ranges, range)
- && (target.contains(range) == false || queryType != QueryType.WITHIN))) {
- allDocsMatch = false;
- }
- } else {
- allDocsMatch = false;
+ boolean allDocsMatch = false;
+ if (values.getDocCount() == reader.maxDoc()
+ && compareRange(values.getMinPackedValue(), values.getMaxPackedValue()) == Relation.CELL_INSIDE_QUERY) {
+ allDocsMatch = true;
}
DocIdSetIterator iterator = allDocsMatch == true ?
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/14320a58/lucene/core/src/test/org/apache/lucene/search/TestDoubleRangeFieldQueries.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDoubleRangeFieldQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestDoubleRangeFieldQueries.java
index 49ca710..a5cbed6 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestDoubleRangeFieldQueries.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestDoubleRangeFieldQueries.java
@@ -31,11 +31,18 @@ public class TestDoubleRangeFieldQueries extends BaseRangeFieldQueryTestCase {
private static final String FIELD_NAME = "doubleRangeField";
private double nextDoubleInternal() {
- if (rarely()) {
- return random().nextBoolean() ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY;
+ switch (random().nextInt(5)) {
+ case 0:
+ return Double.NEGATIVE_INFINITY;
+ case 1:
+ return Double.POSITIVE_INFINITY;
+ default:
+ if (random().nextBoolean()) {
+ return random().nextDouble();
+ } else {
+ return (random().nextInt(15) - 7) / 3d;
+ }
}
- double max = Double.MAX_VALUE / 2;
- return (max + max) * random().nextDouble() - max;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/14320a58/lucene/core/src/test/org/apache/lucene/search/TestFloatRangeFieldQueries.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestFloatRangeFieldQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestFloatRangeFieldQueries.java
index 6dc5907..6eda4de 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestFloatRangeFieldQueries.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestFloatRangeFieldQueries.java
@@ -31,11 +31,18 @@ public class TestFloatRangeFieldQueries extends BaseRangeFieldQueryTestCase {
private static final String FIELD_NAME = "floatRangeField";
private float nextFloatInternal() {
- if (rarely()) {
- return random().nextBoolean() ? Float.NEGATIVE_INFINITY : Float.POSITIVE_INFINITY;
+ switch (random().nextInt(5)) {
+ case 0:
+ return Float.NEGATIVE_INFINITY;
+ case 1:
+ return Float.POSITIVE_INFINITY;
+ default:
+ if (random().nextBoolean()) {
+ return random().nextFloat();
+ } else {
+ return (random().nextInt(15) - 7) / 3f;
+ }
}
- float max = Float.MAX_VALUE / 2;
- return (max + max) * random().nextFloat() - max;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/14320a58/lucene/core/src/test/org/apache/lucene/search/TestIntRangeFieldQueries.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestIntRangeFieldQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestIntRangeFieldQueries.java
index 14771c9..ecbd55b 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestIntRangeFieldQueries.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestIntRangeFieldQueries.java
@@ -23,6 +23,7 @@ import org.apache.lucene.document.IntRange;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.TestUtil;
/**
* Random testing for IntRange Queries.
@@ -31,11 +32,25 @@ public class TestIntRangeFieldQueries extends BaseRangeFieldQueryTestCase {
private static final String FIELD_NAME = "intRangeField";
private int nextIntInternal() {
- if (rarely()) {
- return random().nextBoolean() ? Integer.MAX_VALUE : Integer.MIN_VALUE;
+ switch (random().nextInt(5)) {
+ case 0:
+ return Integer.MIN_VALUE;
+ case 1:
+ return Integer.MAX_VALUE;
+ default:
+ int bpv = random().nextInt(32);
+ switch (bpv) {
+ case 32:
+ return random().nextInt();
+ default:
+ int v = TestUtil.nextInt(random(), 0, (1 << bpv) - 1);
+ if (bpv > 0) {
+ // negative values sometimes
+ v -= 1 << (bpv - 1);
+ }
+ return v;
+ }
}
- int max = Integer.MAX_VALUE / 2;
- return (max + max) * random().nextInt() - max;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/14320a58/lucene/core/src/test/org/apache/lucene/search/TestLongRangeFieldQueries.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestLongRangeFieldQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestLongRangeFieldQueries.java
index 60d7ea3..3ad34b5 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestLongRangeFieldQueries.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestLongRangeFieldQueries.java
@@ -23,6 +23,7 @@ import org.apache.lucene.document.LongRange;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.TestUtil;
/**
* Random testing for LongRange Queries.
@@ -31,11 +32,25 @@ public class TestLongRangeFieldQueries extends BaseRangeFieldQueryTestCase {
private static final String FIELD_NAME = "longRangeField";
private long nextLongInternal() {
- if (rarely()) {
- return random().nextBoolean() ? Long.MAX_VALUE : Long.MIN_VALUE;
+ switch (random().nextInt(5)) {
+ case 0:
+ return Long.MIN_VALUE;
+ case 1:
+ return Long.MAX_VALUE;
+ default:
+ int bpv = random().nextInt(64);
+ switch (bpv) {
+ case 64:
+ return random().nextLong();
+ default:
+ long v = TestUtil.nextLong(random(), 0, (1L << bpv) - 1);
+ if (bpv > 0) {
+ // negative values sometimes
+ v -= 1L << (bpv - 1);
+ }
+ return v;
+ }
}
- long max = Long.MAX_VALUE / 2;
- return (max + max) * random().nextLong() - max;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/14320a58/lucene/misc/src/test/org/apache/lucene/search/TestInetAddressRangeQueries.java
----------------------------------------------------------------------
diff --git a/lucene/misc/src/test/org/apache/lucene/search/TestInetAddressRangeQueries.java b/lucene/misc/src/test/org/apache/lucene/search/TestInetAddressRangeQueries.java
index 252162c..907ba55 100644
--- a/lucene/misc/src/test/org/apache/lucene/search/TestInetAddressRangeQueries.java
+++ b/lucene/misc/src/test/org/apache/lucene/search/TestInetAddressRangeQueries.java
@@ -18,6 +18,7 @@ package org.apache.lucene.search;
import java.net.InetAddress;
import java.net.UnknownHostException;
+import java.util.Arrays;
import org.apache.lucene.document.InetAddressPoint;
import org.apache.lucene.document.InetAddressRange;
@@ -44,8 +45,19 @@ public class TestInetAddressRangeQueries extends BaseRangeFieldQueryTestCase {
/** return random IPv4 or IPv6 address */
private InetAddress nextInetaddress() throws UnknownHostException {
byte[] b = random().nextBoolean() ? new byte[4] : new byte[16];
- random().nextBytes(b);
- return InetAddress.getByAddress(b);
+ switch (random().nextInt(5)) {
+ case 0:
+ return InetAddress.getByAddress(b);
+ case 1:
+ Arrays.fill(b, (byte) 0xff);
+ return InetAddress.getByAddress(b);
+ case 2:
+ Arrays.fill(b, (byte) 42);
+ return InetAddress.getByAddress(b);
+ default:
+ random().nextBytes(b);
+ return InetAddress.getByAddress(b);
+ }
}
@Override
@@ -159,7 +171,7 @@ public class TestInetAddressRangeQueries extends BaseRangeFieldQueryTestCase {
@Override
protected boolean isEqual(Range o) {
IpRange other = (IpRange)o;
- return this.min.equals(other.min) && this.max.equals(other.max);
+ return Arrays.equals(min, other.min) && Arrays.equals(max, other.max);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/14320a58/lucene/test-framework/src/java/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java
index 76de732..7d21eb3 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java
@@ -61,7 +61,9 @@ public abstract class BaseRangeFieldQueryTestCase extends LuceneTestCase {
public void testRandomTiny() throws Exception {
// Make sure single-leaf-node case is OK:
- doTestRandom(10, false);
+ for (int i = 0; i < 10; ++i) {
+ doTestRandom(10, false);
+ }
}
public void testRandomMedium() throws Exception {
[33/44] lucene-solr:jira/solr-8668: SOLR-10731: Fix typo in test
Posted by cp...@apache.org.
SOLR-10731: Fix typo in test
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3cd14476
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3cd14476
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3cd14476
Branch: refs/heads/jira/solr-8668
Commit: 3cd144761a814b927ef02d12bc831f66d43ea4fd
Parents: bdde9a1
Author: Joel Bernstein <jb...@apache.org>
Authored: Wed May 24 20:20:20 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Wed May 24 20:20:20 2017 -0400
----------------------------------------------------------------------
.../apache/solr/client/solrj/io/stream/StreamExpressionTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3cd14476/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index 47b1d31..ab792f9 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -962,7 +962,7 @@ public class StreamExpressionTest extends SolrCloudTestCase {
assertTrue(tuples.size() == 0);
sParams = new ModifiableSolrParams(StreamingTest.mapParams(CommonParams.QT, "/stream"));
- sParams.add("expr", "knn(" + COLLECTIONORALIAS + ", id=\"1\", qf=\"a_t\", r=\"2\", fl=\"id, score\", mintf=\"1\", minwl=\"20\")");
+ sParams.add("expr", "knn(" + COLLECTIONORALIAS + ", id=\"1\", qf=\"a_t\", rows=\"2\", fl=\"id, score\", mintf=\"1\", minwl=\"20\")");
solrStream = new SolrStream(jetty.getBaseUrl().toString() + "/collection1", sParams);
tuples = getTuples(solrStream);
assertTrue(tuples.size() == 0);
[30/44] lucene-solr:jira/solr-8668: SOLR-10684: Add finddelay Stream
Evaluator
Posted by cp...@apache.org.
SOLR-10684: Add finddelay Stream Evaluator
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/47781e39
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/47781e39
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/47781e39
Branch: refs/heads/jira/solr-8668
Commit: 47781e3938af3b60408a6b24c3d2a24fa526a98f
Parents: cd567b9
Author: Joel Bernstein <jb...@apache.org>
Authored: Wed May 24 13:10:37 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Wed May 24 13:10:50 2017 -0400
----------------------------------------------------------------------
.../org/apache/solr/handler/StreamHandler.java | 1 +
.../org/apache/solr/client/solrj/io/Tuple.java | 4 +-
.../solrj/io/stream/FindDelayEvaluator.java | 92 +++++++++++++++++++
.../solrj/io/stream/StreamExpressionTest.java | 94 ++++++++++++++++++++
4 files changed, 190 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/47781e39/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
index c045f20..9880512 100644
--- a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
@@ -185,6 +185,7 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
.withFunctionName("percentile", PercentileEvaluator.class)
.withFunctionName("empiricalDistribution", EmpiricalDistributionEvaluator.class)
.withFunctionName("describe", DescribeEvaluator.class)
+ .withFunctionName("finddelay", FindDelayEvaluator.class)
// metrics
.withFunctionName("min", MinMetric.class)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/47781e39/solr/solrj/src/java/org/apache/solr/client/solrj/io/Tuple.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Tuple.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Tuple.java
index d82c864..1af5f08 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Tuple.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Tuple.java
@@ -90,7 +90,9 @@ public class Tuple implements Cloneable, MapWriter {
}
if(o instanceof Long) {
- return (Long)o;
+ return (Long) o;
+ } else if (o instanceof Number) {
+ return ((Number)o).longValue();
} else {
//Attempt to parse the long
return Long.parseLong(o.toString());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/47781e39/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FindDelayEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FindDelayEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FindDelayEvaluator.java
new file mode 100644
index 0000000..c4f9eec
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FindDelayEvaluator.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.io.stream;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.math3.util.MathArrays;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.eval.ComplexEvaluator;
+import org.apache.solr.client.solrj.io.eval.StreamEvaluator;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
+import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class FindDelayEvaluator extends ComplexEvaluator implements Expressible {
+
+ private static final long serialVersionUID = 1;
+
+ public FindDelayEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ public Number evaluate(Tuple tuple) throws IOException {
+ StreamEvaluator colEval1 = subEvaluators.get(0);
+ StreamEvaluator colEval2 = subEvaluators.get(1);
+
+ List<Number> numbers1 = (List<Number>)colEval1.evaluate(tuple);
+ List<Number> numbers2 = (List<Number>)colEval2.evaluate(tuple);
+ double[] column1 = new double[numbers1.size()];
+ double[] column2 = new double[numbers2.size()];
+
+ for(int i=0; i<numbers1.size(); i++) {
+ column1[i] = numbers1.get(i).doubleValue();
+ }
+
+ //Reverse the second column.
+ //The convolve function will reverse it back.
+ //This allows correlation to be represented using the convolution math.
+ int rIndex=0;
+ for(int i=numbers2.size()-1; i>=0; i--) {
+ column2[rIndex++] = numbers2.get(i).doubleValue();
+ }
+
+ double[] convolution = MathArrays.convolve(column1, column2);
+ double max = -Double.MAX_VALUE;
+ double maxIndex = -1;
+
+ for(int i=0; i< convolution.length; i++) {
+ double abs = Math.abs(convolution[i]);
+ if(abs > max) {
+ max = abs;
+ maxIndex = i;
+ }
+ }
+
+ return (maxIndex+1)-column2.length;
+ }
+
+ @Override
+ public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
+ StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
+ return expression;
+ }
+
+ @Override
+ public Explanation toExplanation(StreamFactory factory) throws IOException {
+ return new Explanation(nodeId.toString())
+ .withExpressionType(ExpressionType.EVALUATOR)
+ .withFunctionName(factory.getFunctionName(getClass()))
+ .withImplementingClass(getClass().getName())
+ .withExpression(toExpression(factory).toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/47781e39/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index f76ed31..04af631 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -5864,6 +5864,100 @@ public class StreamExpressionTest extends SolrCloudTestCase {
assertTrue(prediction == 600.0D);
}
+
+ @Test
+ public void testFinddelay() throws Exception {
+ UpdateRequest updateRequest = new UpdateRequest();
+
+ //Pad column 1 with three zeros.
+ updateRequest.add(id, "10", "price_f", "0.0", "col_s", "a", "order_i", "0");
+ updateRequest.add(id, "11", "price_f", "0.0", "col_s", "a", "order_i", "0");
+ updateRequest.add(id, "12", "price_f", "0.0", "col_s", "a", "order_i", "0");
+ updateRequest.add(id, "1", "price_f", "100.0", "col_s", "a", "order_i", "1");
+ updateRequest.add(id, "2", "price_f", "200.0", "col_s", "a", "order_i", "2");
+ updateRequest.add(id, "3", "price_f", "300.0", "col_s", "a", "order_i", "3");
+ updateRequest.add(id, "4", "price_f", "100.0", "col_s", "a", "order_i", "4");
+ updateRequest.add(id, "5", "price_f", "200.0", "col_s", "a", "order_i", "5");
+ updateRequest.add(id, "6", "price_f", "400.0", "col_s", "a", "order_i", "6");
+ updateRequest.add(id, "7", "price_f", "600.0", "col_s", "a", "order_i", "7");
+
+ updateRequest.add(id, "100", "price_f", "200.0", "col_s", "b", "order_i", "1");
+ updateRequest.add(id, "101", "price_f", "400.0", "col_s", "b", "order_i", "2");
+ updateRequest.add(id, "102", "price_f", "600.0", "col_s", "b", "order_i", "3");
+ updateRequest.add(id, "103", "price_f", "200.0", "col_s", "b", "order_i", "4");
+ updateRequest.add(id, "104", "price_f", "400.0", "col_s", "b", "order_i", "5");
+ updateRequest.add(id, "105", "price_f", "800.0", "col_s", "b", "order_i", "6");
+ updateRequest.add(id, "106", "price_f", "1200.0", "col_s", "b", "order_i", "7");
+
+
+ updateRequest.add(id, "200", "price_f", "-200.0", "col_s", "c", "order_i", "1");
+ updateRequest.add(id, "301", "price_f", "-400.0", "col_s", "c", "order_i", "2");
+ updateRequest.add(id, "402", "price_f", "-600.0", "col_s", "c", "order_i", "3");
+ updateRequest.add(id, "503", "price_f", "-200.0", "col_s", "c", "order_i", "4");
+ updateRequest.add(id, "604", "price_f", "-400.0", "col_s", "c", "order_i", "5");
+ updateRequest.add(id, "705", "price_f", "-800.0", "col_s", "c", "order_i", "6");
+ updateRequest.add(id, "806", "price_f", "-1200.0", "col_s", "c", "order_i", "7");
+ updateRequest.commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ String expr1 = "search("+COLLECTIONORALIAS+", q=\"col_s:a\", fl=\"price_f, order_i\", sort=\"order_i asc\")";
+ String expr2 = "search("+COLLECTIONORALIAS+", q=\"col_s:b\", fl=\"price_f, order_i\", sort=\"order_i asc\")";
+
+ String cexpr = "let(a="+expr1+", b="+expr2+", c=col(a, price_f), d=col(b, price_f), tuple(delay=finddelay(c, d)))";
+
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ Tuple tuple = tuples.get(0);
+ long delay = tuple.getLong("delay");
+ assert(delay == 3);
+
+ expr1 = "search("+COLLECTIONORALIAS+", q=\"col_s:a\", fq=\"id:(1 2 3 4 5 6 7)\", fl=\"price_f, order_i\", sort=\"order_i asc\")";
+ expr2 = "search("+COLLECTIONORALIAS+", q=\"col_s:b\", fl=\"price_f, order_i\", sort=\"order_i asc\")";
+
+ cexpr = "let(a="+expr1+", b="+expr2+", c=col(a, price_f), d=col(b, price_f), tuple(delay=finddelay(c, d)))";
+
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+
+ solrStream = new SolrStream(url, paramsLoc);
+
+ solrStream.setStreamContext(context);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ tuple = tuples.get(0);
+ delay = tuple.getLong("delay");
+ assert(delay == 0);
+
+ //Test negative correlation.
+ expr1 = "search("+COLLECTIONORALIAS+", q=\"col_s:a\", fq=\"id:(1 2 3 4 5 6 7 11 12)\",fl=\"price_f, order_i\", sort=\"order_i asc\")";
+ expr2 = "search("+COLLECTIONORALIAS+", q=\"col_s:c\", fl=\"price_f, order_i\", sort=\"order_i asc\")";
+
+ cexpr = "let(a="+expr1+", b="+expr2+", c=col(a, price_f), d=col(b, price_f), tuple(delay=finddelay(c, d)))";
+
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+
+ solrStream = new SolrStream(url, paramsLoc);
+
+ solrStream.setStreamContext(context);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ tuple = tuples.get(0);
+ delay = tuple.getLong("delay");
+ assert(delay == 2);
+ }
+
+
@Test
public void testDescribe() throws Exception {
UpdateRequest updateRequest = new UpdateRequest();
[41/44] lucene-solr:jira/solr-8668: SOLR-10659: Remove
ResponseBuilder.getSortSpec use in SearchGroupShardResponseProcessor. (Judith
Silverman via Christine Poerschke)
Posted by cp...@apache.org.
SOLR-10659: Remove ResponseBuilder.getSortSpec use in SearchGroupShardResponseProcessor. (Judith Silverman via Christine Poerschke)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6ba1834b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6ba1834b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6ba1834b
Branch: refs/heads/jira/solr-8668
Commit: 6ba1834bc35d5cf322e7ba30dbc86e4d273eebb7
Parents: 2bb6e2c
Author: Christine Poerschke <cp...@apache.org>
Authored: Thu May 25 14:10:55 2017 +0100
Committer: Christine Poerschke <cp...@apache.org>
Committed: Thu May 25 16:46:46 2017 +0100
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +++
.../responseprocessor/SearchGroupShardResponseProcessor.java | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ba1834b/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 0ddf6c0..f6f5481 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -256,6 +256,9 @@ Other Changes
* SOLR-10438: Assign explicit useDocValuesAsStored values to all points field types in
schema-point.xml/TestPointFields. (hossman, Steve Rowe)
+* SOLR-10659: Remove ResponseBuilder.getSortSpec use in SearchGroupShardResponseProcessor.
+ (Judith Silverman via Christine Poerschke)
+
================== 6.6.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6ba1834b/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java b/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java
index ab13f72..18896e0 100644
--- a/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java
+++ b/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java
@@ -52,7 +52,7 @@ public class SearchGroupShardResponseProcessor implements ShardResponseProcessor
*/
@Override
public void process(ResponseBuilder rb, ShardRequest shardRequest) {
- SortSpec ss = rb.getSortSpec();
+ SortSpec groupSortSpec = rb.getGroupingSpec().getGroupSortSpec();
Sort groupSort = rb.getGroupingSpec().getGroupSort();
final String[] fields = rb.getGroupingSpec().getFields();
Sort withinGroupSort = rb.getGroupingSpec().getSortWithinGroup();
@@ -144,7 +144,7 @@ public class SearchGroupShardResponseProcessor implements ShardResponseProcessor
rb.firstPhaseElapsedTime = maxElapsedTime;
for (String groupField : commandSearchGroups.keySet()) {
List<Collection<SearchGroup<BytesRef>>> topGroups = commandSearchGroups.get(groupField);
- Collection<SearchGroup<BytesRef>> mergedTopGroups = SearchGroup.merge(topGroups, ss.getOffset(), ss.getCount(), groupSort);
+ Collection<SearchGroup<BytesRef>> mergedTopGroups = SearchGroup.merge(topGroups, groupSortSpec.getOffset(), groupSortSpec.getCount(), groupSort);
if (mergedTopGroups == null) {
continue;
}
[42/44] lucene-solr:jira/solr-8668: Ref Guide: fix atom example for
SOLR-7383
Posted by cp...@apache.org.
Ref Guide: fix atom example for SOLR-7383
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b3024d67
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b3024d67
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b3024d67
Branch: refs/heads/jira/solr-8668
Commit: b3024d67cae0f2c9bbfb9bdf897c9b43d6ab8926
Parents: 6ba1834
Author: Cassandra Targett <ct...@apache.org>
Authored: Thu May 25 10:53:17 2017 -0500
Committer: Cassandra Targett <ct...@apache.org>
Committed: Thu May 25 10:53:17 2017 -0500
----------------------------------------------------------------------
...ng-structured-data-store-data-with-the-data-import-handler.adoc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b3024d67/solr/solr-ref-guide/src/uploading-structured-data-store-data-with-the-data-import-handler.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/uploading-structured-data-store-data-with-the-data-import-handler.adoc b/solr/solr-ref-guide/src/uploading-structured-data-store-data-with-the-data-import-handler.adoc
index 9004789..33d2362 100644
--- a/solr/solr-ref-guide/src/uploading-structured-data-store-data-with-the-data-import-handler.adoc
+++ b/solr/solr-ref-guide/src/uploading-structured-data-store-data-with-the-data-import-handler.adoc
@@ -473,7 +473,7 @@ Here is an example from the `atom` collection in the `dih` example (data-config
<document>
<entity name="stackoverflow"
- url="http://stackoverflow.com/feeds/tag/solr"
+ url="https://stackoverflow.com/feeds/tag/solr"
processor="XPathEntityProcessor"
forEach="/feed|/feed/entry"
transformer="HTMLStripTransformer,RegexTransformer">
[26/44] lucene-solr:jira/solr-8668: SOLR-10731: Fix precommit
Posted by cp...@apache.org.
SOLR-10731: Fix precommit
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/f4872c92
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/f4872c92
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/f4872c92
Branch: refs/heads/jira/solr-8668
Commit: f4872c9296b509951a30e9d9618a3730f44bbefb
Parents: e2284bb
Author: Joel Bernstein <jb...@apache.org>
Authored: Wed May 24 08:06:01 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Wed May 24 08:06:01 2017 -0400
----------------------------------------------------------------------
.../src/java/org/apache/solr/client/solrj/io/stream/KnnStream.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f4872c92/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/KnnStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/KnnStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/KnnStream.java
index f0a188a..301d017 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/KnnStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/KnnStream.java
@@ -25,7 +25,6 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Set;
import java.util.stream.Collectors;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
[15/44] lucene-solr:jira/solr-8668: Fix javadocs warnings
Posted by cp...@apache.org.
Fix javadocs warnings
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/4f16beb0
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/4f16beb0
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/4f16beb0
Branch: refs/heads/jira/solr-8668
Commit: 4f16beb05369a9855925ba51e181896f4ffe264a
Parents: 28b8696
Author: Ishan Chattopadhyaya <is...@apache.org>
Authored: Wed May 24 00:57:32 2017 +0530
Committer: Ishan Chattopadhyaya <is...@apache.org>
Committed: Wed May 24 00:57:32 2017 +0530
----------------------------------------------------------------------
solr/core/src/java/org/apache/solr/handler/AnalyzeEvaluator.java | 1 +
.../apache/solr/client/solrj/io/eval/AbsoluteValueEvaluator.java | 3 ---
.../java/org/apache/solr/client/solrj/io/eval/AddEvaluator.java | 3 ---
.../java/org/apache/solr/client/solrj/io/eval/AndEvaluator.java | 3 ---
.../org/apache/solr/client/solrj/io/eval/ArcCosineEvaluator.java | 3 ---
.../org/apache/solr/client/solrj/io/eval/ArcSineEvaluator.java | 3 ---
.../org/apache/solr/client/solrj/io/eval/ArcTangentEvaluator.java | 3 ---
.../org/apache/solr/client/solrj/io/eval/BooleanEvaluator.java | 3 ---
.../org/apache/solr/client/solrj/io/eval/CeilingEvaluator.java | 3 ---
.../org/apache/solr/client/solrj/io/eval/CoalesceEvaluator.java | 3 ---
.../apache/solr/client/solrj/io/eval/ConditionalEvaluator.java | 3 ---
.../org/apache/solr/client/solrj/io/eval/CosineEvaluator.java | 3 ---
.../org/apache/solr/client/solrj/io/eval/CubedRootEvaluator.java | 3 ---
.../org/apache/solr/client/solrj/io/eval/DivideEvaluator.java | 3 ---
.../org/apache/solr/client/solrj/io/eval/EqualsEvaluator.java | 3 ---
.../apache/solr/client/solrj/io/eval/ExclusiveOrEvaluator.java | 3 ---
.../java/org/apache/solr/client/solrj/io/eval/FieldEvaluator.java | 3 ---
.../java/org/apache/solr/client/solrj/io/eval/FloorEvaluator.java | 3 ---
.../solr/client/solrj/io/eval/GreaterThanEqualToEvaluator.java | 3 ---
.../apache/solr/client/solrj/io/eval/GreaterThanEvaluator.java | 3 ---
.../solr/client/solrj/io/eval/HyperbolicCosineEvaluator.java | 3 ---
.../apache/solr/client/solrj/io/eval/HyperbolicSineEvaluator.java | 3 ---
.../solr/client/solrj/io/eval/HyperbolicTangentEvaluator.java | 3 ---
.../org/apache/solr/client/solrj/io/eval/IfThenElseEvaluator.java | 3 ---
.../solr/client/solrj/io/eval/LessThanEqualToEvaluator.java | 3 ---
.../org/apache/solr/client/solrj/io/eval/LessThanEvaluator.java | 3 ---
.../org/apache/solr/client/solrj/io/eval/ModuloEvaluator.java | 3 ---
.../org/apache/solr/client/solrj/io/eval/MultiplyEvaluator.java | 3 ---
.../org/apache/solr/client/solrj/io/eval/NaturalLogEvaluator.java | 3 ---
.../java/org/apache/solr/client/solrj/io/eval/NotEvaluator.java | 3 ---
.../org/apache/solr/client/solrj/io/eval/NumberEvaluator.java | 3 ---
.../java/org/apache/solr/client/solrj/io/eval/OrEvaluator.java | 3 ---
.../java/org/apache/solr/client/solrj/io/eval/PowerEvaluator.java | 3 ---
.../org/apache/solr/client/solrj/io/eval/RawValueEvaluator.java | 3 ---
.../java/org/apache/solr/client/solrj/io/eval/RoundEvaluator.java | 3 ---
.../org/apache/solr/client/solrj/io/eval/SimpleEvaluator.java | 3 ---
.../java/org/apache/solr/client/solrj/io/eval/SineEvaluator.java | 3 ---
.../org/apache/solr/client/solrj/io/eval/SquareRootEvaluator.java | 3 ---
.../org/apache/solr/client/solrj/io/eval/StreamEvaluator.java | 3 ---
.../org/apache/solr/client/solrj/io/eval/SubtractEvaluator.java | 3 ---
.../org/apache/solr/client/solrj/io/eval/TangentEvaluator.java | 3 ---
41 files changed, 1 insertion(+), 120 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/core/src/java/org/apache/solr/handler/AnalyzeEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/AnalyzeEvaluator.java b/solr/core/src/java/org/apache/solr/handler/AnalyzeEvaluator.java
index 392930f..c92f093 100644
--- a/solr/core/src/java/org/apache/solr/handler/AnalyzeEvaluator.java
+++ b/solr/core/src/java/org/apache/solr/handler/AnalyzeEvaluator.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.solr.handler;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AbsoluteValueEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AbsoluteValueEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AbsoluteValueEvaluator.java
index 38b3bb5..7ad4a16 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AbsoluteValueEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AbsoluteValueEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AddEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AddEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AddEvaluator.java
index 94a9280..fee7d8c 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AddEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AddEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AndEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AndEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AndEvaluator.java
index 290bd98..a4d968b 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AndEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/AndEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ArcCosineEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ArcCosineEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ArcCosineEvaluator.java
index 0c8e383..ee76542 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ArcCosineEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ArcCosineEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ArcSineEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ArcSineEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ArcSineEvaluator.java
index ed95165..23d860c 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ArcSineEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ArcSineEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ArcTangentEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ArcTangentEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ArcTangentEvaluator.java
index 9325b41..65e0174 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ArcTangentEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ArcTangentEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/BooleanEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/BooleanEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/BooleanEvaluator.java
index 908562f..447ab86 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/BooleanEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/BooleanEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CeilingEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CeilingEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CeilingEvaluator.java
index e2ccc8f..368385e 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CeilingEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CeilingEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CoalesceEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CoalesceEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CoalesceEvaluator.java
index 8a6eda4..a9f08b0 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CoalesceEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CoalesceEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ConditionalEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ConditionalEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ConditionalEvaluator.java
index 025bfae..9c534ed 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ConditionalEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ConditionalEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CosineEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CosineEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CosineEvaluator.java
index 6adbb81..f8fcf6a 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CosineEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CosineEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CubedRootEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CubedRootEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CubedRootEvaluator.java
index 4cd9277..9b7c952 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CubedRootEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CubedRootEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/DivideEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/DivideEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/DivideEvaluator.java
index f21a7f3..de32c6e 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/DivideEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/DivideEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/EqualsEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/EqualsEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/EqualsEvaluator.java
index c467a16..d2e6a9f 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/EqualsEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/EqualsEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ExclusiveOrEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ExclusiveOrEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ExclusiveOrEvaluator.java
index e63cab0..66ea2b0 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ExclusiveOrEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ExclusiveOrEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/FieldEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/FieldEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/FieldEvaluator.java
index 501e9d5..4d3c568 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/FieldEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/FieldEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/FloorEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/FloorEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/FloorEvaluator.java
index 0191a8e..96a221e 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/FloorEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/FloorEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GreaterThanEqualToEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GreaterThanEqualToEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GreaterThanEqualToEvaluator.java
index ad79e82..80cde67 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GreaterThanEqualToEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GreaterThanEqualToEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GreaterThanEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GreaterThanEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GreaterThanEvaluator.java
index 0b0e6e3..65f844a 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GreaterThanEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GreaterThanEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/HyperbolicCosineEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/HyperbolicCosineEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/HyperbolicCosineEvaluator.java
index 4e973a4..28f5e37 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/HyperbolicCosineEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/HyperbolicCosineEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/HyperbolicSineEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/HyperbolicSineEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/HyperbolicSineEvaluator.java
index 5bf4a38..ed1be4f 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/HyperbolicSineEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/HyperbolicSineEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/HyperbolicTangentEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/HyperbolicTangentEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/HyperbolicTangentEvaluator.java
index 89aacd1..4f6b5e1 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/HyperbolicTangentEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/HyperbolicTangentEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/IfThenElseEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/IfThenElseEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/IfThenElseEvaluator.java
index 346b743..d3e23ab 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/IfThenElseEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/IfThenElseEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LessThanEqualToEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LessThanEqualToEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LessThanEqualToEvaluator.java
index cb2fc7a..1590b8f 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LessThanEqualToEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LessThanEqualToEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LessThanEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LessThanEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LessThanEvaluator.java
index 40796b8..af389ae 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LessThanEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LessThanEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ModuloEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ModuloEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ModuloEvaluator.java
index 928754b..a965440 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ModuloEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ModuloEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MultiplyEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MultiplyEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MultiplyEvaluator.java
index 44b0b26..636624c 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MultiplyEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MultiplyEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/NaturalLogEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/NaturalLogEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/NaturalLogEvaluator.java
index 19709e6..5280600 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/NaturalLogEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/NaturalLogEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/NotEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/NotEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/NotEvaluator.java
index da2eeff..90054c5 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/NotEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/NotEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/NumberEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/NumberEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/NumberEvaluator.java
index d7c26b0..5434c01 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/NumberEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/NumberEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/OrEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/OrEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/OrEvaluator.java
index 1cd9df8..0c1886c 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/OrEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/OrEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PowerEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PowerEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PowerEvaluator.java
index a8245b6..415ecc8 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PowerEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PowerEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/RawValueEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/RawValueEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/RawValueEvaluator.java
index 1751380..e83d481 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/RawValueEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/RawValueEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/RoundEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/RoundEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/RoundEvaluator.java
index a34cdf4..959ad39 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/RoundEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/RoundEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SimpleEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SimpleEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SimpleEvaluator.java
index 4a095f8..356c495 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SimpleEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SimpleEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.util.UUID;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SineEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SineEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SineEvaluator.java
index 1e2fbb5..c5ac397 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SineEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SineEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SquareRootEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SquareRootEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SquareRootEvaluator.java
index 74b9d81..a39fdb2 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SquareRootEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SquareRootEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/StreamEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/StreamEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/StreamEvaluator.java
index e82d5d3..0b0d509 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/StreamEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/StreamEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SubtractEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SubtractEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SubtractEvaluator.java
index 3bf62b7..f555947 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SubtractEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SubtractEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f16beb0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/TangentEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/TangentEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/TangentEvaluator.java
index d2a0476..525ff1e 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/TangentEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/TangentEvaluator.java
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- *
- */
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
[37/44] lucene-solr:jira/solr-8668: SOLR-10746: Move all Stream
Evaluators to the eval package
Posted by cp...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CorrelationEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CorrelationEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CorrelationEvaluator.java
deleted file mode 100644
index 75d0c11..0000000
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CorrelationEvaluator.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.client.solrj.io.stream;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.math3.stat.correlation.PearsonsCorrelation;
-import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.eval.ComplexEvaluator;
-import org.apache.solr.client.solrj.io.eval.StreamEvaluator;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
-import org.apache.solr.client.solrj.io.stream.expr.Expressible;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
-import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-
-public class CorrelationEvaluator extends ComplexEvaluator implements Expressible {
-
- private static final long serialVersionUID = 1;
-
- public CorrelationEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
- super(expression, factory);
- }
-
- public Number evaluate(Tuple tuple) throws IOException {
- StreamEvaluator colEval1 = subEvaluators.get(0);
- StreamEvaluator colEval2 = subEvaluators.get(1);
-
- List<Number> numbers1 = (List<Number>)colEval1.evaluate(tuple);
- List<Number> numbers2 = (List<Number>)colEval2.evaluate(tuple);
- double[] column1 = new double[numbers1.size()];
- double[] column2 = new double[numbers2.size()];
-
- for(int i=0; i<numbers1.size(); i++) {
- column1[i] = numbers1.get(i).doubleValue();
- }
-
- for(int i=0; i<numbers2.size(); i++) {
- column2[i] = numbers2.get(i).doubleValue();
- }
-
- PearsonsCorrelation pearsonsCorrelation = new PearsonsCorrelation();
-
- return pearsonsCorrelation.correlation(column1, column2);
- }
-
- @Override
- public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
- StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
- return expression;
- }
-
- @Override
- public Explanation toExplanation(StreamFactory factory) throws IOException {
- return new Explanation(nodeId.toString())
- .withExpressionType(ExpressionType.EVALUATOR)
- .withFunctionName(factory.getFunctionName(getClass()))
- .withImplementingClass(getClass().getName())
- .withExpression(toExpression(factory).toString());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CovarianceEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CovarianceEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CovarianceEvaluator.java
deleted file mode 100644
index 0e3ffcc..0000000
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CovarianceEvaluator.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.client.solrj.io.stream;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.math3.stat.correlation.Covariance;
-import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.eval.ComplexEvaluator;
-import org.apache.solr.client.solrj.io.eval.StreamEvaluator;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
-import org.apache.solr.client.solrj.io.stream.expr.Expressible;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
-import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-
-public class CovarianceEvaluator extends ComplexEvaluator implements Expressible {
-
- private static final long serialVersionUID = 1;
-
- public CovarianceEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
- super(expression, factory);
- }
-
- public Number evaluate(Tuple tuple) throws IOException {
- StreamEvaluator colEval1 = subEvaluators.get(0);
- StreamEvaluator colEval2 = subEvaluators.get(1);
-
- List<Number> numbers1 = (List<Number>)colEval1.evaluate(tuple);
- List<Number> numbers2 = (List<Number>)colEval2.evaluate(tuple);
- double[] column1 = new double[numbers1.size()];
- double[] column2 = new double[numbers2.size()];
-
- for(int i=0; i<numbers1.size(); i++) {
- column1[i] = numbers1.get(i).doubleValue();
- }
-
- for(int i=0; i<numbers2.size(); i++) {
- column2[i] = numbers2.get(i).doubleValue();
- }
-
- Covariance covariance = new Covariance();
-
- return covariance.covariance(column1, column2);
- }
-
- @Override
- public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
- StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
- return expression;
- }
-
- @Override
- public Explanation toExplanation(StreamFactory factory) throws IOException {
- return new Explanation(nodeId.toString())
- .withExpressionType(ExpressionType.EVALUATOR)
- .withFunctionName(factory.getFunctionName(getClass()))
- .withImplementingClass(getClass().getName())
- .withExpression(toExpression(factory).toString());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DescribeEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DescribeEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DescribeEvaluator.java
deleted file mode 100644
index e086ebf..0000000
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DescribeEvaluator.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.client.solrj.io.stream;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
-import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.eval.ComplexEvaluator;
-import org.apache.solr.client.solrj.io.eval.StreamEvaluator;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
-import org.apache.solr.client.solrj.io.stream.expr.Expressible;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
-import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-
-public class DescribeEvaluator extends ComplexEvaluator implements Expressible {
-
- private static final long serialVersionUID = 1;
-
- public DescribeEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
- super(expression, factory);
- }
-
- public Tuple evaluate(Tuple tuple) throws IOException {
-
- if(subEvaluators.size() != 1) {
- throw new IOException("describe expects 1 column as a parameters");
- }
-
- StreamEvaluator colEval = subEvaluators.get(0);
-
- List<Number> numbers = (List<Number>)colEval.evaluate(tuple);
- DescriptiveStatistics descriptiveStatistics = new DescriptiveStatistics();
-
- for(Number n : numbers) {
- descriptiveStatistics.addValue(n.doubleValue());
- }
-
-
- Map map = new HashMap();
-
- map.put("max", descriptiveStatistics.getMax());
- map.put("mean", descriptiveStatistics.getMean());
- map.put("min", descriptiveStatistics.getMin());
- map.put("stdev", descriptiveStatistics.getStandardDeviation());
- map.put("sum", descriptiveStatistics.getSum());
- map.put("N", descriptiveStatistics.getN());
- map.put("var", descriptiveStatistics.getVariance());
- map.put("kurtosis", descriptiveStatistics.getKurtosis());
- map.put("skewness", descriptiveStatistics.getSkewness());
- map.put("popVar", descriptiveStatistics.getPopulationVariance());
- map.put("geometricMean", descriptiveStatistics.getGeometricMean());
- map.put("sumsq", descriptiveStatistics.getSumsq());
-
- return new Tuple(map);
- }
-
-
- @Override
- public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
- StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
- return expression;
- }
-
- @Override
- public Explanation toExplanation(StreamFactory factory) throws IOException {
- return new Explanation(nodeId.toString())
- .withExpressionType(ExpressionType.EVALUATOR)
- .withFunctionName(factory.getFunctionName(getClass()))
- .withImplementingClass(getClass().getName())
- .withExpression(toExpression(factory).toString());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DistanceEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DistanceEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DistanceEvaluator.java
deleted file mode 100644
index ef45d29..0000000
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/DistanceEvaluator.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.client.solrj.io.stream;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.math3.ml.distance.EuclideanDistance;
-import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.eval.ComplexEvaluator;
-import org.apache.solr.client.solrj.io.eval.StreamEvaluator;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
-import org.apache.solr.client.solrj.io.stream.expr.Expressible;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
-import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-
-public class DistanceEvaluator extends ComplexEvaluator implements Expressible {
-
- private static final long serialVersionUID = 1;
-
- public DistanceEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
- super(expression, factory);
- }
-
- public Number evaluate(Tuple tuple) throws IOException {
- StreamEvaluator colEval1 = subEvaluators.get(0);
- StreamEvaluator colEval2 = subEvaluators.get(1);
-
- List<Number> numbers1 = (List<Number>)colEval1.evaluate(tuple);
- List<Number> numbers2 = (List<Number>)colEval2.evaluate(tuple);
- double[] column1 = new double[numbers1.size()];
- double[] column2 = new double[numbers2.size()];
-
- for(int i=0; i<numbers1.size(); i++) {
- column1[i] = numbers1.get(i).doubleValue();
- }
-
- for(int i=0; i<numbers2.size(); i++) {
- column2[i] = numbers2.get(i).doubleValue();
- }
-
- EuclideanDistance distance = new EuclideanDistance();
- return distance.compute(column1, column2);
- }
-
- @Override
- public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
- StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
- return expression;
- }
-
- @Override
- public Explanation toExplanation(StreamFactory factory) throws IOException {
- return new Explanation(nodeId.toString())
- .withExpressionType(ExpressionType.EVALUATOR)
- .withFunctionName(factory.getFunctionName(getClass()))
- .withImplementingClass(getClass().getName())
- .withExpression(toExpression(factory).toString());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/EmpiricalDistributionEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/EmpiricalDistributionEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/EmpiricalDistributionEvaluator.java
deleted file mode 100644
index 46d08f5..0000000
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/EmpiricalDistributionEvaluator.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.client.solrj.io.stream;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Arrays;
-
-import org.apache.commons.math3.random.EmpiricalDistribution;
-import org.apache.commons.math3.stat.descriptive.StatisticalSummary;
-import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.eval.ComplexEvaluator;
-import org.apache.solr.client.solrj.io.eval.StreamEvaluator;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
-import org.apache.solr.client.solrj.io.stream.expr.Expressible;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
-import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-
-public class EmpiricalDistributionEvaluator extends ComplexEvaluator implements Expressible {
-
- private static final long serialVersionUID = 1;
-
- public EmpiricalDistributionEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
- super(expression, factory);
- }
-
- public Tuple evaluate(Tuple tuple) throws IOException {
-
- if(subEvaluators.size() != 1) {
- throw new IOException("Empirical dist expects 1 column as a parameters");
- }
-
- StreamEvaluator colEval1 = subEvaluators.get(0);
-
- List<Number> numbers1 = (List<Number>)colEval1.evaluate(tuple);
- double[] column1 = new double[numbers1.size()];
-
- for(int i=0; i<numbers1.size(); i++) {
- column1[i] = numbers1.get(i).doubleValue();
- }
-
- Arrays.sort(column1);
- EmpiricalDistribution empiricalDistribution = new EmpiricalDistribution();
- empiricalDistribution.load(column1);
-
- Map map = new HashMap();
- StatisticalSummary statisticalSummary = empiricalDistribution.getSampleStats();
-
- map.put("max", statisticalSummary.getMax());
- map.put("mean", statisticalSummary.getMean());
- map.put("min", statisticalSummary.getMin());
- map.put("stdev", statisticalSummary.getStandardDeviation());
- map.put("sum", statisticalSummary.getSum());
- map.put("N", statisticalSummary.getN());
- map.put("var", statisticalSummary.getVariance());
-
- return new EmpiricalDistributionTuple(empiricalDistribution, column1, map);
- }
-
- public static class EmpiricalDistributionTuple extends Tuple {
-
- private EmpiricalDistribution empiricalDistribution;
- private double[] backingArray;
-
- public EmpiricalDistributionTuple(EmpiricalDistribution empiricalDistribution, double[] backingArray, Map map) {
- super(map);
- this.empiricalDistribution = empiricalDistribution;
- this.backingArray = backingArray;
- }
-
- public double percentile(double d) {
- int slot = Arrays.binarySearch(backingArray, d);
-
- if(slot == 0) {
- return 0.0;
- }
-
- if(slot < 0) {
- if(slot == -1) {
- return 0.0D;
- } else {
- //Not a direct hit
- slot = Math.abs(slot);
- --slot;
- if(slot == backingArray.length) {
- return 1.0D;
- } else {
- return (this.empiricalDistribution.cumulativeProbability(backingArray[slot]));
- }
- }
- } else {
- return this.empiricalDistribution.cumulativeProbability(backingArray[slot]);
- }
- }
- }
-
- @Override
- public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
- StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
- return expression;
- }
-
- @Override
- public Explanation toExplanation(StreamFactory factory) throws IOException {
- return new Explanation(nodeId.toString())
- .withExpressionType(ExpressionType.EVALUATOR)
- .withFunctionName(factory.getFunctionName(getClass()))
- .withImplementingClass(getClass().getName())
- .withExpression(toExpression(factory).toString());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FindDelayEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FindDelayEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FindDelayEvaluator.java
deleted file mode 100644
index c4f9eec..0000000
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FindDelayEvaluator.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.client.solrj.io.stream;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.math3.util.MathArrays;
-import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.eval.ComplexEvaluator;
-import org.apache.solr.client.solrj.io.eval.StreamEvaluator;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
-import org.apache.solr.client.solrj.io.stream.expr.Expressible;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
-import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-
-public class FindDelayEvaluator extends ComplexEvaluator implements Expressible {
-
- private static final long serialVersionUID = 1;
-
- public FindDelayEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
- super(expression, factory);
- }
-
- public Number evaluate(Tuple tuple) throws IOException {
- StreamEvaluator colEval1 = subEvaluators.get(0);
- StreamEvaluator colEval2 = subEvaluators.get(1);
-
- List<Number> numbers1 = (List<Number>)colEval1.evaluate(tuple);
- List<Number> numbers2 = (List<Number>)colEval2.evaluate(tuple);
- double[] column1 = new double[numbers1.size()];
- double[] column2 = new double[numbers2.size()];
-
- for(int i=0; i<numbers1.size(); i++) {
- column1[i] = numbers1.get(i).doubleValue();
- }
-
- //Reverse the second column.
- //The convolve function will reverse it back.
- //This allows correlation to be represented using the convolution math.
- int rIndex=0;
- for(int i=numbers2.size()-1; i>=0; i--) {
- column2[rIndex++] = numbers2.get(i).doubleValue();
- }
-
- double[] convolution = MathArrays.convolve(column1, column2);
- double max = -Double.MAX_VALUE;
- double maxIndex = -1;
-
- for(int i=0; i< convolution.length; i++) {
- double abs = Math.abs(convolution[i]);
- if(abs > max) {
- max = abs;
- maxIndex = i;
- }
- }
-
- return (maxIndex+1)-column2.length;
- }
-
- @Override
- public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
- StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
- return expression;
- }
-
- @Override
- public Explanation toExplanation(StreamFactory factory) throws IOException {
- return new Explanation(nodeId.toString())
- .withExpressionType(ExpressionType.EVALUATOR)
- .withFunctionName(factory.getFunctionName(getClass()))
- .withImplementingClass(getClass().getName())
- .withExpression(toExpression(factory).toString());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/LengthEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/LengthEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/LengthEvaluator.java
deleted file mode 100644
index 3ff0db3..0000000
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/LengthEvaluator.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.client.solrj.io.stream;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.eval.ComplexEvaluator;
-import org.apache.solr.client.solrj.io.eval.StreamEvaluator;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
-import org.apache.solr.client.solrj.io.stream.expr.Expressible;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
-import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-
-public class LengthEvaluator extends ComplexEvaluator implements Expressible {
-
- private static final long serialVersionUID = 1;
-
- public LengthEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
- super(expression, factory);
- }
-
- public Number evaluate(Tuple tuple) throws IOException {
- StreamEvaluator colEval1 = subEvaluators.get(0);
- List<Number> numbers = (List<Number>)colEval1.evaluate(tuple);
- return numbers.size();
- }
-
- @Override
- public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
- StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
- return expression;
- }
-
- @Override
- public Explanation toExplanation(StreamFactory factory) throws IOException {
- return new Explanation(nodeId.toString())
- .withExpressionType(ExpressionType.EVALUATOR)
- .withFunctionName(factory.getFunctionName(getClass()))
- .withImplementingClass(getClass().getName())
- .withExpression(toExpression(factory).toString());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/NormalizeEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/NormalizeEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/NormalizeEvaluator.java
deleted file mode 100644
index e011933..0000000
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/NormalizeEvaluator.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.client.solrj.io.stream;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.math3.stat.StatUtils;
-import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.eval.ComplexEvaluator;
-import org.apache.solr.client.solrj.io.eval.StreamEvaluator;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
-import org.apache.solr.client.solrj.io.stream.expr.Expressible;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
-import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-
-public class NormalizeEvaluator extends ComplexEvaluator implements Expressible {
-
- private static final long serialVersionUID = 1;
-
- public NormalizeEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
- super(expression, factory);
- }
-
- public List<Number> evaluate(Tuple tuple) throws IOException {
- StreamEvaluator colEval1 = subEvaluators.get(0);
-
- List<Number> numbers1 = (List<Number>)colEval1.evaluate(tuple);
- double[] column1 = new double[numbers1.size()];
-
- for(int i=0; i<numbers1.size(); i++) {
- column1[i] = numbers1.get(i).doubleValue();
- }
-
- double[] normalized = StatUtils.normalize(column1);
-
- List<Number> normalizeList = new ArrayList(normalized.length);
- for(double d : normalized) {
- normalizeList.add(d);
- }
-
- return normalizeList;
- }
-
- @Override
- public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
- StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
- return expression;
- }
-
- @Override
- public Explanation toExplanation(StreamFactory factory) throws IOException {
- return new Explanation(nodeId.toString())
- .withExpressionType(ExpressionType.EVALUATOR)
- .withFunctionName(factory.getFunctionName(getClass()))
- .withImplementingClass(getClass().getName())
- .withExpression(toExpression(factory).toString());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/PercentileEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/PercentileEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/PercentileEvaluator.java
deleted file mode 100644
index 2bf4d60..0000000
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/PercentileEvaluator.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.client.solrj.io.stream;
-
-import java.io.IOException;
-
-import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.eval.ComplexEvaluator;
-import org.apache.solr.client.solrj.io.eval.StreamEvaluator;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
-import org.apache.solr.client.solrj.io.stream.expr.Expressible;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
-import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-
-public class PercentileEvaluator extends ComplexEvaluator implements Expressible {
-
- private static final long serialVersionUID = 1;
-
- public PercentileEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
- super(expression, factory);
- }
-
- public Number evaluate(Tuple tuple) throws IOException {
-
- if(subEvaluators.size() != 2) {
- throw new IOException("Percentile expects 2 parameters: a regression result and a number");
- }
-
- StreamEvaluator r = subEvaluators.get(0);
- StreamEvaluator d = subEvaluators.get(1);
-
- EmpiricalDistributionEvaluator.EmpiricalDistributionTuple e = (EmpiricalDistributionEvaluator.EmpiricalDistributionTuple)r.evaluate(tuple);
- Number n = (Number)d.evaluate(tuple);
- return e.percentile(n.doubleValue());
- }
-
- @Override
- public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
- StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
- return expression;
- }
-
- @Override
- public Explanation toExplanation(StreamFactory factory) throws IOException {
- return new Explanation(nodeId.toString())
- .withExpressionType(ExpressionType.EVALUATOR)
- .withFunctionName(factory.getFunctionName(getClass()))
- .withImplementingClass(getClass().getName())
- .withExpression(toExpression(factory).toString());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/PredictEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/PredictEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/PredictEvaluator.java
deleted file mode 100644
index 3c3ab84..0000000
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/PredictEvaluator.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.client.solrj.io.stream;
-
-import java.io.IOException;
-
-import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.eval.ComplexEvaluator;
-import org.apache.solr.client.solrj.io.eval.StreamEvaluator;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
-import org.apache.solr.client.solrj.io.stream.expr.Expressible;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
-import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-
-public class PredictEvaluator extends ComplexEvaluator implements Expressible {
-
- private static final long serialVersionUID = 1;
-
- public PredictEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
- super(expression, factory);
- }
-
- public Number evaluate(Tuple tuple) throws IOException {
-
- if(subEvaluators.size() != 2) {
- throw new IOException("Predict expects 2 parameters: a regression result and a number");
- }
-
- StreamEvaluator r = subEvaluators.get(0);
- StreamEvaluator d = subEvaluators.get(1);
-
- RegressionEvaluator.RegressionTuple rt= (RegressionEvaluator.RegressionTuple)r.evaluate(tuple);
- Number n = (Number)d.evaluate(tuple);
- return rt.predict(n.doubleValue());
- }
-
- @Override
- public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
- StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
- return expression;
- }
-
- @Override
- public Explanation toExplanation(StreamFactory factory) throws IOException {
- return new Explanation(nodeId.toString())
- .withExpressionType(ExpressionType.EVALUATOR)
- .withFunctionName(factory.getFunctionName(getClass()))
- .withImplementingClass(getClass().getName())
- .withExpression(toExpression(factory).toString());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/RankEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/RankEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/RankEvaluator.java
deleted file mode 100644
index 2084928..0000000
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/RankEvaluator.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.client.solrj.io.stream;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.math3.stat.ranking.NaturalRanking;
-import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.eval.ComplexEvaluator;
-import org.apache.solr.client.solrj.io.eval.StreamEvaluator;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
-import org.apache.solr.client.solrj.io.stream.expr.Expressible;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
-import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-
-public class RankEvaluator extends ComplexEvaluator implements Expressible {
-
- private static final long serialVersionUID = 1;
-
- public RankEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
- super(expression, factory);
- }
-
- public List<Number> evaluate(Tuple tuple) throws IOException {
- StreamEvaluator colEval = subEvaluators.get(0);
-
- List<Number> numbers = (List<Number>)colEval.evaluate(tuple);
- double[] values = new double[numbers.size()];
- for(int i=0; i<numbers.size(); i++) {
- values[i] = numbers.get(i).doubleValue();
- }
-
- NaturalRanking rank = new NaturalRanking();
- double[] ranked = rank.rank(values);
- List<Number> rankedList = new ArrayList();
- for(int i=0; i<numbers.size(); i++) {
- rankedList.add(ranked[i]);
- }
-
- return rankedList;
- }
-
- @Override
- public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
- StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
- return expression;
- }
-
- @Override
- public Explanation toExplanation(StreamFactory factory) throws IOException {
- return new Explanation(nodeId.toString())
- .withExpressionType(ExpressionType.EVALUATOR)
- .withFunctionName(factory.getFunctionName(getClass()))
- .withImplementingClass(getClass().getName())
- .withExpression(toExpression(factory).toString());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/RegressionEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/RegressionEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/RegressionEvaluator.java
deleted file mode 100644
index 5306193..0000000
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/RegressionEvaluator.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.client.solrj.io.stream;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.math3.stat.regression.SimpleRegression;
-import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.eval.ComplexEvaluator;
-import org.apache.solr.client.solrj.io.eval.StreamEvaluator;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
-import org.apache.solr.client.solrj.io.stream.expr.Expressible;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
-import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-
-public class RegressionEvaluator extends ComplexEvaluator implements Expressible {
-
- private static final long serialVersionUID = 1;
-
- public RegressionEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
- super(expression, factory);
- }
-
- public Tuple evaluate(Tuple tuple) throws IOException {
-
- if(subEvaluators.size() != 2) {
- throw new IOException("Regress expects 2 columns as parameters");
- }
-
- StreamEvaluator colEval1 = subEvaluators.get(0);
- StreamEvaluator colEval2 = subEvaluators.get(1);
-
- List<Number> numbers1 = (List<Number>)colEval1.evaluate(tuple);
- List<Number> numbers2 = (List<Number>)colEval2.evaluate(tuple);
- double[] column1 = new double[numbers1.size()];
- double[] column2 = new double[numbers2.size()];
-
- for(int i=0; i<numbers1.size(); i++) {
- column1[i] = numbers1.get(i).doubleValue();
- }
-
- for(int i=0; i<numbers2.size(); i++) {
- column2[i] = numbers2.get(i).doubleValue();
- }
-
- SimpleRegression regression = new SimpleRegression();
- for(int i=0; i<column1.length; i++) {
- regression.addData(column1[i], column2[i]);
- }
-
- Map map = new HashMap();
- map.put("slope", regression.getSlope());
- map.put("intercept", regression.getIntercept());
- map.put("R", regression.getR());
- map.put("N", regression.getN());
- map.put("regressionSumSquares", regression.getRegressionSumSquares());
- map.put("slopeConfidenceInterval", regression.getSlopeConfidenceInterval());
- map.put("interceptStdErr", regression.getInterceptStdErr());
- map.put("totalSumSquares", regression.getTotalSumSquares());
- map.put("significance", regression.getSignificance());
- map.put("meanSquareError", regression.getMeanSquareError());
- return new RegressionTuple(regression, map);
- }
-
- public static class RegressionTuple extends Tuple {
-
- private SimpleRegression simpleRegression;
-
- public RegressionTuple(SimpleRegression simpleRegression, Map map) {
- super(map);
- this.simpleRegression = simpleRegression;
- }
-
- public double predict(double d) {
- return this.simpleRegression.predict(d);
- }
- }
-
- @Override
- public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
- StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
- return expression;
- }
-
- @Override
- public Explanation toExplanation(StreamFactory factory) throws IOException {
- return new Explanation(nodeId.toString())
- .withExpressionType(ExpressionType.EVALUATOR)
- .withFunctionName(factory.getFunctionName(getClass()))
- .withImplementingClass(getClass().getName())
- .withExpression(toExpression(factory).toString());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ReverseEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ReverseEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ReverseEvaluator.java
deleted file mode 100644
index 5518ed0..0000000
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ReverseEvaluator.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.client.solrj.io.stream;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.eval.ComplexEvaluator;
-import org.apache.solr.client.solrj.io.eval.StreamEvaluator;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
-import org.apache.solr.client.solrj.io.stream.expr.Expressible;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
-import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-
-public class ReverseEvaluator extends ComplexEvaluator implements Expressible {
-
- private static final long serialVersionUID = 1;
-
- public ReverseEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
- super(expression, factory);
- }
-
- public List<Number> evaluate(Tuple tuple) throws IOException {
- StreamEvaluator colEval1 = subEvaluators.get(0);
-
- List<Number> numbers1 = (List<Number>)colEval1.evaluate(tuple);
- List<Number> rev = new ArrayList();
- for(int i=numbers1.size()-1; i>=0; i--) {
- rev.add(numbers1.get(i));
- }
-
- return rev;
- }
-
- @Override
- public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
- StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
- return expression;
- }
-
- @Override
- public Explanation toExplanation(StreamFactory factory) throws IOException {
- return new Explanation(nodeId.toString())
- .withExpressionType(ExpressionType.EVALUATOR)
- .withFunctionName(factory.getFunctionName(getClass()))
- .withImplementingClass(getClass().getName())
- .withExpression(toExpression(factory).toString());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ScaleEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ScaleEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ScaleEvaluator.java
deleted file mode 100644
index 04722aa..0000000
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ScaleEvaluator.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.client.solrj.io.stream;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.math3.util.MathArrays;
-import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.eval.ComplexEvaluator;
-import org.apache.solr.client.solrj.io.eval.StreamEvaluator;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
-import org.apache.solr.client.solrj.io.stream.expr.Expressible;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
-import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-
-public class ScaleEvaluator extends ComplexEvaluator implements Expressible {
-
- private static final long serialVersionUID = 1;
-
- public ScaleEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
- super(expression, factory);
- }
-
- public List<Number> evaluate(Tuple tuple) throws IOException {
- StreamEvaluator numEval = subEvaluators.get(0);
- StreamEvaluator colEval1 = subEvaluators.get(1);
-
- Number num = (Number)numEval.evaluate(tuple);
- List<Number> numbers1 = (List<Number>)colEval1.evaluate(tuple);
- double[] column1 = new double[numbers1.size()];
-
- for(int i=0; i<numbers1.size(); i++) {
- column1[i] = numbers1.get(i).doubleValue();
- }
-
- double[] scaled = MathArrays.scale(num.doubleValue(), column1);
-
- List<Number> scaledList = new ArrayList(scaled.length);
- for(double d : scaled) {
- scaledList.add(d);
- }
-
- return scaledList;
- }
-
- @Override
- public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
- StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
- return expression;
- }
-
- @Override
- public Explanation toExplanation(StreamFactory factory) throws IOException {
- return new Explanation(nodeId.toString())
- .withExpressionType(ExpressionType.EVALUATOR)
- .withFunctionName(factory.getFunctionName(getClass()))
- .withImplementingClass(getClass().getName())
- .withExpression(toExpression(factory).toString());
- }
-}
\ No newline at end of file
[09/44] lucene-solr:jira/solr-8668: LUCENE-7815: Removed the
PostingsHighlighter
Posted by cp...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java b/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java
deleted file mode 100644
index 4b7e66b..0000000
--- a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java
+++ /dev/null
@@ -1,1185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.postingshighlight;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.StoredField;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.queries.CustomScoreQuery;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.PhraseQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.LuceneTestCase;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
-import java.text.BreakIterator;
-import java.util.Arrays;
-import java.util.Map;
-
-public class TestPostingsHighlighter extends LuceneTestCase {
-
- public void testBasics() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");
- iw.addDocument(doc);
- body.setStringValue("Highlighting the first term. Hope it works.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- Query query = new TermQuery(new Term("body", "highlighting"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("Just a test <b>highlighting</b> from postings. ", snippets[0]);
- assertEquals("<b>Highlighting</b> the first term. ", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testFormatWithMatchExceedingContentLength2() throws Exception {
-
- String bodyText = "123 TEST 01234 TEST";
-
- String[] snippets = formatWithMatchExceedingContentLength(bodyText);
-
- assertEquals(1, snippets.length);
- assertEquals("123 <b>TEST</b> 01234 TE", snippets[0]);
- }
-
- public void testFormatWithMatchExceedingContentLength3() throws Exception {
-
- String bodyText = "123 5678 01234 TEST TEST";
-
- String[] snippets = formatWithMatchExceedingContentLength(bodyText);
-
- assertEquals(1, snippets.length);
- assertEquals("123 5678 01234 TE", snippets[0]);
- }
-
- public void testFormatWithMatchExceedingContentLength() throws Exception {
-
- String bodyText = "123 5678 01234 TEST";
-
- String[] snippets = formatWithMatchExceedingContentLength(bodyText);
-
- assertEquals(1, snippets.length);
- // LUCENE-5166: no snippet
- assertEquals("123 5678 01234 TE", snippets[0]);
- }
-
- private String[] formatWithMatchExceedingContentLength(String bodyText) throws IOException {
-
- int maxLength = 17;
-
- final Analyzer analyzer = new MockAnalyzer(random());
-
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- final FieldType fieldType = new FieldType(TextField.TYPE_STORED);
- fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- final Field body = new Field("body", bodyText, fieldType);
-
- Document doc = new Document();
- doc.add(body);
-
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
-
- Query query = new TermQuery(new Term("body", "test"));
-
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(1, topDocs.totalHits);
-
- PostingsHighlighter highlighter = new PostingsHighlighter(maxLength);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
-
-
- ir.close();
- dir.close();
- return snippets;
- }
-
- // simple test highlighting last word.
- public void testHighlightLastWord() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- Query query = new TermQuery(new Term("body", "test"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(1, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(1, snippets.length);
- assertEquals("This is a <b>test</b>", snippets[0]);
-
- ir.close();
- dir.close();
- }
-
- // simple test with one sentence documents.
- public void testOneSentence() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- Query query = new TermQuery(new Term("body", "test"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- // simple test with multiple values that make a result longer than maxLength.
- public void testMaxLengthWithMultivalue() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Document doc = new Document();
-
- for(int i = 0; i < 3 ; i++) {
- Field body = new Field("body", "", offsetsType);
- body.setStringValue("This is a multivalued field");
- doc.add(body);
- }
-
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter(40);
- Query query = new TermQuery(new Term("body", "field"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(1, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(1, snippets.length);
- assertTrue("Snippet should have maximum 40 characters plus the pre and post tags",
- snippets[0].length() == (40 + "<b></b>".length()));
-
- ir.close();
- dir.close();
- }
-
- public void testMultipleFields() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Field title = new Field("title", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
- doc.add(title);
-
- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");
- title.setStringValue("I am hoping for the best.");
- iw.addDocument(doc);
- body.setStringValue("Highlighting the first term. Hope it works.");
- title.setStringValue("But best may not be good enough.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- BooleanQuery.Builder query = new BooleanQuery.Builder();
- query.add(new TermQuery(new Term("body", "highlighting")), BooleanClause.Occur.SHOULD);
- query.add(new TermQuery(new Term("title", "best")), BooleanClause.Occur.SHOULD);
- TopDocs topDocs = searcher.search(query.build(), 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- Map<String,String[]> snippets = highlighter.highlightFields(new String [] { "body", "title" }, query.build(), searcher, topDocs);
- assertEquals(2, snippets.size());
- assertEquals("Just a test <b>highlighting</b> from postings. ", snippets.get("body")[0]);
- assertEquals("<b>Highlighting</b> the first term. ", snippets.get("body")[1]);
- assertEquals("I am hoping for the <b>best</b>.", snippets.get("title")[0]);
- assertEquals("But <b>best</b> may not be good enough.", snippets.get("title")[1]);
- ir.close();
- dir.close();
- }
-
- public void testMultipleTerms() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");
- iw.addDocument(doc);
- body.setStringValue("Highlighting the first term. Hope it works.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- BooleanQuery.Builder query = new BooleanQuery.Builder();
- query.add(new TermQuery(new Term("body", "highlighting")), BooleanClause.Occur.SHOULD);
- query.add(new TermQuery(new Term("body", "just")), BooleanClause.Occur.SHOULD);
- query.add(new TermQuery(new Term("body", "first")), BooleanClause.Occur.SHOULD);
- TopDocs topDocs = searcher.search(query.build(), 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query.build(), searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("<b>Just</b> a test <b>highlighting</b> from postings. ", snippets[0]);
- assertEquals("<b>Highlighting</b> the <b>first</b> term. ", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testMultiplePassages() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");
- iw.addDocument(doc);
- body.setStringValue("This test is another test. Not a good sentence. Test test test test.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- Query query = new TermQuery(new Term("body", "test"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs, 2);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>. Just a <b>test</b> highlighting from postings. ", snippets[0]);
- assertEquals("This <b>test</b> is another <b>test</b>. ... <b>Test</b> <b>test</b> <b>test</b> <b>test</b>.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testUserFailedToIndexOffsets() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType positionsType = new FieldType(TextField.TYPE_STORED);
- positionsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
- Field body = new Field("body", "", positionsType);
- Field title = new StringField("title", "", Field.Store.YES);
- Document doc = new Document();
- doc.add(body);
- doc.add(title);
-
- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");
- title.setStringValue("test");
- iw.addDocument(doc);
- body.setStringValue("This test is another test. Not a good sentence. Test test test test.");
- title.setStringValue("test");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- Query query = new TermQuery(new Term("body", "test"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- expectThrows(IllegalArgumentException.class, () -> {
- highlighter.highlight("body", query, searcher, topDocs, 2);
- });
-
- expectThrows(IllegalArgumentException.class, () -> {
- highlighter.highlight("title", new TermQuery(new Term("title", "test")), searcher, topDocs, 2);
- fail("did not hit expected exception");
- });
-
- ir.close();
- dir.close();
- }
-
- public void testBuddhism() throws Exception {
- String text = "This eight-volume set brings together seminal papers in Buddhist studies from a vast " +
- "range of academic disciplines published over the last forty years. With a new introduction " +
- "by the editor, this collection is a unique and unrivalled research resource for both " +
- "student and scholar. Coverage includes: - Buddhist origins; early history of Buddhism in " +
- "South and Southeast Asia - early Buddhist Schools and Doctrinal History; Theravada Doctrine " +
- "- the Origins and nature of Mahayana Buddhism; some Mahayana religious topics - Abhidharma " +
- "and Madhyamaka - Yogacara, the Epistemological tradition, and Tathagatagarbha - Tantric " +
- "Buddhism (Including China and Japan); Buddhism in Nepal and Tibet - Buddhism in South and " +
- "Southeast Asia, and - Buddhism in China, East Asia, and Japan.";
- Directory dir = newDirectory();
- Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, analyzer);
-
- FieldType positionsType = new FieldType(TextField.TYPE_STORED);
- positionsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", text, positionsType);
- Document document = new Document();
- document.add(body);
- iw.addDocument(document);
- IndexReader ir = iw.getReader();
- iw.close();
- IndexSearcher searcher = newSearcher(ir);
- PhraseQuery query = new PhraseQuery("body", "buddhist", "origins");
- TopDocs topDocs = searcher.search(query, 10);
- assertEquals(1, topDocs.totalHits);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs, 2);
- assertEquals(1, snippets.length);
- assertTrue(snippets[0].contains("<b>Buddhist</b> <b>origins</b>"));
- ir.close();
- dir.close();
- }
-
- public void testCuriousGeorge() throws Exception {
- String text = "It’s the formula for success for preschoolers—Curious George and fire trucks! " +
- "Curious George and the Firefighters is a story based on H. A. and Margret Rey’s " +
- "popular primate and painted in the original watercolor and charcoal style. " +
- "Firefighters are a famously brave lot, but can they withstand a visit from one curious monkey?";
- Directory dir = newDirectory();
- Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, analyzer);
- FieldType positionsType = new FieldType(TextField.TYPE_STORED);
- positionsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", text, positionsType);
- Document document = new Document();
- document.add(body);
- iw.addDocument(document);
- IndexReader ir = iw.getReader();
- iw.close();
- IndexSearcher searcher = newSearcher(ir);
- PhraseQuery query = new PhraseQuery("body", "curious", "george");
- TopDocs topDocs = searcher.search(query, 10);
- assertEquals(1, topDocs.totalHits);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs, 2);
- assertEquals(1, snippets.length);
- assertFalse(snippets[0].contains("<b>Curious</b>Curious"));
- ir.close();
- dir.close();
- }
-
- public void testCambridgeMA() throws Exception {
- BufferedReader r = new BufferedReader(new InputStreamReader(
- this.getClass().getResourceAsStream("CambridgeMA.utf8"), StandardCharsets.UTF_8));
- String text = r.readLine();
- r.close();
- Directory dir = newDirectory();
- Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, analyzer);
- FieldType positionsType = new FieldType(TextField.TYPE_STORED);
- positionsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", text, positionsType);
- Document document = new Document();
- document.add(body);
- iw.addDocument(document);
- IndexReader ir = iw.getReader();
- iw.close();
- IndexSearcher searcher = newSearcher(ir);
- BooleanQuery.Builder query = new BooleanQuery.Builder();
- query.add(new TermQuery(new Term("body", "porter")), BooleanClause.Occur.SHOULD);
- query.add(new TermQuery(new Term("body", "square")), BooleanClause.Occur.SHOULD);
- query.add(new TermQuery(new Term("body", "massachusetts")), BooleanClause.Occur.SHOULD);
- TopDocs topDocs = searcher.search(query.build(), 10);
- assertEquals(1, topDocs.totalHits);
- PostingsHighlighter highlighter = new PostingsHighlighter(Integer.MAX_VALUE-1);
- String snippets[] = highlighter.highlight("body", query.build(), searcher, topDocs, 2);
- assertEquals(1, snippets.length);
- assertTrue(snippets[0].contains("<b>Square</b>"));
- assertTrue(snippets[0].contains("<b>Porter</b>"));
- ir.close();
- dir.close();
- }
-
- public void testPassageRanking() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test. Just highlighting from postings. This is also a much sillier test. Feel free to test test test test test test test.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- Query query = new TermQuery(new Term("body", "test"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(1, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs, 2);
- assertEquals(1, snippets.length);
- assertEquals("This is a <b>test</b>. ... Feel free to <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b>.", snippets[0]);
-
- ir.close();
- dir.close();
- }
-
- public void testBooleanMustNot() throws Exception {
- Directory dir = newDirectory();
- Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, analyzer);
- FieldType positionsType = new FieldType(TextField.TYPE_STORED);
- positionsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "This sentence has both terms. This sentence has only terms.", positionsType);
- Document document = new Document();
- document.add(body);
- iw.addDocument(document);
- IndexReader ir = iw.getReader();
- iw.close();
- IndexSearcher searcher = newSearcher(ir);
- BooleanQuery.Builder query = new BooleanQuery.Builder();
- query.add(new TermQuery(new Term("body", "terms")), BooleanClause.Occur.SHOULD);
- BooleanQuery.Builder query2 = new BooleanQuery.Builder();
- query.add(query2.build(), BooleanClause.Occur.SHOULD);
- query2.add(new TermQuery(new Term("body", "both")), BooleanClause.Occur.MUST_NOT);
- TopDocs topDocs = searcher.search(query.build(), 10);
- assertEquals(1, topDocs.totalHits);
- PostingsHighlighter highlighter = new PostingsHighlighter(Integer.MAX_VALUE-1);
- String snippets[] = highlighter.highlight("body", query.build(), searcher, topDocs, 2);
- assertEquals(1, snippets.length);
- assertFalse(snippets[0].contains("<b>both</b>"));
- ir.close();
- dir.close();
- }
-
- public void testHighlightAllText() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test. Just highlighting from postings. This is also a much sillier test. Feel free to test test test test test test test.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter(10000) {
- @Override
- protected BreakIterator getBreakIterator(String field) {
- return new WholeBreakIterator();
- }
- };
- Query query = new TermQuery(new Term("body", "test"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(1, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs, 2);
- assertEquals(1, snippets.length);
- assertEquals("This is a <b>test</b>. Just highlighting from postings. This is also a much sillier <b>test</b>. Feel free to <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b>.", snippets[0]);
-
- ir.close();
- dir.close();
- }
-
- public void testSpecificDocIDs() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");
- iw.addDocument(doc);
- body.setStringValue("Highlighting the first term. Hope it works.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- Query query = new TermQuery(new Term("body", "highlighting"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- ScoreDoc[] hits = topDocs.scoreDocs;
- int[] docIDs = new int[2];
- docIDs[0] = hits[0].doc;
- docIDs[1] = hits[1].doc;
- String snippets[] = highlighter.highlightFields(new String[] {"body"}, query, searcher, docIDs, new int[] { 1 }).get("body");
- assertEquals(2, snippets.length);
- assertEquals("Just a test <b>highlighting</b> from postings. ", snippets[0]);
- assertEquals("<b>Highlighting</b> the first term. ", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testCustomFieldValueSource() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- Document doc = new Document();
-
- FieldType offsetsType = new FieldType(TextField.TYPE_NOT_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- final String text = "This is a test. Just highlighting from postings. This is also a much sillier test. Feel free to test test test test test test test.";
- Field body = new Field("body", text, offsetsType);
- doc.add(body);
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
-
- PostingsHighlighter highlighter = new PostingsHighlighter(10000) {
- @Override
- protected String[][] loadFieldValues(IndexSearcher searcher, String[] fields, int[] docids, int maxLength) throws IOException {
- assert fields.length == 1;
- assert docids.length == 1;
- String[][] contents = new String[1][1];
- contents[0][0] = text;
- return contents;
- }
-
- @Override
- protected BreakIterator getBreakIterator(String field) {
- return new WholeBreakIterator();
- }
- };
-
- Query query = new TermQuery(new Term("body", "test"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(1, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs, 2);
- assertEquals(1, snippets.length);
- assertEquals("This is a <b>test</b>. Just highlighting from postings. This is also a much sillier <b>test</b>. Feel free to <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b>.", snippets[0]);
-
- ir.close();
- dir.close();
- }
-
- /** Make sure highlighter returns first N sentences if
- * there were no hits. */
- public void testEmptyHighlights() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Document doc = new Document();
-
- Field body = new Field("body", "test this is. another sentence this test has. far away is that planet.", offsetsType);
- doc.add(body);
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- Query query = new TermQuery(new Term("body", "highlighting"));
- int[] docIDs = new int[] {0};
- String snippets[] = highlighter.highlightFields(new String[] {"body"}, query, searcher, docIDs, new int[] { 2 }).get("body");
- assertEquals(1, snippets.length);
- assertEquals("test this is. another sentence this test has. ", snippets[0]);
-
- ir.close();
- dir.close();
- }
-
- /** Make sure highlighter we can customize how emtpy
- * highlight is returned. */
- public void testCustomEmptyHighlights() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Document doc = new Document();
-
- Field body = new Field("body", "test this is. another sentence this test has. far away is that planet.", offsetsType);
- doc.add(body);
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- public Passage[] getEmptyHighlight(String fieldName, BreakIterator bi, int maxPassages) {
- return new Passage[0];
- }
- };
- Query query = new TermQuery(new Term("body", "highlighting"));
- int[] docIDs = new int[] {0};
- String snippets[] = highlighter.highlightFields(new String[] {"body"}, query, searcher, docIDs, new int[] { 2 }).get("body");
- assertEquals(1, snippets.length);
- assertNull(snippets[0]);
-
- ir.close();
- dir.close();
- }
-
- /** Make sure highlighter returns whole text when there
- * are no hits and BreakIterator is null. */
- public void testEmptyHighlightsWhole() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Document doc = new Document();
-
- Field body = new Field("body", "test this is. another sentence this test has. far away is that planet.", offsetsType);
- doc.add(body);
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter(10000) {
- @Override
- protected BreakIterator getBreakIterator(String field) {
- return new WholeBreakIterator();
- }
- };
- Query query = new TermQuery(new Term("body", "highlighting"));
- int[] docIDs = new int[] {0};
- String snippets[] = highlighter.highlightFields(new String[] {"body"}, query, searcher, docIDs, new int[] { 2 }).get("body");
- assertEquals(1, snippets.length);
- assertEquals("test this is. another sentence this test has. far away is that planet.", snippets[0]);
-
- ir.close();
- dir.close();
- }
-
- /** Make sure highlighter is OK with entirely missing
- * field. */
- public void testFieldIsMissing() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Document doc = new Document();
-
- Field body = new Field("body", "test this is. another sentence this test has. far away is that planet.", offsetsType);
- doc.add(body);
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- Query query = new TermQuery(new Term("bogus", "highlighting"));
- int[] docIDs = new int[] {0};
- String snippets[] = highlighter.highlightFields(new String[] {"bogus"}, query, searcher, docIDs, new int[] { 2 }).get("bogus");
- assertEquals(1, snippets.length);
- assertNull(snippets[0]);
-
- ir.close();
- dir.close();
- }
-
- public void testFieldIsJustSpace() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
-
- Document doc = new Document();
- doc.add(new Field("body", " ", offsetsType));
- doc.add(new Field("id", "id", offsetsType));
- iw.addDocument(doc);
-
- doc = new Document();
- doc.add(new Field("body", "something", offsetsType));
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- int docID = searcher.search(new TermQuery(new Term("id", "id")), 1).scoreDocs[0].doc;
-
- Query query = new TermQuery(new Term("body", "highlighting"));
- int[] docIDs = new int[1];
- docIDs[0] = docID;
- String snippets[] = highlighter.highlightFields(new String[] {"body"}, query, searcher, docIDs, new int[] { 2 }).get("body");
- assertEquals(1, snippets.length);
- assertEquals(" ", snippets[0]);
-
- ir.close();
- dir.close();
- }
-
- public void testFieldIsEmptyString() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
-
- Document doc = new Document();
- doc.add(new Field("body", "", offsetsType));
- doc.add(new Field("id", "id", offsetsType));
- iw.addDocument(doc);
-
- doc = new Document();
- doc.add(new Field("body", "something", offsetsType));
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- int docID = searcher.search(new TermQuery(new Term("id", "id")), 1).scoreDocs[0].doc;
-
- Query query = new TermQuery(new Term("body", "highlighting"));
- int[] docIDs = new int[1];
- docIDs[0] = docID;
- String snippets[] = highlighter.highlightFields(new String[] {"body"}, query, searcher, docIDs, new int[] { 2 }).get("body");
- assertEquals(1, snippets.length);
- assertNull(snippets[0]);
-
- ir.close();
- dir.close();
- }
-
- public void testMultipleDocs() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
-
- int numDocs = atLeast(100);
- for(int i=0;i<numDocs;i++) {
- Document doc = new Document();
- String content = "the answer is " + i;
- if ((i & 1) == 0) {
- content += " some more terms";
- }
- doc.add(new Field("body", content, offsetsType));
- doc.add(newStringField("id", ""+i, Field.Store.YES));
- iw.addDocument(doc);
-
- if (random().nextInt(10) == 2) {
- iw.commit();
- }
- }
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- Query query = new TermQuery(new Term("body", "answer"));
- TopDocs hits = searcher.search(query, numDocs);
- assertEquals(numDocs, hits.totalHits);
-
- String snippets[] = highlighter.highlight("body", query, searcher, hits);
- assertEquals(numDocs, snippets.length);
- for(int hit=0;hit<numDocs;hit++) {
- Document doc = searcher.doc(hits.scoreDocs[hit].doc);
- int id = Integer.parseInt(doc.get("id"));
- String expected = "the <b>answer</b> is " + id;
- if ((id & 1) == 0) {
- expected += " some more terms";
- }
- assertEquals(expected, snippets[hit]);
- }
-
- ir.close();
- dir.close();
- }
-
- public void testMultipleSnippetSizes() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Field title = new Field("title", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
- doc.add(title);
-
- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");
- title.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- BooleanQuery.Builder query = new BooleanQuery.Builder();
- query.add(new TermQuery(new Term("body", "test")), BooleanClause.Occur.SHOULD);
- query.add(new TermQuery(new Term("title", "test")), BooleanClause.Occur.SHOULD);
- Map<String,String[]> snippets = highlighter.highlightFields(new String[] { "title", "body" }, query.build(), searcher, new int[] { 0 }, new int[] { 1, 2 });
- String titleHighlight = snippets.get("title")[0];
- String bodyHighlight = snippets.get("body")[0];
- assertEquals("This is a <b>test</b>. ", titleHighlight);
- assertEquals("This is a <b>test</b>. Just a <b>test</b> highlighting from postings. ", bodyHighlight);
- ir.close();
- dir.close();
- }
-
- public void testEncode() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test. Just a test highlighting from <i>postings</i>. Feel free to ignore.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected PassageFormatter getFormatter(String field) {
- return new DefaultPassageFormatter("<b>", "</b>", "... ", true);
- }
- };
- Query query = new TermQuery(new Term("body", "highlighting"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(1, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(1, snippets.length);
- assertEquals("Just a test <b>highlighting</b> from <i>postings</i>. ", snippets[0]);
-
- ir.close();
- dir.close();
- }
-
- /** customizing the gap separator to force a sentence break */
- public void testGapSeparator() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Document doc = new Document();
-
- Field body1 = new Field("body", "", offsetsType);
- body1.setStringValue("This is a multivalued field");
- doc.add(body1);
-
- Field body2 = new Field("body", "", offsetsType);
- body2.setStringValue("This is something different");
- doc.add(body2);
-
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected char getMultiValuedSeparator(String field) {
- assert field.equals("body");
- return '\u2029';
- }
- };
- Query query = new TermQuery(new Term("body", "field"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(1, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(1, snippets.length);
- assertEquals("This is a multivalued <b>field</b>\u2029", snippets[0]);
-
- ir.close();
- dir.close();
- }
-
- // LUCENE-4906
- public void testObjectFormatter() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected PassageFormatter getFormatter(String field) {
- return new PassageFormatter() {
- PassageFormatter defaultFormatter = new DefaultPassageFormatter();
-
- @Override
- public String[] format(Passage passages[], String content) {
- // Just turns the String snippet into a length 2
- // array of String
- return new String[] {"blah blah", defaultFormatter.format(passages, content).toString()};
- }
- };
- }
- };
-
- Query query = new TermQuery(new Term("body", "highlighting"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(1, topDocs.totalHits);
- int[] docIDs = new int[1];
- docIDs[0] = topDocs.scoreDocs[0].doc;
- Map<String,Object[]> snippets = highlighter.highlightFieldsAsObjects(new String[]{"body"}, query, searcher, docIDs, new int[] {1});
- Object[] bodySnippets = snippets.get("body");
- assertEquals(1, bodySnippets.length);
- assertTrue(Arrays.equals(new String[] {"blah blah", "Just a test <b>highlighting</b> from postings. "}, (String[]) bodySnippets[0]));
-
- ir.close();
- dir.close();
- }
-
- public void testFieldSometimesMissingFromSegment() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "foo", offsetsType);
- Document doc = new Document();
- doc.add(body);
- iw.addDocument(doc);
-
- // Make a 2nd segment where body is only stored:
- iw.commit();
- doc = new Document();
- doc.add(new StoredField("body", "foo"));
- iw.addDocument(doc);
-
- IndexReader ir = DirectoryReader.open(iw.w);
- iw.close();
-
- IndexSearcher searcher = new IndexSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- Query query = new MatchAllDocsQuery();
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("foo", snippets[0]);
- assertNull(snippets[1]);
- ir.close();
- dir.close();
- }
-
- public void testCustomScoreQueryHighlight() throws Exception{
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This piece of text refers to Kennedy at the beginning then has a longer piece of text that is very long in the middle and finally ends with another reference to Kennedy");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- TermQuery termQuery = new TermQuery(new Term("body", "very"));
- PostingsHighlighter highlighter = new PostingsHighlighter();
- CustomScoreQuery query = new CustomScoreQuery(termQuery);
-
- IndexSearcher searcher = newSearcher(ir);
- TopDocs hits = searcher.search(query, 10);
- assertEquals(1, hits.totalHits);
-
- String snippets[] = highlighter.highlight("body", query, searcher, hits);
- assertEquals(1, snippets.length);
- assertEquals("This piece of text refers to Kennedy at the beginning then has a longer piece of text that is <b>very</b> long in the middle and finally ends with another reference to Kennedy",
- snippets[0]);
-
- ir.close();
- dir.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java b/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java
deleted file mode 100644
index 7a693d9..0000000
--- a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java
+++ /dev/null
@@ -1,324 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.postingshighlight;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Random;
-
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.TestUtil;
-
-public class TestPostingsHighlighterRanking extends LuceneTestCase {
- /**
- * indexes a bunch of gibberish, and then highlights top(n).
- * asserts that top(n) highlights is a subset of top(n+1) up to some max N
- */
- // TODO: this only tests single-valued fields. we should also index multiple values per field!
- public void testRanking() throws Exception {
- // number of documents: we will check each one
- final int numDocs = atLeast(100);
- // number of top-N snippets, we will check 1 .. N
- final int maxTopN = 5;
- // maximum number of elements to put in a sentence.
- final int maxSentenceLength = 10;
- // maximum number of sentences in a document
- final int maxNumSentences = 20;
-
- Directory dir = newDirectory();
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- Document document = new Document();
- Field id = new StringField("id", "", Field.Store.NO);
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- document.add(id);
- document.add(body);
-
- for (int i = 0; i < numDocs; i++) {
- StringBuilder bodyText = new StringBuilder();
- int numSentences = TestUtil.nextInt(random(), 1, maxNumSentences);
- for (int j = 0; j < numSentences; j++) {
- bodyText.append(newSentence(random(), maxSentenceLength));
- }
- body.setStringValue(bodyText.toString());
- id.setStringValue(Integer.toString(i));
- iw.addDocument(document);
- }
-
- IndexReader ir = iw.getReader();
- IndexSearcher searcher = newSearcher(ir);
- for (int i = 0; i < numDocs; i++) {
- checkDocument(searcher, i, maxTopN);
- }
- iw.close();
- ir.close();
- dir.close();
- }
-
- private void checkDocument(IndexSearcher is, int doc, int maxTopN) throws IOException {
- for (int ch = 'a'; ch <= 'z'; ch++) {
- Term term = new Term("body", "" + (char)ch);
- // check a simple term query
- checkQuery(is, new TermQuery(term), doc, maxTopN);
- // check a boolean query
- BooleanQuery.Builder bq = new BooleanQuery.Builder();
- bq.add(new TermQuery(term), BooleanClause.Occur.SHOULD);
- Term nextTerm = new Term("body", "" + (char)(ch+1));
- bq.add(new TermQuery(nextTerm), BooleanClause.Occur.SHOULD);
- checkQuery(is, bq.build(), doc, maxTopN);
- }
- }
-
- private void checkQuery(IndexSearcher is, Query query, int doc, int maxTopN) throws IOException {
- for (int n = 1; n < maxTopN; n++) {
- final FakePassageFormatter f1 = new FakePassageFormatter();
- PostingsHighlighter p1 = new PostingsHighlighter(Integer.MAX_VALUE-1) {
- @Override
- protected PassageFormatter getFormatter(String field) {
- assertEquals("body", field);
- return f1;
- }
- };
-
- final FakePassageFormatter f2 = new FakePassageFormatter();
- PostingsHighlighter p2 = new PostingsHighlighter(Integer.MAX_VALUE-1) {
- @Override
- protected PassageFormatter getFormatter(String field) {
- assertEquals("body", field);
- return f2;
- }
- };
-
- BooleanQuery.Builder bq = new BooleanQuery.Builder();
- bq.add(query, BooleanClause.Occur.MUST);
- bq.add(new TermQuery(new Term("id", Integer.toString(doc))), BooleanClause.Occur.MUST);
- TopDocs td = is.search(bq.build(), 1);
- p1.highlight("body", bq.build(), is, td, n);
- p2.highlight("body", bq.build(), is, td, n+1);
- assertTrue(f2.seen.containsAll(f1.seen));
- }
- }
-
- /**
- * returns a new random sentence, up to maxSentenceLength "words" in length.
- * each word is a single character (a-z). The first one is capitalized.
- */
- private String newSentence(Random r, int maxSentenceLength) {
- StringBuilder sb = new StringBuilder();
- int numElements = TestUtil.nextInt(r, 1, maxSentenceLength);
- for (int i = 0; i < numElements; i++) {
- if (sb.length() > 0) {
- sb.append(' ');
- sb.append((char) TestUtil.nextInt(r, 'a', 'z'));
- } else {
- // capitalize the first word to help breakiterator
- sb.append((char) TestUtil.nextInt(r, 'A', 'Z'));
- }
- }
- sb.append(". "); // finalize sentence
- return sb.toString();
- }
-
- /**
- * a fake formatter that doesn't actually format passages.
- * instead it just collects them for asserts!
- */
- static class FakePassageFormatter extends PassageFormatter {
- HashSet<Pair> seen = new HashSet<>();
-
- @Override
- public String format(Passage passages[], String content) {
- for (Passage p : passages) {
- // verify some basics about the passage
- assertTrue(p.getScore() >= 0);
- assertTrue(p.getNumMatches() > 0);
- assertTrue(p.getStartOffset() >= 0);
- assertTrue(p.getStartOffset() <= content.length());
- assertTrue(p.getEndOffset() >= p.getStartOffset());
- assertTrue(p.getEndOffset() <= content.length());
- // we use a very simple analyzer. so we can assert the matches are correct
- int lastMatchStart = -1;
- for (int i = 0; i < p.getNumMatches(); i++) {
- BytesRef term = p.getMatchTerms()[i];
- int matchStart = p.getMatchStarts()[i];
- assertTrue(matchStart >= 0);
- // must at least start within the passage
- assertTrue(matchStart < p.getEndOffset());
- int matchEnd = p.getMatchEnds()[i];
- assertTrue(matchEnd >= 0);
- // always moving forward
- assertTrue(matchStart >= lastMatchStart);
- lastMatchStart = matchStart;
- // single character terms
- assertEquals(matchStart+1, matchEnd);
- // and the offsets must be correct...
- assertEquals(1, term.length);
- assertEquals((char)term.bytes[term.offset], Character.toLowerCase(content.charAt(matchStart)));
- }
- // record just the start/end offset for simplicity
- seen.add(new Pair(p.getStartOffset(), p.getEndOffset()));
- }
- return "bogus!!!!!!";
- }
- }
-
- static class Pair {
- final int start;
- final int end;
-
- Pair(int start, int end) {
- this.start = start;
- this.end = end;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + end;
- result = prime * result + start;
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- Pair other = (Pair) obj;
- if (end != other.end) {
- return false;
- }
- if (start != other.start) {
- return false;
- }
- return true;
- }
-
- @Override
- public String toString() {
- return "Pair [start=" + start + ", end=" + end + "]";
- }
- }
-
- /** sets b=0 to disable passage length normalization */
- public void testCustomB() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test. This test is a better test but the sentence is excruiatingly long, " +
- "you have no idea how painful it was for me to type this long sentence into my IDE.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter(10000) {
- @Override
- protected PassageScorer getScorer(String field) {
- return new PassageScorer(1.2f, 0, 87);
- }
- };
- Query query = new TermQuery(new Term("body", "test"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(1, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs, 1);
- assertEquals(1, snippets.length);
- assertTrue(snippets[0].startsWith("This <b>test</b> is a better <b>test</b>"));
-
- ir.close();
- dir.close();
- }
-
- /** sets k1=0 for simple coordinate-level match (# of query terms present) */
- public void testCustomK1() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This has only foo foo. " +
- "On the other hand this sentence contains both foo and bar. " +
- "This has only bar bar bar bar bar bar bar bar bar bar bar bar.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter(10000) {
- @Override
- protected PassageScorer getScorer(String field) {
- return new PassageScorer(0, 0.75f, 87);
- }
- };
- BooleanQuery.Builder query = new BooleanQuery.Builder();
- query.add(new TermQuery(new Term("body", "foo")), BooleanClause.Occur.SHOULD);
- query.add(new TermQuery(new Term("body", "bar")), BooleanClause.Occur.SHOULD);
- TopDocs topDocs = searcher.search(query.build(), 10, Sort.INDEXORDER);
- assertEquals(1, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query.build(), searcher, topDocs, 1);
- assertEquals(1, snippets.length);
- assertTrue(snippets[0].startsWith("On the other hand"));
-
- ir.close();
- dir.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/LengthGoalBreakIteratorTest.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/LengthGoalBreakIteratorTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/LengthGoalBreakIteratorTest.java
index 4dd30e2..2434f52 100644
--- a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/LengthGoalBreakIteratorTest.java
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/LengthGoalBreakIteratorTest.java
@@ -23,7 +23,6 @@ import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.postingshighlight.CustomSeparatorBreakIterator;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.QueryBuilder;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestCustomSeparatorBreakIterator.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestCustomSeparatorBreakIterator.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestCustomSeparatorBreakIterator.java
new file mode 100644
index 0000000..cacb9cb
--- /dev/null
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestCustomSeparatorBreakIterator.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search.uhighlight;
+
+import java.text.BreakIterator;
+import java.util.Locale;
+
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import org.apache.lucene.util.LuceneTestCase;
+
+import static org.apache.lucene.search.uhighlight.TestWholeBreakIterator.assertSameBreaks;
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class TestCustomSeparatorBreakIterator extends LuceneTestCase {
+
+ private static final Character[] SEPARATORS = new Character[]{' ', '\u0000', 8233};
+
+ public void testBreakOnCustomSeparator() throws Exception {
+ Character separator = randomSeparator();
+ BreakIterator bi = new CustomSeparatorBreakIterator(separator);
+ String source = "this" + separator + "is" + separator + "the" + separator + "first" + separator + "sentence";
+ bi.setText(source);
+ assertThat(bi.current(), equalTo(0));
+ assertThat(bi.first(), equalTo(0));
+ assertThat(source.substring(bi.current(), bi.next()), equalTo("this" + separator));
+ assertThat(source.substring(bi.current(), bi.next()), equalTo("is" + separator));
+ assertThat(source.substring(bi.current(), bi.next()), equalTo("the" + separator));
+ assertThat(source.substring(bi.current(), bi.next()), equalTo("first" + separator));
+ assertThat(source.substring(bi.current(), bi.next()), equalTo("sentence"));
+ assertThat(bi.next(), equalTo(BreakIterator.DONE));
+
+ assertThat(bi.last(), equalTo(source.length()));
+ int current = bi.current();
+ assertThat(source.substring(bi.previous(), current), equalTo("sentence"));
+ current = bi.current();
+ assertThat(source.substring(bi.previous(), current), equalTo("first" + separator));
+ current = bi.current();
+ assertThat(source.substring(bi.previous(), current), equalTo("the" + separator));
+ current = bi.current();
+ assertThat(source.substring(bi.previous(), current), equalTo("is" + separator));
+ current = bi.current();
+ assertThat(source.substring(bi.previous(), current), equalTo("this" + separator));
+ assertThat(bi.previous(), equalTo(BreakIterator.DONE));
+ assertThat(bi.current(), equalTo(0));
+
+ assertThat(source.substring(0, bi.following(9)), equalTo("this" + separator + "is" + separator + "the" + separator));
+
+ assertThat(source.substring(0, bi.preceding(9)), equalTo("this" + separator + "is" + separator));
+
+ assertThat(bi.first(), equalTo(0));
+ assertThat(source.substring(0, bi.next(3)), equalTo("this" + separator + "is" + separator + "the" + separator));
+ }
+
+ public void testSingleSentences() throws Exception {
+ BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
+ BreakIterator actual = new CustomSeparatorBreakIterator(randomSeparator());
+ assertSameBreaks("a", expected, actual);
+ assertSameBreaks("ab", expected, actual);
+ assertSameBreaks("abc", expected, actual);
+ assertSameBreaks("", expected, actual);
+ }
+
+ public void testSliceEnd() throws Exception {
+ BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
+ BreakIterator actual = new CustomSeparatorBreakIterator(randomSeparator());
+ assertSameBreaks("a000", 0, 1, expected, actual);
+ assertSameBreaks("ab000", 0, 1, expected, actual);
+ assertSameBreaks("abc000", 0, 1, expected, actual);
+ assertSameBreaks("000", 0, 0, expected, actual);
+ }
+
+ public void testSliceStart() throws Exception {
+ BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
+ BreakIterator actual = new CustomSeparatorBreakIterator(randomSeparator());
+ assertSameBreaks("000a", 3, 1, expected, actual);
+ assertSameBreaks("000ab", 3, 2, expected, actual);
+ assertSameBreaks("000abc", 3, 3, expected, actual);
+ assertSameBreaks("000", 3, 0, expected, actual);
+ }
+
+ public void testSliceMiddle() throws Exception {
+ BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
+ BreakIterator actual = new CustomSeparatorBreakIterator(randomSeparator());
+ assertSameBreaks("000a000", 3, 1, expected, actual);
+ assertSameBreaks("000ab000", 3, 2, expected, actual);
+ assertSameBreaks("000abc000", 3, 3, expected, actual);
+ assertSameBreaks("000000", 3, 0, expected, actual);
+ }
+
+ /** the current position must be ignored, initial position is always first() */
+ public void testFirstPosition() throws Exception {
+ BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
+ BreakIterator actual = new CustomSeparatorBreakIterator(randomSeparator());
+ assertSameBreaks("000ab000", 3, 2, 4, expected, actual);
+ }
+
+ private static char randomSeparator() {
+ return RandomPicks.randomFrom(random(), SEPARATORS);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.java
index ddf8a92..3ffe95a 100644
--- a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.java
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.java
@@ -49,7 +49,6 @@ import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.postingshighlight.WholeBreakIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestWholeBreakIterator.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestWholeBreakIterator.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestWholeBreakIterator.java
new file mode 100644
index 0000000..8e5b2ff
--- /dev/null
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestWholeBreakIterator.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search.uhighlight;
+
+import java.text.BreakIterator;
+import java.text.CharacterIterator;
+import java.text.StringCharacterIterator;
+import java.util.Locale;
+
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestWholeBreakIterator extends LuceneTestCase {
+
+ /** For single sentences, we know WholeBreakIterator should break the same as a sentence iterator */
+ public void testSingleSentences() throws Exception {
+ BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
+ BreakIterator actual = new WholeBreakIterator();
+ assertSameBreaks("a", expected, actual);
+ assertSameBreaks("ab", expected, actual);
+ assertSameBreaks("abc", expected, actual);
+ assertSameBreaks("", expected, actual);
+ }
+
+ public void testSliceEnd() throws Exception {
+ BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
+ BreakIterator actual = new WholeBreakIterator();
+ assertSameBreaks("a000", 0, 1, expected, actual);
+ assertSameBreaks("ab000", 0, 1, expected, actual);
+ assertSameBreaks("abc000", 0, 1, expected, actual);
+ assertSameBreaks("000", 0, 0, expected, actual);
+ }
+
+ public void testSliceStart() throws Exception {
+ BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
+ BreakIterator actual = new WholeBreakIterator();
+ assertSameBreaks("000a", 3, 1, expected, actual);
+ assertSameBreaks("000ab", 3, 2, expected, actual);
+ assertSameBreaks("000abc", 3, 3, expected, actual);
+ assertSameBreaks("000", 3, 0, expected, actual);
+ }
+
+ public void testSliceMiddle() throws Exception {
+ BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
+ BreakIterator actual = new WholeBreakIterator();
+ assertSameBreaks("000a000", 3, 1, expected, actual);
+ assertSameBreaks("000ab000", 3, 2, expected, actual);
+ assertSameBreaks("000abc000", 3, 3, expected, actual);
+ assertSameBreaks("000000", 3, 0, expected, actual);
+ }
+
+ /** the current position must be ignored, initial position is always first() */
+ public void testFirstPosition() throws Exception {
+ BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
+ BreakIterator actual = new WholeBreakIterator();
+ assertSameBreaks("000ab000", 3, 2, 4, expected, actual);
+ }
+
+ public static void assertSameBreaks(String text, BreakIterator expected, BreakIterator actual) {
+ assertSameBreaks(new StringCharacterIterator(text),
+ new StringCharacterIterator(text),
+ expected,
+ actual);
+ }
+
+ public static void assertSameBreaks(String text, int offset, int length, BreakIterator expected, BreakIterator actual) {
+ assertSameBreaks(text, offset, length, offset, expected, actual);
+ }
+
+ public static void assertSameBreaks(String text, int offset, int length, int current, BreakIterator expected, BreakIterator actual) {
+ assertSameBreaks(new StringCharacterIterator(text, offset, offset+length, current),
+ new StringCharacterIterator(text, offset, offset+length, current),
+ expected,
+ actual);
+ }
+
+ /** Asserts that two breakiterators break the text the same way */
+ public static void assertSameBreaks(CharacterIterator one, CharacterIterator two, BreakIterator expected, BreakIterator actual) {
+ expected.setText(one);
+ actual.setText(two);
+
+ assertEquals(expected.current(), actual.current());
+
+ // next()
+ int v = expected.current();
+ while (v != BreakIterator.DONE) {
+ assertEquals(v = expected.next(), actual.next());
+ assertEquals(expected.current(), actual.current());
+ }
+
+ // first()
+ assertEquals(expected.first(), actual.first());
+ assertEquals(expected.current(), actual.current());
+ // last()
+ assertEquals(expected.last(), actual.last());
+ assertEquals(expected.current(), actual.current());
+
+ // previous()
+ v = expected.current();
+ while (v != BreakIterator.DONE) {
+ assertEquals(v = expected.previous(), actual.previous());
+ assertEquals(expected.current(), actual.current());
+ }
+
+ // following()
+ for (int i = one.getBeginIndex(); i <= one.getEndIndex(); i++) {
+ expected.first();
+ actual.first();
+ assertEquals(expected.following(i), actual.following(i));
+ assertEquals(expected.current(), actual.current());
+ }
+
+ // preceding()
+ for (int i = one.getBeginIndex(); i <= one.getEndIndex(); i++) {
+ expected.last();
+ actual.last();
+ assertEquals(expected.preceding(i), actual.preceding(i));
+ assertEquals(expected.current(), actual.current());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java b/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java
index e9c842c..bd32fc1 100644
--- a/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java
+++ b/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java
@@ -28,13 +28,13 @@ import java.util.function.Predicate;
import org.apache.lucene.document.Document;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.postingshighlight.CustomSeparatorBreakIterator;
-import org.apache.lucene.search.postingshighlight.WholeBreakIterator;
+import org.apache.lucene.search.uhighlight.CustomSeparatorBreakIterator;
import org.apache.lucene.search.uhighlight.DefaultPassageFormatter;
import org.apache.lucene.search.uhighlight.LengthGoalBreakIterator;
import org.apache.lucene.search.uhighlight.PassageFormatter;
import org.apache.lucene.search.uhighlight.PassageScorer;
import org.apache.lucene.search.uhighlight.UnifiedHighlighter;
+import org.apache.lucene.search.uhighlight.WholeBreakIterator;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.HighlightParams;
import org.apache.solr.common.params.SolrParams;
[38/44] lucene-solr:jira/solr-8668: SOLR-10746: Move all Stream
Evaluators to the eval package
Posted by cp...@apache.org.
SOLR-10746: Move all Stream Evaluators to the eval package
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7fef7e33
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7fef7e33
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7fef7e33
Branch: refs/heads/jira/solr-8668
Commit: 7fef7e3374da8ea79616d56d583465dea7ead20b
Parents: 6d80320
Author: Joel Bernstein <jb...@apache.org>
Authored: Thu May 25 10:32:28 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Thu May 25 10:32:48 2017 -0400
----------------------------------------------------------------------
.../client/solrj/io/eval/ColumnEvaluator.java | 76 +++++++++++
.../solrj/io/eval/ConvolutionEvaluator.java | 80 ++++++++++++
.../client/solrj/io/eval/CopyOfEvaluator.java | 82 ++++++++++++
.../solrj/io/eval/CopyOfRangeEvaluator.java | 83 ++++++++++++
.../solrj/io/eval/CorrelationEvaluator.java | 75 +++++++++++
.../solrj/io/eval/CovarianceEvaluator.java | 75 +++++++++++
.../client/solrj/io/eval/DescribeEvaluator.java | 90 +++++++++++++
.../client/solrj/io/eval/DistanceEvaluator.java | 75 +++++++++++
.../io/eval/EmpiricalDistributionEvaluator.java | 127 ++++++++++++++++++
.../solrj/io/eval/FindDelayEvaluator.java | 90 +++++++++++++
.../client/solrj/io/eval/LengthEvaluator.java | 58 +++++++++
.../solrj/io/eval/NormalizeEvaluator.java | 74 +++++++++++
.../solrj/io/eval/PercentileEvaluator.java | 66 ++++++++++
.../client/solrj/io/eval/PredictEvaluator.java | 66 ++++++++++
.../client/solrj/io/eval/RankEvaluator.java | 73 +++++++++++
.../solrj/io/eval/RegressionEvaluator.java | 110 ++++++++++++++++
.../client/solrj/io/eval/ReverseEvaluator.java | 65 ++++++++++
.../client/solrj/io/eval/ScaleEvaluator.java | 76 +++++++++++
.../client/solrj/io/stream/ColumnEvaluator.java | 77 -----------
.../solrj/io/stream/ConvolutionEvaluator.java | 82 ------------
.../client/solrj/io/stream/CopyOfEvaluator.java | 84 ------------
.../solrj/io/stream/CopyOfRangeEvaluator.java | 85 ------------
.../solrj/io/stream/CorrelationEvaluator.java | 77 -----------
.../solrj/io/stream/CovarianceEvaluator.java | 77 -----------
.../solrj/io/stream/DescribeEvaluator.java | 92 -------------
.../solrj/io/stream/DistanceEvaluator.java | 77 -----------
.../stream/EmpiricalDistributionEvaluator.java | 129 -------------------
.../solrj/io/stream/FindDelayEvaluator.java | 92 -------------
.../client/solrj/io/stream/LengthEvaluator.java | 60 ---------
.../solrj/io/stream/NormalizeEvaluator.java | 76 -----------
.../solrj/io/stream/PercentileEvaluator.java | 68 ----------
.../solrj/io/stream/PredictEvaluator.java | 68 ----------
.../client/solrj/io/stream/RankEvaluator.java | 75 -----------
.../solrj/io/stream/RegressionEvaluator.java | 112 ----------------
.../solrj/io/stream/ReverseEvaluator.java | 67 ----------
.../client/solrj/io/stream/ScaleEvaluator.java | 78 -----------
36 files changed, 1441 insertions(+), 1476 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ColumnEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ColumnEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ColumnEvaluator.java
new file mode 100644
index 0000000..dbb17b0
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ColumnEvaluator.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
+import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class ColumnEvaluator extends SimpleEvaluator implements Expressible {
+
+ private static final long serialVersionUID = 1;
+ private String name;
+ private String fieldName;
+ ;
+ public ColumnEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ String name = factory.getValueOperand(expression, 0);
+ String fieldName = factory.getValueOperand(expression, 1);
+ init(name, fieldName);
+ }
+
+ private void init(String name, String fieldName) {
+ this.name = name;
+ this.fieldName = fieldName;
+ }
+
+ public List<Number> evaluate(Tuple tuple) throws IOException {
+ List<Tuple> tuples = (List<Tuple>)tuple.get(name);
+ List<Number> column = new ArrayList(tuples.size());
+ for(Tuple t : tuples) {
+ Object o = t.get(fieldName);
+ if(o instanceof Number) {
+ column.add((Number)o);
+ } else {
+ throw new IOException("Found non-numeric in column:"+o.toString());
+ }
+ }
+ return column;
+ }
+
+ @Override
+ public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
+ StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
+ return expression;
+ }
+
+ @Override
+ public Explanation toExplanation(StreamFactory factory) throws IOException {
+ return new Explanation(nodeId.toString())
+ .withExpressionType(ExpressionType.EVALUATOR)
+ .withFunctionName(factory.getFunctionName(getClass()))
+ .withImplementingClass(getClass().getName())
+ .withExpression(toExpression(factory).toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ConvolutionEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ConvolutionEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ConvolutionEvaluator.java
new file mode 100644
index 0000000..6ca178d
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ConvolutionEvaluator.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.ArrayList;
+
+import org.apache.commons.math3.util.MathArrays;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
+import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class ConvolutionEvaluator extends ComplexEvaluator implements Expressible {
+
+ private static final long serialVersionUID = 1;
+
+ public ConvolutionEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ public List<Number> evaluate(Tuple tuple) throws IOException {
+ StreamEvaluator colEval1 = subEvaluators.get(0);
+ StreamEvaluator colEval2 = subEvaluators.get(1);
+
+ List<Number> numbers1 = (List<Number>)colEval1.evaluate(tuple);
+ List<Number> numbers2 = (List<Number>)colEval2.evaluate(tuple);
+ double[] column1 = new double[numbers1.size()];
+ double[] column2 = new double[numbers2.size()];
+
+ for(int i=0; i<numbers1.size(); i++) {
+ column1[i] = numbers1.get(i).doubleValue();
+ }
+
+ for(int i=0; i<numbers2.size(); i++) {
+ column2[i] = numbers2.get(i).doubleValue();
+ }
+
+ double[] conArray = MathArrays.convolve(column1, column2);
+ List<Number> conList = new ArrayList();
+ for(double d :conArray) {
+ conList.add(d);
+ }
+
+ return conList;
+ }
+
+ @Override
+ public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
+ StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
+ return expression;
+ }
+
+ @Override
+ public Explanation toExplanation(StreamFactory factory) throws IOException {
+ return new Explanation(nodeId.toString())
+ .withExpressionType(ExpressionType.EVALUATOR)
+ .withFunctionName(factory.getFunctionName(getClass()))
+ .withImplementingClass(getClass().getName())
+ .withExpression(toExpression(factory).toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CopyOfEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CopyOfEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CopyOfEvaluator.java
new file mode 100644
index 0000000..d379c41
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CopyOfEvaluator.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
+import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class CopyOfEvaluator extends ComplexEvaluator implements Expressible {
+
+ private static final long serialVersionUID = 1;
+
+ public CopyOfEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ public List<Number> evaluate(Tuple tuple) throws IOException {
+ StreamEvaluator colEval1 = subEvaluators.get(0);
+
+ List<Number> numbers1 = (List<Number>)colEval1.evaluate(tuple);
+ double[] vals = new double[numbers1.size()];
+
+ for(int i=0; i<vals.length; i++) {
+ vals[i] = numbers1.get(i).doubleValue();
+ }
+
+ if(subEvaluators.size() == 2) {
+ StreamEvaluator lengthEval = subEvaluators.get(1);
+ Number lengthNum = (Number)lengthEval.evaluate(tuple);
+ int length = lengthNum.intValue();
+ vals = Arrays.copyOf(vals, length);
+ } else {
+ vals = Arrays.copyOf(vals, vals.length);
+ }
+
+ List<Number> copyOf = new ArrayList(vals.length);
+
+ for(int i=0; i<vals.length; i++) {
+ copyOf.add(vals[i]);
+ }
+
+ return copyOf;
+ }
+
+ @Override
+ public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
+ StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
+ return expression;
+ }
+
+ @Override
+ public Explanation toExplanation(StreamFactory factory) throws IOException {
+ return new Explanation(nodeId.toString())
+ .withExpressionType(ExpressionType.EVALUATOR)
+ .withFunctionName(factory.getFunctionName(getClass()))
+ .withImplementingClass(getClass().getName())
+ .withExpression(toExpression(factory).toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CopyOfRangeEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CopyOfRangeEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CopyOfRangeEvaluator.java
new file mode 100644
index 0000000..f1e56dd
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CopyOfRangeEvaluator.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
+import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class CopyOfRangeEvaluator extends ComplexEvaluator implements Expressible {
+
+ private static final long serialVersionUID = 1;
+
+ public CopyOfRangeEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ public List<Number> evaluate(Tuple tuple) throws IOException {
+ StreamEvaluator colEval1 = subEvaluators.get(0);
+
+ List<Number> numbers1 = (List<Number>)colEval1.evaluate(tuple);
+ double[] vals = new double[numbers1.size()];
+
+ for(int i=0; i<vals.length; i++) {
+ vals[i] = numbers1.get(i).doubleValue();
+ }
+
+ StreamEvaluator startIndexEval = subEvaluators.get(1);
+ Number startIndexNum = (Number)startIndexEval.evaluate(tuple);
+ int startIndex = startIndexNum.intValue();
+
+ StreamEvaluator endIndexEval = subEvaluators.get(2);
+ Number endIndexNum = (Number)endIndexEval.evaluate(tuple);
+ int endIndex = endIndexNum.intValue();
+
+ vals = Arrays.copyOfRange(vals, startIndex, endIndex);
+
+ List<Number> copyOf = new ArrayList(vals.length);
+
+ for(int i=0; i<vals.length; i++) {
+ copyOf.add(vals[i]);
+ }
+
+ return copyOf;
+ }
+
+ @Override
+ public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
+ StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
+ return expression;
+ }
+
+ @Override
+ public Explanation toExplanation(StreamFactory factory) throws IOException {
+ return new Explanation(nodeId.toString())
+ .withExpressionType(ExpressionType.EVALUATOR)
+ .withFunctionName(factory.getFunctionName(getClass()))
+ .withImplementingClass(getClass().getName())
+ .withExpression(toExpression(factory).toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CorrelationEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CorrelationEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CorrelationEvaluator.java
new file mode 100644
index 0000000..fc3d8c3
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CorrelationEvaluator.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.math3.stat.correlation.PearsonsCorrelation;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
+import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class CorrelationEvaluator extends ComplexEvaluator implements Expressible {
+
+ private static final long serialVersionUID = 1;
+
+ public CorrelationEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ public Number evaluate(Tuple tuple) throws IOException {
+ StreamEvaluator colEval1 = subEvaluators.get(0);
+ StreamEvaluator colEval2 = subEvaluators.get(1);
+
+ List<Number> numbers1 = (List<Number>)colEval1.evaluate(tuple);
+ List<Number> numbers2 = (List<Number>)colEval2.evaluate(tuple);
+ double[] column1 = new double[numbers1.size()];
+ double[] column2 = new double[numbers2.size()];
+
+ for(int i=0; i<numbers1.size(); i++) {
+ column1[i] = numbers1.get(i).doubleValue();
+ }
+
+ for(int i=0; i<numbers2.size(); i++) {
+ column2[i] = numbers2.get(i).doubleValue();
+ }
+
+ PearsonsCorrelation pearsonsCorrelation = new PearsonsCorrelation();
+
+ return pearsonsCorrelation.correlation(column1, column2);
+ }
+
+ @Override
+ public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
+ StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
+ return expression;
+ }
+
+ @Override
+ public Explanation toExplanation(StreamFactory factory) throws IOException {
+ return new Explanation(nodeId.toString())
+ .withExpressionType(ExpressionType.EVALUATOR)
+ .withFunctionName(factory.getFunctionName(getClass()))
+ .withImplementingClass(getClass().getName())
+ .withExpression(toExpression(factory).toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CovarianceEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CovarianceEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CovarianceEvaluator.java
new file mode 100644
index 0000000..7a6de68
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CovarianceEvaluator.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.math3.stat.correlation.Covariance;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
+import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class CovarianceEvaluator extends ComplexEvaluator implements Expressible {
+
+ private static final long serialVersionUID = 1;
+
+ public CovarianceEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ public Number evaluate(Tuple tuple) throws IOException {
+ StreamEvaluator colEval1 = subEvaluators.get(0);
+ StreamEvaluator colEval2 = subEvaluators.get(1);
+
+ List<Number> numbers1 = (List<Number>)colEval1.evaluate(tuple);
+ List<Number> numbers2 = (List<Number>)colEval2.evaluate(tuple);
+ double[] column1 = new double[numbers1.size()];
+ double[] column2 = new double[numbers2.size()];
+
+ for(int i=0; i<numbers1.size(); i++) {
+ column1[i] = numbers1.get(i).doubleValue();
+ }
+
+ for(int i=0; i<numbers2.size(); i++) {
+ column2[i] = numbers2.get(i).doubleValue();
+ }
+
+ Covariance covariance = new Covariance();
+
+ return covariance.covariance(column1, column2);
+ }
+
+ @Override
+ public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
+ StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
+ return expression;
+ }
+
+ @Override
+ public Explanation toExplanation(StreamFactory factory) throws IOException {
+ return new Explanation(nodeId.toString())
+ .withExpressionType(ExpressionType.EVALUATOR)
+ .withFunctionName(factory.getFunctionName(getClass()))
+ .withImplementingClass(getClass().getName())
+ .withExpression(toExpression(factory).toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/DescribeEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/DescribeEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/DescribeEvaluator.java
new file mode 100644
index 0000000..196afe5
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/DescribeEvaluator.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
+import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class DescribeEvaluator extends ComplexEvaluator implements Expressible {
+
+ private static final long serialVersionUID = 1;
+
+ public DescribeEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ public Tuple evaluate(Tuple tuple) throws IOException {
+
+ if(subEvaluators.size() != 1) {
+ throw new IOException("describe expects 1 column as a parameters");
+ }
+
+ StreamEvaluator colEval = subEvaluators.get(0);
+
+ List<Number> numbers = (List<Number>)colEval.evaluate(tuple);
+ DescriptiveStatistics descriptiveStatistics = new DescriptiveStatistics();
+
+ for(Number n : numbers) {
+ descriptiveStatistics.addValue(n.doubleValue());
+ }
+
+
+ Map map = new HashMap();
+
+ map.put("max", descriptiveStatistics.getMax());
+ map.put("mean", descriptiveStatistics.getMean());
+ map.put("min", descriptiveStatistics.getMin());
+ map.put("stdev", descriptiveStatistics.getStandardDeviation());
+ map.put("sum", descriptiveStatistics.getSum());
+ map.put("N", descriptiveStatistics.getN());
+ map.put("var", descriptiveStatistics.getVariance());
+ map.put("kurtosis", descriptiveStatistics.getKurtosis());
+ map.put("skewness", descriptiveStatistics.getSkewness());
+ map.put("popVar", descriptiveStatistics.getPopulationVariance());
+ map.put("geometricMean", descriptiveStatistics.getGeometricMean());
+ map.put("sumsq", descriptiveStatistics.getSumsq());
+
+ return new Tuple(map);
+ }
+
+
+ @Override
+ public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
+ StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
+ return expression;
+ }
+
+ @Override
+ public Explanation toExplanation(StreamFactory factory) throws IOException {
+ return new Explanation(nodeId.toString())
+ .withExpressionType(ExpressionType.EVALUATOR)
+ .withFunctionName(factory.getFunctionName(getClass()))
+ .withImplementingClass(getClass().getName())
+ .withExpression(toExpression(factory).toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/DistanceEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/DistanceEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/DistanceEvaluator.java
new file mode 100644
index 0000000..201da4b
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/DistanceEvaluator.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.math3.ml.distance.EuclideanDistance;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
+import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class DistanceEvaluator extends ComplexEvaluator implements Expressible {
+
+ private static final long serialVersionUID = 1;
+
+ public DistanceEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ public Number evaluate(Tuple tuple) throws IOException {
+ StreamEvaluator colEval1 = subEvaluators.get(0);
+ StreamEvaluator colEval2 = subEvaluators.get(1);
+
+ List<Number> numbers1 = (List<Number>)colEval1.evaluate(tuple);
+ List<Number> numbers2 = (List<Number>)colEval2.evaluate(tuple);
+ double[] column1 = new double[numbers1.size()];
+ double[] column2 = new double[numbers2.size()];
+
+ for(int i=0; i<numbers1.size(); i++) {
+ column1[i] = numbers1.get(i).doubleValue();
+ }
+
+ for(int i=0; i<numbers2.size(); i++) {
+ column2[i] = numbers2.get(i).doubleValue();
+ }
+
+ EuclideanDistance distance = new EuclideanDistance();
+ return distance.compute(column1, column2);
+ }
+
+ @Override
+ public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
+ StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
+ return expression;
+ }
+
+ @Override
+ public Explanation toExplanation(StreamFactory factory) throws IOException {
+ return new Explanation(nodeId.toString())
+ .withExpressionType(ExpressionType.EVALUATOR)
+ .withFunctionName(factory.getFunctionName(getClass()))
+ .withImplementingClass(getClass().getName())
+ .withExpression(toExpression(factory).toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/EmpiricalDistributionEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/EmpiricalDistributionEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/EmpiricalDistributionEvaluator.java
new file mode 100644
index 0000000..6885352
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/EmpiricalDistributionEvaluator.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Arrays;
+
+import org.apache.commons.math3.random.EmpiricalDistribution;
+import org.apache.commons.math3.stat.descriptive.StatisticalSummary;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
+import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class EmpiricalDistributionEvaluator extends ComplexEvaluator implements Expressible {
+
+ private static final long serialVersionUID = 1;
+
+ public EmpiricalDistributionEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ public Tuple evaluate(Tuple tuple) throws IOException {
+
+ if(subEvaluators.size() != 1) {
+ throw new IOException("Empirical dist expects 1 column as a parameters");
+ }
+
+ StreamEvaluator colEval1 = subEvaluators.get(0);
+
+ List<Number> numbers1 = (List<Number>)colEval1.evaluate(tuple);
+ double[] column1 = new double[numbers1.size()];
+
+ for(int i=0; i<numbers1.size(); i++) {
+ column1[i] = numbers1.get(i).doubleValue();
+ }
+
+ Arrays.sort(column1);
+ EmpiricalDistribution empiricalDistribution = new EmpiricalDistribution();
+ empiricalDistribution.load(column1);
+
+ Map map = new HashMap();
+ StatisticalSummary statisticalSummary = empiricalDistribution.getSampleStats();
+
+ map.put("max", statisticalSummary.getMax());
+ map.put("mean", statisticalSummary.getMean());
+ map.put("min", statisticalSummary.getMin());
+ map.put("stdev", statisticalSummary.getStandardDeviation());
+ map.put("sum", statisticalSummary.getSum());
+ map.put("N", statisticalSummary.getN());
+ map.put("var", statisticalSummary.getVariance());
+
+ return new EmpiricalDistributionTuple(empiricalDistribution, column1, map);
+ }
+
+ public static class EmpiricalDistributionTuple extends Tuple {
+
+ private EmpiricalDistribution empiricalDistribution;
+ private double[] backingArray;
+
+ public EmpiricalDistributionTuple(EmpiricalDistribution empiricalDistribution, double[] backingArray, Map map) {
+ super(map);
+ this.empiricalDistribution = empiricalDistribution;
+ this.backingArray = backingArray;
+ }
+
+ public double percentile(double d) {
+ int slot = Arrays.binarySearch(backingArray, d);
+
+ if(slot == 0) {
+ return 0.0;
+ }
+
+ if(slot < 0) {
+ if(slot == -1) {
+ return 0.0D;
+ } else {
+ //Not a direct hit
+ slot = Math.abs(slot);
+ --slot;
+ if(slot == backingArray.length) {
+ return 1.0D;
+ } else {
+ return (this.empiricalDistribution.cumulativeProbability(backingArray[slot]));
+ }
+ }
+ } else {
+ return this.empiricalDistribution.cumulativeProbability(backingArray[slot]);
+ }
+ }
+ }
+
+ @Override
+ public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
+ StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
+ return expression;
+ }
+
+ @Override
+ public Explanation toExplanation(StreamFactory factory) throws IOException {
+ return new Explanation(nodeId.toString())
+ .withExpressionType(ExpressionType.EVALUATOR)
+ .withFunctionName(factory.getFunctionName(getClass()))
+ .withImplementingClass(getClass().getName())
+ .withExpression(toExpression(factory).toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/FindDelayEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/FindDelayEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/FindDelayEvaluator.java
new file mode 100644
index 0000000..c5a9b8a
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/FindDelayEvaluator.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.math3.util.MathArrays;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
+import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class FindDelayEvaluator extends ComplexEvaluator implements Expressible {
+
+ private static final long serialVersionUID = 1;
+
+ public FindDelayEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ public Number evaluate(Tuple tuple) throws IOException {
+ StreamEvaluator colEval1 = subEvaluators.get(0);
+ StreamEvaluator colEval2 = subEvaluators.get(1);
+
+ List<Number> numbers1 = (List<Number>)colEval1.evaluate(tuple);
+ List<Number> numbers2 = (List<Number>)colEval2.evaluate(tuple);
+ double[] column1 = new double[numbers1.size()];
+ double[] column2 = new double[numbers2.size()];
+
+ for(int i=0; i<numbers1.size(); i++) {
+ column1[i] = numbers1.get(i).doubleValue();
+ }
+
+ //Reverse the second column.
+ //The convolve function will reverse it back.
+ //This allows correlation to be represented using the convolution math.
+ int rIndex=0;
+ for(int i=numbers2.size()-1; i>=0; i--) {
+ column2[rIndex++] = numbers2.get(i).doubleValue();
+ }
+
+ double[] convolution = MathArrays.convolve(column1, column2);
+ double max = -Double.MAX_VALUE;
+ double maxIndex = -1;
+
+ for(int i=0; i< convolution.length; i++) {
+ double abs = Math.abs(convolution[i]);
+ if(abs > max) {
+ max = abs;
+ maxIndex = i;
+ }
+ }
+
+ return (maxIndex+1)-column2.length;
+ }
+
+ @Override
+ public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
+ StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
+ return expression;
+ }
+
+ @Override
+ public Explanation toExplanation(StreamFactory factory) throws IOException {
+ return new Explanation(nodeId.toString())
+ .withExpressionType(ExpressionType.EVALUATOR)
+ .withFunctionName(factory.getFunctionName(getClass()))
+ .withImplementingClass(getClass().getName())
+ .withExpression(toExpression(factory).toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LengthEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LengthEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LengthEvaluator.java
new file mode 100644
index 0000000..da55ee4
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LengthEvaluator.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
+import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class LengthEvaluator extends ComplexEvaluator implements Expressible {
+
+ private static final long serialVersionUID = 1;
+
+ public LengthEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ public Number evaluate(Tuple tuple) throws IOException {
+ StreamEvaluator colEval1 = subEvaluators.get(0);
+ List<Number> numbers = (List<Number>)colEval1.evaluate(tuple);
+ return numbers.size();
+ }
+
+ @Override
+ public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
+ StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
+ return expression;
+ }
+
+ @Override
+ public Explanation toExplanation(StreamFactory factory) throws IOException {
+ return new Explanation(nodeId.toString())
+ .withExpressionType(ExpressionType.EVALUATOR)
+ .withFunctionName(factory.getFunctionName(getClass()))
+ .withImplementingClass(getClass().getName())
+ .withExpression(toExpression(factory).toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/NormalizeEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/NormalizeEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/NormalizeEvaluator.java
new file mode 100644
index 0000000..c85ac20
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/NormalizeEvaluator.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.math3.stat.StatUtils;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
+import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class NormalizeEvaluator extends ComplexEvaluator implements Expressible {
+
+ private static final long serialVersionUID = 1;
+
+ public NormalizeEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ public List<Number> evaluate(Tuple tuple) throws IOException {
+ StreamEvaluator colEval1 = subEvaluators.get(0);
+
+ List<Number> numbers1 = (List<Number>)colEval1.evaluate(tuple);
+ double[] column1 = new double[numbers1.size()];
+
+ for(int i=0; i<numbers1.size(); i++) {
+ column1[i] = numbers1.get(i).doubleValue();
+ }
+
+ double[] normalized = StatUtils.normalize(column1);
+
+ List<Number> normalizeList = new ArrayList(normalized.length);
+ for(double d : normalized) {
+ normalizeList.add(d);
+ }
+
+ return normalizeList;
+ }
+
+ @Override
+ public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
+ StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
+ return expression;
+ }
+
+ @Override
+ public Explanation toExplanation(StreamFactory factory) throws IOException {
+ return new Explanation(nodeId.toString())
+ .withExpressionType(ExpressionType.EVALUATOR)
+ .withFunctionName(factory.getFunctionName(getClass()))
+ .withImplementingClass(getClass().getName())
+ .withExpression(toExpression(factory).toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PercentileEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PercentileEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PercentileEvaluator.java
new file mode 100644
index 0000000..19d423d
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PercentileEvaluator.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
+import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class PercentileEvaluator extends ComplexEvaluator implements Expressible {
+
+ private static final long serialVersionUID = 1;
+
+ public PercentileEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ public Number evaluate(Tuple tuple) throws IOException {
+
+ if(subEvaluators.size() != 2) {
+ throw new IOException("Percentile expects 2 parameters: a regression result and a number");
+ }
+
+ StreamEvaluator r = subEvaluators.get(0);
+ StreamEvaluator d = subEvaluators.get(1);
+
+ EmpiricalDistributionEvaluator.EmpiricalDistributionTuple e = (EmpiricalDistributionEvaluator.EmpiricalDistributionTuple)r.evaluate(tuple);
+ Number n = (Number)d.evaluate(tuple);
+ return e.percentile(n.doubleValue());
+ }
+
+ @Override
+ public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
+ StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
+ return expression;
+ }
+
+ @Override
+ public Explanation toExplanation(StreamFactory factory) throws IOException {
+ return new Explanation(nodeId.toString())
+ .withExpressionType(ExpressionType.EVALUATOR)
+ .withFunctionName(factory.getFunctionName(getClass()))
+ .withImplementingClass(getClass().getName())
+ .withExpression(toExpression(factory).toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PredictEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PredictEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PredictEvaluator.java
new file mode 100644
index 0000000..0d1e763
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PredictEvaluator.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
+import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class PredictEvaluator extends ComplexEvaluator implements Expressible {
+
+ private static final long serialVersionUID = 1;
+
+ public PredictEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ public Number evaluate(Tuple tuple) throws IOException {
+
+ if(subEvaluators.size() != 2) {
+ throw new IOException("Predict expects 2 parameters: a regression result and a number");
+ }
+
+ StreamEvaluator r = subEvaluators.get(0);
+ StreamEvaluator d = subEvaluators.get(1);
+
+ RegressionEvaluator.RegressionTuple rt= (RegressionEvaluator.RegressionTuple)r.evaluate(tuple);
+ Number n = (Number)d.evaluate(tuple);
+ return rt.predict(n.doubleValue());
+ }
+
+ @Override
+ public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
+ StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
+ return expression;
+ }
+
+ @Override
+ public Explanation toExplanation(StreamFactory factory) throws IOException {
+ return new Explanation(nodeId.toString())
+ .withExpressionType(ExpressionType.EVALUATOR)
+ .withFunctionName(factory.getFunctionName(getClass()))
+ .withImplementingClass(getClass().getName())
+ .withExpression(toExpression(factory).toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/RankEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/RankEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/RankEvaluator.java
new file mode 100644
index 0000000..8a22e94
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/RankEvaluator.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.math3.stat.ranking.NaturalRanking;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
+import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class RankEvaluator extends ComplexEvaluator implements Expressible {
+
+ private static final long serialVersionUID = 1;
+
+ public RankEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ public List<Number> evaluate(Tuple tuple) throws IOException {
+ StreamEvaluator colEval = subEvaluators.get(0);
+
+ List<Number> numbers = (List<Number>)colEval.evaluate(tuple);
+ double[] values = new double[numbers.size()];
+ for(int i=0; i<numbers.size(); i++) {
+ values[i] = numbers.get(i).doubleValue();
+ }
+
+ NaturalRanking rank = new NaturalRanking();
+ double[] ranked = rank.rank(values);
+ List<Number> rankedList = new ArrayList();
+ for(int i=0; i<numbers.size(); i++) {
+ rankedList.add(ranked[i]);
+ }
+
+ return rankedList;
+ }
+
+ @Override
+ public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
+ StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
+ return expression;
+ }
+
+ @Override
+ public Explanation toExplanation(StreamFactory factory) throws IOException {
+ return new Explanation(nodeId.toString())
+ .withExpressionType(ExpressionType.EVALUATOR)
+ .withFunctionName(factory.getFunctionName(getClass()))
+ .withImplementingClass(getClass().getName())
+ .withExpression(toExpression(factory).toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/RegressionEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/RegressionEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/RegressionEvaluator.java
new file mode 100644
index 0000000..42a6955
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/RegressionEvaluator.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.math3.stat.regression.SimpleRegression;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
+import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class RegressionEvaluator extends ComplexEvaluator implements Expressible {
+
+ private static final long serialVersionUID = 1;
+
+ public RegressionEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ public Tuple evaluate(Tuple tuple) throws IOException {
+
+ if(subEvaluators.size() != 2) {
+ throw new IOException("Regress expects 2 columns as parameters");
+ }
+
+ StreamEvaluator colEval1 = subEvaluators.get(0);
+ StreamEvaluator colEval2 = subEvaluators.get(1);
+
+ List<Number> numbers1 = (List<Number>)colEval1.evaluate(tuple);
+ List<Number> numbers2 = (List<Number>)colEval2.evaluate(tuple);
+ double[] column1 = new double[numbers1.size()];
+ double[] column2 = new double[numbers2.size()];
+
+ for(int i=0; i<numbers1.size(); i++) {
+ column1[i] = numbers1.get(i).doubleValue();
+ }
+
+ for(int i=0; i<numbers2.size(); i++) {
+ column2[i] = numbers2.get(i).doubleValue();
+ }
+
+ SimpleRegression regression = new SimpleRegression();
+ for(int i=0; i<column1.length; i++) {
+ regression.addData(column1[i], column2[i]);
+ }
+
+ Map map = new HashMap();
+ map.put("slope", regression.getSlope());
+ map.put("intercept", regression.getIntercept());
+ map.put("R", regression.getR());
+ map.put("N", regression.getN());
+ map.put("regressionSumSquares", regression.getRegressionSumSquares());
+ map.put("slopeConfidenceInterval", regression.getSlopeConfidenceInterval());
+ map.put("interceptStdErr", regression.getInterceptStdErr());
+ map.put("totalSumSquares", regression.getTotalSumSquares());
+ map.put("significance", regression.getSignificance());
+ map.put("meanSquareError", regression.getMeanSquareError());
+ return new RegressionTuple(regression, map);
+ }
+
+ public static class RegressionTuple extends Tuple {
+
+ private SimpleRegression simpleRegression;
+
+ public RegressionTuple(SimpleRegression simpleRegression, Map map) {
+ super(map);
+ this.simpleRegression = simpleRegression;
+ }
+
+ public double predict(double d) {
+ return this.simpleRegression.predict(d);
+ }
+ }
+
+ @Override
+ public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
+ StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
+ return expression;
+ }
+
+ @Override
+ public Explanation toExplanation(StreamFactory factory) throws IOException {
+ return new Explanation(nodeId.toString())
+ .withExpressionType(ExpressionType.EVALUATOR)
+ .withFunctionName(factory.getFunctionName(getClass()))
+ .withImplementingClass(getClass().getName())
+ .withExpression(toExpression(factory).toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ReverseEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ReverseEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ReverseEvaluator.java
new file mode 100644
index 0000000..016e995
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ReverseEvaluator.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
+import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class ReverseEvaluator extends ComplexEvaluator implements Expressible {
+
+ private static final long serialVersionUID = 1;
+
+ public ReverseEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ public List<Number> evaluate(Tuple tuple) throws IOException {
+ StreamEvaluator colEval1 = subEvaluators.get(0);
+
+ List<Number> numbers1 = (List<Number>)colEval1.evaluate(tuple);
+ List<Number> rev = new ArrayList();
+ for(int i=numbers1.size()-1; i>=0; i--) {
+ rev.add(numbers1.get(i));
+ }
+
+ return rev;
+ }
+
+ @Override
+ public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
+ StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
+ return expression;
+ }
+
+ @Override
+ public Explanation toExplanation(StreamFactory factory) throws IOException {
+ return new Explanation(nodeId.toString())
+ .withExpressionType(ExpressionType.EVALUATOR)
+ .withFunctionName(factory.getFunctionName(getClass()))
+ .withImplementingClass(getClass().getName())
+ .withExpression(toExpression(factory).toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ScaleEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ScaleEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ScaleEvaluator.java
new file mode 100644
index 0000000..8ff2a7c
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ScaleEvaluator.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.math3.util.MathArrays;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
+import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class ScaleEvaluator extends ComplexEvaluator implements Expressible {
+
+ private static final long serialVersionUID = 1;
+
+ public ScaleEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ public List<Number> evaluate(Tuple tuple) throws IOException {
+ StreamEvaluator numEval = subEvaluators.get(0);
+ StreamEvaluator colEval1 = subEvaluators.get(1);
+
+ Number num = (Number)numEval.evaluate(tuple);
+ List<Number> numbers1 = (List<Number>)colEval1.evaluate(tuple);
+ double[] column1 = new double[numbers1.size()];
+
+ for(int i=0; i<numbers1.size(); i++) {
+ column1[i] = numbers1.get(i).doubleValue();
+ }
+
+ double[] scaled = MathArrays.scale(num.doubleValue(), column1);
+
+ List<Number> scaledList = new ArrayList(scaled.length);
+ for(double d : scaled) {
+ scaledList.add(d);
+ }
+
+ return scaledList;
+ }
+
+ @Override
+ public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
+ StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
+ return expression;
+ }
+
+ @Override
+ public Explanation toExplanation(StreamFactory factory) throws IOException {
+ return new Explanation(nodeId.toString())
+ .withExpressionType(ExpressionType.EVALUATOR)
+ .withFunctionName(factory.getFunctionName(getClass()))
+ .withImplementingClass(getClass().getName())
+ .withExpression(toExpression(factory).toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ColumnEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ColumnEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ColumnEvaluator.java
deleted file mode 100644
index 3e56837..0000000
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ColumnEvaluator.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.client.solrj.io.stream;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.eval.SimpleEvaluator;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
-import org.apache.solr.client.solrj.io.stream.expr.Expressible;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
-import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-
-public class ColumnEvaluator extends SimpleEvaluator implements Expressible {
-
- private static final long serialVersionUID = 1;
- private String name;
- private String fieldName;
- ;
- public ColumnEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
- String name = factory.getValueOperand(expression, 0);
- String fieldName = factory.getValueOperand(expression, 1);
- init(name, fieldName);
- }
-
- private void init(String name, String fieldName) {
- this.name = name;
- this.fieldName = fieldName;
- }
-
- public List<Number> evaluate(Tuple tuple) throws IOException {
- List<Tuple> tuples = (List<Tuple>)tuple.get(name);
- List<Number> column = new ArrayList(tuples.size());
- for(Tuple t : tuples) {
- Object o = t.get(fieldName);
- if(o instanceof Number) {
- column.add((Number)o);
- } else {
- throw new IOException("Found non-numeric in column:"+o.toString());
- }
- }
- return column;
- }
-
- @Override
- public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
- StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
- return expression;
- }
-
- @Override
- public Explanation toExplanation(StreamFactory factory) throws IOException {
- return new Explanation(nodeId.toString())
- .withExpressionType(ExpressionType.EVALUATOR)
- .withFunctionName(factory.getFunctionName(getClass()))
- .withImplementingClass(getClass().getName())
- .withExpression(toExpression(factory).toString());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ConvolutionEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ConvolutionEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ConvolutionEvaluator.java
deleted file mode 100644
index 6d6e3e3..0000000
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ConvolutionEvaluator.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.client.solrj.io.stream;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.ArrayList;
-
-import org.apache.commons.math3.util.MathArrays;
-import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.eval.ComplexEvaluator;
-import org.apache.solr.client.solrj.io.eval.StreamEvaluator;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
-import org.apache.solr.client.solrj.io.stream.expr.Expressible;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
-import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-
-public class ConvolutionEvaluator extends ComplexEvaluator implements Expressible {
-
- private static final long serialVersionUID = 1;
-
- public ConvolutionEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
- super(expression, factory);
- }
-
- public List<Number> evaluate(Tuple tuple) throws IOException {
- StreamEvaluator colEval1 = subEvaluators.get(0);
- StreamEvaluator colEval2 = subEvaluators.get(1);
-
- List<Number> numbers1 = (List<Number>)colEval1.evaluate(tuple);
- List<Number> numbers2 = (List<Number>)colEval2.evaluate(tuple);
- double[] column1 = new double[numbers1.size()];
- double[] column2 = new double[numbers2.size()];
-
- for(int i=0; i<numbers1.size(); i++) {
- column1[i] = numbers1.get(i).doubleValue();
- }
-
- for(int i=0; i<numbers2.size(); i++) {
- column2[i] = numbers2.get(i).doubleValue();
- }
-
- double[] conArray = MathArrays.convolve(column1, column2);
- List<Number> conList = new ArrayList();
- for(double d :conArray) {
- conList.add(d);
- }
-
- return conList;
- }
-
- @Override
- public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
- StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
- return expression;
- }
-
- @Override
- public Explanation toExplanation(StreamFactory factory) throws IOException {
- return new Explanation(nodeId.toString())
- .withExpressionType(ExpressionType.EVALUATOR)
- .withFunctionName(factory.getFunctionName(getClass()))
- .withImplementingClass(getClass().getName())
- .withExpression(toExpression(factory).toString());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CopyOfEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CopyOfEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CopyOfEvaluator.java
deleted file mode 100644
index 2380e8f..0000000
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CopyOfEvaluator.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.client.solrj.io.stream;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.eval.ComplexEvaluator;
-import org.apache.solr.client.solrj.io.eval.StreamEvaluator;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
-import org.apache.solr.client.solrj.io.stream.expr.Expressible;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
-import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-
-public class CopyOfEvaluator extends ComplexEvaluator implements Expressible {
-
- private static final long serialVersionUID = 1;
-
- public CopyOfEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
- super(expression, factory);
- }
-
- public List<Number> evaluate(Tuple tuple) throws IOException {
- StreamEvaluator colEval1 = subEvaluators.get(0);
-
- List<Number> numbers1 = (List<Number>)colEval1.evaluate(tuple);
- double[] vals = new double[numbers1.size()];
-
- for(int i=0; i<vals.length; i++) {
- vals[i] = numbers1.get(i).doubleValue();
- }
-
- if(subEvaluators.size() == 2) {
- StreamEvaluator lengthEval = subEvaluators.get(1);
- Number lengthNum = (Number)lengthEval.evaluate(tuple);
- int length = lengthNum.intValue();
- vals = Arrays.copyOf(vals, length);
- } else {
- vals = Arrays.copyOf(vals, vals.length);
- }
-
- List<Number> copyOf = new ArrayList(vals.length);
-
- for(int i=0; i<vals.length; i++) {
- copyOf.add(vals[i]);
- }
-
- return copyOf;
- }
-
- @Override
- public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
- StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
- return expression;
- }
-
- @Override
- public Explanation toExplanation(StreamFactory factory) throws IOException {
- return new Explanation(nodeId.toString())
- .withExpressionType(ExpressionType.EVALUATOR)
- .withFunctionName(factory.getFunctionName(getClass()))
- .withImplementingClass(getClass().getName())
- .withExpression(toExpression(factory).toString());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fef7e33/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CopyOfRangeEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CopyOfRangeEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CopyOfRangeEvaluator.java
deleted file mode 100644
index c4c1e77..0000000
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CopyOfRangeEvaluator.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.client.solrj.io.stream;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.eval.ComplexEvaluator;
-import org.apache.solr.client.solrj.io.eval.StreamEvaluator;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation;
-import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
-import org.apache.solr.client.solrj.io.stream.expr.Expressible;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
-import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-
-public class CopyOfRangeEvaluator extends ComplexEvaluator implements Expressible {
-
- private static final long serialVersionUID = 1;
-
- public CopyOfRangeEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
- super(expression, factory);
- }
-
- public List<Number> evaluate(Tuple tuple) throws IOException {
- StreamEvaluator colEval1 = subEvaluators.get(0);
-
- List<Number> numbers1 = (List<Number>)colEval1.evaluate(tuple);
- double[] vals = new double[numbers1.size()];
-
- for(int i=0; i<vals.length; i++) {
- vals[i] = numbers1.get(i).doubleValue();
- }
-
- StreamEvaluator startIndexEval = subEvaluators.get(1);
- Number startIndexNum = (Number)startIndexEval.evaluate(tuple);
- int startIndex = startIndexNum.intValue();
-
- StreamEvaluator endIndexEval = subEvaluators.get(2);
- Number endIndexNum = (Number)endIndexEval.evaluate(tuple);
- int endIndex = endIndexNum.intValue();
-
- vals = Arrays.copyOfRange(vals, startIndex, endIndex);
-
- List<Number> copyOf = new ArrayList(vals.length);
-
- for(int i=0; i<vals.length; i++) {
- copyOf.add(vals[i]);
- }
-
- return copyOf;
- }
-
- @Override
- public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
- StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
- return expression;
- }
-
- @Override
- public Explanation toExplanation(StreamFactory factory) throws IOException {
- return new Explanation(nodeId.toString())
- .withExpressionType(ExpressionType.EVALUATOR)
- .withFunctionName(factory.getFunctionName(getClass()))
- .withImplementingClass(getClass().getName())
- .withExpression(toExpression(factory).toString());
- }
-}
\ No newline at end of file
[25/44] lucene-solr:jira/solr-8668: SOLR-10731: Add tests for params
Posted by cp...@apache.org.
SOLR-10731: Add tests for params
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e2284bbe
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e2284bbe
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e2284bbe
Branch: refs/heads/jira/solr-8668
Commit: e2284bbedc1f9550bda2ce7e37d93da45ac4c9f6
Parents: cc87181
Author: Joel Bernstein <jb...@apache.org>
Authored: Wed May 24 07:58:35 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Wed May 24 07:59:02 2017 -0400
----------------------------------------------------------------------
.../solrj/io/stream/StreamExpressionTest.java | 22 ++++++++++++++++++--
1 file changed, 20 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e2284bbe/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index dc3a380..f76ed31 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -920,7 +920,6 @@ public class StreamExpressionTest extends SolrCloudTestCase {
}
}
-
@Test
public void testKnnStream() throws Exception {
@@ -941,7 +940,26 @@ public class StreamExpressionTest extends SolrCloudTestCase {
SolrStream solrStream = new SolrStream(jetty.getBaseUrl().toString() + "/collection1", sParams);
List<Tuple> tuples = getTuples(solrStream);
assertTrue(tuples.size() == 3);
- assertOrder(tuples,2,3,4);
+ assertOrder(tuples, 2, 3, 4);
+
+ sParams = new ModifiableSolrParams(StreamingTest.mapParams(CommonParams.QT, "/stream"));
+ sParams.add("expr", "knn(" + COLLECTIONORALIAS + ", id=\"1\", qf=\"a_t\", rows=\"4\", fl=\"id, score\", mintf=\"1\", maxdf=\"0\")");
+ solrStream = new SolrStream(jetty.getBaseUrl().toString() + "/collection1", sParams);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 0);
+
+ sParams = new ModifiableSolrParams(StreamingTest.mapParams(CommonParams.QT, "/stream"));
+ sParams.add("expr", "knn(" + COLLECTIONORALIAS + ", id=\"1\", qf=\"a_t\", rows=\"4\", fl=\"id, score\", mintf=\"1\", maxwl=\"1\")");
+ solrStream = new SolrStream(jetty.getBaseUrl().toString() + "/collection1", sParams);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 0);
+
+ sParams = new ModifiableSolrParams(StreamingTest.mapParams(CommonParams.QT, "/stream"));
+ sParams.add("expr", "knn(" + COLLECTIONORALIAS + ", id=\"1\", qf=\"a_t\", rows=\"2\", fl=\"id, score\", mintf=\"1\", minwl=\"20\")");
+ solrStream = new SolrStream(jetty.getBaseUrl().toString() + "/collection1", sParams);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 0);
+
} finally {
cache.close();
}
[21/44] lucene-solr:jira/solr-8668: added docs
Posted by cp...@apache.org.
added docs
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8f4d72a7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8f4d72a7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8f4d72a7
Branch: refs/heads/jira/solr-8668
Commit: 8f4d72a7d22f72755a5de4f1d9341a7ffe74949c
Parents: 872ed81
Author: Noble Paul <no...@apache.org>
Authored: Wed May 24 15:59:00 2017 +0930
Committer: Noble Paul <no...@apache.org>
Committed: Wed May 24 15:59:00 2017 +0930
----------------------------------------------------------------------
.../apache/solr/request/SolrQueryRequest.java | 42 ++++++++++++--------
1 file changed, 26 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8f4d72a7/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java b/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java
index 5bad237..2c5090b 100644
--- a/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java
+++ b/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java
@@ -33,58 +33,58 @@ import java.util.Map;
/**
* <p>Container for a request to execute a query.</p>
* <p><code>SolrQueryRequest</code> is not thread safe.</p>
- *
+ *
*
*/
public interface SolrQueryRequest extends AutoCloseable {
/** returns the current request parameters */
- public SolrParams getParams();
+ SolrParams getParams();
/** Change the parameters for this request. This does not affect
* the original parameters returned by getOriginalParams()
*/
- public void setParams(SolrParams params);
-
+ void setParams(SolrParams params);
+
/** A Collection of ContentStreams passed to the request
*/
- public Iterable<ContentStream> getContentStreams();
+ Iterable<ContentStream> getContentStreams();
/** Returns the original request parameters. As this
* does not normally include configured defaults
* it's more suitable for logging.
*/
- public SolrParams getOriginalParams();
+ SolrParams getOriginalParams();
/**
* Generic information associated with this request that may be both read and updated.
*/
- public Map<Object,Object> getContext();
+ Map<Object,Object> getContext();
/**
* This method should be called when all uses of this request are
* finished, so that resources can be freed.
*/
- public void close();
+ void close();
/** The start time of this request in milliseconds.
* Use this only if you need the absolute system time at the start of the request,
* getRequestTimer() provides a more accurate mechanism for timing purposes.
*/
- public long getStartTime();
+ long getStartTime();
/** The timer for this request, created when the request started being processed */
- public RTimerTree getRequestTimer();
+ RTimerTree getRequestTimer();
/** The index searcher associated with this request */
- public SolrIndexSearcher getSearcher();
+ SolrIndexSearcher getSearcher();
/** The solr core (coordinator, etc) associated with this request */
- public SolrCore getCore();
+ SolrCore getCore();
/** The schema snapshot from core.getLatestSchema() at request creation. */
public IndexSchema getSchema();
-
+
/** Replaces the current schema snapshot with the latest from the core. */
public void updateSchemaToLatest();
@@ -97,20 +97,30 @@ public interface SolrQueryRequest extends AutoCloseable {
/** Returns any associated JSON (or null if none) in deserialized generic form.
* Java classes used to represent the JSON are as follows: Map, List, String, Long, Double, Boolean
*/
- public Map<String,Object> getJSON();
+ Map<String,Object> getJSON();
- public void setJSON(Map<String,Object> json);
+ void setJSON(Map<String, Object> json);
- public Principal getUserPrincipal();
+ Principal getUserPrincipal();
default String getPath() {
return (String) getContext().get("path");
}
+ /** Only for V2 API.
+ * Returns a map of path segments and their values . For example ,
+ * if the path is configured as /path/{segment1}/{segment2} and a reguest is made
+ * as /path/x/y the returned map would contain {segment1:x ,segment2:y}
+ */
default Map<String, String> getPathTemplateValues() {
return Collections.emptyMap();
}
+ /** Only for v2 API
+ * if the request contains a command payload, it's parsed and returned as a list of
+ * CommandOperation objects
+ * @param validateInput , If true it is validated against the json schema spec
+ */
default List<CommandOperation> getCommands(boolean validateInput) {
return Collections.emptyList();
}
[12/44] lucene-solr:jira/solr-8668: LUCENE-7815: Removed the
PostingsHighlighter
Posted by cp...@apache.org.
LUCENE-7815: Removed the PostingsHighlighter
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0d3c73ea
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0d3c73ea
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0d3c73ea
Branch: refs/heads/jira/solr-8668
Commit: 0d3c73eaa2dd26af73461fd6ec3494bc12edbe8a
Parents: 14320a5
Author: David Smiley <ds...@apache.org>
Authored: Tue May 23 14:39:51 2017 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Tue May 23 14:39:51 2017 -0400
----------------------------------------------------------------------
lucene/CHANGES.txt | 4 +
lucene/benchmark/conf/highlighters-postings.alg | 4 +-
.../tasks/SearchTravRetHighlightTask.java | 30 -
.../DefaultPassageFormatter.java | 137 --
.../MultiTermHighlighting.java | 282 -----
.../search/postingshighlight/Passage.java | 159 ---
.../postingshighlight/PassageFormatter.java | 40 -
.../search/postingshighlight/PassageScorer.java | 104 --
.../postingshighlight/PostingsHighlighter.java | 820 ------------
.../search/postingshighlight/package-info.java | 21 -
.../CustomSeparatorBreakIterator.java | 150 +++
.../search/uhighlight/WholeBreakIterator.java | 116 ++
.../search/postingshighlight/CambridgeMA.utf8 | 1 -
.../TestMultiTermHighlighting.java | 884 -------------
.../TestPostingsHighlighter.java | 1185 ------------------
.../TestPostingsHighlighterRanking.java | 324 -----
.../uhighlight/LengthGoalBreakIteratorTest.java | 1 -
.../TestCustomSeparatorBreakIterator.java | 114 ++
.../uhighlight/TestUnifiedHighlighter.java | 1 -
.../uhighlight/TestWholeBreakIterator.java | 134 ++
.../solr/highlight/UnifiedSolrHighlighter.java | 4 +-
21 files changed, 522 insertions(+), 3993 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 3951cea..dadba8b 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -53,6 +53,10 @@ API Changes
* LUCENE-7741: DoubleValuesSource now has an explain() method (Alan Woodward,
Adrien Grand)
+* LUCENE-7815: Removed the PostingsHighlighter; you should use the UnifiedHighlighter
+ instead, which derived from the UH. WholeBreakIterator and
+ CustomSeparatorBreakIterator were moved to UH's package. (David Smiley)
+
Bug Fixes
* LUCENE-7626: IndexWriter will no longer accept broken token offsets
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/benchmark/conf/highlighters-postings.alg
----------------------------------------------------------------------
diff --git a/lucene/benchmark/conf/highlighters-postings.alg b/lucene/benchmark/conf/highlighters-postings.alg
index 610908f..2560dad 100644
--- a/lucene/benchmark/conf/highlighters-postings.alg
+++ b/lucene/benchmark/conf/highlighters-postings.alg
@@ -38,7 +38,7 @@ file.query.maker.file=conf/query-terms.txt
log.queries=false
log.step.SearchTravRetHighlight=-1
-highlighter=HlImpl:NONE:SH_A:UH_A:PH_P:UH_P:UH_PV
+highlighter=HlImpl:NONE:SH_A:UH_A:UH_P:UH_PV
{ "Populate"
CreateIndex
@@ -60,6 +60,6 @@ highlighter=HlImpl:NONE:SH_A:UH_A:PH_P:UH_P:UH_PV
CloseReader
NewRound
-} : 6
+} : 5
RepSumByPrefRound HL
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetHighlightTask.java
----------------------------------------------------------------------
diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetHighlightTask.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetHighlightTask.java
index f36854d..d90d3a7 100644
--- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetHighlightTask.java
+++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetHighlightTask.java
@@ -42,7 +42,6 @@ import org.apache.lucene.search.highlight.Highlighter;
import org.apache.lucene.search.highlight.QueryScorer;
import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
import org.apache.lucene.search.highlight.TokenSources;
-import org.apache.lucene.search.postingshighlight.PostingsHighlighter;
import org.apache.lucene.search.uhighlight.UnifiedHighlighter;
import org.apache.lucene.search.vectorhighlight.BoundaryScanner;
import org.apache.lucene.search.vectorhighlight.BreakIteratorBoundaryScanner;
@@ -133,8 +132,6 @@ public class SearchTravRetHighlightTask extends SearchTravTask {
case "UH_P": hlImpl = new UnifiedHLImpl(UnifiedHighlighter.OffsetSource.POSTINGS); break;
case "UH_PV": hlImpl = new UnifiedHLImpl(UnifiedHighlighter.OffsetSource.POSTINGS_WITH_TERM_VECTORS); break;
- case "PH_P": hlImpl = new PostingsHLImpl(); break;
-
default: throw new Exception("unrecognized highlighter type: " + type + " (try 'UH')");
}
}
@@ -224,33 +221,6 @@ public class SearchTravRetHighlightTask extends SearchTravTask {
return clone;
}
- private class PostingsHLImpl implements HLImpl {
- PostingsHighlighter highlighter;
- String[] fields = hlFields.toArray(new String[hlFields.size()]);
- int[] maxPassages;
- PostingsHLImpl() {
- highlighter = new PostingsHighlighter(maxDocCharsToAnalyze) {
- @Override
- protected Analyzer getIndexAnalyzer(String field) { // thus support wildcards
- return analyzer;
- }
-
- @Override
- protected BreakIterator getBreakIterator(String field) {
- return BreakIterator.getSentenceInstance(Locale.ENGLISH);
- }
- };
- maxPassages = new int[hlFields.size()];
- Arrays.fill(maxPassages, maxFrags);
- }
-
- @Override
- public void withTopDocs(IndexSearcher searcher, Query q, TopDocs hits) throws Exception {
- Map<String, String[]> result = highlighter.highlightFields(fields, q, searcher, hits, maxPassages);
- preventOptimizeAway = result.size();
- }
- }
-
private class UnifiedHLImpl implements HLImpl {
UnifiedHighlighter highlighter;
IndexSearcher lastSearcher;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/DefaultPassageFormatter.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/DefaultPassageFormatter.java b/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/DefaultPassageFormatter.java
deleted file mode 100644
index 73822c8..0000000
--- a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/DefaultPassageFormatter.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.postingshighlight;
-
-/**
- * Creates a formatted snippet from the top passages.
- * <p>
- * The default implementation marks the query terms as bold, and places
- * ellipses between unconnected passages.
- */
-public class DefaultPassageFormatter extends PassageFormatter {
- /** text that will appear before highlighted terms */
- protected final String preTag;
- /** text that will appear after highlighted terms */
- protected final String postTag;
- /** text that will appear between two unconnected passages */
- protected final String ellipsis;
- /** true if we should escape for html */
- protected final boolean escape;
-
- /**
- * Creates a new DefaultPassageFormatter with the default tags.
- */
- public DefaultPassageFormatter() {
- this("<b>", "</b>", "... ", false);
- }
-
- /**
- * Creates a new DefaultPassageFormatter with custom tags.
- * @param preTag text which should appear before a highlighted term.
- * @param postTag text which should appear after a highlighted term.
- * @param ellipsis text which should be used to connect two unconnected passages.
- * @param escape true if text should be html-escaped
- */
- public DefaultPassageFormatter(String preTag, String postTag, String ellipsis, boolean escape) {
- if (preTag == null || postTag == null || ellipsis == null) {
- throw new NullPointerException();
- }
- this.preTag = preTag;
- this.postTag = postTag;
- this.ellipsis = ellipsis;
- this.escape = escape;
- }
-
- @Override
- public String format(Passage passages[], String content) {
- StringBuilder sb = new StringBuilder();
- int pos = 0;
- for (Passage passage : passages) {
- // don't add ellipsis if it's the first one, or if it's connected.
- if (passage.startOffset > pos && pos > 0) {
- sb.append(ellipsis);
- }
- pos = passage.startOffset;
- for (int i = 0; i < passage.numMatches; i++) {
- int start = passage.matchStarts[i];
- int end = passage.matchEnds[i];
- // it's possible to have overlapping terms
- if (start > pos) {
- append(sb, content, pos, start);
- }
- if (end > pos) {
- sb.append(preTag);
- append(sb, content, Math.max(pos, start), end);
- sb.append(postTag);
- pos = end;
- }
- }
- // it's possible a "term" from the analyzer could span a sentence boundary.
- append(sb, content, pos, Math.max(pos, passage.endOffset));
- pos = passage.endOffset;
- }
- return sb.toString();
- }
-
- /**
- * Appends original text to the response.
- * @param dest resulting text, possibly transformed or encoded
- * @param content original text content
- * @param start index of the first character in content
- * @param end index of the character following the last character in content
- */
- protected void append(StringBuilder dest, String content, int start, int end) {
- if (escape) {
- // note: these are the rules from owasp.org
- for (int i = start; i < end; i++) {
- char ch = content.charAt(i);
- switch(ch) {
- case '&':
- dest.append("&");
- break;
- case '<':
- dest.append("<");
- break;
- case '>':
- dest.append(">");
- break;
- case '"':
- dest.append(""");
- break;
- case '\'':
- dest.append("'");
- break;
- case '/':
- dest.append("/");
- break;
- default:
- if (ch >= 0x30 && ch <= 0x39 || ch >= 0x41 && ch <= 0x5A || ch >= 0x61 && ch <= 0x7A) {
- dest.append(ch);
- } else if (ch < 0xff) {
- dest.append("&#");
- dest.append((int)ch);
- dest.append(";");
- } else {
- dest.append(ch);
- }
- }
- }
- } else {
- dest.append(content, start, end);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/MultiTermHighlighting.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/MultiTermHighlighting.java b/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/MultiTermHighlighting.java
deleted file mode 100644
index c9733d3..0000000
--- a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/MultiTermHighlighting.java
+++ /dev/null
@@ -1,282 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.postingshighlight;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.List;
-
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.AutomatonQuery;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.ConstantScoreQuery;
-import org.apache.lucene.search.DisjunctionMaxQuery;
-import org.apache.lucene.search.FuzzyQuery;
-import org.apache.lucene.search.PrefixQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermRangeQuery;
-import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper;
-import org.apache.lucene.search.spans.SpanNearQuery;
-import org.apache.lucene.search.spans.SpanNotQuery;
-import org.apache.lucene.search.spans.SpanOrQuery;
-import org.apache.lucene.search.spans.SpanPositionCheckQuery;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.CharsRef;
-import org.apache.lucene.util.UnicodeUtil;
-import org.apache.lucene.util.automaton.Automata;
-import org.apache.lucene.util.automaton.Automaton;
-import org.apache.lucene.util.automaton.CharacterRunAutomaton;
-import org.apache.lucene.util.automaton.LevenshteinAutomata;
-import org.apache.lucene.util.automaton.Operations;
-
-/**
- * Support for highlighting multiterm queries in PostingsHighlighter.
- */
-class MultiTermHighlighting {
-
- /**
- * Extracts all MultiTermQueries for {@code field}, and returns equivalent
- * automata that will match terms.
- */
- static CharacterRunAutomaton[] extractAutomata(Query query, String field) {
- List<CharacterRunAutomaton> list = new ArrayList<>();
- if (query instanceof BooleanQuery) {
- for (BooleanClause clause : (BooleanQuery) query) {
- if (!clause.isProhibited()) {
- list.addAll(Arrays.asList(extractAutomata(clause.getQuery(), field)));
- }
- }
- } else if (query instanceof ConstantScoreQuery) {
- list.addAll(Arrays.asList(extractAutomata(((ConstantScoreQuery) query).getQuery(), field)));
- } else if (query instanceof DisjunctionMaxQuery) {
- for (Query sub : ((DisjunctionMaxQuery) query).getDisjuncts()) {
- list.addAll(Arrays.asList(extractAutomata(sub, field)));
- }
- } else if (query instanceof SpanOrQuery) {
- for (Query sub : ((SpanOrQuery) query).getClauses()) {
- list.addAll(Arrays.asList(extractAutomata(sub, field)));
- }
- } else if (query instanceof SpanNearQuery) {
- for (Query sub : ((SpanNearQuery) query).getClauses()) {
- list.addAll(Arrays.asList(extractAutomata(sub, field)));
- }
- } else if (query instanceof SpanNotQuery) {
- list.addAll(Arrays.asList(extractAutomata(((SpanNotQuery) query).getInclude(), field)));
- } else if (query instanceof SpanPositionCheckQuery) {
- list.addAll(Arrays.asList(extractAutomata(((SpanPositionCheckQuery) query).getMatch(), field)));
- } else if (query instanceof SpanMultiTermQueryWrapper) {
- list.addAll(Arrays.asList(extractAutomata(((SpanMultiTermQueryWrapper<?>) query).getWrappedQuery(), field)));
- } else if (query instanceof PrefixQuery) {
- final PrefixQuery pq = (PrefixQuery) query;
- Term prefix = pq.getPrefix();
- if (prefix.field().equals(field)) {
- list.add(new CharacterRunAutomaton(Operations.concatenate(Automata.makeString(prefix.text()),
- Automata.makeAnyString())) {
- @Override
- public String toString() {
- return pq.toString();
- }
- });
- }
- } else if (query instanceof FuzzyQuery) {
- final FuzzyQuery fq = (FuzzyQuery) query;
- if (fq.getField().equals(field)) {
- String utf16 = fq.getTerm().text();
- int termText[] = new int[utf16.codePointCount(0, utf16.length())];
- for (int cp, i = 0, j = 0; i < utf16.length(); i += Character.charCount(cp)) {
- termText[j++] = cp = utf16.codePointAt(i);
- }
- int termLength = termText.length;
- int prefixLength = Math.min(fq.getPrefixLength(), termLength);
- String suffix = UnicodeUtil.newString(termText, prefixLength, termText.length - prefixLength);
- LevenshteinAutomata builder = new LevenshteinAutomata(suffix, fq.getTranspositions());
- String prefix = UnicodeUtil.newString(termText, 0, prefixLength);
- Automaton automaton = builder.toAutomaton(fq.getMaxEdits(), prefix);
- list.add(new CharacterRunAutomaton(automaton) {
- @Override
- public String toString() {
- return fq.toString();
- }
- });
- }
- } else if (query instanceof TermRangeQuery) {
- final TermRangeQuery tq = (TermRangeQuery) query;
- if (tq.getField().equals(field)) {
- final CharsRef lowerBound;
- if (tq.getLowerTerm() == null) {
- lowerBound = null;
- } else {
- lowerBound = new CharsRef(tq.getLowerTerm().utf8ToString());
- }
-
- final CharsRef upperBound;
- if (tq.getUpperTerm() == null) {
- upperBound = null;
- } else {
- upperBound = new CharsRef(tq.getUpperTerm().utf8ToString());
- }
-
- final boolean includeLower = tq.includesLower();
- final boolean includeUpper = tq.includesUpper();
- final CharsRef scratch = new CharsRef();
- final Comparator<CharsRef> comparator = CharsRef.getUTF16SortedAsUTF8Comparator();
-
- // this is *not* an automaton, but it's very simple
- list.add(new CharacterRunAutomaton(Automata.makeEmpty()) {
- @Override
- public boolean run(char[] s, int offset, int length) {
- scratch.chars = s;
- scratch.offset = offset;
- scratch.length = length;
-
- if (lowerBound != null) {
- int cmp = comparator.compare(scratch, lowerBound);
- if (cmp < 0 || (!includeLower && cmp == 0)) {
- return false;
- }
- }
-
- if (upperBound != null) {
- int cmp = comparator.compare(scratch, upperBound);
- if (cmp > 0 || (!includeUpper && cmp == 0)) {
- return false;
- }
- }
- return true;
- }
-
- @Override
- public String toString() {
- return tq.toString();
- }
- });
- }
- } else if (query instanceof AutomatonQuery) {
- final AutomatonQuery aq = (AutomatonQuery) query;
- if (aq.getField().equals(field)) {
- list.add(new CharacterRunAutomaton(aq.getAutomaton()) {
- @Override
- public String toString() {
- return aq.toString();
- }
- });
- }
- }
- return list.toArray(new CharacterRunAutomaton[list.size()]);
- }
-
- /**
- * Returns a "fake" DocsAndPositionsEnum over the tokenstream, returning offsets where {@code matchers}
- * matches tokens.
- * <p>
- * This is solely used internally by PostingsHighlighter: <b>DO NOT USE THIS METHOD!</b>
- */
- static PostingsEnum getDocsEnum(final TokenStream ts, final CharacterRunAutomaton[] matchers) throws IOException {
- final CharTermAttribute charTermAtt = ts.addAttribute(CharTermAttribute.class);
- final OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
- ts.reset();
-
- // TODO: we could use CachingWrapperFilter, (or consume twice) to allow us to have a true freq()
- // but this would have a performance cost for likely little gain in the user experience, it
- // would only serve to make this method less bogus.
- // instead, we always return freq() = Integer.MAX_VALUE and let PH terminate based on offset...
-
- return new PostingsEnum() {
- int currentDoc = -1;
- int currentMatch = -1;
- int currentStartOffset = -1;
- int currentEndOffset = -1;
- TokenStream stream = ts;
-
- final BytesRef matchDescriptions[] = new BytesRef[matchers.length];
-
- @Override
- public int nextPosition() throws IOException {
- if (stream != null) {
- while (stream.incrementToken()) {
- for (int i = 0; i < matchers.length; i++) {
- if (matchers[i].run(charTermAtt.buffer(), 0, charTermAtt.length())) {
- currentStartOffset = offsetAtt.startOffset();
- currentEndOffset = offsetAtt.endOffset();
- currentMatch = i;
- return 0;
- }
- }
- }
- stream.end();
- stream.close();
- stream = null;
- }
- // exhausted
- currentStartOffset = currentEndOffset = Integer.MAX_VALUE;
- return Integer.MAX_VALUE;
- }
-
- @Override
- public int freq() throws IOException {
- return Integer.MAX_VALUE; // lie
- }
-
- @Override
- public int startOffset() throws IOException {
- assert currentStartOffset >= 0;
- return currentStartOffset;
- }
-
- @Override
- public int endOffset() throws IOException {
- assert currentEndOffset >= 0;
- return currentEndOffset;
- }
-
- @Override
- public BytesRef getPayload() throws IOException {
- if (matchDescriptions[currentMatch] == null) {
- matchDescriptions[currentMatch] = new BytesRef(matchers[currentMatch].toString());
- }
- return matchDescriptions[currentMatch];
- }
-
- @Override
- public int docID() {
- return currentDoc;
- }
-
- @Override
- public int nextDoc() throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public int advance(int target) throws IOException {
- return currentDoc = target;
- }
-
- @Override
- public long cost() {
- return 0;
- }
- };
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/Passage.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/Passage.java b/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/Passage.java
deleted file mode 100644
index 50aebea..0000000
--- a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/Passage.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.postingshighlight;
-
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.InPlaceMergeSorter;
-import org.apache.lucene.util.RamUsageEstimator;
-
-/**
- * Represents a passage (typically a sentence of the document).
- * <p>
- * A passage contains {@link #getNumMatches} highlights from the query,
- * and the offsets and query terms that correspond with each match.
- * @lucene.experimental
- */
-public final class Passage {
- int startOffset = -1;
- int endOffset = -1;
- float score = 0.0f;
-
- int matchStarts[] = new int[8];
- int matchEnds[] = new int[8];
- BytesRef matchTerms[] = new BytesRef[8];
- int numMatches = 0;
-
- void addMatch(int startOffset, int endOffset, BytesRef term) {
- assert startOffset >= this.startOffset && startOffset <= this.endOffset;
- if (numMatches == matchStarts.length) {
- int newLength = ArrayUtil.oversize(numMatches+1, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
- int newMatchStarts[] = new int[newLength];
- int newMatchEnds[] = new int[newLength];
- BytesRef newMatchTerms[] = new BytesRef[newLength];
- System.arraycopy(matchStarts, 0, newMatchStarts, 0, numMatches);
- System.arraycopy(matchEnds, 0, newMatchEnds, 0, numMatches);
- System.arraycopy(matchTerms, 0, newMatchTerms, 0, numMatches);
- matchStarts = newMatchStarts;
- matchEnds = newMatchEnds;
- matchTerms = newMatchTerms;
- }
- assert matchStarts.length == matchEnds.length && matchEnds.length == matchTerms.length;
- matchStarts[numMatches] = startOffset;
- matchEnds[numMatches] = endOffset;
- matchTerms[numMatches] = term;
- numMatches++;
- }
-
- void sort() {
- final int starts[] = matchStarts;
- final int ends[] = matchEnds;
- final BytesRef terms[] = matchTerms;
- new InPlaceMergeSorter() {
- @Override
- protected void swap(int i, int j) {
- int temp = starts[i];
- starts[i] = starts[j];
- starts[j] = temp;
-
- temp = ends[i];
- ends[i] = ends[j];
- ends[j] = temp;
-
- BytesRef tempTerm = terms[i];
- terms[i] = terms[j];
- terms[j] = tempTerm;
- }
-
- @Override
- protected int compare(int i, int j) {
- return Integer.compare(starts[i], starts[j]);
- }
-
- }.sort(0, numMatches);
- }
-
- void reset() {
- startOffset = endOffset = -1;
- score = 0.0f;
- numMatches = 0;
- }
-
- /**
- * Start offset of this passage.
- * @return start index (inclusive) of the passage in the
- * original content: always >= 0.
- */
- public int getStartOffset() {
- return startOffset;
- }
-
- /**
- * End offset of this passage.
- * @return end index (exclusive) of the passage in the
- * original content: always >= {@link #getStartOffset()}
- */
- public int getEndOffset() {
- return endOffset;
- }
-
- /**
- * Passage's score.
- */
- public float getScore() {
- return score;
- }
-
- /**
- * Number of term matches available in
- * {@link #getMatchStarts}, {@link #getMatchEnds},
- * {@link #getMatchTerms}
- */
- public int getNumMatches() {
- return numMatches;
- }
-
- /**
- * Start offsets of the term matches, in increasing order.
- * <p>
- * Only {@link #getNumMatches} are valid. Note that these
- * offsets are absolute (not relative to {@link #getStartOffset()}).
- */
- public int[] getMatchStarts() {
- return matchStarts;
- }
-
- /**
- * End offsets of the term matches, corresponding with {@link #getMatchStarts}.
- * <p>
- * Only {@link #getNumMatches} are valid. Note that it's possible that an end offset
- * could exceed beyond the bounds of the passage ({@link #getEndOffset()}), if the
- * Analyzer produced a term which spans a passage boundary.
- */
- public int[] getMatchEnds() {
- return matchEnds;
- }
-
- /**
- * BytesRef (term text) of the matches, corresponding with {@link #getMatchStarts()}.
- * <p>
- * Only {@link #getNumMatches()} are valid.
- */
- public BytesRef[] getMatchTerms() {
- return matchTerms;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PassageFormatter.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PassageFormatter.java b/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PassageFormatter.java
deleted file mode 100644
index f1596c1..0000000
--- a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PassageFormatter.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.postingshighlight;
-
-/**
- * Creates a formatted snippet from the top passages.
- *
- * @lucene.experimental
- */
-public abstract class PassageFormatter {
-
- /**
- * Formats the top <code>passages</code> from <code>content</code>
- * into a human-readable text snippet.
- *
- * @param passages top-N passages for the field. Note these are sorted in
- * the order that they appear in the document for convenience.
- * @param content content for the field.
- * @return formatted highlight. Note that for the
- * non-expert APIs in {@link PostingsHighlighter} that
- * return String, the toString method on the Object
- * returned by this method is used to compute the string.
- */
- public abstract Object format(Passage passages[], String content);
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PassageScorer.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PassageScorer.java b/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PassageScorer.java
deleted file mode 100644
index 1f74f7a..0000000
--- a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PassageScorer.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.postingshighlight;
-
-/**
- * Ranks passages found by {@link PostingsHighlighter}.
- * <p>
- * Each passage is scored as a miniature document within the document.
- * The final score is computed as {@link #norm} * ∑ ({@link #weight} * {@link #tf}).
- * The default implementation is {@link #norm} * BM25.
- * @lucene.experimental
- */
-public class PassageScorer {
-
- // TODO: this formula is completely made up. It might not provide relevant snippets!
-
- /** BM25 k1 parameter, controls term frequency normalization */
- final float k1;
- /** BM25 b parameter, controls length normalization. */
- final float b;
- /** A pivot used for length normalization. */
- final float pivot;
-
- /**
- * Creates PassageScorer with these default values:
- * <ul>
- * <li>{@code k1 = 1.2},
- * <li>{@code b = 0.75}.
- * <li>{@code pivot = 87}
- * </ul>
- */
- public PassageScorer() {
- // 1.2 and 0.75 are well-known bm25 defaults (but maybe not the best here) ?
- // 87 is typical average english sentence length.
- this(1.2f, 0.75f, 87f);
- }
-
- /**
- * Creates PassageScorer with specified scoring parameters
- * @param k1 Controls non-linear term frequency normalization (saturation).
- * @param b Controls to what degree passage length normalizes tf values.
- * @param pivot Pivot value for length normalization (some rough idea of average sentence length in characters).
- */
- public PassageScorer(float k1, float b, float pivot) {
- this.k1 = k1;
- this.b = b;
- this.pivot = pivot;
- }
-
- /**
- * Computes term importance, given its in-document statistics.
- *
- * @param contentLength length of document in characters
- * @param totalTermFreq number of time term occurs in document
- * @return term importance
- */
- public float weight(int contentLength, int totalTermFreq) {
- // approximate #docs from content length
- float numDocs = 1 + contentLength / pivot;
- // numDocs not numDocs - docFreq (ala DFR), since we approximate numDocs
- return (k1 + 1) * (float) Math.log(1 + (numDocs + 0.5D)/(totalTermFreq + 0.5D));
- }
-
- /**
- * Computes term weight, given the frequency within the passage
- * and the passage's length.
- *
- * @param freq number of occurrences of within this passage
- * @param passageLen length of the passage in characters.
- * @return term weight
- */
- public float tf(int freq, int passageLen) {
- float norm = k1 * ((1 - b) + b * (passageLen / pivot));
- return freq / (freq + norm);
- }
-
- /**
- * Normalize a passage according to its position in the document.
- * <p>
- * Typically passages towards the beginning of the document are
- * more useful for summarizing the contents.
- * <p>
- * The default implementation is <code>1 + 1/log(pivot + passageStart)</code>
- * @param passageStart start offset of the passage
- * @return a boost value multiplied into the passage's core.
- */
- public float norm(int passageStart) {
- return 1 + 1/(float)Math.log(pivot + passageStart);
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java b/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java
deleted file mode 100644
index e4d3667..0000000
--- a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java
+++ /dev/null
@@ -1,820 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.postingshighlight;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.text.BreakIterator;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.PriorityQueue;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexReaderContext;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.MultiReader;
-import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.ReaderUtil;
-import org.apache.lucene.index.StoredFieldVisitor;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.InPlaceMergeSorter;
-import org.apache.lucene.util.UnicodeUtil;
-import org.apache.lucene.util.automaton.CharacterRunAutomaton;
-
-/**
- * Simple highlighter that does not analyze fields nor use
- * term vectors. Instead it requires
- * {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}.
- * <p>
- * PostingsHighlighter treats the single original document as the whole corpus, and then scores individual
- * passages as if they were documents in this corpus. It uses a {@link BreakIterator} to find
- * passages in the text; by default it breaks using {@link BreakIterator#getSentenceInstance(Locale)
- * getSentenceInstance(Locale.ROOT)}. It then iterates in parallel (merge sorting by offset) through
- * the positions of all terms from the query, coalescing those hits that occur in a single passage
- * into a {@link Passage}, and then scores each Passage using a separate {@link PassageScorer}.
- * Passages are finally formatted into highlighted snippets with a {@link PassageFormatter}.
- * <p>
- * You can customize the behavior by subclassing this highlighter, some important hooks:
- * <ul>
- * <li>{@link #getBreakIterator(String)}: Customize how the text is divided into passages.
- * <li>{@link #getScorer(String)}: Customize how passages are ranked.
- * <li>{@link #getFormatter(String)}: Customize how snippets are formatted.
- * <li>{@link #getIndexAnalyzer(String)}: Enable highlighting of MultiTermQuerys such as {@code WildcardQuery}.
- * </ul>
- * <p>
- * <b>WARNING</b>: The code is very new and probably still has some exciting bugs!
- * <p>
- * Example usage:
- * <pre class="prettyprint">
- * // configure field with offsets at index time
- * FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- * offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- * Field body = new Field("body", "foobar", offsetsType);
- *
- * // retrieve highlights at query time
- * PostingsHighlighter highlighter = new PostingsHighlighter();
- * Query query = new TermQuery(new Term("body", "highlighting"));
- * TopDocs topDocs = searcher.search(query, n);
- * String highlights[] = highlighter.highlight("body", query, searcher, topDocs);
- * </pre>
- * <p>
- * This is thread-safe, and can be used across different readers.
- * @lucene.experimental
- */
-public class PostingsHighlighter {
-
- // TODO: maybe allow re-analysis for tiny fields? currently we require offsets,
- // but if the analyzer is really fast and the field is tiny, this might really be
- // unnecessary.
-
- /** for rewriting: we don't want slow processing from MTQs */
- private static final IndexSearcher EMPTY_INDEXSEARCHER;
- static {
- try {
- IndexReader emptyReader = new MultiReader();
- EMPTY_INDEXSEARCHER = new IndexSearcher(emptyReader);
- EMPTY_INDEXSEARCHER.setQueryCache(null);
- } catch (IOException bogus) {
- throw new RuntimeException(bogus);
- }
- }
-
- /** Default maximum content size to process. Typically snippets
- * closer to the beginning of the document better summarize its content */
- public static final int DEFAULT_MAX_LENGTH = 10000;
-
- private final int maxLength;
-
- /** Set the first time {@link #getFormatter} is called,
- * and then reused. */
- private PassageFormatter defaultFormatter;
-
- /** Set the first time {@link #getScorer} is called,
- * and then reused. */
- private PassageScorer defaultScorer;
-
- /**
- * Creates a new highlighter with {@link #DEFAULT_MAX_LENGTH}.
- */
- public PostingsHighlighter() {
- this(DEFAULT_MAX_LENGTH);
- }
-
- /**
- * Creates a new highlighter, specifying maximum content length.
- * @param maxLength maximum content size to process.
- * @throws IllegalArgumentException if <code>maxLength</code> is negative or <code>Integer.MAX_VALUE</code>
- */
- public PostingsHighlighter(int maxLength) {
- if (maxLength < 0 || maxLength == Integer.MAX_VALUE) {
- // two reasons: no overflow problems in BreakIterator.preceding(offset+1),
- // our sentinel in the offsets queue uses this value to terminate.
- throw new IllegalArgumentException("maxLength must be < Integer.MAX_VALUE");
- }
- this.maxLength = maxLength;
- }
-
- /** Returns the {@link BreakIterator} to use for
- * dividing text into passages. This returns
- * {@link BreakIterator#getSentenceInstance(Locale)} by default;
- * subclasses can override to customize. */
- protected BreakIterator getBreakIterator(String field) {
- return BreakIterator.getSentenceInstance(Locale.ROOT);
- }
-
- /** Returns the {@link PassageFormatter} to use for
- * formatting passages into highlighted snippets. This
- * returns a new {@code PassageFormatter} by default;
- * subclasses can override to customize. */
- protected PassageFormatter getFormatter(String field) {
- if (defaultFormatter == null) {
- defaultFormatter = new DefaultPassageFormatter();
- }
- return defaultFormatter;
- }
-
- /** Returns the {@link PassageScorer} to use for
- * ranking passages. This
- * returns a new {@code PassageScorer} by default;
- * subclasses can override to customize. */
- protected PassageScorer getScorer(String field) {
- if (defaultScorer == null) {
- defaultScorer = new PassageScorer();
- }
- return defaultScorer;
- }
-
- /**
- * Highlights the top passages from a single field.
- *
- * @param field field name to highlight.
- * Must have a stored string value and also be indexed with offsets.
- * @param query query to highlight.
- * @param searcher searcher that was previously used to execute the query.
- * @param topDocs TopDocs containing the summary result documents to highlight.
- * @return Array of formatted snippets corresponding to the documents in <code>topDocs</code>.
- * If no highlights were found for a document, the
- * first sentence for the field will be returned.
- * @throws IOException if an I/O error occurred during processing
- * @throws IllegalArgumentException if <code>field</code> was indexed without
- * {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}
- */
- public String[] highlight(String field, Query query, IndexSearcher searcher, TopDocs topDocs) throws IOException {
- return highlight(field, query, searcher, topDocs, 1);
- }
-
- /**
- * Highlights the top-N passages from a single field.
- *
- * @param field field name to highlight.
- * Must have a stored string value and also be indexed with offsets.
- * @param query query to highlight.
- * @param searcher searcher that was previously used to execute the query.
- * @param topDocs TopDocs containing the summary result documents to highlight.
- * @param maxPassages The maximum number of top-N ranked passages used to
- * form the highlighted snippets.
- * @return Array of formatted snippets corresponding to the documents in <code>topDocs</code>.
- * If no highlights were found for a document, the
- * first {@code maxPassages} sentences from the
- * field will be returned.
- * @throws IOException if an I/O error occurred during processing
- * @throws IllegalArgumentException if <code>field</code> was indexed without
- * {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}
- */
- public String[] highlight(String field, Query query, IndexSearcher searcher, TopDocs topDocs, int maxPassages) throws IOException {
- Map<String,String[]> res = highlightFields(new String[] { field }, query, searcher, topDocs, new int[] { maxPassages });
- return res.get(field);
- }
-
- /**
- * Highlights the top passages from multiple fields.
- * <p>
- * Conceptually, this behaves as a more efficient form of:
- * <pre class="prettyprint">
- * Map m = new HashMap();
- * for (String field : fields) {
- * m.put(field, highlight(field, query, searcher, topDocs));
- * }
- * return m;
- * </pre>
- *
- * @param fields field names to highlight.
- * Must have a stored string value and also be indexed with offsets.
- * @param query query to highlight.
- * @param searcher searcher that was previously used to execute the query.
- * @param topDocs TopDocs containing the summary result documents to highlight.
- * @return Map keyed on field name, containing the array of formatted snippets
- * corresponding to the documents in <code>topDocs</code>.
- * If no highlights were found for a document, the
- * first sentence from the field will be returned.
- * @throws IOException if an I/O error occurred during processing
- * @throws IllegalArgumentException if <code>field</code> was indexed without
- * {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}
- */
- public Map<String,String[]> highlightFields(String fields[], Query query, IndexSearcher searcher, TopDocs topDocs) throws IOException {
- int maxPassages[] = new int[fields.length];
- Arrays.fill(maxPassages, 1);
- return highlightFields(fields, query, searcher, topDocs, maxPassages);
- }
-
- /**
- * Highlights the top-N passages from multiple fields.
- * <p>
- * Conceptually, this behaves as a more efficient form of:
- * <pre class="prettyprint">
- * Map m = new HashMap();
- * for (String field : fields) {
- * m.put(field, highlight(field, query, searcher, topDocs, maxPassages));
- * }
- * return m;
- * </pre>
- *
- * @param fields field names to highlight.
- * Must have a stored string value and also be indexed with offsets.
- * @param query query to highlight.
- * @param searcher searcher that was previously used to execute the query.
- * @param topDocs TopDocs containing the summary result documents to highlight.
- * @param maxPassages The maximum number of top-N ranked passages per-field used to
- * form the highlighted snippets.
- * @return Map keyed on field name, containing the array of formatted snippets
- * corresponding to the documents in <code>topDocs</code>.
- * If no highlights were found for a document, the
- * first {@code maxPassages} sentences from the
- * field will be returned.
- * @throws IOException if an I/O error occurred during processing
- * @throws IllegalArgumentException if <code>field</code> was indexed without
- * {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}
- */
- public Map<String,String[]> highlightFields(String fields[], Query query, IndexSearcher searcher, TopDocs topDocs, int maxPassages[]) throws IOException {
- final ScoreDoc scoreDocs[] = topDocs.scoreDocs;
- int docids[] = new int[scoreDocs.length];
- for (int i = 0; i < docids.length; i++) {
- docids[i] = scoreDocs[i].doc;
- }
-
- return highlightFields(fields, query, searcher, docids, maxPassages);
- }
-
- /**
- * Highlights the top-N passages from multiple fields,
- * for the provided int[] docids.
- *
- * @param fieldsIn field names to highlight.
- * Must have a stored string value and also be indexed with offsets.
- * @param query query to highlight.
- * @param searcher searcher that was previously used to execute the query.
- * @param docidsIn containing the document IDs to highlight.
- * @param maxPassagesIn The maximum number of top-N ranked passages per-field used to
- * form the highlighted snippets.
- * @return Map keyed on field name, containing the array of formatted snippets
- * corresponding to the documents in <code>docidsIn</code>.
- * If no highlights were found for a document, the
- * first {@code maxPassages} from the field will
- * be returned.
- * @throws IOException if an I/O error occurred during processing
- * @throws IllegalArgumentException if <code>field</code> was indexed without
- * {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}
- */
- public Map<String,String[]> highlightFields(String fieldsIn[], Query query, IndexSearcher searcher, int[] docidsIn, int maxPassagesIn[]) throws IOException {
- Map<String,String[]> snippets = new HashMap<>();
- for(Map.Entry<String,Object[]> ent : highlightFieldsAsObjects(fieldsIn, query, searcher, docidsIn, maxPassagesIn).entrySet()) {
- Object[] snippetObjects = ent.getValue();
- String[] snippetStrings = new String[snippetObjects.length];
- snippets.put(ent.getKey(), snippetStrings);
- for(int i=0;i<snippetObjects.length;i++) {
- Object snippet = snippetObjects[i];
- if (snippet != null) {
- snippetStrings[i] = snippet.toString();
- }
- }
- }
-
- return snippets;
- }
-
- /**
- * Expert: highlights the top-N passages from multiple fields,
- * for the provided int[] docids, to custom Object as
- * returned by the {@link PassageFormatter}. Use
- * this API to render to something other than String.
- *
- * @param fieldsIn field names to highlight.
- * Must have a stored string value and also be indexed with offsets.
- * @param query query to highlight.
- * @param searcher searcher that was previously used to execute the query.
- * @param docidsIn containing the document IDs to highlight.
- * @param maxPassagesIn The maximum number of top-N ranked passages per-field used to
- * form the highlighted snippets.
- * @return Map keyed on field name, containing the array of formatted snippets
- * corresponding to the documents in <code>docidsIn</code>.
- * If no highlights were found for a document, the
- * first {@code maxPassages} from the field will
- * be returned.
- * @throws IOException if an I/O error occurred during processing
- * @throws IllegalArgumentException if <code>field</code> was indexed without
- * {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}
- */
- protected Map<String,Object[]> highlightFieldsAsObjects(String fieldsIn[], Query query, IndexSearcher searcher, int[] docidsIn, int maxPassagesIn[]) throws IOException {
- if (fieldsIn.length < 1) {
- throw new IllegalArgumentException("fieldsIn must not be empty");
- }
- if (fieldsIn.length != maxPassagesIn.length) {
- throw new IllegalArgumentException("invalid number of maxPassagesIn");
- }
- SortedSet<Term> queryTerms = new TreeSet<>();
- EMPTY_INDEXSEARCHER.createNormalizedWeight(query, false).extractTerms(queryTerms);
-
- IndexReaderContext readerContext = searcher.getIndexReader().getContext();
- List<LeafReaderContext> leaves = readerContext.leaves();
-
- // Make our own copies because we sort in-place:
- int[] docids = new int[docidsIn.length];
- System.arraycopy(docidsIn, 0, docids, 0, docidsIn.length);
- final String fields[] = new String[fieldsIn.length];
- System.arraycopy(fieldsIn, 0, fields, 0, fieldsIn.length);
- final int maxPassages[] = new int[maxPassagesIn.length];
- System.arraycopy(maxPassagesIn, 0, maxPassages, 0, maxPassagesIn.length);
-
- // sort for sequential io
- Arrays.sort(docids);
- new InPlaceMergeSorter() {
-
- @Override
- protected void swap(int i, int j) {
- String tmp = fields[i];
- fields[i] = fields[j];
- fields[j] = tmp;
- int tmp2 = maxPassages[i];
- maxPassages[i] = maxPassages[j];
- maxPassages[j] = tmp2;
- }
-
- @Override
- protected int compare(int i, int j) {
- return fields[i].compareTo(fields[j]);
- }
-
- }.sort(0, fields.length);
-
- // pull stored data:
- String[][] contents = loadFieldValues(searcher, fields, docids, maxLength);
-
- Map<String,Object[]> highlights = new HashMap<>();
- for (int i = 0; i < fields.length; i++) {
- String field = fields[i];
- int numPassages = maxPassages[i];
- Term floor = new Term(field, "");
- Term ceiling = new Term(field, UnicodeUtil.BIG_TERM);
- SortedSet<Term> fieldTerms = queryTerms.subSet(floor, ceiling);
- // TODO: should we have some reasonable defaults for term pruning? (e.g. stopwords)
-
- // Strip off the redundant field:
- BytesRef terms[] = new BytesRef[fieldTerms.size()];
- int termUpto = 0;
- for(Term term : fieldTerms) {
- terms[termUpto++] = term.bytes();
- }
- Map<Integer,Object> fieldHighlights = highlightField(field, contents[i], getBreakIterator(field), terms, docids, leaves, numPassages, query);
-
- Object[] result = new Object[docids.length];
- for (int j = 0; j < docidsIn.length; j++) {
- result[j] = fieldHighlights.get(docidsIn[j]);
- }
- highlights.put(field, result);
- }
- return highlights;
- }
-
- /** Loads the String values for each field X docID to be
- * highlighted. By default this loads from stored
- * fields, but a subclass can change the source. This
- * method should allocate the String[fields.length][docids.length]
- * and fill all values. The returned Strings must be
- * identical to what was indexed. */
- protected String[][] loadFieldValues(IndexSearcher searcher, String[] fields, int[] docids, int maxLength) throws IOException {
- String contents[][] = new String[fields.length][docids.length];
- char valueSeparators[] = new char[fields.length];
- for (int i = 0; i < fields.length; i++) {
- valueSeparators[i] = getMultiValuedSeparator(fields[i]);
- }
- LimitedStoredFieldVisitor visitor = new LimitedStoredFieldVisitor(fields, valueSeparators, maxLength);
- for (int i = 0; i < docids.length; i++) {
- searcher.doc(docids[i], visitor);
- for (int j = 0; j < fields.length; j++) {
- contents[j][i] = visitor.getValue(j).toString();
- }
- visitor.reset();
- }
- return contents;
- }
-
- /**
- * Returns the logical separator between values for multi-valued fields.
- * The default value is a space character, which means passages can span across values,
- * but a subclass can override, for example with {@code U+2029 PARAGRAPH SEPARATOR (PS)}
- * if each value holds a discrete passage for highlighting.
- */
- protected char getMultiValuedSeparator(String field) {
- return ' ';
- }
-
- /**
- * Returns the analyzer originally used to index the content for {@code field}.
- * <p>
- * This is used to highlight some MultiTermQueries.
- * @return Analyzer or null (the default, meaning no special multi-term processing)
- */
- protected Analyzer getIndexAnalyzer(String field) {
- return null;
- }
-
- private Map<Integer,Object> highlightField(String field, String contents[], BreakIterator bi, BytesRef terms[], int[] docids, List<LeafReaderContext> leaves, int maxPassages, Query query) throws IOException {
- Map<Integer,Object> highlights = new HashMap<>();
-
- PassageFormatter fieldFormatter = getFormatter(field);
- if (fieldFormatter == null) {
- throw new NullPointerException("PassageFormatter must not be null");
- }
-
- // check if we should do any multiterm processing
- Analyzer analyzer = getIndexAnalyzer(field);
- CharacterRunAutomaton automata[] = new CharacterRunAutomaton[0];
- if (analyzer != null) {
- automata = MultiTermHighlighting.extractAutomata(query, field);
- }
-
- // resize 'terms', where the last term is the multiterm matcher
- if (automata.length > 0) {
- BytesRef newTerms[] = new BytesRef[terms.length + 1];
- System.arraycopy(terms, 0, newTerms, 0, terms.length);
- terms = newTerms;
- }
-
- // we are processing in increasing docid order, so we only need to reinitialize stuff on segment changes
- // otherwise, we will just advance() existing enums to the new document in the same segment.
- PostingsEnum postings[] = null;
- TermsEnum termsEnum = null;
- int lastLeaf = -1;
-
- for (int i = 0; i < docids.length; i++) {
- String content = contents[i];
- if (content.length() == 0) {
- continue; // nothing to do
- }
- bi.setText(content);
- int doc = docids[i];
- int leaf = ReaderUtil.subIndex(doc, leaves);
- LeafReaderContext subContext = leaves.get(leaf);
- LeafReader r = subContext.reader();
-
- assert leaf >= lastLeaf; // increasing order
-
- // if the segment has changed, we must initialize new enums.
- if (leaf != lastLeaf) {
- Terms t = r.terms(field);
- if (t != null) {
- if (!t.hasOffsets()) {
- // no offsets available
- throw new IllegalArgumentException("field '" + field + "' was indexed without offsets, cannot highlight");
- }
- termsEnum = t.iterator();
- postings = new PostingsEnum[terms.length];
- } else {
- termsEnum = null;
- }
- }
- if (termsEnum == null) {
- continue; // no terms for this field, nothing to do
- }
-
- // if there are multi-term matches, we have to initialize the "fake" enum for each document
- if (automata.length > 0) {
- PostingsEnum dp = MultiTermHighlighting.getDocsEnum(analyzer.tokenStream(field, content), automata);
- dp.advance(doc - subContext.docBase);
- postings[terms.length-1] = dp; // last term is the multiterm matcher
- }
-
- Passage passages[] = highlightDoc(field, terms, content.length(), bi, doc - subContext.docBase, termsEnum, postings, maxPassages);
-
- if (passages.length == 0) {
- // no passages were returned, so ask for a default summary
- passages = getEmptyHighlight(field, bi, maxPassages);
- }
-
- if (passages.length > 0) {
- highlights.put(doc, fieldFormatter.format(passages, content));
- }
-
- lastLeaf = leaf;
- }
-
- return highlights;
- }
-
- // algorithm: treat sentence snippets as miniature documents
- // we can intersect these with the postings lists via BreakIterator.preceding(offset),s
- // score each sentence as norm(sentenceStartOffset) * sum(weight * tf(freq))
- private Passage[] highlightDoc(String field, BytesRef terms[], int contentLength, BreakIterator bi, int doc,
- TermsEnum termsEnum, PostingsEnum[] postings, int n) throws IOException {
- PassageScorer scorer = getScorer(field);
- if (scorer == null) {
- throw new NullPointerException("PassageScorer must not be null");
- }
- PriorityQueue<OffsetsEnum> pq = new PriorityQueue<>();
- float weights[] = new float[terms.length];
- // initialize postings
- for (int i = 0; i < terms.length; i++) {
- PostingsEnum de = postings[i];
- int pDoc;
- if (de == EMPTY) {
- continue;
- } else if (de == null) {
- postings[i] = EMPTY; // initially
- if (!termsEnum.seekExact(terms[i])) {
- continue; // term not found
- }
- de = postings[i] = termsEnum.postings(null, PostingsEnum.OFFSETS);
- assert de != null;
- pDoc = de.advance(doc);
- } else {
- pDoc = de.docID();
- if (pDoc < doc) {
- pDoc = de.advance(doc);
- }
- }
-
- if (doc == pDoc) {
- weights[i] = scorer.weight(contentLength, de.freq());
- de.nextPosition();
- pq.add(new OffsetsEnum(de, i));
- }
- }
-
- pq.add(new OffsetsEnum(EMPTY, Integer.MAX_VALUE)); // a sentinel for termination
-
- PriorityQueue<Passage> passageQueue = new PriorityQueue<>(n, new Comparator<Passage>() {
- @Override
- public int compare(Passage left, Passage right) {
- if (left.score < right.score) {
- return -1;
- } else if (left.score > right.score) {
- return 1;
- } else {
- return left.startOffset - right.startOffset;
- }
- }
- });
- Passage current = new Passage();
-
- OffsetsEnum off;
- while ((off = pq.poll()) != null) {
- final PostingsEnum dp = off.dp;
- int start = dp.startOffset();
- assert start >= 0;
- int end = dp.endOffset();
- // LUCENE-5166: this hit would span the content limit... however more valid
- // hits may exist (they are sorted by start). so we pretend like we never
- // saw this term, it won't cause a passage to be added to passageQueue or anything.
- assert EMPTY.startOffset() == Integer.MAX_VALUE;
- if (start < contentLength && end > contentLength) {
- continue;
- }
- if (start >= current.endOffset) {
- if (current.startOffset >= 0) {
- // finalize current
- current.score *= scorer.norm(current.startOffset);
- // new sentence: first add 'current' to queue
- if (passageQueue.size() == n && current.score < passageQueue.peek().score) {
- current.reset(); // can't compete, just reset it
- } else {
- passageQueue.offer(current);
- if (passageQueue.size() > n) {
- current = passageQueue.poll();
- current.reset();
- } else {
- current = new Passage();
- }
- }
- }
- // if we exceed limit, we are done
- if (start >= contentLength) {
- Passage passages[] = new Passage[passageQueue.size()];
- passageQueue.toArray(passages);
- for (Passage p : passages) {
- p.sort();
- }
- // sort in ascending order
- Arrays.sort(passages, new Comparator<Passage>() {
- @Override
- public int compare(Passage left, Passage right) {
- return left.startOffset - right.startOffset;
- }
- });
- return passages;
- }
- // advance breakiterator
- assert BreakIterator.DONE < 0;
- current.startOffset = Math.max(bi.preceding(start+1), 0);
- current.endOffset = Math.min(bi.next(), contentLength);
- }
- int tf = 0;
- while (true) {
- tf++;
- BytesRef term = terms[off.id];
- if (term == null) {
- // multitermquery match, pull from payload
- term = off.dp.getPayload();
- assert term != null;
- }
- current.addMatch(start, end, term);
- if (off.pos == dp.freq()) {
- break; // removed from pq
- } else {
- off.pos++;
- dp.nextPosition();
- start = dp.startOffset();
- end = dp.endOffset();
- }
- if (start >= current.endOffset || end > contentLength) {
- pq.offer(off);
- break;
- }
- }
- current.score += weights[off.id] * scorer.tf(tf, current.endOffset - current.startOffset);
- }
-
- // Dead code but compiler disagrees:
- assert false;
- return null;
- }
-
- /** Called to summarize a document when no hits were
- * found. By default this just returns the first
- * {@code maxPassages} sentences; subclasses can override
- * to customize. */
- protected Passage[] getEmptyHighlight(String fieldName, BreakIterator bi, int maxPassages) {
- // BreakIterator should be un-next'd:
- List<Passage> passages = new ArrayList<>();
- int pos = bi.current();
- assert pos == 0;
- while (passages.size() < maxPassages) {
- int next = bi.next();
- if (next == BreakIterator.DONE) {
- break;
- }
- Passage passage = new Passage();
- passage.score = Float.NaN;
- passage.startOffset = pos;
- passage.endOffset = next;
- passages.add(passage);
- pos = next;
- }
-
- return passages.toArray(new Passage[passages.size()]);
- }
-
- private static class OffsetsEnum implements Comparable<OffsetsEnum> {
- PostingsEnum dp;
- int pos;
- int id;
-
- OffsetsEnum(PostingsEnum dp, int id) throws IOException {
- this.dp = dp;
- this.id = id;
- this.pos = 1;
- }
-
- @Override
- public int compareTo(OffsetsEnum other) {
- try {
- int off = dp.startOffset();
- int otherOff = other.dp.startOffset();
- if (off == otherOff) {
- return id - other.id;
- } else {
- return Integer.compare(off, otherOff);
- }
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- private static final PostingsEnum EMPTY = new PostingsEnum() {
-
- @Override
- public int nextPosition() throws IOException { return -1; }
-
- @Override
- public int startOffset() throws IOException { return Integer.MAX_VALUE; }
-
- @Override
- public int endOffset() throws IOException { return Integer.MAX_VALUE; }
-
- @Override
- public BytesRef getPayload() throws IOException { return null; }
-
- @Override
- public int freq() throws IOException { return 0; }
-
- @Override
- public int docID() { return NO_MORE_DOCS; }
-
- @Override
- public int nextDoc() throws IOException { return NO_MORE_DOCS; }
-
- @Override
- public int advance(int target) throws IOException { return NO_MORE_DOCS; }
-
- @Override
- public long cost() { return 0; }
- };
-
- private static class LimitedStoredFieldVisitor extends StoredFieldVisitor {
- private final String fields[];
- private final char valueSeparators[];
- private final int maxLength;
- private final StringBuilder builders[];
- private int currentField = -1;
-
- public LimitedStoredFieldVisitor(String fields[], char valueSeparators[], int maxLength) {
- assert fields.length == valueSeparators.length;
- this.fields = fields;
- this.valueSeparators = valueSeparators;
- this.maxLength = maxLength;
- builders = new StringBuilder[fields.length];
- for (int i = 0; i < builders.length; i++) {
- builders[i] = new StringBuilder();
- }
- }
-
- @Override
- public void stringField(FieldInfo fieldInfo, byte[] bytes) throws IOException {
- String value = new String(bytes, StandardCharsets.UTF_8);
- assert currentField >= 0;
- StringBuilder builder = builders[currentField];
- if (builder.length() > 0 && builder.length() < maxLength) {
- builder.append(valueSeparators[currentField]);
- }
- if (builder.length() + value.length() > maxLength) {
- builder.append(value, 0, maxLength - builder.length());
- } else {
- builder.append(value);
- }
- }
-
- @Override
- public Status needsField(FieldInfo fieldInfo) throws IOException {
- currentField = Arrays.binarySearch(fields, fieldInfo.name);
- if (currentField < 0) {
- return Status.NO;
- } else if (builders[currentField].length() > maxLength) {
- return fields.length == 1 ? Status.STOP : Status.NO;
- }
- return Status.YES;
- }
-
- String getValue(int i) {
- return builders[i].toString();
- }
-
- void reset() {
- currentField = -1;
- for (int i = 0; i < fields.length; i++) {
- builders[i].setLength(0);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/package-info.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/package-info.java b/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/package-info.java
deleted file mode 100644
index 10013c2..0000000
--- a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Highlighter implementation that uses offsets from postings lists.
- */
-package org.apache.lucene.search.postingshighlight;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/CustomSeparatorBreakIterator.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/CustomSeparatorBreakIterator.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/CustomSeparatorBreakIterator.java
new file mode 100644
index 0000000..7119aed
--- /dev/null
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/CustomSeparatorBreakIterator.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search.uhighlight;
+
+import java.text.BreakIterator;
+import java.text.CharacterIterator;
+
+/**
+ * A {@link BreakIterator} that breaks the text whenever a certain separator, provided as a constructor argument, is found.
+ */
+public final class CustomSeparatorBreakIterator extends BreakIterator {
+
+ private final char separator;
+ private CharacterIterator text;
+ private int current;
+
+ public CustomSeparatorBreakIterator(char separator) {
+ this.separator = separator;
+ }
+
+ @Override
+ public int current() {
+ return current;
+ }
+
+ @Override
+ public int first() {
+ text.setIndex(text.getBeginIndex());
+ return current = text.getIndex();
+ }
+
+ @Override
+ public int last() {
+ text.setIndex(text.getEndIndex());
+ return current = text.getIndex();
+ }
+
+ @Override
+ public int next() {
+ if (text.getIndex() == text.getEndIndex()) {
+ return DONE;
+ } else {
+ return advanceForward();
+ }
+ }
+
+ private int advanceForward() {
+ char c;
+ while ((c = text.next()) != CharacterIterator.DONE) {
+ if (c == separator) {
+ return current = text.getIndex() + 1;
+ }
+ }
+ assert text.getIndex() == text.getEndIndex();
+ return current = text.getIndex();
+ }
+
+ @Override
+ public int following(int pos) {
+ if (pos < text.getBeginIndex() || pos > text.getEndIndex()) {
+ throw new IllegalArgumentException("offset out of bounds");
+ } else if (pos == text.getEndIndex()) {
+ // this conflicts with the javadocs, but matches actual behavior (Oracle has a bug in something)
+ // https://bugs.openjdk.java.net/browse/JDK-8015110
+ text.setIndex(text.getEndIndex());
+ current = text.getIndex();
+ return DONE;
+ } else {
+ text.setIndex(pos);
+ current = text.getIndex();
+ return advanceForward();
+ }
+ }
+
+ @Override
+ public int previous() {
+ if (text.getIndex() == text.getBeginIndex()) {
+ return DONE;
+ } else {
+ return advanceBackward();
+ }
+ }
+
+ private int advanceBackward() {
+ char c;
+ while ((c = text.previous()) != CharacterIterator.DONE) {
+ if (c == separator) {
+ return current = text.getIndex() + 1;
+ }
+ }
+ assert text.getIndex() == text.getBeginIndex();
+ return current = text.getIndex();
+ }
+
+ @Override
+ public int preceding(int pos) {
+ if (pos < text.getBeginIndex() || pos > text.getEndIndex()) {
+ throw new IllegalArgumentException("offset out of bounds");
+ } else if (pos == text.getBeginIndex()) {
+ // this conflicts with the javadocs, but matches actual behavior (Oracle has a bug in something)
+ // https://bugs.openjdk.java.net/browse/JDK-8015110
+ text.setIndex(text.getBeginIndex());
+ current = text.getIndex();
+ return DONE;
+ } else {
+ text.setIndex(pos);
+ current = text.getIndex();
+ return advanceBackward();
+ }
+ }
+
+ @Override
+ public int next(int n) {
+ if (n < 0) {
+ for (int i = 0; i < -n; i++) {
+ previous();
+ }
+ } else {
+ for (int i = 0; i < n; i++) {
+ next();
+ }
+ }
+ return current();
+ }
+
+ @Override
+ public CharacterIterator getText() {
+ return text;
+ }
+
+ @Override
+ public void setText(CharacterIterator newText) {
+ text = newText;
+ current = text.getBeginIndex();
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/WholeBreakIterator.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/WholeBreakIterator.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/WholeBreakIterator.java
new file mode 100644
index 0000000..37f48aa
--- /dev/null
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/WholeBreakIterator.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search.uhighlight;
+
+import java.text.BreakIterator;
+import java.text.CharacterIterator;
+
+/** Just produces one single fragment for the entire text */
+public final class WholeBreakIterator extends BreakIterator {
+ private CharacterIterator text;
+ private int start;
+ private int end;
+ private int current;
+
+ @Override
+ public int current() {
+ return current;
+ }
+
+ @Override
+ public int first() {
+ return (current = start);
+ }
+
+ @Override
+ public int following(int pos) {
+ if (pos < start || pos > end) {
+ throw new IllegalArgumentException("offset out of bounds");
+ } else if (pos == end) {
+ // this conflicts with the javadocs, but matches actual behavior (Oracle has a bug in something)
+ // https://bugs.openjdk.java.net/browse/JDK-8015110
+ current = end;
+ return DONE;
+ } else {
+ return last();
+ }
+ }
+
+ @Override
+ public CharacterIterator getText() {
+ return text;
+ }
+
+ @Override
+ public int last() {
+ return (current = end);
+ }
+
+ @Override
+ public int next() {
+ if (current == end) {
+ return DONE;
+ } else {
+ return last();
+ }
+ }
+
+ @Override
+ public int next(int n) {
+ if (n < 0) {
+ for (int i = 0; i < -n; i++) {
+ previous();
+ }
+ } else {
+ for (int i = 0; i < n; i++) {
+ next();
+ }
+ }
+ return current();
+ }
+
+ @Override
+ public int preceding(int pos) {
+ if (pos < start || pos > end) {
+ throw new IllegalArgumentException("offset out of bounds");
+ } else if (pos == start) {
+ // this conflicts with the javadocs, but matches actual behavior (Oracle has a bug in something)
+ // https://bugs.openjdk.java.net/browse/JDK-8015110
+ current = start;
+ return DONE;
+ } else {
+ return first();
+ }
+ }
+
+ @Override
+ public int previous() {
+ if (current == start) {
+ return DONE;
+ } else {
+ return first();
+ }
+ }
+
+ @Override
+ public void setText(CharacterIterator newText) {
+ start = newText.getBeginIndex();
+ end = newText.getEndIndex();
+ text = newText;
+ current = start;
+ }
+}
[35/44] lucene-solr git commit: LUCENE-7841: Normalize ґ to г in Ukrainian analyzer
Posted by cp...@apache.org.
LUCENE-7841: Normalize ґ to г in Ukrainian analyzer
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/effd3766
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/effd3766
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/effd3766
Branch: refs/heads/jira/solr-8668
Commit: effd3766e7fc501a04a6cea08a4c803fa7197768
Parents: 10d3aa2
Author: Dawid Weiss <dw...@apache.org>
Authored: Thu May 25 14:04:28 2017 +0200
Committer: Dawid Weiss <dw...@apache.org>
Committed: Thu May 25 14:09:36 2017 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 4 ++++
.../lucene/analysis/uk/UkrainianMorfologikAnalyzer.java | 2 ++
.../apache/lucene/analysis/uk/TestUkrainianAnalyzer.java | 9 ++++++++-
lucene/ivy-versions.properties | 2 +-
lucene/licenses/morfologik-ukrainian-search-3.7.5.jar.sha1 | 1 -
lucene/licenses/morfologik-ukrainian-search-3.7.6.jar.sha1 | 1 +
6 files changed, 16 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/effd3766/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index fe1be6e..a45c11a 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -109,6 +109,10 @@ Other
from methods that don't declare them ("sneaky throw" hack). (Robert Muir,
Uwe Schindler, Dawid Weiss)
+Improvements
+
+* LUCENE-7841: Normalize ґ to г in Ukrainian analyzer. (Andriy Rysin via Dawid Weiss)
+
======================= Lucene 6.6.0 =======================
New Features
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/effd3766/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/uk/UkrainianMorfologikAnalyzer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/uk/UkrainianMorfologikAnalyzer.java b/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/uk/UkrainianMorfologikAnalyzer.java
index 6955fe3..cd502fd 100644
--- a/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/uk/UkrainianMorfologikAnalyzer.java
+++ b/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/uk/UkrainianMorfologikAnalyzer.java
@@ -116,6 +116,8 @@ public final class UkrainianMorfologikAnalyzer extends StopwordAnalyzerBase {
// ignored characters
builder.add("\u0301", "");
builder.add("\u00AD", "");
+ builder.add("ґ", "г");
+ builder.add("Ґ", "Г");
NormalizeCharMap normMap = builder.build();
reader = new MappingCharFilter(normMap, reader);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/effd3766/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/uk/TestUkrainianAnalyzer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/uk/TestUkrainianAnalyzer.java b/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/uk/TestUkrainianAnalyzer.java
index 15b247d..e9a0102 100644
--- a/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/uk/TestUkrainianAnalyzer.java
+++ b/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/uk/TestUkrainianAnalyzer.java
@@ -52,10 +52,17 @@ public class TestUkrainianAnalyzer extends BaseTokenStreamTestCase {
public void testCapsTokenStream() throws Exception {
Analyzer a = new UkrainianMorfologikAnalyzer();
assertAnalyzesTo(a, "Цих Чайковського і Ґете.",
- new String[] { "Чайковське", "Чайковський", "Ґете" });
+ new String[] { "Чайковське", "Чайковський", "Гете" });
a.close();
}
+ public void testCharNormalization() throws Exception {
+ Analyzer a = new UkrainianMorfologikAnalyzer();
+ assertAnalyzesTo(a, "Ґюмрі та Гюмрі.",
+ new String[] { "Гюмрі", "Гюмрі" });
+ a.close();
+ }
+
public void testSampleSentence() throws Exception {
Analyzer a = new UkrainianMorfologikAnalyzer();
assertAnalyzesTo(a, "Це — проект генерування словника з тегами частин мови для української мови.",
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/effd3766/lucene/ivy-versions.properties
----------------------------------------------------------------------
diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties
index 3c45b2c..3318c21 100644
--- a/lucene/ivy-versions.properties
+++ b/lucene/ivy-versions.properties
@@ -276,7 +276,7 @@ org.slf4j.version = 1.7.7
/org.tukaani/xz = 1.5
/rome/rome = 1.0
-ua.net.nlp.morfologik-ukrainian-search.version = 3.7.5
+ua.net.nlp.morfologik-ukrainian-search.version = 3.7.6
/ua.net.nlp/morfologik-ukrainian-search = ${ua.net.nlp.morfologik-ukrainian-search.version}
/xerces/xercesImpl = 2.9.1
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/effd3766/lucene/licenses/morfologik-ukrainian-search-3.7.5.jar.sha1
----------------------------------------------------------------------
diff --git a/lucene/licenses/morfologik-ukrainian-search-3.7.5.jar.sha1 b/lucene/licenses/morfologik-ukrainian-search-3.7.5.jar.sha1
deleted file mode 100644
index 8794e71..0000000
--- a/lucene/licenses/morfologik-ukrainian-search-3.7.5.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2b8c8fbd740164d220ca7d18605b8b2092e163e9
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/effd3766/lucene/licenses/morfologik-ukrainian-search-3.7.6.jar.sha1
----------------------------------------------------------------------
diff --git a/lucene/licenses/morfologik-ukrainian-search-3.7.6.jar.sha1 b/lucene/licenses/morfologik-ukrainian-search-3.7.6.jar.sha1
new file mode 100644
index 0000000..6f0b86c
--- /dev/null
+++ b/lucene/licenses/morfologik-ukrainian-search-3.7.6.jar.sha1
@@ -0,0 +1 @@
+8d2c4bf006f59227bcba8885b4602b3a8b5bd799
[05/44] lucene-solr:jira/solr-8668: SOLR-10729: Deprecated LatLonType,
GeoHashField, SpatialPointVectorFieldType,
and SpatialTermQueryPrefixTreeFieldType
Posted by cp...@apache.org.
SOLR-10729: Deprecated LatLonType, GeoHashField, SpatialPointVectorFieldType, and SpatialTermQueryPrefixTreeFieldType
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/92ed8b42
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/92ed8b42
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/92ed8b42
Branch: refs/heads/jira/solr-8668
Commit: 92ed8b4220d1427a0c07eecaab422c2d22a2ce8c
Parents: 2218ded
Author: David Smiley <ds...@apache.org>
Authored: Tue May 23 11:09:32 2017 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Tue May 23 11:09:32 2017 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 7 +++++++
solr/core/src/java/org/apache/solr/schema/GeoHashField.java | 3 +++
solr/core/src/java/org/apache/solr/schema/LatLonType.java | 3 +++
.../org/apache/solr/schema/SpatialPointVectorFieldType.java | 3 ++-
.../solr/schema/SpatialTermQueryPrefixTreeFieldType.java | 3 ++-
5 files changed, 17 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/92ed8b42/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index e29952b..951b9d5 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -79,6 +79,9 @@ Upgrading from Solr 6.x
* The PostingsSolrHighlighter is deprecated. Furthermore, it now internally works via a re-configuration
of the UnifiedSolrHighlighter.
+* Deprecated LatLonType, GeoHashField, SpatialPointVectorFieldType, and SpatialTermQueryPrefixTreeFieldType.
+ Instead, switch to LatLonPointSpatialField or SpatialRecursivePrefixTreeFieldType or RptWithGeometrySpatialField.
+
New Features
----------------------
* SOLR-9857, SOLR-9858: Collect aggregated metrics from nodes and shard leaders in overseer. (ab)
@@ -504,6 +507,10 @@ Other Changes
* SOLR-10644: solr.in.sh installed by install script should be writable by solr user (janhoy)
+* SOLR-10729: Deprecated LatLonType, GeoHashField, SpatialPointVectorFieldType, and SpatialTermQueryPrefixTreeFieldType.
+ Instead, switch to LatLonPointSpatialField or SpatialRecursivePrefixTreeFieldType or RptWithGeometrySpatialField.
+ (David Smiley)
+
================== 6.5.1 ==================
Bug Fixes
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/92ed8b42/solr/core/src/java/org/apache/solr/schema/GeoHashField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/GeoHashField.java b/solr/core/src/java/org/apache/solr/schema/GeoHashField.java
index 3f0d765..842333f 100644
--- a/solr/core/src/java/org/apache/solr/schema/GeoHashField.java
+++ b/solr/core/src/java/org/apache/solr/schema/GeoHashField.java
@@ -40,7 +40,10 @@ import org.locationtech.spatial4j.shape.Point;
* This is a class that represents a <a
* href="http://en.wikipedia.org/wiki/Geohash">Geohash</a> field. The field is
* provided as a lat/lon pair and is internally represented as a string.
+ *
+ * @deprecated use {@link LatLonPointSpatialField} instead
*/
+@Deprecated
public class GeoHashField extends FieldType implements SpatialQueryable {
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/92ed8b42/solr/core/src/java/org/apache/solr/schema/LatLonType.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/LatLonType.java b/solr/core/src/java/org/apache/solr/schema/LatLonType.java
index 8c4e19a..6898369 100644
--- a/solr/core/src/java/org/apache/solr/schema/LatLonType.java
+++ b/solr/core/src/java/org/apache/solr/schema/LatLonType.java
@@ -54,7 +54,10 @@ import org.locationtech.spatial4j.shape.Rectangle;
/**
* Represents a Latitude/Longitude as a 2 dimensional point. Latitude is <b>always</b> specified first.
+ *
+ * @deprecated use {@link LatLonPointSpatialField} instead
*/
+@Deprecated
public class LatLonType extends AbstractSubTypeFieldType implements SpatialQueryable {
protected static final int LAT = 0;
protected static final int LON = 1;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/92ed8b42/solr/core/src/java/org/apache/solr/schema/SpatialPointVectorFieldType.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/SpatialPointVectorFieldType.java b/solr/core/src/java/org/apache/solr/schema/SpatialPointVectorFieldType.java
index 5c7734e..ef05f18 100644
--- a/solr/core/src/java/org/apache/solr/schema/SpatialPointVectorFieldType.java
+++ b/solr/core/src/java/org/apache/solr/schema/SpatialPointVectorFieldType.java
@@ -25,8 +25,9 @@ import org.apache.lucene.spatial.vector.PointVectorStrategy;
/**
* @see PointVectorStrategy
- * @lucene.experimental
+ * @deprecated use {@link LatLonPointSpatialField} instead
*/
+@Deprecated
public class SpatialPointVectorFieldType extends AbstractSpatialFieldType<PointVectorStrategy> implements SchemaAware {
protected String numberFieldName = "tdouble";//in example schema defaults to non-zero precision step -- a good choice
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/92ed8b42/solr/core/src/java/org/apache/solr/schema/SpatialTermQueryPrefixTreeFieldType.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/SpatialTermQueryPrefixTreeFieldType.java b/solr/core/src/java/org/apache/solr/schema/SpatialTermQueryPrefixTreeFieldType.java
index 1779f01..203a7b8 100644
--- a/solr/core/src/java/org/apache/solr/schema/SpatialTermQueryPrefixTreeFieldType.java
+++ b/solr/core/src/java/org/apache/solr/schema/SpatialTermQueryPrefixTreeFieldType.java
@@ -20,8 +20,9 @@ import org.apache.lucene.spatial.prefix.TermQueryPrefixTreeStrategy;
/**
* @see TermQueryPrefixTreeStrategy
- * @lucene.experimental
+ * @deprecated use {@link org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy} or {@link RptWithGeometrySpatialField}
*/
+@Deprecated
public class SpatialTermQueryPrefixTreeFieldType extends AbstractSpatialPrefixTreeFieldType<TermQueryPrefixTreeStrategy> {
@Override
[13/44] lucene-solr:jira/solr-8668: updated changelog to reflect
backport to 6.6 branch
Posted by cp...@apache.org.
updated changelog to reflect backport to 6.6 branch
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6a82b42b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6a82b42b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6a82b42b
Branch: refs/heads/jira/solr-8668
Commit: 6a82b42b93382ca2f7fab3a4e242d7f620633684
Parents: 0d3c73e
Author: Martijn van Groningen <ma...@gmail.com>
Authored: Tue May 23 21:07:43 2017 +0200
Committer: Martijn van Groningen <mv...@apache.org>
Committed: Tue May 23 21:07:43 2017 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6a82b42b/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index dadba8b..879d754 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -105,12 +105,7 @@ Other
* LUCENE-7800: Remove code that potentially rethrows checked exceptions
from methods that don't declare them ("sneaky throw" hack). (Robert Muir,
- Uwe Schindler, Dawid Weiss)
-
-Bug Fixes
-
-* LUCENE-7810: Fix equals() and hashCode() methods of several join queries.
- (Hossman, Adrien Grand, Martijn van Groningen)
+ Uwe Schindler, Dawid Weiss)
======================= Lucene 6.6.0 =======================
@@ -162,6 +157,9 @@ Bug Fixes
* LUCENE-7847: Fixed all-docs-match optimization of range queries on range
fields. (Adrien Grand)
+* LUCENE-7810: Fix equals() and hashCode() methods of several join queries.
+ (Hossman, Adrien Grand, Martijn van Groningen)
+
Improvements
* LUCENE-7782: OfflineSorter now passes the total number of items it
[34/44] lucene-solr:jira/solr-8668: SOLR-7383: Switch to https for
StackOverflow Using http no longer works
Posted by cp...@apache.org.
SOLR-7383: Switch to https for StackOverflow
Using http no longer works
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/10d3aa23
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/10d3aa23
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/10d3aa23
Branch: refs/heads/jira/solr-8668
Commit: 10d3aa23fe3d12c652de8c8e3f434e8c83bdbe0d
Parents: 3cd1447
Author: Alexandre Rafalovitch <ar...@apache.org>
Authored: Thu May 25 00:42:43 2017 -0400
Committer: Alexandre Rafalovitch <ar...@apache.org>
Committed: Thu May 25 00:42:43 2017 -0400
----------------------------------------------------------------------
solr/example/example-DIH/solr/atom/conf/atom-data-config.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/10d3aa23/solr/example/example-DIH/solr/atom/conf/atom-data-config.xml
----------------------------------------------------------------------
diff --git a/solr/example/example-DIH/solr/atom/conf/atom-data-config.xml b/solr/example/example-DIH/solr/atom/conf/atom-data-config.xml
index 53b5060..b7de812 100644
--- a/solr/example/example-DIH/solr/atom/conf/atom-data-config.xml
+++ b/solr/example/example-DIH/solr/atom/conf/atom-data-config.xml
@@ -3,7 +3,7 @@
<document>
<entity name="stackoverflow"
- url="http://stackoverflow.com/feeds/tag/solr"
+ url="https://stackoverflow.com/feeds/tag/solr"
processor="XPathEntityProcessor"
forEach="/feed|/feed/entry"
transformer="HTMLStripTransformer,RegexTransformer">
[43/44] lucene-solr:jira/solr-8668: SOLR-10233:
ChaosMonkeySafeLeaderWithPullReplicasTest - Catch SolrException while waiting
for the cluster to be ready
Posted by cp...@apache.org.
SOLR-10233: ChaosMonkeySafeLeaderWithPullReplicasTest - Catch SolrException while waiting for the cluster to be ready
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1e4d2052
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1e4d2052
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1e4d2052
Branch: refs/heads/jira/solr-8668
Commit: 1e4d2052e6ce10b4012eda8802a8d32ccadeeba3
Parents: b3024d6
Author: Tomas Fernandez Lobbe <tf...@apache.org>
Authored: Thu May 25 09:37:16 2017 -0700
Committer: Tomas Fernandez Lobbe <tf...@apache.org>
Committed: Thu May 25 09:37:16 2017 -0700
----------------------------------------------------------------------
.../solr/cloud/ChaosMonkeySafeLeaderWithPullReplicasTest.java | 3 ++-
.../src/java/org/apache/solr/cloud/SolrCloudTestCase.java | 6 +++++-
2 files changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1e4d2052/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPullReplicasTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPullReplicasTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPullReplicasTest.java
index f2e8845..e4859c0 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPullReplicasTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPullReplicasTest.java
@@ -27,6 +27,7 @@ import org.apache.solr.SolrTestCaseJ4.SuppressObjectReleaseTracker;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
@@ -234,7 +235,7 @@ public class ChaosMonkeySafeLeaderWithPullReplicasTest extends AbstractFullDistr
try {
del("*:*");
break;
- } catch (SolrServerException e) {
+ } catch (SolrServerException | SolrException e) {
// cluster may not be up yet
e.printStackTrace();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1e4d2052/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
index 180cf6e..42c4577 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
@@ -22,11 +22,13 @@ import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Predicate;
@@ -256,13 +258,15 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {
*/
protected void waitForState(String message, String collection, CollectionStatePredicate predicate) {
AtomicReference<DocCollection> state = new AtomicReference<>();
+ AtomicReference<Set<String>> liveNodesLastSeen = new AtomicReference<>();
try {
cluster.getSolrClient().waitForState(collection, DEFAULT_TIMEOUT, TimeUnit.SECONDS, (n, c) -> {
state.set(c);
+ liveNodesLastSeen.set(n);
return predicate.matches(n, c);
});
} catch (Exception e) {
- fail(message + "\n" + e.getMessage() + "\nLast available state: " + state.get());
+ fail(message + "\n" + e.getMessage() + "\nLive Nodes: " + Arrays.toString(liveNodesLastSeen.get().toArray()) + "\nLast available state: " + state.get());
}
}
[02/44] lucene-solr:jira/solr-8668: LUCENE-7810: Fix equals() and
hashCode() methods of several join queries.
Posted by cp...@apache.org.
LUCENE-7810: Fix equals() and hashCode() methods of several join queries.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/85c1319c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/85c1319c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/85c1319c
Branch: refs/heads/jira/solr-8668
Commit: 85c1319c7678e9deaccb7b7add50fa3f465bd44f
Parents: 08d2457
Author: Martijn van Groningen <ma...@gmail.com>
Authored: Mon May 22 17:56:08 2017 +0200
Committer: Martijn van Groningen <mv...@apache.org>
Committed: Tue May 23 10:45:54 2017 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 5 +
.../lucene/search/join/GlobalOrdinalsQuery.java | 6 +-
.../join/GlobalOrdinalsWithScoreQuery.java | 18 +-
.../org/apache/lucene/search/join/JoinUtil.java | 23 +-
.../join/PointInSetIncludingScoreQuery.java | 10 +-
.../search/join/TermsIncludingScoreQuery.java | 91 +++----
.../apache/lucene/search/join/TermsQuery.java | 39 +--
.../apache/lucene/search/join/TestJoinUtil.java | 246 +++++++++++++++++++
.../search/join/TestScoreJoinQPNoScore.java | 9 +
9 files changed, 357 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/85c1319c/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 43b13b1..2138321 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -103,6 +103,11 @@ Other
from methods that don't declare them ("sneaky throw" hack). (Robert Muir,
Uwe Schindler, Dawid Weiss)
+Bug Fixes
+
+* LUCENE-7810: Fix equals() and hashCode() methods of several join queries.
+ (Hossman, Adrien Grand, Martijn van Groningen)
+
======================= Lucene 6.6.0 =======================
New Features
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/85c1319c/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java
index 5aaca1a..aacda2d 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java
@@ -20,7 +20,6 @@ import java.io.IOException;
import java.util.Set;
import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.SortedDocValues;
@@ -51,13 +50,14 @@ final class GlobalOrdinalsQuery extends Query {
// id of the context rather than the context itself in order not to hold references to index readers
private final Object indexReaderContextId;
- GlobalOrdinalsQuery(LongBitSet foundOrds, String joinField, MultiDocValues.OrdinalMap globalOrds, Query toQuery, Query fromQuery, IndexReaderContext context) {
+ GlobalOrdinalsQuery(LongBitSet foundOrds, String joinField, MultiDocValues.OrdinalMap globalOrds, Query toQuery,
+ Query fromQuery, Object indexReaderContextId) {
this.foundOrds = foundOrds;
this.joinField = joinField;
this.globalOrds = globalOrds;
this.toQuery = toQuery;
this.fromQuery = fromQuery;
- this.indexReaderContextId = context.id();
+ this.indexReaderContextId = indexReaderContextId;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/85c1319c/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreQuery.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreQuery.java
index 5e614ea..1c80bf3 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreQuery.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreQuery.java
@@ -20,7 +20,6 @@ import java.io.IOException;
import java.util.Set;
import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.SortedDocValues;
@@ -45,21 +44,25 @@ final class GlobalOrdinalsWithScoreQuery extends Query {
private final Query toQuery;
// just for hashcode and equals:
+ private final ScoreMode scoreMode;
private final Query fromQuery;
private final int min;
private final int max;
// id of the context rather than the context itself in order not to hold references to index readers
private final Object indexReaderContextId;
- GlobalOrdinalsWithScoreQuery(GlobalOrdinalsWithScoreCollector collector, String joinField, MultiDocValues.OrdinalMap globalOrds, Query toQuery, Query fromQuery, int min, int max, IndexReaderContext context) {
+ GlobalOrdinalsWithScoreQuery(GlobalOrdinalsWithScoreCollector collector, ScoreMode scoreMode, String joinField,
+ MultiDocValues.OrdinalMap globalOrds, Query toQuery, Query fromQuery, int min, int max,
+ Object indexReaderContextId) {
this.collector = collector;
this.joinField = joinField;
this.globalOrds = globalOrds;
this.toQuery = toQuery;
+ this.scoreMode = scoreMode;
this.fromQuery = fromQuery;
this.min = min;
this.max = max;
- this.indexReaderContextId = context.id();
+ this.indexReaderContextId = indexReaderContextId;
}
@Override
@@ -67,6 +70,13 @@ final class GlobalOrdinalsWithScoreQuery extends Query {
if (searcher.getTopReaderContext().id() != indexReaderContextId) {
throw new IllegalStateException("Creating the weight against a different index reader than this query has been built for.");
}
+ boolean doNoMinMax = min <= 0 && max == Integer.MAX_VALUE;
+ if (needsScores == false && doNoMinMax) {
+ // We don't need scores then quickly change the query to not uses the scores:
+ GlobalOrdinalsQuery globalOrdinalsQuery = new GlobalOrdinalsQuery(collector.collectedOrds, joinField, globalOrds,
+ toQuery, fromQuery, indexReaderContextId);
+ return globalOrdinalsQuery.createWeight(searcher, false, boost);
+ }
return new W(this, toQuery.createWeight(searcher, false, 1f));
}
@@ -79,6 +89,7 @@ final class GlobalOrdinalsWithScoreQuery extends Query {
private boolean equalsTo(GlobalOrdinalsWithScoreQuery other) {
return min == other.min &&
max == other.max &&
+ scoreMode.equals(other.scoreMode) &&
joinField.equals(other.joinField) &&
fromQuery.equals(other.fromQuery) &&
toQuery.equals(other.toQuery) &&
@@ -88,6 +99,7 @@ final class GlobalOrdinalsWithScoreQuery extends Query {
@Override
public int hashCode() {
int result = classHash();
+ result = 31 * result + scoreMode.hashCode();
result = 31 * result + joinField.hashCode();
result = 31 * result + toQuery.hashCode();
result = 31 * result + fromQuery.hashCode();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/85c1319c/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java b/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java
index 537b224..d7e0ae8 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java
@@ -104,7 +104,7 @@ public final class JoinUtil {
termsWithScoreCollector = GenericTermsCollector.createCollectorSV(svFunction, scoreMode);
}
- return createJoinQuery(multipleValuesPerDocument, toField, fromQuery, fromSearcher, scoreMode, termsWithScoreCollector);
+ return createJoinQuery(multipleValuesPerDocument, toField, fromQuery, fromField, fromSearcher, scoreMode, termsWithScoreCollector);
}
/**
@@ -362,7 +362,7 @@ public final class JoinUtil {
encoded.length = bytesPerDim;
if (needsScore) {
- return new PointInSetIncludingScoreQuery(fromQuery, multipleValuesPerDocument, toField, bytesPerDim, stream) {
+ return new PointInSetIncludingScoreQuery(scoreMode, fromQuery, multipleValuesPerDocument, toField, bytesPerDim, stream) {
@Override
protected String toString(byte[] value) {
@@ -379,25 +379,26 @@ public final class JoinUtil {
}
}
- private static Query createJoinQuery(boolean multipleValuesPerDocument, String toField, Query fromQuery,
- IndexSearcher fromSearcher, ScoreMode scoreMode, final GenericTermsCollector collector)
- throws IOException {
+ private static Query createJoinQuery(boolean multipleValuesPerDocument, String toField, Query fromQuery, String fromField,
+ IndexSearcher fromSearcher, ScoreMode scoreMode, final GenericTermsCollector collector) throws IOException {
fromSearcher.search(fromQuery, collector);
-
switch (scoreMode) {
case None:
- return new TermsQuery(toField, fromQuery, collector.getCollectedTerms());
+ return new TermsQuery(toField, collector.getCollectedTerms(), fromField, fromQuery, fromSearcher.getTopReaderContext().id());
case Total:
case Max:
case Min:
case Avg:
return new TermsIncludingScoreQuery(
+ scoreMode,
toField,
multipleValuesPerDocument,
collector.getCollectedTerms(),
collector.getScoresPerTerm(),
- fromQuery
+ fromField,
+ fromQuery,
+ fromSearcher.getTopReaderContext().id()
);
default:
throw new IllegalArgumentException(String.format(Locale.ROOT, "Score mode %s isn't supported.", scoreMode));
@@ -507,7 +508,8 @@ public final class JoinUtil {
if (min <= 0 && max == Integer.MAX_VALUE) {
GlobalOrdinalsCollector globalOrdinalsCollector = new GlobalOrdinalsCollector(joinField, ordinalMap, valueCount);
searcher.search(rewrittenFromQuery, globalOrdinalsCollector);
- return new GlobalOrdinalsQuery(globalOrdinalsCollector.getCollectorOrdinals(), joinField, ordinalMap, rewrittenToQuery, rewrittenFromQuery, searcher.getTopReaderContext());
+ return new GlobalOrdinalsQuery(globalOrdinalsCollector.getCollectorOrdinals(), joinField, ordinalMap, rewrittenToQuery,
+ rewrittenFromQuery, searcher.getTopReaderContext().id());
} else {
globalOrdinalsWithScoreCollector = new GlobalOrdinalsWithScoreCollector.NoScore(joinField, ordinalMap, valueCount, min, max);
break;
@@ -516,7 +518,8 @@ public final class JoinUtil {
throw new IllegalArgumentException(String.format(Locale.ROOT, "Score mode %s isn't supported.", scoreMode));
}
searcher.search(rewrittenFromQuery, globalOrdinalsWithScoreCollector);
- return new GlobalOrdinalsWithScoreQuery(globalOrdinalsWithScoreCollector, joinField, ordinalMap, rewrittenToQuery, rewrittenFromQuery, min, max, searcher.getTopReaderContext());
+ return new GlobalOrdinalsWithScoreQuery(globalOrdinalsWithScoreCollector, scoreMode, joinField, ordinalMap, rewrittenToQuery,
+ rewrittenFromQuery, min, max, searcher.getTopReaderContext().id());
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/85c1319c/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java
index 70c28d5..3130ae6 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/PointInSetIncludingScoreQuery.java
@@ -66,6 +66,7 @@ abstract class PointInSetIncludingScoreQuery extends Query {
}
};
+ final ScoreMode scoreMode;
final Query originalQuery;
final boolean multipleValuesPerDocument;
final PrefixCodedTerms sortedPackedPoints;
@@ -81,8 +82,9 @@ abstract class PointInSetIncludingScoreQuery extends Query {
}
- PointInSetIncludingScoreQuery(Query originalQuery, boolean multipleValuesPerDocument, String field, int bytesPerDim,
- Stream packedPoints) {
+ PointInSetIncludingScoreQuery(ScoreMode scoreMode, Query originalQuery, boolean multipleValuesPerDocument,
+ String field, int bytesPerDim, Stream packedPoints) {
+ this.scoreMode = scoreMode;
this.originalQuery = originalQuery;
this.multipleValuesPerDocument = multipleValuesPerDocument;
this.field = field;
@@ -276,6 +278,7 @@ abstract class PointInSetIncludingScoreQuery extends Query {
@Override
public final int hashCode() {
int hash = classHash();
+ hash = 31 * hash + scoreMode.hashCode();
hash = 31 * hash + field.hashCode();
hash = 31 * hash + originalQuery.hashCode();
hash = 31 * hash + sortedPackedPointsHashCode;
@@ -290,7 +293,8 @@ abstract class PointInSetIncludingScoreQuery extends Query {
}
private boolean equalsTo(PointInSetIncludingScoreQuery other) {
- return other.field.equals(field) &&
+ return other.scoreMode.equals(scoreMode) &&
+ other.field.equals(field) &&
other.originalQuery.equals(originalQuery) &&
other.bytesPerDim == bytesPerDim &&
other.sortedPackedPointsHashCode == sortedPackedPointsHashCode &&
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/85c1319c/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java
index 28d3044..cd3beaf 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java
@@ -17,11 +17,10 @@
package org.apache.lucene.search.join;
import java.io.IOException;
-import java.io.PrintStream;
import java.util.Locale;
+import java.util.Objects;
import java.util.Set;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
@@ -40,48 +39,36 @@ import org.apache.lucene.util.FixedBitSet;
class TermsIncludingScoreQuery extends Query {
- final String field;
- final boolean multipleValuesPerDocument;
- final BytesRefHash terms;
- final float[] scores;
- final int[] ords;
- final Query originalQuery;
- final Query unwrittenOriginalQuery;
-
- TermsIncludingScoreQuery(String field, boolean multipleValuesPerDocument, BytesRefHash terms, float[] scores, Query originalQuery) {
- this.field = field;
+ private final ScoreMode scoreMode;
+ private final String toField;
+ private final boolean multipleValuesPerDocument;
+ private final BytesRefHash terms;
+ private final float[] scores;
+ private final int[] ords;
+
+ // These fields are used for equals() and hashcode() only
+ private final Query fromQuery;
+ private final String fromField;
+ // id of the context rather than the context itself in order not to hold references to index readers
+ private final Object topReaderContextId;
+
+ TermsIncludingScoreQuery(ScoreMode scoreMode, String toField, boolean multipleValuesPerDocument, BytesRefHash terms, float[] scores,
+ String fromField, Query fromQuery, Object indexReaderContextId) {
+ this.scoreMode = scoreMode;
+ this.toField = toField;
this.multipleValuesPerDocument = multipleValuesPerDocument;
this.terms = terms;
this.scores = scores;
- this.originalQuery = originalQuery;
this.ords = terms.sort();
- this.unwrittenOriginalQuery = originalQuery;
- }
- private TermsIncludingScoreQuery(String field, boolean multipleValuesPerDocument, BytesRefHash terms, float[] scores, int[] ords, Query originalQuery, Query unwrittenOriginalQuery) {
- this.field = field;
- this.multipleValuesPerDocument = multipleValuesPerDocument;
- this.terms = terms;
- this.scores = scores;
- this.originalQuery = originalQuery;
- this.ords = ords;
- this.unwrittenOriginalQuery = unwrittenOriginalQuery;
+ this.fromField = fromField;
+ this.fromQuery = fromQuery;
+ this.topReaderContextId = indexReaderContextId;
}
@Override
public String toString(String string) {
- return String.format(Locale.ROOT, "TermsIncludingScoreQuery{field=%s;originalQuery=%s}", field, unwrittenOriginalQuery);
- }
-
- @Override
- public Query rewrite(IndexReader reader) throws IOException {
- final Query originalQueryRewrite = originalQuery.rewrite(reader);
- if (originalQueryRewrite != originalQuery) {
- return new TermsIncludingScoreQuery(field, multipleValuesPerDocument, terms, scores,
- ords, originalQueryRewrite, originalQuery);
- } else {
- return super.rewrite(reader);
- }
+ return String.format(Locale.ROOT, "TermsIncludingScoreQuery{field=%s;fromQuery=%s}", toField, fromQuery);
}
@Override
@@ -91,21 +78,25 @@ class TermsIncludingScoreQuery extends Query {
}
private boolean equalsTo(TermsIncludingScoreQuery other) {
- return field.equals(other.field) &&
- unwrittenOriginalQuery.equals(other.unwrittenOriginalQuery);
+ return Objects.equals(scoreMode, other.scoreMode) &&
+ Objects.equals(toField, other.toField) &&
+ Objects.equals(fromField, other.fromField) &&
+ Objects.equals(fromQuery, other.fromQuery) &&
+ Objects.equals(topReaderContextId, other.topReaderContextId);
}
@Override
public int hashCode() {
- final int prime = 31;
- int result = classHash();
- result += prime * field.hashCode();
- result += prime * unwrittenOriginalQuery.hashCode();
- return result;
+ return classHash() + Objects.hash(scoreMode, toField, fromField, fromQuery, topReaderContextId);
}
@Override
public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException {
+ if (needsScores == false) {
+ // We don't need scores then quickly change the query:
+ TermsQuery termsQuery = new TermsQuery(toField, terms, fromField, fromQuery, topReaderContextId);
+ return searcher.rewrite(termsQuery).createWeight(searcher, false, boost);
+ }
return new Weight(TermsIncludingScoreQuery.this) {
@Override
@@ -113,7 +104,7 @@ class TermsIncludingScoreQuery extends Query {
@Override
public Explanation explain(LeafReaderContext context, int doc) throws IOException {
- Terms terms = context.reader().terms(field);
+ Terms terms = context.reader().terms(toField);
if (terms != null) {
TermsEnum segmentTermsEnum = terms.iterator();
BytesRef spare = new BytesRef();
@@ -133,7 +124,7 @@ class TermsIncludingScoreQuery extends Query {
@Override
public Scorer scorer(LeafReaderContext context) throws IOException {
- Terms terms = context.reader().terms(field);
+ Terms terms = context.reader().terms(toField);
if (terms == null) {
return null;
}
@@ -151,7 +142,7 @@ class TermsIncludingScoreQuery extends Query {
};
}
-
+
class SVInOrderScorer extends Scorer {
final DocIdSetIterator matchingDocsIterator;
@@ -238,14 +229,4 @@ class TermsIncludingScoreQuery extends Query {
}
}
- void dump(PrintStream out){
- out.println(field+":");
- final BytesRef ref = new BytesRef();
- for (int i = 0; i < terms.size(); i++) {
- terms.get(ords[i], ref);
- out.print(ref+" "+ref.utf8ToString()+" ");
- out.println(" score="+scores[ords[i]]);
- out.println("");
- }
- }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/85c1319c/lucene/join/src/java/org/apache/lucene/search/join/TermsQuery.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/TermsQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/TermsQuery.java
index 63561c3..3ff0a5c 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/TermsQuery.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/TermsQuery.java
@@ -16,6 +16,9 @@
*/
package org.apache.lucene.search.join;
+import java.io.IOException;
+import java.util.Objects;
+
import org.apache.lucene.index.FilteredTermsEnum;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
@@ -25,8 +28,6 @@ import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash;
-import java.io.IOException;
-
/**
* A query that has an array of terms from a specific field. This query will match documents have one or more terms in
* the specified field that match with the terms specified in the array.
@@ -37,17 +38,25 @@ class TermsQuery extends MultiTermQuery {
private final BytesRefHash terms;
private final int[] ords;
- private final Query fromQuery; // Used for equals() only
+
+ // These fields are used for equals() and hashcode() only
+ private final String fromField;
+ private final Query fromQuery;
+ // id of the context rather than the context itself in order not to hold references to index readers
+ private final Object indexReaderContextId;
/**
- * @param field The field that should contain terms that are specified in the previous parameter
- * @param terms The terms that matching documents should have. The terms must be sorted by natural order.
+ * @param toField The field that should contain terms that are specified in the next parameter.
+ * @param terms The terms that matching documents should have. The terms must be sorted by natural order.
+ * @param indexReaderContextId Refers to the top level index reader used to create the set of terms in the previous parameter.
*/
- TermsQuery(String field, Query fromQuery, BytesRefHash terms) {
- super(field);
- this.fromQuery = fromQuery;
+ TermsQuery(String toField, BytesRefHash terms, String fromField, Query fromQuery, Object indexReaderContextId) {
+ super(toField);
this.terms = terms;
ords = terms.sort();
+ this.fromField = fromField;
+ this.fromQuery = fromQuery;
+ this.indexReaderContextId = indexReaderContextId;
}
@Override
@@ -63,6 +72,7 @@ class TermsQuery extends MultiTermQuery {
public String toString(String string) {
return "TermsQuery{" +
"field=" + field +
+ "fromQuery=" + fromQuery.toString(field) +
'}';
}
@@ -77,18 +87,15 @@ class TermsQuery extends MultiTermQuery {
}
TermsQuery other = (TermsQuery) obj;
- if (!fromQuery.equals(other.fromQuery)) {
- return false;
- }
- return true;
+ return Objects.equals(field, other.field) &&
+ Objects.equals(fromField, other.fromField) &&
+ Objects.equals(fromQuery, other.fromQuery) &&
+ Objects.equals(indexReaderContextId, other.indexReaderContextId);
}
@Override
public int hashCode() {
- final int prime = 31;
- int result = super.hashCode();
- result += prime * fromQuery.hashCode();
- return result;
+ return classHash() + Objects.hash(field, fromField, fromQuery, indexReaderContextId);
}
static class SeekingTermSetTermsEnum extends FilteredTermsEnum {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/85c1319c/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
index 72b6bf5..c95e144 100644
--- a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
@@ -21,6 +21,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -922,6 +923,251 @@ public class TestJoinUtil extends LuceneTestCase {
dir.close();
}
+ public void testEquals() throws Exception {
+ final int numDocs = atLeast(random(), 50);
+ try (final Directory dir = newDirectory()) {
+ try (final RandomIndexWriter w = new RandomIndexWriter(random(), dir,
+ newIndexWriterConfig(new MockAnalyzer(random()))
+ .setMergePolicy(newLogMergePolicy()))) {
+ boolean multiValued = random().nextBoolean();
+ String joinField = multiValued ? "mvField" : "svField";
+ for (int id = 0; id < numDocs; id++) {
+ Document doc = new Document();
+ doc.add(new TextField("id", "" + id, Field.Store.NO));
+ doc.add(new TextField("name", "name" + (id % 7), Field.Store.NO));
+ if (multiValued) {
+ int numValues = 1 + random().nextInt(2);
+ for (int i = 0; i < numValues; i++) {
+ doc.add(new SortedSetDocValuesField(joinField, new BytesRef("" + random().nextInt(13))));
+ }
+ } else {
+ doc.add(new SortedDocValuesField(joinField, new BytesRef("" + random().nextInt(13))));
+ }
+ w.addDocument(doc);
+ }
+
+ Set<ScoreMode> scoreModes = EnumSet.allOf(ScoreMode.class);
+ ScoreMode scoreMode1 = RandomPicks.randomFrom(random(), scoreModes);
+ scoreModes.remove(scoreMode1);
+ ScoreMode scoreMode2 = RandomPicks.randomFrom(random(), scoreModes);
+
+ final Query x;
+ try (IndexReader r = w.getReader()) {
+ IndexSearcher indexSearcher = new IndexSearcher(r);
+ x = JoinUtil.createJoinQuery(joinField, multiValued, joinField,
+ new TermQuery(new Term("name", "name5")),
+ indexSearcher, scoreMode1);
+ assertEquals("identical calls to createJoinQuery",
+ x, JoinUtil.createJoinQuery(joinField, multiValued, joinField,
+ new TermQuery(new Term("name", "name5")),
+ indexSearcher, scoreMode1));
+
+ assertFalse("score mode (" + scoreMode1 + " != " + scoreMode2 + "), but queries are equal",
+ x.equals(JoinUtil.createJoinQuery(joinField, multiValued, joinField,
+ new TermQuery(new Term("name", "name5")),
+ indexSearcher, scoreMode2)));
+
+
+ assertFalse("from fields (joinField != \"other_field\") but queries equals",
+ x.equals(JoinUtil.createJoinQuery(joinField, multiValued, "other_field",
+ new TermQuery(new Term("name", "name5")),
+ indexSearcher, scoreMode1)));
+
+ assertFalse("from fields (\"other_field\" != joinField) but queries equals",
+ x.equals(JoinUtil.createJoinQuery("other_field", multiValued, joinField,
+ new TermQuery(new Term("name", "name5")),
+ indexSearcher, scoreMode1)));
+
+ assertFalse("fromQuery (name:name5 != name:name6) but queries equals",
+ x.equals(JoinUtil.createJoinQuery("other_field", multiValued, joinField,
+ new TermQuery(new Term("name", "name6")),
+ indexSearcher, scoreMode1)));
+ }
+
+ for (int i = 0; i < 13; i++) {
+ Document doc = new Document();
+ doc.add(new TextField("id", "new_id" , Field.Store.NO));
+ doc.add(new TextField("name", "name5", Field.Store.NO));
+ if (multiValued) {
+ int numValues = 1 + random().nextInt(2);
+ for (int j = 0; j < numValues; j++) {
+ doc.add(new SortedSetDocValuesField(joinField, new BytesRef("" + i)));
+ }
+ } else {
+ doc.add(new SortedDocValuesField(joinField, new BytesRef("" + i)));
+ }
+ w.addDocument(doc);
+ }
+ try (IndexReader r = w.getReader()) {
+ IndexSearcher indexSearcher = new IndexSearcher(r);
+ assertFalse("Query shouldn't be equal, because different index readers ",
+ x.equals(JoinUtil.createJoinQuery(joinField, multiValued, joinField,
+ new TermQuery(new Term("name", "name5")),
+ indexSearcher, scoreMode1)));
+ }
+ }
+ }
+ }
+
+ public void testEquals_globalOrdinalsJoin() throws Exception {
+ final int numDocs = atLeast(random(), 50);
+ try (final Directory dir = newDirectory()) {
+ try (final RandomIndexWriter w = new RandomIndexWriter(random(), dir,
+ newIndexWriterConfig(new MockAnalyzer(random()))
+ .setMergePolicy(newLogMergePolicy()))) {
+ String joinField = "field";
+ for (int id = 0; id < numDocs; id++) {
+ Document doc = new Document();
+ doc.add(new TextField("id", "" + id, Field.Store.NO));
+ doc.add(new TextField("name", "name" + (id % 7), Field.Store.NO));
+ doc.add(new SortedDocValuesField(joinField, new BytesRef("" + random().nextInt(13))));
+ w.addDocument(doc);
+ }
+
+ Set<ScoreMode> scoreModes = EnumSet.allOf(ScoreMode.class);
+ ScoreMode scoreMode1 = RandomPicks.randomFrom(random(), scoreModes);
+ scoreModes.remove(scoreMode1);
+ ScoreMode scoreMode2 = RandomPicks.randomFrom(random(), scoreModes);
+
+ final Query x;
+ try (IndexReader r = w.getReader()) {
+ SortedDocValues[] values = new SortedDocValues[r.leaves().size()];
+ for (int i = 0; i < values.length; i++) {
+ LeafReader leafReader = r.leaves().get(i).reader();
+ values[i] = DocValues.getSorted(leafReader, joinField);
+ }
+ MultiDocValues.OrdinalMap ordinalMap = MultiDocValues.OrdinalMap.build(
+ null, values, PackedInts.DEFAULT
+ );
+ IndexSearcher indexSearcher = new IndexSearcher(r);
+ x = JoinUtil.createJoinQuery(joinField, new TermQuery(new Term("name", "name5")), new MatchAllDocsQuery(),
+ indexSearcher, scoreMode1, ordinalMap);
+ assertEquals("identical calls to createJoinQuery",
+ x, JoinUtil.createJoinQuery(joinField, new TermQuery(new Term("name", "name5")), new MatchAllDocsQuery(),
+ indexSearcher, scoreMode1, ordinalMap));
+
+ assertFalse("score mode (" + scoreMode1 + " != " + scoreMode2 + "), but queries are equal",
+ x.equals(JoinUtil.createJoinQuery(joinField, new TermQuery(new Term("name", "name5")), new MatchAllDocsQuery(),
+ indexSearcher, scoreMode2, ordinalMap)));
+ assertFalse("fromQuery (name:name5 != name:name6) but queries equals",
+ x.equals(JoinUtil.createJoinQuery(joinField, new TermQuery(new Term("name", "name6")), new MatchAllDocsQuery(),
+ indexSearcher, scoreMode1, ordinalMap)));
+ }
+
+ for (int i = 0; i < 13; i++) {
+ Document doc = new Document();
+ doc.add(new TextField("id", "new_id" , Field.Store.NO));
+ doc.add(new TextField("name", "name5", Field.Store.NO));
+ doc.add(new SortedDocValuesField(joinField, new BytesRef("" + i)));
+ w.addDocument(doc);
+ }
+ try (IndexReader r = w.getReader()) {
+ SortedDocValues[] values = new SortedDocValues[r.leaves().size()];
+ for (int i = 0; i < values.length; i++) {
+ LeafReader leafReader = r.leaves().get(i).reader();
+ values[i] = DocValues.getSorted(leafReader, joinField);
+ }
+ MultiDocValues.OrdinalMap ordinalMap = MultiDocValues.OrdinalMap.build(
+ null, values, PackedInts.DEFAULT
+ );
+ IndexSearcher indexSearcher = new IndexSearcher(r);
+ assertFalse("Query shouldn't be equal, because different index readers ",
+ x.equals(JoinUtil.createJoinQuery(joinField, new TermQuery(new Term("name", "name5")), new MatchAllDocsQuery(),
+ indexSearcher, scoreMode1, ordinalMap)));
+ }
+ }
+ }
+ }
+
+ public void testEquals_numericJoin() throws Exception {
+ final int numDocs = atLeast(random(), 50);
+ try (final Directory dir = newDirectory()) {
+ try (final RandomIndexWriter w = new RandomIndexWriter(random(), dir,
+ newIndexWriterConfig(new MockAnalyzer(random()))
+ .setMergePolicy(newLogMergePolicy()))) {
+ boolean multiValued = random().nextBoolean();
+ String joinField = multiValued ? "mvField" : "svField";
+ for (int id = 0; id < numDocs; id++) {
+ Document doc = new Document();
+ doc.add(new TextField("id", "" + id, Field.Store.NO));
+ doc.add(new TextField("name", "name" + (id % 7), Field.Store.NO));
+ if (multiValued) {
+ int numValues = 1 + random().nextInt(2);
+ for (int i = 0; i < numValues; i++) {
+ doc.add(new IntPoint(joinField, random().nextInt(13)));
+ doc.add(new SortedNumericDocValuesField(joinField, random().nextInt(13)));
+ }
+ } else {
+ doc.add(new IntPoint(joinField, random().nextInt(13)));
+ doc.add(new NumericDocValuesField(joinField, random().nextInt(13)));
+ }
+ w.addDocument(doc);
+ }
+
+ Set<ScoreMode> scoreModes = EnumSet.allOf(ScoreMode.class);
+ ScoreMode scoreMode1 = scoreModes.toArray(new ScoreMode[0])[random().nextInt(scoreModes.size())];
+ scoreModes.remove(scoreMode1);
+ ScoreMode scoreMode2 = scoreModes.toArray(new ScoreMode[0])[random().nextInt(scoreModes.size())];
+
+ final Query x;
+ try (IndexReader r = w.getReader()) {
+ IndexSearcher indexSearcher = new IndexSearcher(r);
+ x = JoinUtil.createJoinQuery(joinField, multiValued, joinField,
+ Integer.class, new TermQuery(new Term("name", "name5")),
+ indexSearcher, scoreMode1);
+ assertEquals("identical calls to createJoinQuery",
+ x, JoinUtil.createJoinQuery(joinField, multiValued, joinField,
+ Integer.class, new TermQuery(new Term("name", "name5")),
+ indexSearcher, scoreMode1));
+
+ assertFalse("score mode (" + scoreMode1 + " != " + scoreMode2 + "), but queries are equal",
+ x.equals(JoinUtil.createJoinQuery(joinField, multiValued, joinField,
+ Integer.class, new TermQuery(new Term("name", "name5")),
+ indexSearcher, scoreMode2)));
+
+ assertFalse("from fields (joinField != \"other_field\") but queries equals",
+ x.equals(JoinUtil.createJoinQuery(joinField, multiValued, "other_field",
+ Integer.class, new TermQuery(new Term("name", "name5")),
+ indexSearcher, scoreMode1)));
+
+ assertFalse("from fields (\"other_field\" != joinField) but queries equals",
+ x.equals(JoinUtil.createJoinQuery("other_field", multiValued, joinField,
+ Integer.class, new TermQuery(new Term("name", "name5")),
+ indexSearcher, scoreMode1)));
+
+ assertFalse("fromQuery (name:name5 != name:name6) but queries equals",
+ x.equals(JoinUtil.createJoinQuery("other_field", multiValued, joinField,
+ Integer.class, new TermQuery(new Term("name", "name6")),
+ indexSearcher, scoreMode1)));
+ }
+
+ for (int i = 0; i < 13; i++) {
+ Document doc = new Document();
+ doc.add(new TextField("id", "new_id" , Field.Store.NO));
+ doc.add(new TextField("name", "name5", Field.Store.NO));
+ if (multiValued) {
+ int numValues = 1 + random().nextInt(2);
+ for (int j = 0; j < numValues; j++) {
+ doc.add(new SortedNumericDocValuesField(joinField, i));
+ doc.add(new IntPoint(joinField, i));
+ }
+ } else {
+ doc.add(new NumericDocValuesField(joinField, i));
+ doc.add(new IntPoint(joinField, i));
+ }
+ w.addDocument(doc);
+ }
+ try (IndexReader r = w.getReader()) {
+ IndexSearcher indexSearcher = new IndexSearcher(r);
+ assertFalse("Query shouldn't be equal, because different index readers ",
+ x.equals(JoinUtil.createJoinQuery(joinField, multiValued, joinField,
+ Integer.class, new TermQuery(new Term("name", "name5")),
+ indexSearcher, scoreMode1)));
+ }
+ }
+ }
+ }
+
@Test
@Slow
public void testSingleValueRandomJoin() throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/85c1319c/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPNoScore.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPNoScore.java b/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPNoScore.java
index 0d9801e..38c111a 100644
--- a/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPNoScore.java
+++ b/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPNoScore.java
@@ -158,6 +158,15 @@ public class TestScoreJoinQPNoScore extends SolrTestCaseJ4 {
}
+ public void testNotEquals() throws SyntaxError, IOException{
+ try (SolrQueryRequest req = req("*:*")) {
+ Query x = QParser.getParser("{!join from=dept_id_s to=dept_ss score=none}text_t:develop", req).getQuery();
+ Query y = QParser.getParser("{!join from=dept_ss to=dept_ss score=none}text_t:develop", req).getQuery();
+ assertFalse("diff from fields produce equal queries",
+ x.equals(y));
+ }
+ }
+
public void testJoinQueryType() throws SyntaxError, IOException{
SolrQueryRequest req = null;
try{
[17/44] lucene-solr:jira/solr-8668: LUCENE-7815: Remove more
PostingsHighlighter remnants
Posted by cp...@apache.org.
LUCENE-7815: Remove more PostingsHighlighter remnants
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/85c3ae20
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/85c3ae20
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/85c3ae20
Branch: refs/heads/jira/solr-8668
Commit: 85c3ae2040d175ddc0af2147ccde2c9b7599ef59
Parents: 2319d69
Author: David Smiley <ds...@apache.org>
Authored: Tue May 23 17:13:03 2017 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Tue May 23 17:13:03 2017 -0400
----------------------------------------------------------------------
.../CustomSeparatorBreakIterator.java | 150 -------------------
.../postingshighlight/WholeBreakIterator.java | 116 --------------
.../TestCustomSeparatorBreakIterator.java | 114 --------------
.../TestWholeBreakIterator.java | 134 -----------------
4 files changed, 514 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/85c3ae20/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/CustomSeparatorBreakIterator.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/CustomSeparatorBreakIterator.java b/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/CustomSeparatorBreakIterator.java
deleted file mode 100644
index 9a4c461..0000000
--- a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/CustomSeparatorBreakIterator.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.postingshighlight;
-
-import java.text.BreakIterator;
-import java.text.CharacterIterator;
-
-/**
- * A {@link BreakIterator} that breaks the text whenever a certain separator, provided as a constructor argument, is found.
- */
-public final class CustomSeparatorBreakIterator extends BreakIterator {
-
- private final char separator;
- private CharacterIterator text;
- private int current;
-
- public CustomSeparatorBreakIterator(char separator) {
- this.separator = separator;
- }
-
- @Override
- public int current() {
- return current;
- }
-
- @Override
- public int first() {
- text.setIndex(text.getBeginIndex());
- return current = text.getIndex();
- }
-
- @Override
- public int last() {
- text.setIndex(text.getEndIndex());
- return current = text.getIndex();
- }
-
- @Override
- public int next() {
- if (text.getIndex() == text.getEndIndex()) {
- return DONE;
- } else {
- return advanceForward();
- }
- }
-
- private int advanceForward() {
- char c;
- while ((c = text.next()) != CharacterIterator.DONE) {
- if (c == separator) {
- return current = text.getIndex() + 1;
- }
- }
- assert text.getIndex() == text.getEndIndex();
- return current = text.getIndex();
- }
-
- @Override
- public int following(int pos) {
- if (pos < text.getBeginIndex() || pos > text.getEndIndex()) {
- throw new IllegalArgumentException("offset out of bounds");
- } else if (pos == text.getEndIndex()) {
- // this conflicts with the javadocs, but matches actual behavior (Oracle has a bug in something)
- // https://bugs.openjdk.java.net/browse/JDK-8015110
- text.setIndex(text.getEndIndex());
- current = text.getIndex();
- return DONE;
- } else {
- text.setIndex(pos);
- current = text.getIndex();
- return advanceForward();
- }
- }
-
- @Override
- public int previous() {
- if (text.getIndex() == text.getBeginIndex()) {
- return DONE;
- } else {
- return advanceBackward();
- }
- }
-
- private int advanceBackward() {
- char c;
- while ((c = text.previous()) != CharacterIterator.DONE) {
- if (c == separator) {
- return current = text.getIndex() + 1;
- }
- }
- assert text.getIndex() == text.getBeginIndex();
- return current = text.getIndex();
- }
-
- @Override
- public int preceding(int pos) {
- if (pos < text.getBeginIndex() || pos > text.getEndIndex()) {
- throw new IllegalArgumentException("offset out of bounds");
- } else if (pos == text.getBeginIndex()) {
- // this conflicts with the javadocs, but matches actual behavior (Oracle has a bug in something)
- // https://bugs.openjdk.java.net/browse/JDK-8015110
- text.setIndex(text.getBeginIndex());
- current = text.getIndex();
- return DONE;
- } else {
- text.setIndex(pos);
- current = text.getIndex();
- return advanceBackward();
- }
- }
-
- @Override
- public int next(int n) {
- if (n < 0) {
- for (int i = 0; i < -n; i++) {
- previous();
- }
- } else {
- for (int i = 0; i < n; i++) {
- next();
- }
- }
- return current();
- }
-
- @Override
- public CharacterIterator getText() {
- return text;
- }
-
- @Override
- public void setText(CharacterIterator newText) {
- text = newText;
- current = text.getBeginIndex();
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/85c3ae20/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/WholeBreakIterator.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/WholeBreakIterator.java b/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/WholeBreakIterator.java
deleted file mode 100644
index 7f5ba6a..0000000
--- a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/WholeBreakIterator.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.postingshighlight;
-
-import java.text.BreakIterator;
-import java.text.CharacterIterator;
-
-/** Just produces one single fragment for the entire text */
-public final class WholeBreakIterator extends BreakIterator {
- private CharacterIterator text;
- private int start;
- private int end;
- private int current;
-
- @Override
- public int current() {
- return current;
- }
-
- @Override
- public int first() {
- return (current = start);
- }
-
- @Override
- public int following(int pos) {
- if (pos < start || pos > end) {
- throw new IllegalArgumentException("offset out of bounds");
- } else if (pos == end) {
- // this conflicts with the javadocs, but matches actual behavior (Oracle has a bug in something)
- // https://bugs.openjdk.java.net/browse/JDK-8015110
- current = end;
- return DONE;
- } else {
- return last();
- }
- }
-
- @Override
- public CharacterIterator getText() {
- return text;
- }
-
- @Override
- public int last() {
- return (current = end);
- }
-
- @Override
- public int next() {
- if (current == end) {
- return DONE;
- } else {
- return last();
- }
- }
-
- @Override
- public int next(int n) {
- if (n < 0) {
- for (int i = 0; i < -n; i++) {
- previous();
- }
- } else {
- for (int i = 0; i < n; i++) {
- next();
- }
- }
- return current();
- }
-
- @Override
- public int preceding(int pos) {
- if (pos < start || pos > end) {
- throw new IllegalArgumentException("offset out of bounds");
- } else if (pos == start) {
- // this conflicts with the javadocs, but matches actual behavior (Oracle has a bug in something)
- // https://bugs.openjdk.java.net/browse/JDK-8015110
- current = start;
- return DONE;
- } else {
- return first();
- }
- }
-
- @Override
- public int previous() {
- if (current == start) {
- return DONE;
- } else {
- return first();
- }
- }
-
- @Override
- public void setText(CharacterIterator newText) {
- start = newText.getBeginIndex();
- end = newText.getEndIndex();
- text = newText;
- current = start;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/85c3ae20/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestCustomSeparatorBreakIterator.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestCustomSeparatorBreakIterator.java b/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestCustomSeparatorBreakIterator.java
deleted file mode 100644
index 29e7e97..0000000
--- a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestCustomSeparatorBreakIterator.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.postingshighlight;
-
-import com.carrotsearch.randomizedtesting.generators.RandomPicks;
-import org.apache.lucene.util.LuceneTestCase;
-
-import java.text.BreakIterator;
-import java.util.Locale;
-
-import static org.apache.lucene.search.postingshighlight.TestWholeBreakIterator.assertSameBreaks;
-import static org.hamcrest.CoreMatchers.equalTo;
-
-public class TestCustomSeparatorBreakIterator extends LuceneTestCase {
-
- private static final Character[] SEPARATORS = new Character[]{' ', '\u0000', 8233};
-
- public void testBreakOnCustomSeparator() throws Exception {
- Character separator = randomSeparator();
- BreakIterator bi = new CustomSeparatorBreakIterator(separator);
- String source = "this" + separator + "is" + separator + "the" + separator + "first" + separator + "sentence";
- bi.setText(source);
- assertThat(bi.current(), equalTo(0));
- assertThat(bi.first(), equalTo(0));
- assertThat(source.substring(bi.current(), bi.next()), equalTo("this" + separator));
- assertThat(source.substring(bi.current(), bi.next()), equalTo("is" + separator));
- assertThat(source.substring(bi.current(), bi.next()), equalTo("the" + separator));
- assertThat(source.substring(bi.current(), bi.next()), equalTo("first" + separator));
- assertThat(source.substring(bi.current(), bi.next()), equalTo("sentence"));
- assertThat(bi.next(), equalTo(BreakIterator.DONE));
-
- assertThat(bi.last(), equalTo(source.length()));
- int current = bi.current();
- assertThat(source.substring(bi.previous(), current), equalTo("sentence"));
- current = bi.current();
- assertThat(source.substring(bi.previous(), current), equalTo("first" + separator));
- current = bi.current();
- assertThat(source.substring(bi.previous(), current), equalTo("the" + separator));
- current = bi.current();
- assertThat(source.substring(bi.previous(), current), equalTo("is" + separator));
- current = bi.current();
- assertThat(source.substring(bi.previous(), current), equalTo("this" + separator));
- assertThat(bi.previous(), equalTo(BreakIterator.DONE));
- assertThat(bi.current(), equalTo(0));
-
- assertThat(source.substring(0, bi.following(9)), equalTo("this" + separator + "is" + separator + "the" + separator));
-
- assertThat(source.substring(0, bi.preceding(9)), equalTo("this" + separator + "is" + separator));
-
- assertThat(bi.first(), equalTo(0));
- assertThat(source.substring(0, bi.next(3)), equalTo("this" + separator + "is" + separator + "the" + separator));
- }
-
- public void testSingleSentences() throws Exception {
- BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
- BreakIterator actual = new CustomSeparatorBreakIterator(randomSeparator());
- assertSameBreaks("a", expected, actual);
- assertSameBreaks("ab", expected, actual);
- assertSameBreaks("abc", expected, actual);
- assertSameBreaks("", expected, actual);
- }
-
- public void testSliceEnd() throws Exception {
- BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
- BreakIterator actual = new CustomSeparatorBreakIterator(randomSeparator());
- assertSameBreaks("a000", 0, 1, expected, actual);
- assertSameBreaks("ab000", 0, 1, expected, actual);
- assertSameBreaks("abc000", 0, 1, expected, actual);
- assertSameBreaks("000", 0, 0, expected, actual);
- }
-
- public void testSliceStart() throws Exception {
- BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
- BreakIterator actual = new CustomSeparatorBreakIterator(randomSeparator());
- assertSameBreaks("000a", 3, 1, expected, actual);
- assertSameBreaks("000ab", 3, 2, expected, actual);
- assertSameBreaks("000abc", 3, 3, expected, actual);
- assertSameBreaks("000", 3, 0, expected, actual);
- }
-
- public void testSliceMiddle() throws Exception {
- BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
- BreakIterator actual = new CustomSeparatorBreakIterator(randomSeparator());
- assertSameBreaks("000a000", 3, 1, expected, actual);
- assertSameBreaks("000ab000", 3, 2, expected, actual);
- assertSameBreaks("000abc000", 3, 3, expected, actual);
- assertSameBreaks("000000", 3, 0, expected, actual);
- }
-
- /** the current position must be ignored, initial position is always first() */
- public void testFirstPosition() throws Exception {
- BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
- BreakIterator actual = new CustomSeparatorBreakIterator(randomSeparator());
- assertSameBreaks("000ab000", 3, 2, 4, expected, actual);
- }
-
- private static char randomSeparator() {
- return RandomPicks.randomFrom(random(), SEPARATORS);
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/85c3ae20/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestWholeBreakIterator.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestWholeBreakIterator.java b/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestWholeBreakIterator.java
deleted file mode 100644
index 43249ea..0000000
--- a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestWholeBreakIterator.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.postingshighlight;
-
-import org.apache.lucene.util.LuceneTestCase;
-
-import java.text.BreakIterator;
-import java.text.CharacterIterator;
-import java.text.StringCharacterIterator;
-import java.util.Locale;
-
-public class TestWholeBreakIterator extends LuceneTestCase {
-
- /** For single sentences, we know WholeBreakIterator should break the same as a sentence iterator */
- public void testSingleSentences() throws Exception {
- BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
- BreakIterator actual = new WholeBreakIterator();
- assertSameBreaks("a", expected, actual);
- assertSameBreaks("ab", expected, actual);
- assertSameBreaks("abc", expected, actual);
- assertSameBreaks("", expected, actual);
- }
-
- public void testSliceEnd() throws Exception {
- BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
- BreakIterator actual = new WholeBreakIterator();
- assertSameBreaks("a000", 0, 1, expected, actual);
- assertSameBreaks("ab000", 0, 1, expected, actual);
- assertSameBreaks("abc000", 0, 1, expected, actual);
- assertSameBreaks("000", 0, 0, expected, actual);
- }
-
- public void testSliceStart() throws Exception {
- BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
- BreakIterator actual = new WholeBreakIterator();
- assertSameBreaks("000a", 3, 1, expected, actual);
- assertSameBreaks("000ab", 3, 2, expected, actual);
- assertSameBreaks("000abc", 3, 3, expected, actual);
- assertSameBreaks("000", 3, 0, expected, actual);
- }
-
- public void testSliceMiddle() throws Exception {
- BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
- BreakIterator actual = new WholeBreakIterator();
- assertSameBreaks("000a000", 3, 1, expected, actual);
- assertSameBreaks("000ab000", 3, 2, expected, actual);
- assertSameBreaks("000abc000", 3, 3, expected, actual);
- assertSameBreaks("000000", 3, 0, expected, actual);
- }
-
- /** the current position must be ignored, initial position is always first() */
- public void testFirstPosition() throws Exception {
- BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
- BreakIterator actual = new WholeBreakIterator();
- assertSameBreaks("000ab000", 3, 2, 4, expected, actual);
- }
-
- public static void assertSameBreaks(String text, BreakIterator expected, BreakIterator actual) {
- assertSameBreaks(new StringCharacterIterator(text),
- new StringCharacterIterator(text),
- expected,
- actual);
- }
-
- public static void assertSameBreaks(String text, int offset, int length, BreakIterator expected, BreakIterator actual) {
- assertSameBreaks(text, offset, length, offset, expected, actual);
- }
-
- public static void assertSameBreaks(String text, int offset, int length, int current, BreakIterator expected, BreakIterator actual) {
- assertSameBreaks(new StringCharacterIterator(text, offset, offset+length, current),
- new StringCharacterIterator(text, offset, offset+length, current),
- expected,
- actual);
- }
-
- /** Asserts that two breakiterators break the text the same way */
- public static void assertSameBreaks(CharacterIterator one, CharacterIterator two, BreakIterator expected, BreakIterator actual) {
- expected.setText(one);
- actual.setText(two);
-
- assertEquals(expected.current(), actual.current());
-
- // next()
- int v = expected.current();
- while (v != BreakIterator.DONE) {
- assertEquals(v = expected.next(), actual.next());
- assertEquals(expected.current(), actual.current());
- }
-
- // first()
- assertEquals(expected.first(), actual.first());
- assertEquals(expected.current(), actual.current());
- // last()
- assertEquals(expected.last(), actual.last());
- assertEquals(expected.current(), actual.current());
-
- // previous()
- v = expected.current();
- while (v != BreakIterator.DONE) {
- assertEquals(v = expected.previous(), actual.previous());
- assertEquals(expected.current(), actual.current());
- }
-
- // following()
- for (int i = one.getBeginIndex(); i <= one.getEndIndex(); i++) {
- expected.first();
- actual.first();
- assertEquals(expected.following(i), actual.following(i));
- assertEquals(expected.current(), actual.current());
- }
-
- // preceding()
- for (int i = one.getBeginIndex(); i <= one.getEndIndex(); i++) {
- expected.last();
- actual.last();
- assertEquals(expected.preceding(i), actual.preceding(i));
- assertEquals(expected.current(), actual.current());
- }
- }
-}
[23/44] lucene-solr:jira/solr-8668: SOLR-10731: Add knn Streaming
Expression WIP
Posted by cp...@apache.org.
SOLR-10731: Add knn Streaming Expression WIP
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/89d48dfd
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/89d48dfd
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/89d48dfd
Branch: refs/heads/jira/solr-8668
Commit: 89d48dfd9cfd63dfd54595551ec07e042eba432e
Parents: 65bfa48
Author: Joel Bernstein <jb...@apache.org>
Authored: Tue May 23 19:07:39 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Wed May 24 07:59:01 2017 -0400
----------------------------------------------------------------------
.../org/apache/solr/handler/StreamHandler.java | 1 +
.../solr/client/solrj/io/stream/KnnStream.java | 249 +++++++++++++++++++
.../solrj/io/stream/StreamExpressionTest.java | 40 +++
3 files changed, 290 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/89d48dfd/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
index 9fd7c8b..c045f20 100644
--- a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
@@ -121,6 +121,7 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
.withFunctionName("topic", TopicStream.class)
.withFunctionName("commit", CommitStream.class)
.withFunctionName("random", RandomStream.class)
+ .withFunctionName("knn", KnnStream.class)
// decorator streams
.withFunctionName("merge", MergeStream.class)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/89d48dfd/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/KnnStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/KnnStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/KnnStream.java
new file mode 100644
index 0000000..f0a188a
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/KnnStream.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.io.stream;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.io.SolrClientCache;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.comp.StreamComparator;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
+import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.params.ModifiableSolrParams;
+
+import static org.apache.solr.common.params.CommonParams.Q;
+
+public class KnnStream extends TupleStream implements Expressible {
+
+ private static String[] mltParams = {"qf", "mintf", "mindf", "maxdf", "minwl", "maxwl", "maxqt", "maxntp", "boost"};
+
+ private String zkHost;
+ private Map<String, String> props;
+ private String collection;
+ protected transient SolrClientCache cache;
+ protected transient CloudSolrClient cloudSolrClient;
+ private Iterator<SolrDocument> documentIterator;
+ private String id;
+
+ public KnnStream(String zkHost,
+ String collection,
+ String id,
+ Map<String, String> props) throws IOException {
+ init(zkHost, collection, id, props);
+ }
+
+ public KnnStream(StreamExpression expression, StreamFactory factory) throws IOException{
+ // grab all parameters out
+ String collectionName = factory.getValueOperand(expression, 0);
+ List<StreamExpressionNamedParameter> namedParams = factory.getNamedOperands(expression);
+ StreamExpressionNamedParameter zkHostExpression = factory.getNamedOperand(expression, "zkHost");
+ StreamExpressionNamedParameter idExpression = factory.getNamedOperand(expression, "id");
+ StreamExpressionNamedParameter qfExpression = factory.getNamedOperand(expression, "qf");
+
+
+ // Collection Name
+ if(null == collectionName){
+ throw new IOException(String.format(Locale.ROOT,"invalid expression %s - collectionName expected as first operand",expression));
+ }
+
+ // Named parameters - passed directly to solr as solrparams
+ System.out.println("####Params:"+namedParams.size());
+ if(namedParams.size() < 2){
+ throw new IOException(String.format(Locale.ROOT,"invalid expression %s - at least two named parameters expected. eg. 'id' and 'qf'",expression));
+ }
+
+ // pull out known named params
+ Map<String,String> params = new HashMap<String,String>();
+ for(StreamExpressionNamedParameter namedParam : namedParams){
+ if(!namedParam.getName().equals("zkHost") && !namedParam.getName().equals("id")){
+ params.put(namedParam.getName(), namedParam.getParameter().toString().trim());
+ }
+ }
+
+ String id = null;
+ if(idExpression != null) {
+ id = ((StreamExpressionValue)idExpression.getParameter()).getValue();
+ } else {
+ throw new IOException("id parameter is expected for KnnStream");
+ }
+
+ if(qfExpression == null) {
+ throw new IOException("qf parameter is expected for KnnStream");
+ }
+
+ // zkHost, optional - if not provided then will look into factory list to get
+ String zkHost = null;
+ if(null == zkHostExpression){
+ zkHost = factory.getCollectionZkHost(collectionName);
+ if(zkHost == null) {
+ zkHost = factory.getDefaultZkHost();
+ }
+ }
+ else if(zkHostExpression.getParameter() instanceof StreamExpressionValue){
+ zkHost = ((StreamExpressionValue)zkHostExpression.getParameter()).getValue();
+ }
+ if(null == zkHost){
+ throw new IOException(String.format(Locale.ROOT,"invalid expression %s - zkHost not found for collection '%s'",expression,collectionName));
+ }
+
+ // We've got all the required items
+ init(zkHost, collectionName, id, params);
+ }
+
+ private void init(String zkHost, String collection, String id, Map<String, String> props) throws IOException {
+ this.zkHost = zkHost;
+ this.props = props;
+ this.collection = collection;
+ this.id = id;
+ }
+
+ @Override
+ public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
+ // function name
+ StreamExpression expression = new StreamExpression(factory.getFunctionName(this.getClass()));
+
+ // collection
+ expression.addParameter(collection);
+
+ // parameters
+ for(Entry<String,String> param : props.entrySet()){
+ expression.addParameter(new StreamExpressionNamedParameter(param.getKey(), param.getValue()));
+ }
+
+ // zkHost
+ expression.addParameter(new StreamExpressionNamedParameter("zkHost", zkHost));
+
+ return expression;
+ }
+
+ @Override
+ public Explanation toExplanation(StreamFactory factory) throws IOException {
+
+ StreamExplanation explanation = new StreamExplanation(getStreamNodeId().toString());
+
+ explanation.setFunctionName(factory.getFunctionName(this.getClass()));
+ explanation.setImplementingClass(this.getClass().getName());
+ explanation.setExpressionType(ExpressionType.STREAM_SOURCE);
+ explanation.setExpression(toExpression(factory).toString());
+
+ // child is a datastore so add it at this point
+ StreamExplanation child = new StreamExplanation(getStreamNodeId() + "-datastore");
+ child.setFunctionName(String.format(Locale.ROOT, "solr (%s)", collection));
+ child.setImplementingClass("Solr/Lucene");
+ child.setExpressionType(ExpressionType.DATASTORE);
+ if(null != props){
+ child.setExpression(props.entrySet().stream().map(e -> String.format(Locale.ROOT, "%s=%s", e.getKey(), e.getValue())).collect(Collectors.joining(",")));
+ }
+ explanation.addChild(child);
+
+ return explanation;
+ }
+
+ public void setStreamContext(StreamContext context) {
+ cache = context.getSolrClientCache();
+ }
+
+ public List<TupleStream> children() {
+ List<TupleStream> l = new ArrayList();
+ return l;
+ }
+
+ public void open() throws IOException {
+ cloudSolrClient = cache.getCloudSolrClient(zkHost);
+ ModifiableSolrParams params = getParams(this.props);
+
+ StringBuilder builder = new StringBuilder();
+
+ for(String key : mltParams) {
+ if(params.get(key) != null) {
+ builder.append(" " + key + "=" + params.get(key));
+ params.remove(key);
+ }
+ }
+
+ params.add(Q, "{!mlt"+builder.toString()+"}"+id);
+
+ QueryRequest request = new QueryRequest(params);
+ try {
+ QueryResponse response = request.process(cloudSolrClient, collection);
+ SolrDocumentList docs = response.getResults();
+ documentIterator = docs.iterator();
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ public void close() throws IOException {
+
+ }
+
+ public Tuple read() throws IOException {
+ if(documentIterator.hasNext()) {
+ Map map = new HashMap();
+ SolrDocument doc = documentIterator.next();
+ for(String key : doc.keySet()) {
+ map.put(key, doc.get(key));
+ }
+ return new Tuple(map);
+ } else {
+ Map fields = new HashMap();
+ fields.put("EOF", true);
+ Tuple tuple = new Tuple(fields);
+ return tuple;
+ }
+ }
+
+ private ModifiableSolrParams getParams(Map<String, String> props) {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ for(String key : props.keySet()) {
+ String value = props.get(key);
+ params.add(key, value);
+ }
+ return params;
+ }
+
+ public int getCost() {
+ return 0;
+ }
+
+ @Override
+ public StreamComparator getStreamSort() {
+ return null;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/89d48dfd/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index a881e53..d464989 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -920,6 +920,46 @@ public class StreamExpressionTest extends SolrCloudTestCase {
}
}
+
+ @Test
+ public void testKnnStream() throws Exception {
+
+ UpdateRequest update = new UpdateRequest();
+
+ update.add(id, "1", "a_t", "hello world have a very nice day blah");
+ update.add(id, "2", "a_t", "hello world have a very nice day fancy sky");
+ update.add(id, "3", "a_t", "hello world have a very nice bug out");
+ update.add(id, "4", "a_t", "hello world have a very streaming is fun");
+
+
+ update.commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ StreamExpression expression;
+ TupleStream stream;
+
+ StreamFactory factory = new StreamFactory()
+ .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
+ .withFunctionName("random", RandomStream.class);
+
+
+ StreamContext context = new StreamContext();
+ SolrClientCache cache = new SolrClientCache();
+ try {
+ context.setSolrClientCache(cache);
+
+ ModifiableSolrParams sParams = new ModifiableSolrParams(StreamingTest.mapParams(CommonParams.QT, "/stream"));
+ sParams.add("expr", "knn(" + COLLECTIONORALIAS + ", id=\"1\", qf=\"a_t\", rows=\"4\", fl=\"id\")");
+ JettySolrRunner jetty = cluster.getJettySolrRunner(0);
+ SolrStream solrStream = new SolrStream(jetty.getBaseUrl().toString() + "/collection1", sParams);
+ List<Tuple> tuples = getTuples(solrStream);
+ System.out.println("## Tuples:"+tuples.size());
+ assertTrue(tuples.size() == 4);
+
+ } finally {
+ cache.close();
+ }
+ }
+
@Test
public void testReducerStream() throws Exception {
[10/44] lucene-solr:jira/solr-8668: LUCENE-7815: Removed the
PostingsHighlighter
Posted by cp...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java b/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java
deleted file mode 100644
index 0620bd6..0000000
--- a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java
+++ /dev/null
@@ -1,884 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.postingshighlight;
-
-import java.util.Collections;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.ConstantScoreQuery;
-import org.apache.lucene.search.DisjunctionMaxQuery;
-import org.apache.lucene.search.FuzzyQuery;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.PrefixQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.RegexpQuery;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TermRangeQuery;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.WildcardQuery;
-import org.apache.lucene.search.spans.SpanFirstQuery;
-import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper;
-import org.apache.lucene.search.spans.SpanNearQuery;
-import org.apache.lucene.search.spans.SpanNotQuery;
-import org.apache.lucene.search.spans.SpanOrQuery;
-import org.apache.lucene.search.spans.SpanQuery;
-import org.apache.lucene.search.spans.SpanTermQuery;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.LuceneTestCase;
-
-/**
- * Some tests that override {@link PostingsHighlighter#getIndexAnalyzer} to
- * highlight wilcard, fuzzy, etc queries.
- */
-public class TestMultiTermHighlighting extends LuceneTestCase {
-
- public void testWildcards() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- Query query = new WildcardQuery(new Term("body", "te*"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- // wrong field
- BooleanQuery.Builder bq = new BooleanQuery.Builder();
- bq.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
- bq.add(new WildcardQuery(new Term("bogus", "te*")), BooleanClause.Occur.SHOULD);
- topDocs = searcher.search(bq.build(), 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", bq.build(), searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a test.", snippets[0]);
- assertEquals("Test a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testOnePrefix() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- Query query = new PrefixQuery(new Term("body", "te"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- // wrong field
- BooleanQuery.Builder bq = new BooleanQuery.Builder();
- bq.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
- bq.add(new PrefixQuery(new Term("bogus", "te")), BooleanClause.Occur.SHOULD);
- topDocs = searcher.search(bq.build(), 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", bq.build(), searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a test.", snippets[0]);
- assertEquals("Test a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testOneRegexp() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- Query query = new RegexpQuery(new Term("body", "te.*"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- // wrong field
- BooleanQuery.Builder bq = new BooleanQuery.Builder();
- bq.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
- bq.add(new RegexpQuery(new Term("bogus", "te.*")), BooleanClause.Occur.SHOULD);
- topDocs = searcher.search(bq.build(), 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", bq.build(), searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a test.", snippets[0]);
- assertEquals("Test a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testOneFuzzy() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- Query query = new FuzzyQuery(new Term("body", "tets"), 1);
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- // with prefix
- query = new FuzzyQuery(new Term("body", "tets"), 1, 2);
- topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- // wrong field
- BooleanQuery.Builder bq = new BooleanQuery.Builder();
- bq.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
- bq.add(new FuzzyQuery(new Term("bogus", "tets"), 1), BooleanClause.Occur.SHOULD);
- topDocs = searcher.search(bq.build(), 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", bq.build(), searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a test.", snippets[0]);
- assertEquals("Test a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testRanges() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- Query query = TermRangeQuery.newStringRange("body", "ta", "tf", true, true);
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- // null start
- query = TermRangeQuery.newStringRange("body", null, "tf", true, true);
- topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This <b>is</b> <b>a</b> <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> <b>a</b> <b>one</b> <b>sentence</b> <b>document</b>.", snippets[1]);
-
- // null end
- query = TermRangeQuery.newStringRange("body", "ta", null, true, true);
- topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("<b>This</b> is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- // exact start inclusive
- query = TermRangeQuery.newStringRange("body", "test", "tf", true, true);
- topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- // exact end inclusive
- query = TermRangeQuery.newStringRange("body", "ta", "test", true, true);
- topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- // exact start exclusive
- BooleanQuery.Builder bq = new BooleanQuery.Builder();
- bq.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
- bq.add(TermRangeQuery.newStringRange("body", "test", "tf", false, true), BooleanClause.Occur.SHOULD);
- topDocs = searcher.search(bq.build(), 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", bq.build(), searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a test.", snippets[0]);
- assertEquals("Test a one sentence document.", snippets[1]);
-
- // exact end exclusive
- bq = new BooleanQuery.Builder();
- bq.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
- bq.add(TermRangeQuery.newStringRange("body", "ta", "test", true, false), BooleanClause.Occur.SHOULD);
- topDocs = searcher.search(bq.build(), 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", bq.build(), searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a test.", snippets[0]);
- assertEquals("Test a one sentence document.", snippets[1]);
-
- // wrong field
- bq = new BooleanQuery.Builder();
- bq.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
- bq.add(TermRangeQuery.newStringRange("bogus", "ta", "tf", true, true), BooleanClause.Occur.SHOULD);
- topDocs = searcher.search(bq.build(), 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", bq.build(), searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a test.", snippets[0]);
- assertEquals("Test a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testWildcardInBoolean() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- BooleanQuery.Builder query = new BooleanQuery.Builder();
- query.add(new WildcardQuery(new Term("body", "te*")), BooleanClause.Occur.SHOULD);
- TopDocs topDocs = searcher.search(query.build(), 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query.build(), searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- // must not
- query = new BooleanQuery.Builder();
- query.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
- query.add(new WildcardQuery(new Term("bogus", "te*")), BooleanClause.Occur.MUST_NOT);
- topDocs = searcher.search(query.build(), 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", query.build(), searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a test.", snippets[0]);
- assertEquals("Test a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testWildcardInFiltered() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- Query query = new BooleanQuery.Builder()
- .add(new WildcardQuery(new Term("body", "te*")), Occur.MUST)
- .add(new TermQuery(new Term("body", "test")), Occur.FILTER)
- .build();
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testWildcardInConstantScore() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- ConstantScoreQuery query = new ConstantScoreQuery(new WildcardQuery(new Term("body", "te*")));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testWildcardInDisjunctionMax() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- DisjunctionMaxQuery query = new DisjunctionMaxQuery(
- Collections.singleton(new WildcardQuery(new Term("body", "te*"))), 0);
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testSpanWildcard() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- Query query = new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*")));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testSpanOr() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- SpanQuery childQuery = new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*")));
- Query query = new SpanOrQuery(new SpanQuery[] { childQuery });
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testSpanNear() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- SpanQuery childQuery = new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*")));
- Query query = new SpanNearQuery(new SpanQuery[] { childQuery, childQuery }, 0, false);
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testSpanNot() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- SpanQuery include = new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*")));
- SpanQuery exclude = new SpanTermQuery(new Term("body", "bogus"));
- Query query = new SpanNotQuery(include, exclude);
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testSpanPositionCheck() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- SpanQuery childQuery = new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*")));
- Query query = new SpanFirstQuery(childQuery, 1000000);
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- /** Runs a query with two MTQs and confirms the formatter
- * can tell which query matched which hit. */
- public void testWhichMTQMatched() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- BooleanQuery.Builder query = new BooleanQuery.Builder();
- query.add(new WildcardQuery(new Term("body", "te*")), BooleanClause.Occur.SHOULD);
- query.add(new WildcardQuery(new Term("body", "one")), BooleanClause.Occur.SHOULD);
- query.add(new WildcardQuery(new Term("body", "se*")), BooleanClause.Occur.SHOULD);
- TopDocs topDocs = searcher.search(query.build(), 10, Sort.INDEXORDER);
- assertEquals(1, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query.build(), searcher, topDocs);
- assertEquals(1, snippets.length);
-
- // Default formatter just bolds each hit:
- assertEquals("<b>Test</b> a <b>one</b> <b>sentence</b> document.", snippets[0]);
-
- // Now use our own formatter, that also stuffs the
- // matching term's text into the result:
- highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
-
- @Override
- protected PassageFormatter getFormatter(String field) {
- return new PassageFormatter() {
-
- @Override
- public Object format(Passage passages[], String content) {
- // Copied from DefaultPassageFormatter, but
- // tweaked to include the matched term:
- StringBuilder sb = new StringBuilder();
- int pos = 0;
- for (Passage passage : passages) {
- // don't add ellipsis if it's the first one, or if it's connected.
- if (passage.startOffset > pos && pos > 0) {
- sb.append("... ");
- }
- pos = passage.startOffset;
- for (int i = 0; i < passage.numMatches; i++) {
- int start = passage.matchStarts[i];
- int end = passage.matchEnds[i];
- // it's possible to have overlapping terms
- if (start > pos) {
- sb.append(content, pos, start);
- }
- if (end > pos) {
- sb.append("<b>");
- sb.append(content, Math.max(pos, start), end);
- sb.append('(');
- sb.append(passage.getMatchTerms()[i].utf8ToString());
- sb.append(')');
- sb.append("</b>");
- pos = end;
- }
- }
- // it's possible a "term" from the analyzer could span a sentence boundary.
- sb.append(content, pos, Math.max(pos, passage.endOffset));
- pos = passage.endOffset;
- }
- return sb.toString();
- }
- };
- }
- };
-
- assertEquals(1, topDocs.totalHits);
- snippets = highlighter.highlight("body", query.build(), searcher, topDocs);
- assertEquals(1, snippets.length);
-
- // Default formatter bolds each hit:
- assertEquals("<b>Test(body:te*)</b> a <b>one(body:one)</b> <b>sentence(body:se*)</b> document.", snippets[0]);
-
- ir.close();
- dir.close();
- }
-}
[24/44] lucene-solr:jira/solr-8668: SOLR-10731: Add first test case
Posted by cp...@apache.org.
SOLR-10731: Add first test case
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/cc87181c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/cc87181c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/cc87181c
Branch: refs/heads/jira/solr-8668
Commit: cc87181c54f10dec809dc9dbd876fa750f1d6461
Parents: 89d48df
Author: Joel Bernstein <jb...@apache.org>
Authored: Tue May 23 20:05:08 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Wed May 24 07:59:01 2017 -0400
----------------------------------------------------------------------
.../solrj/io/stream/StreamExpressionTest.java | 23 +++++---------------
1 file changed, 5 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cc87181c/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index d464989..dc3a380 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -925,36 +925,23 @@ public class StreamExpressionTest extends SolrCloudTestCase {
public void testKnnStream() throws Exception {
UpdateRequest update = new UpdateRequest();
-
update.add(id, "1", "a_t", "hello world have a very nice day blah");
- update.add(id, "2", "a_t", "hello world have a very nice day fancy sky");
- update.add(id, "3", "a_t", "hello world have a very nice bug out");
update.add(id, "4", "a_t", "hello world have a very streaming is fun");
-
-
+ update.add(id, "3", "a_t", "hello world have a very nice bug out");
+ update.add(id, "2", "a_t", "hello world have a very nice day fancy sky");
update.commit(cluster.getSolrClient(), COLLECTIONORALIAS);
- StreamExpression expression;
- TupleStream stream;
-
- StreamFactory factory = new StreamFactory()
- .withCollectionZkHost(COLLECTIONORALIAS, cluster.getZkServer().getZkAddress())
- .withFunctionName("random", RandomStream.class);
-
-
StreamContext context = new StreamContext();
SolrClientCache cache = new SolrClientCache();
try {
context.setSolrClientCache(cache);
-
ModifiableSolrParams sParams = new ModifiableSolrParams(StreamingTest.mapParams(CommonParams.QT, "/stream"));
- sParams.add("expr", "knn(" + COLLECTIONORALIAS + ", id=\"1\", qf=\"a_t\", rows=\"4\", fl=\"id\")");
+ sParams.add("expr", "knn(" + COLLECTIONORALIAS + ", id=\"1\", qf=\"a_t\", rows=\"4\", fl=\"id, score\", mintf=\"1\")");
JettySolrRunner jetty = cluster.getJettySolrRunner(0);
SolrStream solrStream = new SolrStream(jetty.getBaseUrl().toString() + "/collection1", sParams);
List<Tuple> tuples = getTuples(solrStream);
- System.out.println("## Tuples:"+tuples.size());
- assertTrue(tuples.size() == 4);
-
+ assertTrue(tuples.size() == 3);
+ assertOrder(tuples,2,3,4);
} finally {
cache.close();
}
[03/44] lucene-solr:jira/solr-8668: SOLR-10727:
SolrIndexSearcher.numDocs empty docSet optimization
Posted by cp...@apache.org.
SOLR-10727: SolrIndexSearcher.numDocs empty docSet optimization
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/715d8b0c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/715d8b0c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/715d8b0c
Branch: refs/heads/jira/solr-8668
Commit: 715d8b0ccf42d30661a7964c33bd27c9fec3d18e
Parents: 85c1319
Author: David Smiley <ds...@apache.org>
Authored: Tue May 23 10:51:00 2017 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Tue May 23 10:51:00 2017 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +++
solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java | 3 +++
2 files changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/715d8b0c/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 799df74..7a604f4 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -148,6 +148,9 @@ Optimizations
string fields from the FieldCache, resulting in up to 56% better throughput for those cases.
(yonik)
+* SOLR-10727: Avoid polluting the filter cache for certain types of faceting (typically ranges) when
+ the base docset is empty. (David Smiley)
+
Other Changes
----------------------
* SOLR-10236: Removed FieldType.getNumericType(). Use getNumberType() instead. (Tomás Fernández Löbbe)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/715d8b0c/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
index 9d63aad..599fd24 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
@@ -2033,6 +2033,9 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
* If there is a low-level I/O error.
*/
public int numDocs(Query a, DocSet b) throws IOException {
+ if (b.size() == 0) {
+ return 0;
+ }
if (filterCache != null) {
// Negative query if absolute value different from original
Query absQ = QueryUtils.getAbs(a);
[22/44] lucene-solr:jira/solr-8668: LUCENE-7810: Fix numeric join
equals test failure.
Posted by cp...@apache.org.
LUCENE-7810: Fix numeric join equals test failure.
Numeric join equals isn't based on the index reader, but rather on the collected join values.
In a test failure during the second indexing round no new join values were indexed causing the equals assertion to fail.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/65bfa487
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/65bfa487
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/65bfa487
Branch: refs/heads/jira/solr-8668
Commit: 65bfa48770b33242d80dcce946073cc5698da7b1
Parents: 8f4d72a
Author: Martijn van Groningen <ma...@gmail.com>
Authored: Wed May 24 09:50:23 2017 +0200
Committer: Martijn van Groningen <mv...@apache.org>
Committed: Wed May 24 09:50:36 2017 +0200
----------------------------------------------------------------------
.../src/test/org/apache/lucene/search/join/TestJoinUtil.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65bfa487/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
index c95e144..49dd333 100644
--- a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
@@ -1141,7 +1141,7 @@ public class TestJoinUtil extends LuceneTestCase {
indexSearcher, scoreMode1)));
}
- for (int i = 0; i < 13; i++) {
+ for (int i = 14; i < 26; i++) {
Document doc = new Document();
doc.add(new TextField("id", "new_id" , Field.Store.NO));
doc.add(new TextField("name", "name5", Field.Store.NO));
@@ -1159,7 +1159,7 @@ public class TestJoinUtil extends LuceneTestCase {
}
try (IndexReader r = w.getReader()) {
IndexSearcher indexSearcher = new IndexSearcher(r);
- assertFalse("Query shouldn't be equal, because different index readers ",
+ assertFalse("Query shouldn't be equal, because new join values have been indexed",
x.equals(JoinUtil.createJoinQuery(joinField, multiValued, joinField,
Integer.class, new TermQuery(new Term("name", "name5")),
indexSearcher, scoreMode1)));
[11/44] lucene-solr:jira/solr-8668: LUCENE-7815: Removed the
PostingsHighlighter
Posted by cp...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/CambridgeMA.utf8
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/CambridgeMA.utf8 b/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/CambridgeMA.utf8
deleted file mode 100644
index d60b6fa..0000000
--- a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/CambridgeMA.utf8
+++ /dev/null
@@ -1 +0,0 @@
-{{Distinguish|Cambridge, England}} {{primary sources|date=June 2012}} {{Use mdy dates|date=January 2011}} {{Infobox settlement |official_name = Cambridge, Massachusetts |nickname = |motto = "Boston's Left Bank"<ref>{{cite web|url= http://www.epodunk.com/cgi-bin/genInfo.php?locIndex=2894|title=Profile for Cambridge, Massachusetts, MA|publisher= ePodunk |accessdate= November 1, 2012}}</ref> |image_skyline = CambridgeMACityHall2.jpg |imagesize = 175px |image_caption = Cambridge City Hall |image_seal = |image_flag = |image_map = Cambridge ma highlight.png |mapsize = 250px |map_caption = Location in Middlesex County in Massachusetts |image_map1 = |mapsize1 = |map_caption1 = |coordinates_region = US-MA |subdivision_type = Country |subdivision_name = United States |subdivision_type1 = State |subdivision_name1 = [[Massachusetts]] |subdivision_type2 = [[List of counties in Massachusetts|County]] |subdivision_name2 = [[Middlesex County, Massachusetts|Middlesex]] |established_title = Settled |
established_date = 1630 |established_title2 = Incorporated |established_date2 = 1636 |established_title3 = |established_date3 = |government_type = [[Council-manager government|Council-City Manager]] |leader_title = Mayor |leader_name = Henrietta Davis |leader_title1 = [[City manager|City Manager]] |leader_name1 = [[Robert W. Healy]] |area_magnitude = |area_total_km2 = 18.47 |area_total_sq_mi = 7.13 |area_land_km2 = 16.65 |area_land_sq_mi = 6.43 |area_water_km2 = 1.81 |area_water_sq_mi = 0.70 |population_as_of = 2010 |population_blank2_title = [[Demonym]] |population_blank2 = [[Cantabrigian]] |settlement_type = City |population_total = 105,162 |population_density_km2 = 6,341.98 |population_density_sq_mi = 16,422.08 |elevation_m = 12 |elevation_ft = 40 |timezone = [[Eastern Time Zone|Eastern]] |utc_offset = -5 |timezone_DST = [[Eastern Time Zone|Eastern]] |utc_offset_DST = -4 |coordinates_display = display=inline,title |latd = 42 |latm = 22 |lats = 25 |latNS = N |longd = 71 |longm = 0
6 |longs = 38 |longEW = W |website = [http://www.cambridgema.gov/ www.cambridgema.gov] |postal_code_type = ZIP code |postal_code = 02138, 02139, 02140, 02141, 02142 |area_code = [[Area code 617|617]] / [[Area code 857|857]] |blank_name = [[Federal Information Processing Standard|FIPS code]] |blank_info = 25-11000 |blank1_name = [[Geographic Names Information System|GNIS]] feature ID |blank1_info = 0617365 |footnotes = }} '''Cambridge''' is a city in [[Middlesex County, Massachusetts|Middlesex County]], [[Massachusetts]], [[United States]], in the [[Greater Boston]] area. It was named in honor of the [[University of Cambridge]] in [[England]], an important center of the [[Puritan]] theology embraced by the town's founders.<ref>{{cite book|last=Degler|first=Carl Neumann|title=Out of Our Pasts: The Forces That Shaped Modern America|publisher=HarperCollins|location=New York|year=1984|url=http://books.google.com/books?id=NebLe1ueuGQC&pg=PA18&lpg=PA18&dq=cambridge+university+puritans+newt
owne#v=onepage&q=&f=false|accessdate=September 9, 2009 | isbn=978-0-06-131985-3}}</ref> Cambridge is home to two of the world's most prominent universities, [[Harvard University]] and the [[Massachusetts Institute of Technology]]. According to the [[2010 United States Census]], the city's population was 105,162.<ref name="2010.census.gov">{{cite web|url=http://2010.census.gov/news/releases/operations/cb11-cn104.html |title=Census 2010 News | U.S. Census Bureau Delivers Massachusetts' 2010 Census Population Totals, Including First Look at Race and Hispanic Origin Data for Legislative Redistricting |publisher=2010.census.gov |date=2011-03-22 |accessdate=2012-04-28}}</ref> It is the fifth most populous city in the state, behind [[Boston]], [[Worcester, MA|Worcester]], [[Springfield, MA|Springfield]], and [[Lowell, Massachusetts|Lowell]].<ref name="2010.census.gov"/> Cambridge was one of the two [[county seat]]s of Middlesex County prior to the abolition of county government in 199
7; [[Lowell, Massachusetts|Lowell]] was the other. ==History== {{See also|Timeline of Cambridge, Massachusetts history}} [[File:Formation of Massachusetts towns.svg|thumb|A map showing the original boundaries of Cambridge]] The site for what would become Cambridge was chosen in December 1630, because it was located safely upriver from Boston Harbor, which made it easily defensible from attacks by enemy ships. Also, the water from the local spring was so good that the local Native Americans believed it had medicinal properties.{{Citation needed|date=November 2009}} [[Thomas Dudley]], his daughter [[Anne Bradstreet]] and her husband Simon were among the first settlers of the town. The first houses were built in the spring of 1631. The settlement was initially referred to as "the newe towne".<ref name=drake>{{cite book|last=Drake|first=Samuel Adams|title=History of Middlesex County, Massachusetts|publisher=Estes and Lauriat|location=Boston|year=1880|volume=1|pages=305–16|url=http://b
ooks.google.com/books?id=QGolOAyd9RMC&pg=PA316&lpg=PA305&dq=newetowne&ct=result#PPA305,M1|accessdate=December 26, 2008}}</ref> Official Massachusetts records show the name capitalized as '''Newe Towne''' by 1632.<ref name=public>{{cite book|title=Report on the Custody and Condition of the Public Records of Parishes|publisher=Massachusetts Secretary of the Commonwealth|url=http://books.google.com/books?id=IyYWAAAAYAAJ&pg=RA1-PA298&lpg=RA1-PA298&dq=%22Ordered+That+Newtowne+shall+henceforward+be+called%22|location=Boston|year=1889|page=298|accessdate=December 24, 2008}}</ref> Located at the first convenient [[Charles River]] crossing west of [[Boston]], Newe Towne was one of a number of towns (including Boston, [[Dorchester, Massachusetts|Dorchester]], [[Watertown, Massachusetts|Watertown]], and [[Weymouth, Massachusetts|Weymouth]]) founded by the 700 original [[Puritan]] colonists of the [[Massachusetts Bay Colony]] under governor [[John Winthrop]]. The original village site is in the
heart of today's [[Harvard Square]]. The marketplace where farmers brought in crops from surrounding towns to sell survives today as the small park at the corner of John F. Kennedy (J.F.K.) and Winthrop Streets, then at the edge of a salt marsh, since filled. The town included a much larger area than the present city, with various outlying parts becoming independent towns over the years: [[Newton, Massachusetts|Newton (originally Cambridge Village, then Newtown)]] in 1688,<ref>{{cite book |last= Ritter |first= Priscilla R. |coauthors= Thelma Fleishman |title= Newton, Massachusetts 1679–1779: A Biographical Directory |year= 1982 |publisher= New England Historic Genealogical Society }}</ref> [[Lexington, Massachusetts|Lexington (Cambridge Farms)]] in 1712, and both [[Arlington, Massachusetts|West Cambridge (originally Menotomy)]] and [[Brighton, Massachusetts|Brighton (Little Cambridge)]] in 1807.<ref>{{cite web |url=http://www.brightonbot.com/history.php |title=A Short History of
Allston-Brighton |first=Marchione |last=William P. |author= |authorlink= |coauthors= |date= |month= |year=2011 |work=Brighton-Allston Historical Society |publisher=Brighton Board of Trade |location= |page= |pages= |at= |language= |trans_title= |arxiv= |asin= |bibcode= |doi= |doibroken= |isbn= |issn= |jfm= |jstor= |lccn= |mr= |oclc= |ol= |osti= |pmc = |pmid= |rfc= |ssrn= |zbl= |id= |archiveurl= |archivedate= |deadurl= |accessdate=December 21, 2011 |quote= |ref= |separator= |postscript=}}</ref> Part of West Cambridge joined the new town of [[Belmont, Massachusetts|Belmont]] in 1859, and the rest of West Cambridge was renamed Arlington in 1867; Brighton was annexed by Boston in 1874. In the late 19th century, various schemes for annexing Cambridge itself to the City of Boston were pursued and rejected.<ref>{{cite news |title=ANNEXATION AND ITS FRUITS |author=Staff writer |first= |last= |authorlink= |url=http://query.nytimes.com/gst/abstract.html?res=9901E4DC173BEF34BC4D52DFB766838F669F
DE |agency= |newspaper=[[The New York Times]] |publisher= |isbn= |issn= |pmid= |pmd= |bibcode= |doi= |date=January 15, 1874, Wednesday |page= 4 |pages= |accessdate=|archiveurl=http://query.nytimes.com/mem/archive-free/pdf?res=9901E4DC173BEF34BC4D52DFB766838F669FDE |archivedate=January 15, 1874 |ref= }}</ref><ref>{{cite news |title=BOSTON'S ANNEXATION SCHEMES.; PROPOSAL TO ABSORB CAMBRIDGE AND OTHER NEAR-BY TOWNS |author=Staff writer |first= |last= |authorlink= |url=http://query.nytimes.com/gst/abstract.html?res=9C05E1DC1F39E233A25754C2A9659C94639ED7CF |agency= |newspaper=[[The New York Times]] |publisher= |isbn= |issn= |pmid= |pmd= |bibcode= |doi= |date=March 26, 1892, Wednesday |page= 11 |pages= |accessdate=August 21, 2010|archiveurl=http://query.nytimes.com/mem/archive-free/pdf?res=9C05E1DC1F39E233A25754C2A9659C94639ED7CF |archivedate=March 27, 1892 |ref= }}</ref> In 1636, [[Harvard College]] was founded by the colony to train [[minister (religion)|ministers]] and the new town was
chosen for its site by [[Thomas Dudley]]. By 1638, the name "Newe Towne" had "compacted by usage into 'Newtowne'."<ref name=drake /> In May 1638<ref>{{cite book|title=The Cambridge of Eighteen Hundred and Ninety-six|editor=Arthur Gilman, ed.|publisher=Committee on the Memorial Volume|location=Cambridge|year=1896|page=8}}</ref><ref>{{cite web|author=Harvard News Office |url=http://news.harvard.edu/gazette/2002/05.02/02-history.html |title=''Harvard Gazette'' historical calendar giving May 12, 1638 as date of name change; certain other sources say May 2, 1638 or late 1637 |publisher=News.harvard.edu |date=2002-05-02 |accessdate=2012-04-28}}</ref> the name was changed to '''Cambridge''' in honor of the [[University of Cambridge|university]] in [[Cambridge, England]].<ref>{{cite book |last= Hannah Winthrop Chapter, D.A.R. |title= Historic Guide to Cambridge |edition= Second |year= 1907 |publisher= Hannah Winthrop Chapter, D.A.R. |location= Cambridge, Mass. |pages= 20–21 |quot
e= On October 15, 1637, the Great and General Court passed a vote that: "The college is ordered to bee at Newetowne." In this same year the name of Newetowne was changed to Cambridge, ("It is ordered that Newetowne shall henceforward be called Cambridge") in honor of the university in Cambridge, England, where many of the early settlers were educated. }}</ref> The first president ([[Henry Dunster]]), the first benefactor ([[John Harvard (clergyman)|John Harvard]]), and the first schoolmaster ([[Nathaniel Eaton]]) of Harvard were all Cambridge University alumni, as was the then ruling (and first) governor of the [[Massachusetts Bay Colony]], John Winthrop. In 1629, Winthrop had led the signing of the founding document of the city of Boston, which was known as the [[Cambridge Agreement]], after the university.<ref>{{cite web|url=http://www.winthropsociety.org/doc_cambr.php|publisher=The Winthrop Society|title=Descendants of the Great Migration|accessdate=September 8, 2008}}</ref>
It was Governor Thomas Dudley who, in 1650, signed the charter creating the corporation which still governs Harvard College.<ref>{{cite web|url=http://hul.harvard.edu/huarc/charter.html |title=Harvard Charter of 1650, Harvard University Archives, Harvard University, harvard.edu |publisher=Hul.harvard.edu |date= |accessdate=2012-04-28}}</ref><ref>{{cite book |last1= |first1= |authorlink1= |editor1-first= |editor1-last= |editor1-link= |others= |title=Constitution of the Commonwealth of Massachusetts|url=http://www.mass.gov/legis/const.htm |accessdate=December 13, 2009 |edition= |series= |volume= |date=September 1, 1779 |publisher=The General Court of Massachusetts |location= |isbn= |oclc= |doi= |page= |pages=|chapter=Chapter V: The University at Cambridge, and encouragement of literature, etc. |chapterurl= |ref= |bibcode= }}</ref> [[Image:Washington taking command of the American Army at Cambridge, 1775 - NARA - 532874.tif|thumb|right|George Washington in Cambridge, 1775]] Cambridge
grew slowly as an agricultural village eight miles (13 km) by road from Boston, the capital of the colony. By the [[American Revolution]], most residents lived near the [[Cambridge Common|Common]] and Harvard College, with farms and estates comprising most of the town. Most of the inhabitants were descendants of the original Puritan colonists, but there was also a small elite of [[Anglicans|Anglican]] "worthies" who were not involved in village life, who made their livings from estates, investments, and trade, and lived in mansions along "the Road to Watertown" (today's [[Brattle Street (Cambridge, Massachusetts)|Brattle Street]], still known as [[Tory Row]]). In 1775, [[George Washington]] came up from [[Virginia]] to take command of fledgling volunteer American soldiers camped on the [[Cambridge Common]]—today called the birthplace of the [[U.S. Army]]. (The name of today's nearby Sheraton Commander Hotel refers to that event.) Most of the Tory estates were confiscated afte
r the Revolution. On January 24, 1776, [[Henry Knox]] arrived with artillery captured from [[Fort Ticonderoga]], which enabled Washington to drive the British army out of Boston. [[File:Cambridge 1873 WardMap.jpg|thumb|300px|left|A map of Cambridge from 1873]] Between 1790 and 1840, Cambridge began to grow rapidly, with the construction of the [[West Boston Bridge]] in 1792, that connected Cambridge directly to Boston, making it no longer necessary to travel eight miles (13 km) through the [[Boston Neck]], [[Roxbury, Massachusetts|Roxbury]], and [[Brookline, Massachusetts|Brookline]] to cross the [[Charles River]]. A second bridge, the Canal Bridge, opened in 1809 alongside the new [[Middlesex Canal]]. The new bridges and roads made what were formerly estates and [[marsh]]land into prime industrial and residential districts. In the mid-19th century, Cambridge was the center of a literary revolution when it gave the country a new identity through poetry and literature. Cambridge
was home to the famous Fireside Poets—so called because their poems would often be read aloud by families in front of their evening fires. In their day, the [[Fireside Poets]]—[[Henry Wadsworth Longfellow]], [[James Russell Lowell]], and [[Oliver Wendell Holmes, Sr.|Oliver Wendell Holmes]]—were as popular and influential as rock stars are today.{{Citation needed|date=November 2009}} Soon after, [[Toll road|turnpikes]] were built: the [[Cambridge and Concord Turnpike]] (today's Broadway and Concord Ave.), the [[Middlesex Turnpike (Massachusetts)|Middlesex Turnpike]] (Hampshire St. and [[Massachusetts Avenue (Boston)|Massachusetts Ave.]] northwest of [[Porter Square]]), and what are today's Cambridge, Main, and Harvard Streets were roads to connect various areas of Cambridge to the bridges. In addition, railroads crisscrossed the town during the same era, leading to the development of Porter Square as well as the creation of neighboring town [[Somerville, Massachusetts|Somervil
le]] from the formerly rural parts of [[Charlestown, Massachusetts|Charlestown]]. [[File:Middlesex Canal (Massachusetts) map, 1852.jpg|thumb|1852 Map of Boston area showing Cambridge and rail lines.]] Cambridge was incorporated as a city in 1846. This was despite noticeable tensions between East Cambridge, Cambridgeport, and Old Cambridge that stemmed from differences in in each area's culture, sources of income, and the national origins of the residents.<ref>Cambridge Considered: A Very Brief History of Cambridge, 1800-1900, Part I. http://cambridgeconsidered.blogspot.com/2011/01/very-brief-history-of-cambridge-1800.html</ref> The city's commercial center began to shift from Harvard Square to Central Square, which became the downtown of the city around this time. Between 1850 and 1900, Cambridge took on much of its present character—[[streetcar suburb]]an development along the turnpikes, with working-class and industrial neighborhoods focused on East Cambridge, comfortable middle
-class housing being built on old estates in Cambridgeport and Mid-Cambridge, and upper-class enclaves near Harvard University and on the minor hills of the city. The coming of the railroad to North Cambridge and Northwest Cambridge then led to three major changes in the city: the development of massive brickyards and brickworks between Massachusetts Ave., Concord Ave. and [[Alewife Brook]]; the ice-cutting industry launched by [[Frederic Tudor]] on [[Fresh Pond, Cambridge, Massachusetts|Fresh Pond]]; and the carving up of the last estates into residential subdivisions to provide housing to the thousands of immigrants that arrived to work in the new industries. For many years, the city's largest employer was the [[New England Glass Company]], founded in 1818. By the middle of the 19th century it was the largest and most modern glassworks in the world. In 1888, all production was moved, by [[Edward Libbey|Edward Drummond Libbey]], to [[Toledo, Ohio]], where it continues today under t
he name Owens Illinois. Flint glassware with heavy lead content, produced by that company, is prized by antique glass collectors. There is none on public display in Cambridge, but there is a large collection in the [[Toledo Museum of Art]]. Among the largest businesses located in Cambridge was the firm of [[Carter's Ink Company]], whose neon sign long adorned the [[Charles River]] and which was for many years the largest manufacturer of ink in the world. By 1920, Cambridge was one of the main industrial cities of [[New England]], with nearly 120,000 residents. As industry in New England began to decline during the [[Great Depression]] and after World War II, Cambridge lost much of its industrial base. It also began the transition to being an intellectual, rather than an industrial, center. Harvard University had always been important in the city (both as a landowner and as an institution), but it began to play a more dominant role in the city's life and culture. Also, the move of th
e [[Massachusetts Institute of Technology]] from Boston in 1916 ensured Cambridge's status as an intellectual center of the United States. After the 1950s, the city's population began to decline slowly, as families tended to be replaced by single people and young couples. The 1980s brought a wave of high-technology startups, creating software such as [[Visicalc]] and [[Lotus 1-2-3]], and advanced computers, but many of these companies fell into decline with the fall of the minicomputer and [[DOS]]-based systems. However, the city continues to be home to many startups as well as a thriving biotech industry. By the end of the 20th century, Cambridge had one of the most expensive housing markets in the Northeastern United States. While maintaining much diversity in class, race, and age, it became harder and harder for those who grew up in the city to be able to afford to stay. The end of [[rent control]] in 1994 prompted many Cambridge renters to move to housing that was more affordabl
e, in Somerville and other communities. In 2005, a reassessment of residential property values resulted in a disproportionate number of houses owned by non-affluent people jumping in value relative to other houses, with hundreds having their property tax increased by over 100%; this forced many homeowners in Cambridge to move elsewhere.<ref>Cambridge Chronicle, October 6, 13, 20, 27, 2005</ref> As of 2012, Cambridge's mix of amenities and proximity to Boston has kept housing prices relatively stable. ==Geography== [[File:Charles River Cambridge USA.jpg|thumb|upright|A view from Boston of Harvard's [[Weld Boathouse]] and Cambridge in winter. The [[Charles River]] is in the foreground.]] According to the [[United States Census Bureau]], Cambridge has a total area of {{convert|7.1|sqmi|km2}}, of which {{convert|6.4|sqmi|km2}} of it is land and {{convert|0.7|sqmi|km2}} of it (9.82%) is water. ===Adjacent municipalities=== Cambridge is located in eastern Massachusetts, bordered by: *the
city of [[Boston]] to the south (across the [[Charles River]]) and east *the city of [[Somerville, Massachusetts|Somerville]] to the north *the town of [[Arlington, Massachusetts|Arlington]] to the northwest *the town of [[Belmont, Massachusetts|Belmont]] and *the city of [[Watertown, Massachusetts|Watertown]] to the west The border between Cambridge and the neighboring city of [[Somerville, Massachusetts|Somerville]] passes through densely populated neighborhoods which are connected by the [[Red Line (MBTA)|MBTA Red Line]]. Some of the main squares, [[Inman Square|Inman]], [[Porter Square|Porter]], and to a lesser extent, [[Harvard Square|Harvard]], are very close to the city line, as are Somerville's [[Union Square (Somerville)|Union]] and [[Davis Square]]s. ===Neighborhoods=== ====Squares==== [[File:Centralsquarecambridgemass.jpg|thumb|[[Central Square (Cambridge)|Central Square]]]] [[File:Harvard square 2009j.JPG|thumb|[[Harvard Square]]]] [[File:Cambridge MA Inman Square.jpg|th
umb|[[Inman Square]]]] Cambridge has been called the "City of Squares" by some,<ref>{{cite web|author=No Writer Attributed |url=http://www.thecrimson.com/article/1969/9/18/cambridge-a-city-of-squares-pcambridge/ |title="Cambridge: A City of Squares" Harvard Crimson, Sept. 18, 1969 |publisher=Thecrimson.com |date=1969-09-18 |accessdate=2012-04-28}}</ref><ref>{{cite web|url=http://www.travelwritersmagazine.com/RonBernthal/Cambridge.html |title=Cambridge Journal: Massachusetts City No Longer in Boston's Shadow |publisher=Travelwritersmagazine.com |date= |accessdate=2012-04-28}}</ref> as most of its commercial districts are major street intersections known as [[Town square|squares]]. Each of the squares acts as a neighborhood center. These include: * [[Kendall Square]], formed by the junction of Broadway, Main Street, and Third Street, is also known as '''Technology Square''', a name shared with an office and laboratory building cluster in the neighborhood. Just over the [[Longfellow Br
idge]] from Boston, at the eastern end of the [[Massachusetts Institute of Technology|MIT]] campus, it is served by the [[Kendall (MBTA station)|Kendall/MIT]] station on the [[Massachusetts Bay Transportation Authority|MBTA]] [[Red Line (MBTA)|Red Line]] subway. Most of Cambridge's large office towers are located here, giving the area somewhat of an office park feel. A flourishing [[biotech]] industry has grown up around this area. The "One Kendall Square" complex is nearby, but—confusingly—not actually in Kendall Square. Also, the "Cambridge Center" office complex is located here, and not at the actual center of Cambridge. * [[Central Square (Cambridge)|Central Square]], formed by the junction of Massachusetts Avenue, Prospect Street, and Western Avenue, is well known for its wide variety of ethnic restaurants. As recently as the late 1990s it was rather run-down; it underwent a controversial [[gentrification]] in recent years (in conjunction with the development of the nearby
[[University Park at MIT]]), and continues to grow more expensive. It is served by the [[Central (MBTA station)|Central Station]] stop on the MBTA Red Line subway. '''Lafayette Square''', formed by the junction of Massachusetts Avenue, Columbia Street, Sidney Street, and Main Street, is considered part of the Central Square area. [[Cambridgeport]] is south of Central Square along Magazine Street and Brookline Street. * [[Harvard Square]], formed by the junction of Massachusetts Avenue, Brattle Street, and JFK Street. This is the primary site of [[Harvard University]], and is a major Cambridge shopping area. It is served by a [[Harvard (MBTA station)|Red Line station]]. Harvard Square was originally the northwestern terminus of the Red Line and a major transfer point to streetcars that also operated in a short [[Harvard Bus Tunnel|tunnel]]—which is still a major bus terminal, although the area under the Square was reconfigured dramatically in the 1980s when the Red Line was extende
d. The Harvard Square area includes '''Brattle Square''' and '''Eliot Square'''. A short distance away from the square lies the [[Cambridge Common]], while the neighborhood north of Harvard and east of Massachusetts Avenue is known as Agassiz in honor of the famed scientist [[Louis Agassiz]]. * [[Porter Square]], about a mile north on Massachusetts Avenue from Harvard Square, is formed by the junction of Massachusetts and Somerville Avenues, and includes part of the city of [[Somerville, Massachusetts|Somerville]]. It is served by the [[Porter (MBTA station)|Porter Square Station]], a complex housing a [[Red Line (MBTA)|Red Line]] stop and a [[Fitchburg Line]] [[MBTA commuter rail|commuter rail]] stop. [[Lesley University]]'s University Hall and Porter campus are located at Porter Square. * [[Inman Square]], at the junction of Cambridge and Hampshire streets in Mid-Cambridge. Inman Square is home to many diverse restaurants, bars, music venues and boutiques. The funky street scene s
till holds some urban flair, but was dressed up recently with Victorian streetlights, benches and bus stops. A new community park was installed and is a favorite place to enjoy some takeout food from the nearby restaurants and ice cream parlor. * [[Lechmere Square]], at the junction of Cambridge and First streets, adjacent to the CambridgeSide Galleria shopping mall. Perhaps best known as the northern terminus of the [[Massachusetts Bay Transportation Authority|MBTA]] [[Green Line (MBTA)|Green Line]] subway, at [[Lechmere (MBTA station)|Lechmere Station]]. ====Other neighborhoods==== The residential neighborhoods ([http://www.cambridgema.gov/CPD/publications/neighborhoods.cfm map]) in Cambridge border, but are not defined by the squares. These include: * [[East Cambridge, Massachusetts|East Cambridge]] (Area 1) is bordered on the north by the [[Somerville, Massachusetts|Somerville]] border, on the east by the Charles River, on the south by Broadway and Main Street, and on the west b
y the [[Grand Junction Railroad]] tracks. It includes the [[NorthPoint (Cambridge, Massachusetts)|NorthPoint]] development. * [[Massachusetts Institute of Technology|MIT]] Campus ([[MIT Campus (Area 2), Cambridge|Area 2]]) is bordered on the north by Broadway, on the south and east by the Charles River, and on the west by the Grand Junction Railroad tracks. * [[Wellington-Harrington]] (Area 3) is bordered on the north by the [[Somerville, Massachusetts|Somerville]] border, on the south and west by Hampshire Street, and on the east by the Grand Junction Railroad tracks. Referred to as "Mid-Block".{{clarify|What is? By whom? A full sentence would help.|date=September 2011}} * [[Area 4, Cambridge|Area 4]] is bordered on the north by Hampshire Street, on the south by Massachusetts Avenue, on the west by Prospect Street, and on the east by the Grand Junction Railroad tracks. Residents of Area 4 often refer to their neighborhood simply as "The Port", and refer to the area of Cambridgeport
and Riverside as "The Coast". * [[Cambridgeport]] (Area 5) is bordered on the north by Massachusetts Avenue, on the south by the Charles River, on the west by River Street, and on the east by the Grand Junction Railroad tracks. * [[Mid-Cambridge]] (Area 6) is bordered on the north by Kirkland and Hampshire Streets and the [[Somerville, Massachusetts|Somerville]] border, on the south by Massachusetts Avenue, on the west by Peabody Street, and on the east by Prospect Street. * [[Riverside, Cambridge|Riverside]] (Area 7), an area sometimes referred to as "The Coast," is bordered on the north by Massachusetts Avenue, on the south by the Charles River, on the west by JFK Street, and on the east by River Street. * [[Agassiz, Cambridge, Massachusetts|Agassiz (Harvard North)]] (Area 8) is bordered on the north by the [[Somerville, Massachusetts|Somerville]] border, on the south and east by Kirkland Street, and on the west by Massachusetts Avenue. * [[Peabody, Cambridge, Massachusetts|Peabo
dy]] (Area 9) is bordered on the north by railroad tracks, on the south by Concord Avenue, on the west by railroad tracks, and on the east by Massachusetts Avenue. The Avon Hill sub-neighborhood consists of the higher elevations bounded by Upland Road, Raymond Street, Linnaean Street and Massachusetts Avenue. * Brattle area/[[West Cambridge (neighborhood)|West Cambridge]] (Area 10) is bordered on the north by Concord Avenue and Garden Street, on the south by the Charles River and the [[Watertown, Massachusetts|Watertown]] border, on the west by Fresh Pond and the Collins Branch Library, and on the east by JFK Street. It includes the sub-neighborhoods of Brattle Street (formerly known as [[Tory Row]]) and Huron Village. * [[North Cambridge, Massachusetts|North Cambridge]] (Area 11) is bordered on the north by the [[Arlington, Massachusetts|Arlington]] and [[Somerville, Massachusetts|Somerville]] borders, on the south by railroad tracks, on the west by the [[Belmont, Massachusetts|Bel
mont]] border, and on the east by the [[Somerville, Massachusetts|Somerville]] border. * [[Cambridge Highlands]] (Area 12) is bordered on the north and east by railroad tracks, on the south by Fresh Pond, and on the west by the [[Belmont, Massachusetts|Belmont]] border. * [[Strawberry Hill, Cambridge|Strawberry Hill]] (Area 13) is bordered on the north by Fresh Pond, on the south by the [[Watertown, Massachusetts|Watertown]] border, on the west by the [[Belmont, Massachusetts|Belmont]] border, and on the east by railroad tracks. ===Parks and outdoors=== [[File:Alewife Brook Reservation.jpg|thumb|Alewife Brook Reservation]] Consisting largely of densely built residential space, Cambridge lacks significant tracts of public parkland. This is partly compensated for, however, by the presence of easily accessible open space on the university campuses, including [[Harvard Yard]] and MIT's Great Lawn, as well as the considerable open space of [[Mount Auburn Cemetery]]. At the western edge o
f Cambridge, the cemetery is well known as the first garden cemetery, for its distinguished inhabitants, for its superb landscaping (the oldest planned landscape in the country), and as a first-rate [[arboretum]]. Although known as a Cambridge landmark, much of the cemetery lies within the bounds of Watertown.<ref>http://www2.cambridgema.gov/CityOfCambridge_Content/documents/CambridgeStreetMap18x24_032007.pdf</ref> It is also a significant [[Important Bird Area]] (IBA) in the Greater Boston area. Public parkland includes the esplanade along the Charles River, which mirrors its [[Charles River Esplanade|Boston counterpart]], [[Cambridge Common]], a busy and historic public park immediately adjacent to the Harvard campus, and the [[Alewife Brook Reservation]] and [[Fresh Pond, Cambridge, Massachusetts|Fresh Pond]] in the western part of the city. ==Demographics== {{Historical populations | type=USA | align=right | 1790|2115 | 1800|2453 | 1810|2323 | 1820|3295 | 1830|6072 | 1840|8409 |
1850|15215 | 1860|26060 | 1870|39634 | 1880|52669 | 1890|70028 | 1900|91886 | 1910|104839 | 1920|109694 | 1930|113643 | 1940|110879 | 1950|120740 | 1960|107716 | 1970|100361 | 1980|95322 | 1990|95802 | 2000|101355 | 2010|105162 | footnote= {{Historical populations/Massachusetts municipalities references}}<ref name="1950_Census_Urban_populations_since_1790">{{cite journal | title=1950 Census of Population | volume=1: Number of Inhabitants | at=Section 6, Pages 21-7 through 21-09, Massachusetts Table 4. Population of Urban Places of 10,000 or more from Earliest Census to 1920 | publisher=Bureau of the Census | accessdate=July 12, 2011 | year=1952 | url=http://www2.census.gov/prod2/decennial/documents/23761117v1ch06.pdf}}</ref> }} As of the census{{GR|2}} of 2010, there were 105,162 people, 44,032 households, and 17,420 families residing in the city. The population density was 16,422.08 people per square mile (6,341.98/km²), making Cambridge the fifth most densely populated city in t
he US<ref name=CountyCityDataBook>County and City Data Book: 2000. Washington, DC: US Department of Commerce, Bureau of the Census. Table C-1.</ref> and the second most densely populated city in [[Massachusetts]] behind neighboring [[Somerville, Massachusetts|Somerville]].<ref>[http://www.boston.com/realestate/news/articles/2008/07/13/highest_population_density/ Highest Population Density, The Boston Globe]</ref> There were 47,291 housing units at an average density of 7,354.7 per square mile (2,840.3/km²). The racial makeup of the city was 66.60% [[White (U.S. Census)|White]], 11.70% [[Black (people)|Black]] or [[Race (United States Census)|African American]], 0.20% [[Native American (U.S. Census)|Native American]], 15.10% [[Asian (U.S. Census)|Asian]], 0.01% [[Pacific Islander (U.S. Census)|Pacific Islander]], 2.10% from [[Race (United States Census)|other races]], and 4.30% from two or more races. 7.60% of the population were [[Hispanics in the United States|Hispanic]] or [[Lati
no (U.S. Census)|Latino]] of any race. [[Non-Hispanic Whites]] were 62.1% of the population in 2010,<ref>{{cite web |url=http://quickfacts.census.gov/qfd/states/25/2511000.html |title=Cambridge (city), Massachusetts |work=State & County QuickFacts |publisher=U.S. Census Bureau}}</ref> down from 89.7% in 1970.<ref>{{cite web|title=Massachusetts - Race and Hispanic Origin for Selected Cities and Other Places: Earliest Census to 1990|publisher=U.S. Census Bureau|url=http://www.census.gov/population/www/documentation/twps0076/twps0076.html}}</ref> This rather closely parallels the average [[racial demographics of the United States]] as a whole, although Cambridge has significantly more Asians than the average, and fewer Hispanics and Caucasians. 11.0% were of [[irish people|Irish]], 7.2% English, 6.9% [[italians|Italian]], 5.5% [[West Indian]] and 5.3% [[germans|German]] ancestry according to [[Census 2000]]. 69.4% spoke English, 6.9% Spanish, 3.2% [[Standard Mandarin|Chinese]] or [[Sta
ndard Mandarin|Mandarin]], 3.0% [[portuguese language|Portuguese]], 2.9% [[French-based creole languages|French Creole]], 2.3% French, 1.5% [[korean language|Korean]], and 1.0% [[italian language|Italian]] as their first language. There were 44,032 households out of which 16.9% had children under the age of 18 living with them, 28.9% were married couples living together, 8.4% had a female householder with no husband present, and 60.4% were non-families. 40.7% of all households were made up of individuals and 9.6% had someone living alone who was 65 years of age or older. The average household size was 2.00 and the average family size was 2.76. In the city the population was spread out with 13.3% under the age of 18, 21.2% from 18 to 24, 38.6% from 25 to 44, 17.8% from 45 to 64, and 9.2% who were 65 years of age or older. The median age was 30.5 years. For every 100 females, there were 96.1 males. For every 100 females age 18 and over, there were 94.7 males. The median income for a h
ousehold in the city was $47,979, and the median income for a family was $59,423 (these figures had risen to $58,457 and $79,533 respectively {{as of|2007|alt=as of a 2007 estimate}}<ref>{{cite web|url=http://factfinder.census.gov/servlet/ACSSAFFFacts?_event=Search&geo_id=16000US2418750&_geoContext=01000US%7C04000US24%7C16000US2418750&_street=&_county=cambridge&_cityTown=cambridge&_state=04000US25&_zip=&_lang=en&_sse=on&ActiveGeoDiv=geoSelect&_useEV=&pctxt=fph&pgsl=160&_submenuId=factsheet_1&ds_name=ACS_2007_3YR_SAFF&_ci_nbr=null&qr_name=null®=null%3Anull&_keyword=&_industry= |title=U.S. Census, 2000 |publisher=Factfinder.census.gov |date= |accessdate=2012-04-28}}</ref>). Males had a median income of $43,825 versus $38,489 for females. The per capita income for the city was $31,156. About 8.7% of families and 12.9% of the population were below the poverty line, including 15.1% of those under age 18 and 12.9% of those age 65 or over. Cambridge was ranked as one of the most liberal
cities in America.<ref>{{cite web|author=Aug 16, 2005 12:00 AM |url=http://www.govpro.com/News/Article/31439/ |title=Study Ranks America’s Most Liberal and Conservative Cities |publisher=Govpro.com |date=2005-08-16 |accessdate=2012-04-28}}</ref> Locals living in and near the city jokingly refer to it as "The People's Republic of Cambridge."<ref>[http://www.universalhub.com/glossary/peoples_republic_the.html Wicked Good Guide to Boston English] Accessed February 2, 2009</ref> For 2012, the residential property tax rate in Cambridge is $8.48 per $1,000.<ref>{{cite web|url=http://www.cambridgema.gov/finance/propertytaxinformation/fy12propertytaxinformation.aspx |title=FY12 Property Tax Information - City of Cambridge, Massachusetts |publisher=Cambridgema.gov |date= |accessdate=2012-04-28}}</ref> Cambridge enjoys the highest possible [[bond credit rating]], AAA, with all three Wall Street rating agencies.<ref>http://www.cambridgema.gov/CityOfCambridge_Content/documents/Understanding_
Your_Taxes_2007.pdf</ref> Cambridge is noted for its diverse population, both racially and economically. Residents, known as ''Cantabrigians'', include affluent [[MIT]] and Harvard professors. The first legal applications in America for same-sex marriage licenses were issued at Cambridge's City Hall.<ref>{{cite web|url=http://www.boston.com/news/local/articles/2004/05/17/free_to_marry/ |title=Free to Marry |work=[[The Boston Globe]] |date=2004-05-17 |accessdate=2012-07-18}}</ref> Cambridge is also the birthplace of [[Thailand|Thai]] king [[Bhumibol Adulyadej|Bhumibol Adulyadej (Rama IX)]], who is the world's longest reigning monarch at age 82 (2010), as well as the longest reigning monarch in Thai history. He is also the first king of a foreign country to be born in the United States. ==Government== ===Federal and state representation=== {| class=wikitable ! colspan = 6 | Voter registration and party enrollment {{as of|lc=y|df=US|2008|10|15}}<ref>{{cite web|title = 2008 State Party
Election Party Enrollment Statistics | publisher = Massachusetts Elections Division | format = PDF | accessdate = July 7, 2010 | url = http://www.sec.state.ma.us/ele/elepdf/st_county_town_enroll_breakdown_08.pdf}}</ref> |- ! colspan = 2 | Party ! Number of voters ! Percentage {{American politics/party colors/Democratic/row}} | [[Democratic Party (United States)|Democratic]] | style="text-align:center;"| 37,822 | style="text-align:center;"| 58.43% {{American politics/party colors/Republican/row}} | [[Republican Party (United States)|Republican]] | style="text-align:center;"| 3,280 | style="text-align:center;"| 5.07% {{American politics/party colors/Independent/row}} | Unaffiliated | style="text-align:center;"| 22,935 | style="text-align:center;"| 35.43% {{American politics/party colors/Libertarian/row}} | Minor Parties | style="text-align:center;"| 690 | style="text-align:center;"| 1.07% |- ! colspan = 2 | Total ! style="text-align:center;"| 64,727 ! style="text-align:center;"| 100%
|} Cambridge is part of [[Massachusetts's 8th congressional district]], represented by Democrat [[Mike Capuano]], elected in 1998. The state's senior member of the [[United States Senate]] is Democrat [[John Kerry]], elected in 1984. The state's junior member is Republican [[Scott Brown]], [[United States Senate special election in Massachusetts, 2010|elected in 2010]] to fill the vacancy caused by the death of long-time Democratic Senator [[Ted Kennedy]]. The Governor of Massachusetts is Democrat [[Deval Patrick]], elected in 2006 and re-elected in 2010. On the state level, Cambridge is represented in six districts in the [[Massachusetts House of Representatives]]: the 24th Middlesex (which includes parts of Belmont and Arlington), the 25th and 26th Middlesex (the latter which includes a portion of Somerville), the 29th Middlesex (which includes a small part of Watertown), and the Eighth and Ninth Suffolk (both including parts of the City of Boston). The city is represented in the
[[Massachusetts Senate]] as a part of the "First Suffolk and Middlesex" district (this contains parts of Boston, Revere and Winthrop each in Suffolk County); the "Middlesex, Suffolk and Essex" district, which includes Everett and Somerville, with Boston, Chelsea, and Revere of Suffolk, and Saugus in Essex; and the "Second Suffolk and Middlesex" district, containing parts of the City of Boston in Suffolk county, and Cambridge, Belmont and Watertown in Middlesex county.<ref>{{cite web|url=http://www.malegislature.gov/ |title=Index of Legislative Representation by City and Town, from |publisher=Mass.gov |date= |accessdate=2012-04-28}}</ref> In addition to the [[Cambridge Police Department (Massachusetts)|Cambridge Police Department]], the city is patrolled by the Fifth (Brighton) Barracks of Troop H of the [[Massachusetts State Police]].<ref>[http://www.mass.gov/?pageID=eopsterminal&L=5&L0=Home&L1=Law+Enforcement+%26+Criminal+Justice&L2=Law+Enforcement&L3=State+Police+Troops&L4=Troop+H
&sid=Eeops&b=terminalcontent&f=msp_divisions_field_services_troops_troop_h_msp_field_troop_h_station_h5&csid=Eeops Station H-5, SP Brighton]{{dead link|date=April 2012}}</ref> Due, however, to close proximity, the city also practices functional cooperation with the Fourth (Boston) Barracks of Troop H, as well.<ref>[http://www.mass.gov/?pageID=eopsterminal&L=5&L0=Home&L1=Law+Enforcement+%26+Criminal+Justice&L2=Law+Enforcement&L3=State+Police+Troops&L4=Troop+H&sid=Eeops&b=terminalcontent&f=msp_divisions_field_services_troops_troop_h_msp_field_troop_h_station_h4&csid=Eeops Station H-4, SP Boston]{{dead link|date=April 2012}}</ref> ===City government=== [[File:CambridgeMACityHall1.jpg|thumb|right|[[Cambridge, Massachusetts City Hall|Cambridge City Hall]] in the 1980s]] Cambridge has a city government led by a [[List of mayors of Cambridge, Massachusetts|Mayor]] and nine-member City Council. There is also a six-member School Committee which functions alongside the Superintendent of publi
c schools. The councilors and school committee members are elected every two years using the [[single transferable vote]] (STV) system.<ref>{{cite web|url=http://www.cambridgema.gov/election/Proportional_Representation.cfm |title=Proportional Representation Voting in Cambridge |publisher=Cambridgema.gov |date= |accessdate=2012-04-28}}</ref> Once a laborious process that took several days to complete by hand, ballot sorting and calculations to determine the outcome of elections are now quickly performed by computer, after the ballots have been [[Optical scan voting system|optically scanned]]. The mayor is elected by the city councilors from amongst themselves, and serves as the chair of City Council meetings. The mayor also sits on the School Committee. However, the Mayor is not the Chief Executive of the City. Rather, the City Manager, who is appointed by the City Council, serves in that capacity. Under the City's Plan E form of government the city council does not have the power to
appoint or remove city officials who are under direction of the city manager. The city council and its individual members are also forbidden from giving orders to any subordinate of the city manager.<ref>http://www.cambridgema.gov/CityOfCambridge_Content/documents/planE.pdf</ref> [[Robert W. Healy]] is the City Manager; he has served in the position since 1981. In recent history, the media has highlighted the salary of the City Manager as being one of the highest in the State of Massachusetts.<ref>{{cite news |title=Cambridge city manager's salary almost as much as Obama's pay |url=http://www.wickedlocal.com/cambridge/features/x1837730973/Cambridge-city-managers-salary-almost-as-much-as-Obamas |agency= |newspaper=Wicked Local: Cambridge |publisher= |date=August 11, 2011 |accessdate=December 30, 2011 |quote= |archiveurl= |archivedate= |deadurl= |ref=}}</ref> The city council consists of:<ref>{{cite web|url=http://www.cambridgema.gov/ccouncil/citycouncilmembers.aspx |title=City of Ca
mbridge – City Council Members |publisher=Cambridgema.gov |date= |accessdate=2012-04-28}}</ref>{{Refbegin|3}} *[[Leland Cheung]] (Jan. 2010–present) *Henrietta Davis (Jan. 1996–present)* *Marjorie C. Decker (Jan. 2000–present)<ref>{{cite web |url= http://www.wickedlocal.com/cambridge/news/x738245499/Marjorie-Decker-announces-she-will-run-for-Alice-Wolfs-Cambridge-State-Representative-seat |title= Marjorie Decker announces she will run for Alice Wolf's Cambridge State Representative seat |date= 22 March 2012 |work= Wicked Local Cambridge |publisher= GateHouse Media, Inc. |accessdate= 4 April 2012 }}</ref> *Craig A. Kelley (Jan. 2006–present) *David Maher (Jan. 2000-Jan. 2006, Sept. 2007–present<ref>{{cite web|author=By ewelin, on September 5th, 2007 |url=http://www.cambridgehighlands.com/2007/09/david-p-maher-elected-to-fill-michael-sullivans-vacated-city-council-seat |title=David P. Maher Elected to fill Michael Sullivan’s Vacated City Council Seat • Cambridge Highla
nds Neighborhood Association |publisher=Cambridgehighlands.com |date=2007-09-05 |accessdate=2012-04-28}}</ref>)** *[[Kenneth Reeves]] (Jan. 1990–present)** *[[E. Denise Simmons]] (Jan. 2002–present)** *[[Timothy J. Toomey, Jr.]] (Jan. 1990–present) *Minka vanBeuzekom (Jan. 2012–present){{Refend}} ''* = Current Mayor''<br> ''** = former Mayor'' ===Fire Department=== The city of Cambridge is protected full-time by the 274 professional firefighters of the Cambridge Fire Department. The current Chief of Department is Gerald R. Reardon. The Cambridge Fire Department operates out of eight fire stations, located throughout the city, under the command of two divisions. The CFD also maintains and operates a front-line fire apparatus fleet of eight engines, four ladders, two Non-Transport Paramedic EMS units, a Haz-Mat unit, a Tactical Rescue unit, a Dive Rescue unit, two Marine units, and numerous special, support, and reserve units. John J. Gelinas, Chief of Operations, is in charge
of day to day operation of the department.<ref>{{cite web|url=http://www2.cambridgema.gov/cfd/ |title=City of Cambridge Fire Department |publisher=.cambridgema.gov |date=2005-03-13 |accessdate=2012-06-26}}</ref> The CFD is rated as a Class 1 fire department by the [[Insurance Services Office]] (ISO), and is one of only 32 fire departments so rated, out of 37,000 departments in the United States. The other class 1 departments in New England are in [[Hartford, Connecticut]] and [[Milford, Connecticut]]. Class 1 signifies the highest level of fire protection according to various criteria.<ref>{{cite web|url=http://www2.cambridgema.gov/CFD/Class1FD.cfm |title=Class 1 Fire Department |publisher=.cambridgema.gov |date=1999-07-01 |accessdate=2012-06-26}}</ref> The CFD responds to approximately 15,000 emergency calls annually. {| class=wikitable |- valign=bottom ! Engine Company ! Ladder Company ! Special Unit ! Division ! Address ! Neighborhood |- | Engine 1 || Ladder 1 || || || 491 Broad
way || Harvard Square |- | Engine 2 || Ladder 3 || Squad 2 || || 378 Massachusetts Ave. || Lafayette Square |- | Engine 3 || Ladder 2 || || || 175 Cambridge St. || East Cambridge |- | Engine 4 || || Squad 4 || || 2029 Massachusetts Ave. || Porter Square |- | Engine 5 || || || Division 1 || 1384 Cambridge St. || Inman Square |- | Engine 6 || || || || 176 River St. || Cambridgeport |- | Engine 8 || Ladder 4 || || Division 2 || 113 Garden St. || Taylor Square |- | Engine 9 || || || || 167 Lexington Ave || West Cambridge |- | Maintenance Facility || || || || 100 Smith Pl. || |} ===Water Department=== Cambridge is unusual among cities inside Route 128 in having a non-[[MWRA]] water supply. City water is obtained from [[Hobbs Brook]] (in [[Lincoln, Massachusetts|Lincoln]] and [[Waltham, Massachusetts|Waltham]]), [[Stony Brook (Boston)|Stony Brook]] (Waltham and [[Weston, Massachusetts|Weston]]), and [[Fresh Pond (Cambridge, Massachusetts)|Fresh Pond]] (Cambridge). The city owns over 1200
acres of land in other towns that includes these reservoirs and portions of their watershed.<ref>{{cite web|url=http://www2.cambridgema.gov/CWD/wat_lands.cfm |title=Cambridge Watershed Lands & Facilities |publisher=.cambridgema.gov |date= |accessdate=2012-04-28}}</ref> Water is treated at Fresh Pond, then pumped uphill to an elevation of {{convert|176|ft|m}} [[above sea level]] at the Payson Park Reservoir ([[Belmont, Massachusetts|Belmont]]); From there, the water is redistributed downhill via gravity to individual users in the city.<ref>{{cite web|url=http://www.cambridgema.gov/CityOfCambridge_Content/documents/CWD_March_2010.pdf |title=Water supply system |format=PDF |date= |accessdate=2012-04-28}}</ref><ref>[http://www.cambridgema.gov/CWD/fpfaqs.cfm Is Fresh Pond really used for drinking water?], Cambridge Water Department</ref> ===County government=== Cambridge is a [[county seat]] of [[Middlesex County, Massachusetts]], along with [[Lowell, Massachusetts|Lowell]]. Though the c
ounty government was abolished in 1997, the county still exists as a geographical and political region. The employees of Middlesex County courts, jails, registries, and other county agencies now work directly for the state. At present, the county's registrars of [[Deed]]s and Probate remain in Cambridge; however, the Superior Court and District Attorney have had their base of operations transferred to [[Woburn, Massachusetts|Woburn]]. Third District court has shifted operations to [[Medford, Massachusetts|Medford]], and the Sheriff's office for the county is still awaiting a near-term relocation.<ref>{{cite news | url=http://www.boston.com/news/local/massachusetts/articles/2008/02/14/court_move_a_hassle_for_commuters/ |title=Court move a hassle for commuters |accessdate=July 25, 2009 |first=Eric |last=Moskowitz |authorlink= |coauthors= |date=February 14, 2008 |work=[[Boston Globe|The Boston Globe]] |pages= |archiveurl= |archivedate= |quote=In a little more than a month, Middlesex Su
perior Court will open in Woburn after nearly four decades at the Edward J. Sullivan Courthouse in Cambridge. With it, the court will bring the roughly 500 people who pass through its doors each day – the clerical staff, lawyers, judges, jurors, plaintiffs, defendants, and others who use or work in the system.}}</ref><ref>{{cite news | url=http://www.wickedlocal.com/cambridge/homepage/x135741754/Cambridges-Middlesex-Jail-courts-may-be-shuttered-for-good |title=Cambridge's Middlesex Jail, courts may be shuttered for good |accessdate=July 25, 2009 |first=Charlie |last=Breitrose |authorlink= |coauthors= |date=July 7, 2009 |work=Wicked Local News: Cambridge |pages= |archiveurl= |archivedate= |quote=The courts moved out of the building to allow workers to remove asbestos. Superior Court moved to Woburn in March 2008, and in February, the Third District Court moved to Medford.}}</ref> ==Education== [[File:MIT Main Campus Aerial.jpg|thumb|Aerial view of part of [[MIT]]'s main campus]] [[
File:Dunster House.jpg|thumb|[[Dunster House]], Harvard]] ===Higher education=== Cambridge is perhaps best known as an academic and intellectual center, owing to its colleges and universities, which include: *[[Cambridge College]] *[[Cambridge School of Culinary Arts]] *[[Episcopal Divinity School]] *[[Harvard University]] *[[Hult International Business School]] *[[Lesley University]] *[[Longy School of Music]] *[[Massachusetts Institute of Technology]] *[[Le Cordon Bleu College of Culinary Arts in Boston]] [[Nobel laureates by university affiliation|At least 129]] of the world's total 780 [[Nobel Prize]] winners have been, at some point in their careers, affiliated with universities in Cambridge. The [[American Academy of Arts and Sciences]] is also based in Cambridge. ===Primary and secondary public education=== The Cambridge Public School District encompasses 12 elementary schools that follow a variety of different educational systems and philosophies. All but one of the elementa
ry schools extend up to the [[middle school]] grades as well. The 12 elementary schools are: *[[Amigos School]] *Baldwin School *Cambridgeport School *Fletcher-Maynard Academy *Graham and Parks Alternative School *Haggerty School *Kennedy-Longfellow School *King Open School *Martin Luther King, Jr. School *Morse School (a [[Core Knowledge Foundation|Core Knowledge]] school) *Peabody School *Tobin School (a [[Montessori school]]) There are three public high schools serving Cambridge students, including the [[Cambridge Rindge and Latin School]].<ref>{{cite web|url=http://www.cpsd.us/Web/PubInfo/SchoolsAtAGlance06-07.pdf|title=Cambridge Public Schools at a Glance|format=PDF}}{{dead link|date=June 2012}}</ref> and Community Charter School of Cambridge (www.ccscambridge.org) In 2003, the CRLS, also known as Rindge, came close to losing its educational accreditation when it was placed on probation by the [[New England Association of Schools and Colleges]].<ref name="Crimson MCAS">{{cite w
eb|url=http://www.thecrimson.com/article.aspx?ref=512061|title=School Fights Achievement Gap|publisher=The Harvard Crimson|accessdate=May 14, 2009}}</ref> The school has improved under Principal Chris Saheed, graduation rates hover around 98%, and 70% of students gain college admission. Community Charter School of Cambridge serves 350 students, primarily from Boston and Cambridge, and is a tuition free public charter school with a college preparatory curriculum. All students from the class of 2009 and 2010 gained admission to college. Outside of the main public schools are public charter schools including: [[Benjamin Banneker Charter School]], which serves students in grades K-6,<ref>{{cite web|url=http://www.banneker.org/ |title=The Benjamin Banneker Charter Public School |publisher=Banneker.org |date=2012-03-01 |accessdate=2012-04-28}}</ref> [[Community Charter School of Cambridge]],<ref>{{cite web|url=http://www.ccscambridge.org/ |title=Community Charter School of Cambridge |publ
isher=Ccscambridge.org |date= |accessdate=2012-04-28}}</ref> which is located in Kendall Square and serves students in grades 7–12, and [[Prospect Hill Academy]], a [[charter school]] whose upper school is in [[Central Square (Cambridge)|Central Square]], though it is not a part of the Cambridge Public School District. ===Primary and secondary private education=== [[File:Cambridge Public Library, Cambridge, Massachusetts.JPG|thumb|right|[[Cambridge Public Library]] original building, part of an expanded facility]] There are also many private schools in the city including: <!-- please keep alphabetical --> *[[Boston Archdiocesan Choir School]] (BACS) *[[Buckingham Browne & Nichols]] (BB&N) *[[Cambridge montessori school|Cambridge Montessori School]] (CMS) *Cambridge [[Religious Society of Friends|Friends]] School. Thomas Waring served as founding headmaster of the school. *Fayerweather Street School (FSS)[http://www.fayerweather.org/ ] *[[International School of Boston]] (ISB, form
erly École Bilingue) *[[Matignon High School]] *[[North Cambridge Catholic High School]] (re-branded as Cristo Rey Boston and relocated to Dorchester, MA in 2010) *[[Shady Hill School]] *St. Peter School ==Economy== [[File:Cambridge Skyline.jpg|thumb|Buildings of [[Kendall Square]], center of Cambridge's [[biotech]] economy, seen from the [[Charles River]]]] Manufacturing was an important part of the economy in the late 19th and early 20th century, but educational institutions are the city's biggest employers today. Harvard and [[Massachusetts Institute of Technology|MIT]] together employ about 20,000.<ref name="2008top25">[http://www2.cambridgema.gov/cdd/data/labor/top25/top25_2008.html Top 25 Cambridge Employers: 2008], City of Cambridge</ref> As a cradle of technological innovation, Cambridge was home to technology firms [[Analog Devices]], [[Akamai Technologies|Akamai]], [[BBN Technologies|Bolt, Beranek, and Newman (BBN Technologies)]] (now part of Raytheon), [[General Radio|Ge
neral Radio (later GenRad)]], [[Lotus Development Corporation]] (now part of [[IBM]]), [[Polaroid Corporation|Polaroid]], [[Symbolics]], and [[Thinking Machines]]. In 1996, [[Polaroid Corporation|Polaroid]], [[Arthur D. Little]], and [[Lotus Development Corporation|Lotus]] were top employers with over 1,000 employees in Cambridge, but faded out a few years later. Health care and biotechnology firms such as [[Genzyme]], [[Biogen Idec]], [[Millennium Pharmaceuticals]], [[Sanofi]], [[Pfizer]] and [[Novartis]]<ref>{{cite news |title=Novartis doubles plan for Cambridge |author=Casey Ross and Robert Weisman |first= |last= |authorlink= |authorlink2= |url=http://articles.boston.com/2010-10-27/business/29323650_1_french-drug-maker-astrazeneca-plc-research-operations |agency= |newspaper=[[The Boston Globe]] |publisher= |isbn= |issn= |pmid= |pmd= |bibcode= |doi= |date=October 27, 2010 |page= |pages= |accessdate=April 12, 2011|quote=Already Cambridge’s largest corporate employer, the Swiss fi
rm expects to hire an additional 200 to 300 employees over the next five years, bringing its total workforce in the city to around 2,300. Novartis’s global research operations are headquartered in Cambridge, across Massachusetts Avenue from the site of the new four-acre campus. |archiveurl= |archivedate= |ref=}}</ref> have significant presences in the city. Though headquartered in Switzerland, Novartis continues to expand its operations in Cambridge. Other major biotech and pharmaceutical firms expanding their presence in Cambridge include [[GlaxoSmithKline]], [[AstraZeneca]], [[Shire plc|Shire]], and [[Pfizer]].<ref>{{cite news|title=Novartis Doubles Plan for Cambridge|url=http://www.boston.com/business/healthcare/articles/2010/10/27/novartis_doubles_plan_for_cambridge/|accessdate=23 February 2012 | work=The Boston Globe|first1=Casey|last1=Ross|first2=Robert|last2=Weisman|date=October 27, 2010}}</ref> Most Biotech firms in Cambridge are located around [[Kendall Square]] and [[Eas
t Cambridge, Massachusetts|East Cambridge]], which decades ago were the city's center of manufacturing. A number of biotechnology companies are also located in [[University Park at MIT]], a new development in another former manufacturing area. None of the high technology firms that once dominated the economy was among the 25 largest employers in 2005, but by 2008 high tech companies [[Akamai Technologies|Akamai]] and [[ITA Software]] had grown to be among the largest 25 employers.<ref name="2008top25" /> [[Google]],<ref>{{cite web|url=http://www.google.com/corporate/address.html |title=Google Offices |publisher=Google.com |date= |accessdate=2012-07-18}}</ref> [[IBM Research]], and [[Microsoft Research]] maintain offices in Cambridge. In late January 2012—less than a year after acquiring [[Billerica, Massachusetts|Billerica]]-based analytic database management company, [[Vertica]]—[[Hewlett-Packard]] announced it would also be opening its first offices in Cambridge.<ref>{{cite we
b|last=Huang|first=Gregory|title=Hewlett-Packard Expands to Cambridge via Vertica’s "Big Data" Center|url=http://www.xconomy.com/boston/2012/01/23/hewlett-packard-expands-to-cambridge-via-verticas-big-data-center/?single_page=true}}</ref> Around this same time, e-commerce giants [[Staples Inc.|Staples]]<ref>{{cite web|title=Staples to bring e-commerce office to Cambridge's Kendall Square Read more: Staples to bring e-commerce office to Cambridge's Kendall Square - Cambridge, Massachusetts - Cambridge Chronicle http://www.wickedlocal.com/cambridge/news/x690035936/Staples-to-bring-E-commerce-office-to-Cambridges-Kendall-Square#ixzz1nDY39Who|url=http://www.wickedlocal.com/cambridge/news/x690035936/Staples-to-bring-E-commerce-office-to-Cambridges-Kendall-Square#axzz1kg3no7Zg}}</ref> and [[Amazon.com]]<ref>{{cite web|title=Amazon Seeks Brick-And-Mortar Presence In Boston Area|url=http://www.wbur.org/2011/12/22/amazon-boston}}</ref> said they would be opening research and innovation cen
ters in Kendall Square. Video game developer [[Harmonix Music Systems]] is based in [[Central Square (Cambridge)|Central Square]]. The proximity of Cambridge's universities has also made the city a center for nonprofit groups and think tanks, including the [[National Bureau of Economic Research]], the [[Smithsonian Astrophysical Observatory]], the [[Lincoln Institute of Land Policy]], [[Cultural Survival]], and [[One Laptop per Child]]. In September 2011, an initiative by the City of Cambridge called the "[[Entrepreneur Walk of Fame]]" was launched. It seeks to highlight individuals who have made contributions to innovation in the global business community.<ref>{{cite news |title=Stars of invention |author= |first=Kathleen |last=Pierce |url=http://articles.boston.com/2011-09-16/business/30165912_1_gates-and-jobs-microsoft-granite-stars |agency= |newspaper=The Boston Globe|date=September 16, 2011 |page= |pages= |at= |accessdate=October 1, 2011}}</ref> ===Top employers=== The top ten
employers in the city are:<ref>{{cite web|url=http://cambridgema.gov/citynewsandpublications/news/2012/01/fy11comprehensiveannualfinancialreportnowavailable.aspx |title=City of Cambridge, Massachusetts Comprehensive Annual Financial Report July 1, 2010—June 30, 2011 |publisher=Cambridgema.gov |date=2011-06-30 |accessdate=2012-04-28}}</ref> {| class="wikitable" |- ! # ! Employer ! # of employees |- | 1 |[[Harvard University]] |10,718 |- |2 |[[Massachusetts Institute of Technology]] |7,604 |- |3 |City of Cambridge |2,922 |- |4 |[[Novartis]] Institutes for BioMedical Research |2,095 |- |5 |[[Mount Auburn Hospital]] |1,665 |- |6 |[[Vertex Pharmaceuticals]] |1,600 |- |7 |[[Genzyme]] |1,504 |- |8 |[[Biogen Idec]] |1,350 |- |9 |[[Federal government of the United States|Federal Government]] |1,316 |- |10 |[[Pfizer]] |1,300 |} ==Transportation== {{See also|Boston transportation}} ===Road=== [[File:Harvard Square at Peabody Street and Mass Avenue.jpg|thumb|[[Massachusetts Avenue (Boston)|Ma
ssachusetts Avenue]] in [[Harvard Square]]]] Several major roads lead to Cambridge, including [[Massachusetts State Highway 2|Route 2]], [[Massachusetts State Highway 16|Route 16]] and the [[Massachusetts State Highway 28|McGrath Highway (Route 28)]]. The [[Massachusetts Turnpike]] does not pass through Cambridge, but provides access by an exit in nearby [[Allston, Massachusetts|Allston]]. Both [[U.S. Route 1]] and [[I-93 (MA)]] also provide additional access on the eastern end of Cambridge at Leverett Circle in [[Boston]]. [[Massachusetts State Highway 2A|Route 2A]] runs the length of the city, chiefly along Massachusetts Avenue. The Charles River forms the southern border of Cambridge and is crossed by 11 bridges connecting Cambridge to Boston, including the [[Longfellow Bridge]] and the [[Harvard Bridge]], eight of which are open to motorized road traffic. Cambridge has an irregular street network because many of the roads date from the colonial era. Contrary to popular belief, t
he road system did not evolve from longstanding cow-paths. Roads connected various village settlements with each other and nearby towns, and were shaped by geographic features, most notably streams, hills, and swampy areas. Today, the major "squares" are typically connected by long, mostly straight roads, such as Massachusetts Avenue between [[Harvard Square]] and [[Central Square (Cambridge)|Central Square]], or Hampshire Street between [[Kendall Square]] and [[Inman Square]]. ===Mass transit=== [[File:Central MBTA station.jpg|thumb|[[Central (MBTA)|Central station on the MBTA Red Line]]]] Cambridge is well served by the [[MBTA]], including the [[Porter (MBTA station)|Porter Square stop]] on the regional [[MBTA Commuter Rail|Commuter Rail]], the [[Lechmere (MBTA station)|Lechmere stop]] on the [[Green Line (MBTA)|Green Line]], and five stops on the [[Red Line (MBTA)|Red Line]] ([[Alewife Station (MBTA)|Alewife]], [[Porter (MBTA)|Porter Square]], [[Harvard (MBTA station)|Harvard Squ
are]], [[Central (MBTA station)|Central Square]], and [[Kendall/MIT (MBTA station)|Kendall Square/MIT]]). Alewife Station, the current terminus of the Red Line, has a large multi-story parking garage (at a rate of $7 per day {{as of|lc=y|2009}}).<ref>{{cite web|url=http://www.mbta.com/schedules_and_maps/subway/lines/stations/?stopId=10029 |title=> Schedules & Maps > Subway > Alewife Station |publisher=MBTA |date= |accessdate=2012-04-28}}</ref> The [[Harvard Bus Tunnel]], under Harvard Square, reduces traffic congestion on the surface, and connects to the Red Line underground. This tunnel was originally opened for streetcars in 1912, and served trackless trolleys and buses as the routes were converted. The tunnel was partially reconfigured when the Red Line was extended to Alewife in the early 1980s. Outside of the state-owned transit agency, the city is also served by the Charles River Transportation Management Agency (CRTMA) shuttles which are supported by some of the largest compa
nies operating in city, in addition to the municipal government itself.<ref>{{cite web |url=http://www.charlesrivertma.org/members.htm |title=Charles River TMA Members |author=Staff writer |date=(As of) January 1, 2013 |work=CRTMA |publisher= |language= |trans_title= |type= |archiveurl= |archivedate= |deadurl= |accessdate=January 1, 2013 |quote= |ref= |separator= |postscript=}} </ref> ===Cycling=== Cambridge has several [[bike path]]s, including one along the Charles River,<ref>{{cite web|url=http://www.mass.gov/dcr/parks/metroboston/maps/bikepaths_dudley.gif |title=Dr. Paul Dudley White Bikepath |date= |accessdate=2012-04-28}}</ref> and the [[Cambridge Linear Park|Linear Park]] connecting the [[Minuteman Bikeway]] at Alewife with the [[Somerville Community Path]]. Bike parking is common and there are bike lanes on many streets, although concerns have been expressed regarding the suitability of many of the lanes. On several central MIT streets, bike lanes transfer onto the sidewalk.
Cambridge bans cycling on certain sections of sidewalk where pedestrian traffic is heavy.<ref>{{cite web|url=http://www.cambridgema.gov/cdd/et/bike/bike_ban.html |title=Sidewalk Bicycling Banned Areas – Cambridge Massachusetts |publisher=Cambridgema.gov |date= |accessdate=2012-04-28}}</ref><ref>{{cite web|url=http://www.cambridgema.gov/cdd/et/bike/bike_reg.html |title=Traffic Regulations for Cyclists – Cambridge Massachusetts |publisher=Cambridgema.gov |date=1997-05-01 |accessdate=2012-04-28}}</ref> While ''[[Bicycling Magazine]]'' has rated Boston as one of the worst cities in the nation for bicycling (In their words, for "lousy roads, scarce and unconnected bike lanes and bike-friendly gestures from City Hall that go nowhere—such as hiring a bike coordinator in 2001, only to cut the position two years later"),<ref>[http://www.bicycling.com/article/1,6610,s1-2-16-14593-11,00.html Urban Treasures – bicycling.com]{{dead link|date=April 2012}}</ref> it has listed Cambridge as
an honorable mention as one of the best<ref>[http://www.bicycling.com/article/1,6610,s1-2-16-14593-9,00.html Urban Treasures – bicycling.com]{{dead link|date=April 2012}}</ref> and was called by the magazine "Boston's Great Hope." Cambridge has an active, official bicycle committee. ===Walking=== [[File:Weeks Footbridge Cambridge, MA.jpg|thumb|The [[John W. Weeks Bridge|Weeks Bridge]] provides a pedestrian-only connection between Boston's Allston-Brighton neighborhood and Cambridge over the Charles River]] Walking is a popular activity in Cambridge. Per year 2000 data, of the communities in the U.S. with more than 100,000 residents, Cambridge has the highest percentage of commuters who walk to work.<ref>{{cite web|url=http://www.bikesatwork.com/carfree/census-lookup.php?state_select=ALL_STATES&lower_pop=100000&upper_pop=99999999&sort_num=2&show_rows=25&first_row=0 |title=The Carfree Census Database: Result of search for communities in any state with population over 100,000, sorte
d in descending order by % Pedestrian Commuters |publisher=Bikesatwork.com |date= |accessdate=2012-04-28}}</ref> Cambridge receives a "Walk Score" of 100 out of 100 possible points.<ref>[http://www.walkscore.com/get-score.php?street=cambridge%2C+ma&go=Go Walk Score site] Accessed July 28, 2009</ref> Cambridge's major historic squares have been recently changed into a modern walking landscape, which has sparked a traffic calming program based on the needs of pedestrians rather than of motorists. ===Intercity=== The Boston intercity bus and train stations at [[South Station]], Boston, and [[Logan International Airport]] in [[East Boston]], are accessible by [[Red Line (MBTA)|subway]]. The [[Fitchburg Line]] rail service from [[Porter (MBTA station)|Porter Square]] connects to some western suburbs. Since October 2010, there has also been intercity bus service between [[Alewife (MBTA station)|Alewife Station]] (Cambridge) and [[New York City]].<ref>{{cite web|last=Thomas |first=Sarah |u
rl=http://www.boston.com/yourtown/news/cambridge/2010/10/warren_mbta_welcome_world_wide.html |title=NYC-bound buses will roll from Newton, Cambridge |publisher=Boston.com |date=2010-10-19 |accessdate=2012-04-28}}</ref> ==Media== ===Newspapers=== Cambridge is served by several weekly newspapers. The most prominent is the ''[[Cambridge Chronicle]]'', which is also the oldest surviving weekly paper in the United States. ===Radio=== Cambridge is home to the following commercially licensed and student-run radio stations: {| class=wikitable |- ! [[Callsign]] !! Frequency !! City/town !! Licensee !! Format |- | [[WHRB]] || align=right | 95.3 FM || Cambridge (Harvard) || Harvard Radio Broadcasting Co., Inc. || [[Variety (US radio)|Musical variety]] |- | [[WJIB]] || align=right | 740 AM || Cambridge || Bob Bittner Broadcasting || [[Adult Standards]]/Pop |- | [[WMBR]] || align=right | 88.1 FM || Cambridge (MIT) || Technology Broadcasting Corporation || [[College radio]] |} ===Television=
== Cambridge Community Television (CCTV) has served the Cambridge community since its inception in 1988. CCTV operates Cambridge's public access television facility and programs three television channels, 8, 9, and 96 on the Cambridge cable system (Comcast). ===Social media=== As of 2011, a growing number of social media efforts provide means for participatory engagement with the locality of Cambridge, such as Localocracy<ref>"Localocracy is an online town common where registered voters using real names can weigh in on local issues." [http://cambridge.localocracy.com/ Localocracy Cambridge, Massachusetts]. Accessed 2011-10-01</ref> and [[foursquare (website)|Foursquare]]. ==Culture, art and architecture== [[File:Fogg.jpg|thumb|[[Fogg Museum]], Harvard]] ===Museums=== * [[Harvard Art Museum]], including the [[Busch-Reisinger Museum]], a collection of Germanic art the [[Fogg Art Museum]], a comprehensive collection of Western art, and the [[Arthur M. Sackler Museum]], a collection of
Middle East and Asian art * [[Harvard Museum of Natural History]], including the [[Glass Flowers]] collection * [[Peabody Museum of Archaeology and Ethnology]], Harvard *[[Semitic Museum]], Harvard * [[MIT Museum]] * [[List Visual Arts Center]], MIT ===Public art=== Cambridge has a large and varied collection of permanent public art, both on city property (managed by the Cambridge Arts Council),<ref>{{cite web|url=http://www.cambridgema.gov/CAC/Public/overview.cfm |title=CAC Public Art Program |publisher=Cambridgema.gov |date=2007-03-13 |accessdate=2012-04-28}}</ref> and on the campuses of Harvard<ref>{{cite web|url=http://ofa.fas.harvard.edu/visualarts/pubart.php |title=Office for the Arts at Harvard: Public Art |publisher=Ofa.fas.harvard.edu |date= |accessdate=2012-04-28}}</ref> and MIT.<ref>{{cite web|url=http://listart.mit.edu/map |title=MIT Public Art Collection Map |publisher=Listart.mit.edu |date= |accessdate=2012-04-28}}</ref> Temporary public artworks are displayed as part
of the annual Cambridge River Festival on the banks of the Charles River, during winter celebrations in Harvard and Central Squares, and at university campus sites. Experimental forms of public artistic and cultural expression include the Central Square World's Fair, the Somerville-based annual Honk! Festival,<ref>{{cite web|url=http://honkfest.org/ |title= Honk Fest}}</ref> and [[If This House Could Talk]],<ref>{{cite web|url=http://cambridgehistory.org/discover/ifthishousecouldtalk/index.html |title=The Cambridge Historical Society}}</ref> a neighborhood art and history event. {{or|date=April 2012}} {{Citation needed|date=April 2012}} An active tradition of street musicians and other performers in Harvard Square entertains an audience of tourists and local residents during the warmer months of the year. The performances are coordinated through a public process that has been developed collaboratively by the performers,<ref>{{cite web|url=http://www.buskersadvocates.org/ | title= St
reet Arts & Buskers Advocates}}</ref> city administrators, private organizations and business groups.<ref>{{cite web|url=http://harvardsquare.com/Home/Arts-and-Entertainment/Street-Arts-and-Buskers-Advocates.aspx |title=Street Arts and Buskers Advocates |publisher=Harvardsquare.com |date= |accessdate=2012-04-28}}</ref> [[File:Longfellow National Historic Site, Cambridge, Massachusetts.JPG|thumb|right|The [[Longfellow National Historic Site]]]] [[File:Wfm stata center.jpg|thumb|[[Stata Center]], MIT]] [[File:Simmons Hall, MIT, Cambridge, Massachusetts.JPG|thumb|[[List of MIT undergraduate dormitories|Simmons Hall]], MIT]] ===Architecture=== Despite intensive urbanization during the late 19th century and 20th century, Cambridge has preserved an unusual number of historic buildings, including some dating to the 17th century. The city also contains an abundance of innovative contemporary architecture, largely built by Harvard and MIT. ;Notable historic buildings in the city include: * T
he [[Asa Gray House]] (1810) * [[Austin Hall, Harvard University]] (1882–84) * [[Cambridge, Massachusetts City Hall|Cambridge City Hall]] (1888–89) * [[Cambridge Public Library]] (1888) * [[Christ Church, Cambridge]] (1761) * [[Cooper-Frost-Austin House]] (1689–1817) * [[Elmwood (Cambridge, Massachusetts)|Elmwood House]] (1767), residence of the [[President of Harvard University]] * [[First Church of Christ, Scientist (Cambridge, Massachusetts)|First Church of Christ, Scientist]] (1924–30) * [[The First Parish in Cambridge]] (1833) * [[Harvard-Epworth United Methodist Church]] (1891–93) * [[Harvard Lampoon Building]] (1909) * The [[Hooper-Lee-Nichols House]] (1685–1850) * [[Longfellow National Historic Site]] (1759), former home of poet [[Henry Wadsworth Longfellow]] * [[The Memorial Church of Harvard University]] (1932) * [[Memorial Hall, Harvard University]] (1870–77) * [[Middlesex County Courthouse (Massachusetts)|Middlesex County Courthouse]] (1814–48) * [[Urban
Rowhouse (40-48 Pearl Street, Cambridge, Massachusetts)|Urban Rowhouse]] (1875) * [[spite house|O'Reilly Spite House]] (1908), built to spite a neighbor who would not sell his adjacent land<ref name="existing">Bloom, Jonathan. (February 2, 2003) [[Boston Globe]] ''[http://nl.newsbank.com/nl-search/we/Archives?p_product=BG&p_theme=bg&p_action=search&p_maxdocs=200&p_topdoc=1&p_text_direct-0=0F907F2342522B5D&p_field_direct-0=document_id&p_perpage=10&p_sort=YMD_date:D Existing by the Thinnest of Margins. A Concord Avenue Landmark Gives New Meaning to Cozy.]'' Section: City Weekly; Page 11. Location: 260 Concord Ave, Cambridge, MA 02138.</ref> {{See also|List of Registered Historic Places in Cambridge, Massachusetts}} ;Contemporary architecture: * [[List of MIT undergraduate dormitories#Baker House|Baker House]] dormitory, MIT, by Finnish architect [[Alvar Aalto]], one of only two buildings by Aalto in the US * Harvard Graduate Center/Harkness Commons, by [[The Architects Collaborative]]
(TAC, with [[Walter Gropius]]) * [[Carpenter Center for the Visual Arts]], Harvard, the only building in North America by [[Le Corbusier]] * [[Kresge Auditorium]], MIT, by [[Eero Saarinen]] * [[MIT Chapel]], by [[Eero Saarinen]] * [[Design Research Building]], by [[Benjamin Thompson and Associates]] * [[American Academy of Arts and Sciences]], by [[Kallmann McKinnell and Wood]], also architects of Boston City Hall * [[Arthur M. Sackler Museum]], Harvard, one of the few buildings in the U.S. by [[James Stirling (architect)|James Stirling]], winner of the [[Pritzker Prize]] * [[Stata Center]], MIT, by [[Frank Gehry]] * [[List of MIT undergraduate dormitories#Simmons Hall|Simmons Hall]], MIT, by [[Steven Holl]] ===Music=== <!-- make section generic. NEEDS MORE WORK. remove marketing fluff for Ryles. --> The city has an active music scene from classical performances to the latest popular bands. ==Sister cities== Cambridge has 8 active, official [[Twin towns and sister cities|sister cit
ies]], and an unofficial relationship with [[Cambridge]], England:<ref name="peacom">"A message from the Peace Commission" [http://www.cambridgema.gov/peace/newsandpublications/news/detail.aspx?path=%2fsitecore%2fcontent%2fhome%2fpeace%2fnewsandpublications%2fnews%2f2008%2f02%2finformationoncambridgessistercities].</ref> *{{Flagicon|PRT}} [[Coimbra]], [[Portugal]] *{{Flagicon|CUB}} [[Cienfuegos]], [[Cuba]] *{{Flagicon|ITA}} [[Gaeta]], [[Italy]] *{{Flagicon|IRL}} [[Galway]], [[Republic of Ireland|Ireland]] *{{Flagicon|ARM}} [[Yerevan]], [[Armenia]]<ref>{{cite web|url=http://www.cysca.org/ |title=Cambridge-Yerevan Sister City Association |publisher=Cysca.org |date= |accessdate=2012-04-28}}</ref> *{{Flagicon|SLV}} [[San José Las Flores, Chalatenango|San José Las Flores]], [[El Salvador]] *{{Flagicon|JPN}} [[Tsukuba, Ibaraki|Tsukuba Science City]], Japan *{{Flagicon|POL}} [[Kraków]], [[Poland]] *{{Flagicon|CHN}} [[Haidian District]], [[China]] Ten other official sister city relations
hips are inactive: [[Dublin]], Ireland; [[Ischia]], [[Catania]], and [[Florence]], Italy; [[Kraków]], Poland; [[Santo Domingo Oeste]], Dominican Republic; [[Southwark]], London, England; [[Yuseong]], Daejeon, Korea; and [[Haidian District|Haidian]], Beijing, China.<ref name="peacom"/> There has also been an unofficial relationship with: *{{Flagicon|GBR}} [[Cambridge]], England, UK<ref>{{cite web|url=http://www.cambridgema.gov/peace/newsandpublications/news/detail.aspx?path=%2fsitecore%2fcontent%2fhome%2fpeace%2fnewsandpublications%2fnews%2f2008%2f02%2finformationoncambridgessistercities |title="Sister Cities", Cambridge Peace Commission |publisher=Cambridgema.gov |date=2008-02-15 |accessdate=2012-07-18}}</ref> ==Zip codes== *02138—Harvard Square/West Cambridge *02139—Central Square/Inman Square/MIT *02140—Porter Square/North Cambridge *02141—East Cambridge *02142—Kendall Square ==References== {{reflist|30em}} ==General references== * ''History of Middlesex County, Massach
usetts'', [http://books.google.com/books?id=QGolOAyd9RMC&dq=intitle:History+intitle:of+intitle:Middlesex+intitle:County+intitle:Massachusetts&lr=&num=50&as_brr=0&source=gbs_other_versions_sidebar_s&cad=5 Volume 1 (A-H)], [http://books.google.com/books?id=hNaAnwRMedUC&pg=PA506&dq=intitle:History+intitle:of+intitle:Middlesex+intitle:County+intitle:Massachusetts&lr=&num=50&as_brr=0#PPA3,M1 Volume 2 (L-W)] compiled by Samuel Adams Drake, published 1879–1880. ** [http://books.google.com/books?id=QGolOAyd9RMC&printsec=titlepage#PPA305,M1 Cambridge article] by Rev. Edward Abbott in volume 1, pages 305–358. *Eliot, Samuel Atkins. ''A History of Cambridge, Massachusetts: 1630–1913''. Cambridge: The Cambridge Tribune, 1913. *Hiestand, Emily. "Watershed: An Excursion in Four Parts" The Georgia Review Spring 1998 pages 7–28 *[[Lucius Robinson Paige|Paige, Lucius]]. ''History of Cambridge, Massachusetts: 1630–1877''. Cambridge: The Riverside Press, 1877. *Survey of Architectural Histor
y in Cambridge: Mid Cambridge, 1967, Cambridge Historical Commission, Cambridge, Mass.{{ISBN missing}} *Survey of Architectural History in Cambridge: Cambridgeport, 1971 ISBN 0-262-53013-9, Cambridge Historical Commission, Cambridge, Mass. *Survey of Architectural History in Cambridge: Old Cambridge, 1973 ISBN 0-262-53014-7, Cambridge Historical Commission, Cambridge, Mass. *Survey of Architectural History in Cambridge: Northwest Cambridge, 1977 ISBN 0-262-53032-5, Cambridge Historical Commission, Cambridge, Mass. *Survey of Architectural History in Cambridge: East Cambridge, 1988 (revised) ISBN 0-262-53078-3, Cambridge Historical Commission, Cambridge, Mass. *{{cite book|last=Sinclair|first=Jill|title=Fresh Pond: The History of a Cambridge Landscape|publisher=MIT Press|location=Cambridge, Mass.|date=April 2009|isbn=978-0-262-19591-1 }} *{{cite book|last=Seaburg|first=Alan|title=Cambridge on the Charles|url=http://books.google.com/books?id=c7_oCS782-8C|publisher=Anne Miniver Press|l
ocation=Billerica, Mass.|year=2001|author=Seaburg, A. and Dahill, T. and Rose, C.H.|isbn=978-0-9625794-9-3}} ==External links== {{Commons category}} <!-- for current and future use if material is uploaded --> {{Wikivoyage|Cambridge (Massachusetts)}} {{Portal|Boston}} {{Commons category|Cambridge, Massachusetts}} *{{Official website|http://www.cambridgema.gov/}} *[http://www.cambridge-usa.org/ Cambridge Office for Tourism] *[http://www.city-data.com/city/Cambridge-Massachusetts.html City-Data.com] *[http://www.epodunk.com/cgi-bin/genInfo.php?locIndex=2894 ePodunk: Profile for Cambridge, Massachusetts] *{{dmoz|Regional/North_America/United_States/Massachusetts/Localities/C/Cambridge}} <br/><!--this break is to put visual space between the last information and the following template if needed--> ===Maps=== *[http://www.cambridgema.gov/GIS/FindMapAtlas.cfm Cambridge Maps] *[http://www.cambridgema.gov/GIS City of Cambridge Geographic Information System (GIS)] *[http://www.salemdeeds.com/
atlases_results.asp?ImageType=index&atlastype=MassWorld&atlastown=&atlas=MASSACHUSETTS+1871&atlas_desc=MASSACHUSETTS+1871 ''1871 Atlas of Massachusetts''.] by Wall & Gray. [http://www.salemdeeds.com/atlases_pages.asp?ImageName=PAGE_0010_0011.jpg&atlastype=MassWorld&atlastown=&atlas=MASSACHUSETTS+1871&atlas_desc=MASSACHUSETTS+1871&pageprefix= Map of Massachusetts.] [http://www.salemdeeds.com/atlases_pages.asp?ImageName=PAGE_0044_0045.jpg&atlastype=MassWorld&atlastown=&atlas=MASSACHUSETTS+1871&atlas_desc=MASSACHUSETTS+1871&pageprefix= Map of Middlesex County.] *Dutton, E.P. [http://maps.bpl.org/details_10717/?srch_query=Dutton%2C+E.P.&srch_fields=all&srch_author=on&srch_style=exact&srch_fa=save&srch_ok=Go+Search Chart of Boston Harbor and Massachusetts Bay with Map of Adjacent Country.] Published 1867. A good map of roads and rail lines around Cambridge. *[http://www.citymap.com/cambridge/index.htm Cambridge Citymap – Community, Business, and Visitor Map.] *[http://docs.unh.edu/town
s/CambridgeMassachusettsMapList.htm Old USGS maps of Cambridge area.] {{Greater Boston}} {{Middlesex County, Massachusetts}} {{Massachusetts}} {{New England}} {{Massachusetts cities and mayors of 100,000 population}} [[Category:Cambridge, Massachusetts| ]] [[Category:University towns in the United States]] [[Category:County seats in Massachusetts]] [[Category:Populated places established in 1630]] [[Category:Charles River]] [[Category:Place names of English origin in the United States]] [[af:Cambridge, Massachusetts]] [[ar:كامبريدج، ماساتشوستس]] [[zh-min-nan:Cambridge, Massachusetts]] [[be:Горад Кембрыдж, Масачусетс]] [[be-x-old:Кембрыдж (Масачусэтс)]] [[bg:Кеймбридж (Масачузетс)]] [[br:Cambridge (Massachusetts)]] [[ca:Cambridge (Massachusetts)]] [[cs:Cambridge (Massachusetts)]] [[cy:Cambridge, Massachusetts]] [[da:Cambridge (Massachusetts)]] [[de:Cambridge (Massachusetts)]] [[et:Cambridge (Massachusetts)
]] [[es:Cambridge (Massachusetts)]] [[eo:Kembriĝo (Masaĉuseco)]] [[eu:Cambridge (Massachusetts)]] [[fa:کمبریج (ماساچوست)]] [[fr:Cambridge (Massachusetts)]] [[gd:Cambridge (MA)]] [[ko:케임브리지 (매사추세츠 주)]] [[hy:Քեմբրիջ (Մասաչուսեթս)]] [[id:Cambridge, Massachusetts]] [[it:Cambridge (Massachusetts)]] [[he:קיימברידג' (מסצ'וסטס)]] [[jv:Cambridge, Massachusetts]] [[kk:Кэмбридж (Массачусетс)]] [[kw:Cambridge, Massachusetts]] [[sw:Cambridge, Massachusetts]] [[ht:Cambridge, Massachusetts]] [[la:Cantabrigia (Massachusetta)]] [[lv:Keimbridža]] [[lb:Cambridge (Massachusetts)]] [[hu:Cambridge (Massachusetts)]] [[mr:केंब्रिज, मॅसेच्युसेट्स]] [[ms:Cambridge, Massachusetts]] [[nl:Cambridge (Massachusetts)]] [[ja:ケンブリッジ (マサチューセッツ州)]] [[no:Cambridge (Massachusetts)]] [[pl:Cambridge (Massachusetts)]] [[pt:Cambridge (Massachusetts)]] [[ro:Cambri
dge, Massachusetts]] [[ru:Кембридж (Массачусетс)]] [[scn:Cambridge (Massachusetts), USA]] [[simple:Cambridge, Massachusetts]] [[sk:Cambridge (Massachusetts)]] [[sl:Cambridge, Massachusetts]] [[sr:Кембриџ (Масачусетс)]] [[fi:Cambridge (Massachusetts)]] [[sv:Cambridge, Massachusetts]] [[tl:Cambridge, Massachusetts]] [[ta:கேம்பிரிஜ், மாசசூசெட்ஸ்]] [[th:เคมบริดจ์ (รัฐแมสซาชูเซตส์)]] [[tg:Кембриҷ (Массачусетс)]] [[tr:Cambridge, Massachusetts]] [[uk:Кембридж (Массачусетс)]] [[vi:Cambridge, Massachusetts]] [[vo:Cambridge (Massachusetts)]] [[war:Cambridge, Massachusetts]] [[yi:קעמברידזש, מאסאטשוסעטס]] [[zh:剑桥 (马萨诸塞州)]]
\ No newline at end of file
[31/44] lucene-solr:jira/solr-8668: SOLR-10710: Fix LTR failing tests
Posted by cp...@apache.org.
SOLR-10710: Fix LTR failing tests
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/94731aaa
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/94731aaa
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/94731aaa
Branch: refs/heads/jira/solr-8668
Commit: 94731aaa098aa7c6f14fdf8dd63575e999c3f26e
Parents: 47781e3
Author: Tomas Fernandez Lobbe <tf...@apache.org>
Authored: Wed May 24 13:28:07 2017 -0700
Committer: Tomas Fernandez Lobbe <tf...@apache.org>
Committed: Wed May 24 13:28:07 2017 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../solr/ltr/feature/FieldLengthFeature.java | 3 +-
.../apache/solr/ltr/TestLTRQParserPlugin.java | 11 ++---
.../solr/ltr/TestParallelWeightCreation.java | 5 ++-
.../solr/ltr/TestSelectiveWeightCreation.java | 44 ++++++++++----------
5 files changed, 35 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/94731aaa/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 5b92e3c..436b80e 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -194,6 +194,8 @@ Other Changes
* SOLR-10700: Deprecated and converted the PostingsSolrHighlighter to extend UnifiedSolrHighlighter and thus no
longer use the PostingsHighlighter. It should behave mostly the same. (David Smiley)
+* SOLR-10710: Fix LTR failing tests. (Diego Ceccarelli via Tomás Fernández Löbbe)
+
================== 6.7.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/94731aaa/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/FieldLengthFeature.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/FieldLengthFeature.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/FieldLengthFeature.java
index 4c17aff..00159b9 100644
--- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/FieldLengthFeature.java
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/FieldLengthFeature.java
@@ -76,8 +76,7 @@ public class FieldLengthFeature extends Feature {
static {
NORM_TABLE[0] = 0;
for (int i = 1; i < 256; i++) {
- float norm = SmallFloat.byte315ToFloat((byte) i);
- NORM_TABLE[i] = 1.0f / (norm * norm);
+ NORM_TABLE[i] = SmallFloat.byte4ToInt((byte) i);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/94731aaa/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTRQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTRQParserPlugin.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTRQParserPlugin.java
index d4457a0..decb1c0 100644
--- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTRQParserPlugin.java
+++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTRQParserPlugin.java
@@ -88,15 +88,16 @@ public class TestLTRQParserPlugin extends TestRerankBase {
query.add("rows", "4");
query.add("fv", "true");
- String nonRerankedScore = "0.09271725";
+ // FIXME: design better way to test this, we cannot check an absolute score
+ // String nonRerankedScore = "0.09271725";
// Normal solr order
assertJQ("/query" + query.toQueryString(),
"/response/docs/[0]/id=='9'",
"/response/docs/[1]/id=='8'",
"/response/docs/[2]/id=='7'",
- "/response/docs/[3]/id=='6'",
- "/response/docs/[3]/score=="+nonRerankedScore
+ "/response/docs/[3]/id=='6'"
+ // "/response/docs/[3]/score=="+nonRerankedScore
);
query.add("rq", "{!ltr model=6029760550880411648 reRankDocs=3}");
@@ -106,8 +107,8 @@ public class TestLTRQParserPlugin extends TestRerankBase {
"/response/docs/[0]/id=='7'",
"/response/docs/[1]/id=='8'",
"/response/docs/[2]/id=='9'",
- "/response/docs/[3]/id=='6'",
- "/response/docs/[3]/score=="+nonRerankedScore
+ "/response/docs/[3]/id=='6'"
+ // "/response/docs/[3]/score=="+nonRerankedScore
);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/94731aaa/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestParallelWeightCreation.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestParallelWeightCreation.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestParallelWeightCreation.java
index 630a68c..46330c9 100644
--- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestParallelWeightCreation.java
+++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestParallelWeightCreation.java
@@ -42,8 +42,9 @@ public class TestParallelWeightCreation extends TestRerankBase{
query.add("rows", "4");
query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3}");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='4'");
+ // SOLR-10710, feature based on query with term w3 now scores higher on doc 4, updated
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='4'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='3'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='1'");
aftertest();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/94731aaa/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java
index 7bf8373..cbd0e23 100644
--- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java
+++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java
@@ -210,14 +210,14 @@ public class TestSelectiveWeightCreation extends TestRerankBase {
@Test
public void testSelectiveWeightsRequestFeaturesFromDifferentStore() throws Exception {
- final String docs0fv_sparse = FeatureLoggerTestUtils.toFeatureVector(
- "matchedTitle","1.0", "titlePhraseMatch","0.6103343");
- final String docs0fv_dense = FeatureLoggerTestUtils.toFeatureVector(
- "matchedTitle","1.0", "titlePhraseMatch","0.6103343", "titlePhrasesMatch","0.0");
- final String docs0fv_fstore4= FeatureLoggerTestUtils.toFeatureVector(
- "popularity","3.0", "originalScore","1.0");
-
- final String docs0fv = chooseDefaultFeatureVector(docs0fv_dense, docs0fv_sparse);
+// final String docs0fv_sparse = FeatureLoggerTestUtils.toFeatureVector(
+// "matchedTitle","1.0", "titlePhraseMatch","0.6103343");
+// final String docs0fv_dense = FeatureLoggerTestUtils.toFeatureVector(
+// "matchedTitle","1.0", "titlePhraseMatch","0.6103343", "titlePhrasesMatch","0.0");
+// final String docs0fv_fstore4= FeatureLoggerTestUtils.toFeatureVector(
+// "popularity","3.0", "originalScore","1.0");
+//
+// final String docs0fv = chooseDefaultFeatureVector(docs0fv_dense, docs0fv_sparse);
// extract all features in externalmodel's store (default store)
// rerank using externalmodel (default store)
@@ -227,11 +227,12 @@ public class TestSelectiveWeightCreation extends TestRerankBase {
query.add("rows", "5");
query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3 efi.userTitlePhrase1=w2 efi.userTitlePhrase2=w1}");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='4'");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='1'");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv+"'");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.33873552");
+ // SOLR-10710, feature based on query with term w3 now scores higher on doc 4, updated
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='4'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='3'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='1'");
+ // FIXME design better way to test this, we can't rely on absolute scores
+ // assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv+"'");
// extract all features from fstore4
// rerank using externalmodel (default store)
@@ -240,11 +241,12 @@ public class TestSelectiveWeightCreation extends TestRerankBase {
query.add("fl", "*,score,fv:[fv store=fstore4 efi.myPop=3]");
query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3}");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='4'");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='1'");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv_fstore4+"'");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.33873552");
+ // SOLR-10710, feature based on query with term w3 now scores higher on doc 4, updated
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='4'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='3'");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='1'");
+ // FIXME design better way to test this, we can't rely on absolute scores
+ // assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv_fstore4+"'");
// extract all features from fstore4
// rerank using externalmodel2 (fstore2)
@@ -255,9 +257,9 @@ public class TestSelectiveWeightCreation extends TestRerankBase {
assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='5'");
assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='4'");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='3'");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv_fstore4+"'");
- assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==2.5");
+ assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='3'");
+ // FIXME design better way to test this, we can't rely on absolute scores
+ // assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv_fstore4+"'");
}
}
[27/44] lucene-solr:jira/solr-8668: SOLR-10731: Remove debugging
Posted by cp...@apache.org.
SOLR-10731: Remove debugging
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/851ab0ad
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/851ab0ad
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/851ab0ad
Branch: refs/heads/jira/solr-8668
Commit: 851ab0ad34b57d9655f284e48daca56609fe344f
Parents: f4872c9
Author: Joel Bernstein <jb...@apache.org>
Authored: Wed May 24 08:07:03 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Wed May 24 08:07:03 2017 -0400
----------------------------------------------------------------------
.../src/java/org/apache/solr/client/solrj/io/stream/KnnStream.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/851ab0ad/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/KnnStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/KnnStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/KnnStream.java
index 301d017..1d5f187 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/KnnStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/KnnStream.java
@@ -82,7 +82,6 @@ public class KnnStream extends TupleStream implements Expressible {
}
// Named parameters - passed directly to solr as solrparams
- System.out.println("####Params:"+namedParams.size());
if(namedParams.size() < 2){
throw new IOException(String.format(Locale.ROOT,"invalid expression %s - at least two named parameters expected. eg. 'id' and 'qf'",expression));
}
[40/44] lucene-solr:jira/solr-8668: SOLR-10479: Adds support for
HttpShardHandlerFactory.loadBalancerRequests(MinimumAbsolute|MaximumFraction)
configuration. (Ramsey Haddad, Daniel Collins, Christine Poerschke)
Posted by cp...@apache.org.
SOLR-10479: Adds support for HttpShardHandlerFactory.loadBalancerRequests(MinimumAbsolute|MaximumFraction) configuration. (Ramsey Haddad, Daniel Collins, Christine Poerschke)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2bb6e2ca
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2bb6e2ca
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2bb6e2ca
Branch: refs/heads/jira/solr-8668
Commit: 2bb6e2cacabdcea6c7534595dfc23cd17973a68d
Parents: 0b47126
Author: Christine Poerschke <cp...@apache.org>
Authored: Thu May 25 12:30:30 2017 +0100
Committer: Christine Poerschke <cp...@apache.org>
Committed: Thu May 25 16:46:45 2017 +0100
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../component/HttpShardHandlerFactory.java | 28 ++++-
.../solr-shardhandler-loadBalancerRequests.xml | 23 +++++
.../component/TestHttpShardHandlerFactory.java | 102 +++++++++++++++++++
.../client/solrj/impl/LBHttpSolrClient.java | 50 ++++++++-
5 files changed, 202 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2bb6e2ca/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 436b80e..0ddf6c0 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -227,6 +227,9 @@ New Features
* SOLR-10721: Provide a way to know when Core Discovery is finished and when all async cores are done loading
(Erick Erickson)
+* SOLR-10479: Adds support for HttpShardHandlerFactory.loadBalancerRequests(MinimumAbsolute|MaximumFraction)
+ configuration. (Ramsey Haddad, Daniel Collins, Christine Poerschke)
+
Bug Fixes
----------------------
* SOLR-10723 JSON Facet API: resize() implemented incorrectly for CountSlotAcc, HllAgg.NumericAcc
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2bb6e2ca/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
index e3787cd..73d9707 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
@@ -97,6 +97,8 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
int maximumPoolSize = Integer.MAX_VALUE;
int keepAliveTime = 5;
int queueSize = -1;
+ int permittedLoadBalancerRequestsMinimumAbsolute = 0;
+ float permittedLoadBalancerRequestsMaximumFraction = 1.0f;
boolean accessPolicy = false;
private String scheme = null;
@@ -122,6 +124,12 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
// If the threadpool uses a backing queue, what is its maximum size (-1) to use direct handoff
static final String INIT_SIZE_OF_QUEUE = "sizeOfQueue";
+ // The minimum number of replicas that may be used
+ static final String LOAD_BALANCER_REQUESTS_MIN_ABSOLUTE = "loadBalancerRequestsMinimumAbsolute";
+
+ // The maximum proportion of replicas to be used
+ static final String LOAD_BALANCER_REQUESTS_MAX_FRACTION = "loadBalancerRequestsMaximumFraction";
+
// Configure if the threadpool favours fairness over throughput
static final String INIT_FAIRNESS_POLICY = "fairnessPolicy";
@@ -164,6 +172,16 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
this.maximumPoolSize = getParameter(args, INIT_MAX_POOL_SIZE, maximumPoolSize,sb);
this.keepAliveTime = getParameter(args, MAX_THREAD_IDLE_TIME, keepAliveTime,sb);
this.queueSize = getParameter(args, INIT_SIZE_OF_QUEUE, queueSize,sb);
+ this.permittedLoadBalancerRequestsMinimumAbsolute = getParameter(
+ args,
+ LOAD_BALANCER_REQUESTS_MIN_ABSOLUTE,
+ permittedLoadBalancerRequestsMinimumAbsolute,
+ sb);
+ this.permittedLoadBalancerRequestsMaximumFraction = getParameter(
+ args,
+ LOAD_BALANCER_REQUESTS_MAX_FRACTION,
+ permittedLoadBalancerRequestsMaximumFraction,
+ sb);
this.accessPolicy = getParameter(args, INIT_FAIRNESS_POLICY, accessPolicy,sb);
log.debug("created with {}",sb);
@@ -252,7 +270,15 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
*/
public LBHttpSolrClient.Rsp makeLoadBalancedRequest(final QueryRequest req, List<String> urls)
throws SolrServerException, IOException {
- return loadbalancer.request(new LBHttpSolrClient.Req(req, urls));
+ return loadbalancer.request(newLBHttpSolrClientReq(req, urls));
+ }
+
+ protected LBHttpSolrClient.Req newLBHttpSolrClientReq(final QueryRequest req, List<String> urls) {
+ int numServersToTry = (int)Math.floor(urls.size() * this.permittedLoadBalancerRequestsMaximumFraction);
+ if (numServersToTry < this.permittedLoadBalancerRequestsMinimumAbsolute) {
+ numServersToTry = this.permittedLoadBalancerRequestsMinimumAbsolute;
+ }
+ return new LBHttpSolrClient.Req(req, urls, numServersToTry);
}
/**
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2bb6e2ca/solr/core/src/test-files/solr/solr-shardhandler-loadBalancerRequests.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/solr-shardhandler-loadBalancerRequests.xml b/solr/core/src/test-files/solr/solr-shardhandler-loadBalancerRequests.xml
new file mode 100644
index 0000000..92339d9
--- /dev/null
+++ b/solr/core/src/test-files/solr/solr-shardhandler-loadBalancerRequests.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<solr>
+ <shardHandlerFactory name="shardHandlerFactory" class="solr.HttpShardHandlerFactory">
+ <int name="loadBalancerRequestsMinimumAbsolute">${solr.tests.loadBalancerRequestsMinimumAbsolute:0}</int>
+ <float name="loadBalancerRequestsMaximumFraction">${solr.tests.loadBalancerRequestsMaximumFraction:1.0}</float>
+ </shardHandlerFactory>
+</solr>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2bb6e2ca/solr/core/src/test/org/apache/solr/handler/component/TestHttpShardHandlerFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/component/TestHttpShardHandlerFactory.java b/solr/core/src/test/org/apache/solr/handler/component/TestHttpShardHandlerFactory.java
new file mode 100644
index 0000000..3ffa015
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/component/TestHttpShardHandlerFactory.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.component;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.handler.component.HttpShardHandlerFactory;
+import org.apache.solr.handler.component.ShardHandlerFactory;
+
+import org.junit.BeforeClass;
+import org.junit.AfterClass;
+
+/**
+ * Tests specifying a custom ShardHandlerFactory
+ */
+public class TestHttpShardHandlerFactory extends SolrTestCaseJ4 {
+
+ private static final String LOAD_BALANCER_REQUESTS_MIN_ABSOLUTE = "solr.tests.loadBalancerRequestsMinimumAbsolute";
+ private static final String LOAD_BALANCER_REQUESTS_MAX_FRACTION = "solr.tests.loadBalancerRequestsMaximumFraction";
+
+ private static int expectedLoadBalancerRequestsMinimumAbsolute = 0;
+ private static float expectedLoadBalancerRequestsMaximumFraction = 1.0f;
+
+ @BeforeClass
+ public static void beforeTests() throws Exception {
+ expectedLoadBalancerRequestsMinimumAbsolute = random().nextInt(3); // 0 .. 2
+ expectedLoadBalancerRequestsMaximumFraction = (1+random().nextInt(10))/10f; // 0.1 .. 1.0
+ System.setProperty(LOAD_BALANCER_REQUESTS_MIN_ABSOLUTE, Integer.toString(expectedLoadBalancerRequestsMinimumAbsolute));
+ System.setProperty(LOAD_BALANCER_REQUESTS_MAX_FRACTION, Float.toString(expectedLoadBalancerRequestsMaximumFraction));
+ }
+
+ @AfterClass
+ public static void afterTests() {
+ System.clearProperty(LOAD_BALANCER_REQUESTS_MIN_ABSOLUTE);
+ System.clearProperty(LOAD_BALANCER_REQUESTS_MAX_FRACTION);
+ }
+
+ public void testLoadBalancerRequestsMinMax() throws Exception {
+ final Path home = Paths.get(TEST_HOME());
+ CoreContainer cc = null;
+ ShardHandlerFactory factory = null;
+ try {
+ cc = CoreContainer.createAndLoad(home, home.resolve("solr-shardhandler-loadBalancerRequests.xml"));
+ factory = cc.getShardHandlerFactory();
+
+ // test that factory is HttpShardHandlerFactory with expected url reserve fraction
+ assertTrue(factory instanceof HttpShardHandlerFactory);
+ final HttpShardHandlerFactory httpShardHandlerFactory = ((HttpShardHandlerFactory)factory);
+ assertEquals(expectedLoadBalancerRequestsMinimumAbsolute, httpShardHandlerFactory.permittedLoadBalancerRequestsMinimumAbsolute, 0.0);
+ assertEquals(expectedLoadBalancerRequestsMaximumFraction, httpShardHandlerFactory.permittedLoadBalancerRequestsMaximumFraction, 0.0);
+
+ // create a dummy request and dummy url list
+ final QueryRequest queryRequest = null;
+ final List<String> urls = new ArrayList<>();
+ for (int ii=0; ii<10; ++ii) {
+ urls.add(null);
+ }
+
+ // create LBHttpSolrClient request
+ final LBHttpSolrClient.Req req = httpShardHandlerFactory.newLBHttpSolrClientReq(queryRequest, urls);
+
+ // actual vs. expected test
+ final int actualNumServersToTry = req.getNumServersToTry().intValue();
+ int expectedNumServersToTry = (int)Math.floor(urls.size() * expectedLoadBalancerRequestsMaximumFraction);
+ if (expectedNumServersToTry < expectedLoadBalancerRequestsMinimumAbsolute) {
+ expectedNumServersToTry = expectedLoadBalancerRequestsMinimumAbsolute;
+ }
+ assertEquals("wrong numServersToTry for"
+ + " urls.size="+urls.size()
+ + " expectedLoadBalancerRequestsMinimumAbsolute="+expectedLoadBalancerRequestsMinimumAbsolute
+ + " expectedLoadBalancerRequestsMaximumFraction="+expectedLoadBalancerRequestsMaximumFraction,
+ expectedNumServersToTry,
+ actualNumServersToTry);
+
+ } finally {
+ if (factory != null) factory.close();
+ if (cc != null) cc.shutdown();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2bb6e2ca/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java
index ed6ae7b..8dc2fd9 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java
@@ -185,11 +185,17 @@ public class LBHttpSolrClient extends SolrClient {
protected SolrRequest request;
protected List<String> servers;
protected int numDeadServersToTry;
+ private final Integer numServersToTry;
public Req(SolrRequest request, List<String> servers) {
+ this(request, servers, null);
+ }
+
+ public Req(SolrRequest request, List<String> servers, Integer numServersToTry) {
this.request = request;
this.servers = servers;
this.numDeadServersToTry = servers.size();
+ this.numServersToTry = numServersToTry;
}
public SolrRequest getRequest() {
@@ -209,6 +215,10 @@ public class LBHttpSolrClient extends SolrClient {
public void setNumDeadServersToTry(int numDeadServersToTry) {
this.numDeadServersToTry = numDeadServersToTry;
}
+
+ public Integer getNumServersToTry() {
+ return numServersToTry;
+ }
}
public static class Rsp {
@@ -360,6 +370,9 @@ public class LBHttpSolrClient extends SolrClient {
boolean isNonRetryable = req.request instanceof IsUpdateRequest || ADMIN_PATHS.contains(req.request.getPath());
List<ServerWrapper> skipped = null;
+ final Integer numServersToTry = req.getNumServersToTry();
+ int numServersTried = 0;
+
boolean timeAllowedExceeded = false;
long timeAllowedNano = getTimeAllowedInNanos(req.getRequest());
long timeOutTime = System.nanoTime() + timeAllowedNano;
@@ -387,8 +400,14 @@ public class LBHttpSolrClient extends SolrClient {
}
try {
MDC.put("LBHttpSolrClient.url", serverStr);
+
+ if (numServersToTry != null && numServersTried > numServersToTry.intValue()) {
+ break;
+ }
+
HttpSolrClient client = makeSolrClient(serverStr);
+ ++numServersTried;
ex = doRequest(client, req, rsp, isNonRetryable, false, null);
if (ex == null) {
return rsp; // SUCCESS
@@ -405,8 +424,13 @@ public class LBHttpSolrClient extends SolrClient {
break;
}
+ if (numServersToTry != null && numServersTried > numServersToTry.intValue()) {
+ break;
+ }
+
try {
MDC.put("LBHttpSolrClient.url", wrapper.client.getBaseURL());
+ ++numServersTried;
ex = doRequest(wrapper.client, req, rsp, isNonRetryable, true, wrapper.getKey());
if (ex == null) {
return rsp; // SUCCESS
@@ -422,7 +446,13 @@ public class LBHttpSolrClient extends SolrClient {
if (timeAllowedExceeded) {
solrServerExceptionMessage = "Time allowed to handle this request exceeded";
} else {
- solrServerExceptionMessage = "No live SolrServers available to handle this request";
+ if (numServersToTry != null && numServersTried > numServersToTry.intValue()) {
+ solrServerExceptionMessage = "No live SolrServers available to handle this request:"
+ + " numServersTried="+numServersTried
+ + " numServersToTry="+numServersToTry.intValue();
+ } else {
+ solrServerExceptionMessage = "No live SolrServers available to handle this request";
+ }
}
if (ex == null) {
throw new SolrServerException(solrServerExceptionMessage);
@@ -594,10 +624,16 @@ public class LBHttpSolrClient extends SolrClient {
@Override
public NamedList<Object> request(final SolrRequest request, String collection)
throws SolrServerException, IOException {
+ return request(request, collection, null);
+ }
+
+ public NamedList<Object> request(final SolrRequest request, String collection,
+ final Integer numServersToTry) throws SolrServerException, IOException {
Exception ex = null;
ServerWrapper[] serverList = aliveServerList;
- int maxTries = serverList.length;
+ final int maxTries = (numServersToTry == null ? serverList.length : numServersToTry.intValue());
+ int numServersTried = 0;
Map<String,ServerWrapper> justFailed = null;
boolean timeAllowedExceeded = false;
@@ -612,6 +648,7 @@ public class LBHttpSolrClient extends SolrClient {
ServerWrapper wrapper = serverList[count % serverList.length];
try {
+ ++numServersTried;
return wrapper.client.request(request, collection);
} catch (SolrException e) {
// Server is alive but the request was malformed or invalid
@@ -638,6 +675,7 @@ public class LBHttpSolrClient extends SolrClient {
if (wrapper.standard==false || justFailed!=null && justFailed.containsKey(wrapper.getKey())) continue;
try {
+ ++numServersTried;
NamedList<Object> rsp = wrapper.client.request(request, collection);
// remove from zombie list *before* adding to alive to avoid a race that could lose a server
zombieServers.remove(wrapper.getKey());
@@ -663,7 +701,13 @@ public class LBHttpSolrClient extends SolrClient {
if (timeAllowedExceeded) {
solrServerExceptionMessage = "Time allowed to handle this request exceeded";
} else {
- solrServerExceptionMessage = "No live SolrServers available to handle this request";
+ if (numServersToTry != null && numServersTried > numServersToTry.intValue()) {
+ solrServerExceptionMessage = "No live SolrServers available to handle this request:"
+ + " numServersTried="+numServersTried
+ + " numServersToTry="+numServersToTry.intValue();
+ } else {
+ solrServerExceptionMessage = "No live SolrServers available to handle this request";
+ }
}
if (ex == null) {
throw new SolrServerException(solrServerExceptionMessage);
[29/44] lucene-solr:jira/solr-8668: LUCENE-7540: Upgrade ICU to 59.1
Posted by cp...@apache.org.
LUCENE-7540: Upgrade ICU to 59.1
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/cd567b98
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/cd567b98
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/cd567b98
Branch: refs/heads/jira/solr-8668
Commit: cd567b985ac031f804c8cfb56045d140cdb1dd0d
Parents: fed7343
Author: Jim Ferenczi <ji...@apache.org>
Authored: Wed May 24 18:11:10 2017 +0200
Committer: Jim Ferenczi <ji...@apache.org>
Committed: Wed May 24 18:11:10 2017 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 2 +
.../lucene/analysis/util/UnicodeProps.java | 6 +-
.../icu/src/data/utr30/DiacriticFolding.txt | 5 +
.../icu/src/data/utr30/NativeDigitFolding.txt | 30 +++
lucene/analysis/icu/src/data/utr30/nfc.txt | 22 +-
lucene/analysis/icu/src/data/utr30/nfkc.txt | 5 +-
lucene/analysis/icu/src/data/utr30/nfkc_cf.txt | 225 ++++++++++++++++++-
.../analysis/icu/segmentation/Default.brk | Bin 35264 -> 36768 bytes
.../icu/segmentation/MyanmarSyllable.brk | Bin 19776 -> 20744 bytes
.../org/apache/lucene/analysis/icu/utr30.nrm | Bin 53840 -> 55184 bytes
.../analysis/icu/GenerateUTR30DataFiles.java | 2 +-
lucene/ivy-versions.properties | 2 +-
lucene/licenses/icu4j-56.1.jar.sha1 | 1 -
lucene/licenses/icu4j-59.1.jar.sha1 | 1 +
solr/licenses/icu4j-56.1.jar.sha1 | 1 -
solr/licenses/icu4j-59.1.jar.sha1 | 1 +
16 files changed, 289 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cd567b98/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 879d754..fe1be6e 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -99,6 +99,8 @@ Other
* LUCENE-7753: Make fields static when possible.
(Daniel Jelinski via Adrien Grand)
+* LUCENE-7540: Upgrade ICU to 59.1 (Mike McCandless, Jim Ferenczi)
+
======================= Lucene 6.7.0 =======================
Other
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cd567b98/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/UnicodeProps.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/UnicodeProps.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/UnicodeProps.java
index 75070d1..00ee311 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/UnicodeProps.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/UnicodeProps.java
@@ -24,15 +24,15 @@ import org.apache.lucene.util.SparseFixedBitSet;
/**
* This file contains unicode properties used by various {@link CharTokenizer}s.
- * The data was created using ICU4J v56.1.0.0
+ * The data was created using ICU4J v59.1.0.0
* <p>
- * Unicode version: 8.0.0.0
+ * Unicode version: 9.0.0.0
*/
public final class UnicodeProps {
private UnicodeProps() {}
/** Unicode version that was used to generate this file: {@value} */
- public static final String UNICODE_VERSION = "8.0.0.0";
+ public static final String UNICODE_VERSION = "9.0.0.0";
/** Bitset with Unicode WHITESPACE code points. */
public static final Bits WHITESPACE = createBits(
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cd567b98/lucene/analysis/icu/src/data/utr30/DiacriticFolding.txt
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/data/utr30/DiacriticFolding.txt b/lucene/analysis/icu/src/data/utr30/DiacriticFolding.txt
index 3772daf..eb5b78e 100644
--- a/lucene/analysis/icu/src/data/utr30/DiacriticFolding.txt
+++ b/lucene/analysis/icu/src/data/utr30/DiacriticFolding.txt
@@ -168,11 +168,14 @@ FFE3>
1134D>
11366..1136C>
11370..11374>
+11442>
+11446>
114C2..114C3>
115BF..115C0>
1163F>
116B6..116B7>
1172B>
+11C3F>
16AF0..16AF4>
16F8F..16F9F>
1D167..1D169>
@@ -181,6 +184,8 @@ FFE3>
1D185..1D18B>
1D1AA..1D1AD>
1E8D0..1E8D6>
+1E944..1E946>
+1E948..1E94A>
# Latin script "composed" that do not further decompose, so decompose here
# These are from AsciiFoldingFilter
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cd567b98/lucene/analysis/icu/src/data/utr30/NativeDigitFolding.txt
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/data/utr30/NativeDigitFolding.txt b/lucene/analysis/icu/src/data/utr30/NativeDigitFolding.txt
index 62e6aef..fb8cf1a 100644
--- a/lucene/analysis/icu/src/data/utr30/NativeDigitFolding.txt
+++ b/lucene/analysis/icu/src/data/utr30/NativeDigitFolding.txt
@@ -510,6 +510,16 @@ ABF9>0039 # MEETEI MAYEK DIGIT NINE
112F7>0037 # KHUDAWADI DIGIT SEVEN
112F8>0038 # KHUDAWADI DIGIT EIGHT
112F9>0039 # KHUDAWADI DIGIT NINE
+11450>0030 # NEWA DIGIT ZERO
+11451>0031 # NEWA DIGIT ONE
+11452>0032 # NEWA DIGIT TWO
+11453>0033 # NEWA DIGIT THREE
+11454>0034 # NEWA DIGIT FOUR
+11455>0035 # NEWA DIGIT FIVE
+11456>0036 # NEWA DIGIT SIX
+11457>0037 # NEWA DIGIT SEVEN
+11458>0038 # NEWA DIGIT EIGHT
+11459>0039 # NEWA DIGIT NINE
114D0>0030 # TIRHUTA DIGIT ZERO
114D1>0031 # TIRHUTA DIGIT ONE
114D2>0032 # TIRHUTA DIGIT TWO
@@ -560,6 +570,16 @@ ABF9>0039 # MEETEI MAYEK DIGIT NINE
118E7>0037 # WARANG CITI DIGIT SEVEN
118E8>0038 # WARANG CITI DIGIT EIGHT
118E9>0039 # WARANG CITI DIGIT NINE
+11C50>0030 # BHAIKSUKI DIGIT ZERO
+11C51>0031 # BHAIKSUKI DIGIT ONE
+11C52>0032 # BHAIKSUKI DIGIT TWO
+11C53>0033 # BHAIKSUKI DIGIT THREE
+11C54>0034 # BHAIKSUKI DIGIT FOUR
+11C55>0035 # BHAIKSUKI DIGIT FIVE
+11C56>0036 # BHAIKSUKI DIGIT SIX
+11C57>0037 # BHAIKSUKI DIGIT SEVEN
+11C58>0038 # BHAIKSUKI DIGIT EIGHT
+11C59>0039 # BHAIKSUKI DIGIT NINE
16A60>0030 # MRO DIGIT ZERO
16A61>0031 # MRO DIGIT ONE
16A62>0032 # MRO DIGIT TWO
@@ -580,4 +600,14 @@ ABF9>0039 # MEETEI MAYEK DIGIT NINE
16B57>0037 # PAHAWH HMONG DIGIT SEVEN
16B58>0038 # PAHAWH HMONG DIGIT EIGHT
16B59>0039 # PAHAWH HMONG DIGIT NINE
+1E950>0030 # ADLAM DIGIT ZERO
+1E951>0031 # ADLAM DIGIT ONE
+1E952>0032 # ADLAM DIGIT TWO
+1E953>0033 # ADLAM DIGIT THREE
+1E954>0034 # ADLAM DIGIT FOUR
+1E955>0035 # ADLAM DIGIT FIVE
+1E956>0036 # ADLAM DIGIT SIX
+1E957>0037 # ADLAM DIGIT SEVEN
+1E958>0038 # ADLAM DIGIT EIGHT
+1E959>0039 # ADLAM DIGIT NINE
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cd567b98/lucene/analysis/icu/src/data/utr30/nfc.txt
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/data/utr30/nfc.txt b/lucene/analysis/icu/src/data/utr30/nfc.txt
index 5b7374f..5f9b182 100644
--- a/lucene/analysis/icu/src/data/utr30/nfc.txt
+++ b/lucene/analysis/icu/src/data/utr30/nfc.txt
@@ -1,4 +1,4 @@
-# Copyright (C) 1999-2014, International Business Machines
+# Copyright (C) 1999-2016, International Business Machines
# Corporation and others. All Rights Reserved.
#
# file name: nfc.txt
@@ -7,7 +7,7 @@
#
# Complete data for Unicode NFC normalization.
-* Unicode 7.0.0
+* Unicode 9.0.0
# Canonical_Combining_Class (ccc) values
0300..0314:230
@@ -129,6 +129,8 @@
0825..0827:230
0829..082D:230
0859..085B:220
+08D4..08E1:230
+08E3:220
08E4..08E5:230
08E6:220
08E7..08E8:230
@@ -232,6 +234,7 @@
1DCF:220
1DD0:202
1DD1..1DF5:230
+1DFB:230
1DFC:233
1DFD:220
1DFE:230
@@ -260,7 +263,7 @@
3099..309A:8
A66F:230
A674..A67D:230
-A69F:230
+A69E..A69F:230
A6F0..A6F1:230
A806:9
A8C4:9
@@ -280,6 +283,7 @@ ABED:9
FB1E:26
FE20..FE26:230
FE27..FE2D:220
+FE2E..FE2F:230
101FD:220
102E0:220
10376..1037A:230
@@ -299,6 +303,7 @@ FE27..FE2D:220
11133..11134:9
11173:7
111C0:9
+111CA:7
11235:9
11236:7
112E9:7
@@ -307,6 +312,8 @@ FE27..FE2D:220
1134D:9
11366..1136C:230
11370..11374:230
+11442:9
+11446:7
114C2:9
114C3:7
115BF:9
@@ -314,6 +321,8 @@ FE27..FE2D:220
1163F:9
116B6:9
116B7:7
+1172B:9
+11C3F:9
16AF0..16AF4:1
16B30..16B36:230
1BC9E:1
@@ -326,7 +335,14 @@ FE27..FE2D:220
1D18A..1D18B:220
1D1AA..1D1AD:230
1D242..1D244:230
+1E000..1E006:230
+1E008..1E018:230
+1E01B..1E021:230
+1E023..1E024:230
+1E026..1E02A:230
1E8D0..1E8D6:220
+1E944..1E949:230
+1E94A:7
# Canonical decomposition mappings
00C0>0041 0300 # one-way: diacritic 0300
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cd567b98/lucene/analysis/icu/src/data/utr30/nfkc.txt
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/data/utr30/nfkc.txt b/lucene/analysis/icu/src/data/utr30/nfkc.txt
index fea4129..f51fa5d 100644
--- a/lucene/analysis/icu/src/data/utr30/nfkc.txt
+++ b/lucene/analysis/icu/src/data/utr30/nfkc.txt
@@ -1,4 +1,4 @@
-# Copyright (C) 1999-2014, International Business Machines
+# Copyright (C) 1999-2016, International Business Machines
# Corporation and others. All Rights Reserved.
#
# file name: nfkc.txt
@@ -11,7 +11,7 @@
# to NFKC one-way mappings.
# Use this file as the second gennorm2 input file after nfc.txt.
-* Unicode 7.0.0
+* Unicode 9.0.0
00A0>0020
00A8>0020 0308
@@ -3675,6 +3675,7 @@ FFEE>25CB
1F238>7533
1F239>5272
1F23A>55B6
+1F23B>914D
1F240>3014 672C 3015
1F241>3014 4E09 3015
1F242>3014 4E8C 3015
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cd567b98/lucene/analysis/icu/src/data/utr30/nfkc_cf.txt
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/data/utr30/nfkc_cf.txt b/lucene/analysis/icu/src/data/utr30/nfkc_cf.txt
index b24b4b2..7f33df5 100644
--- a/lucene/analysis/icu/src/data/utr30/nfkc_cf.txt
+++ b/lucene/analysis/icu/src/data/utr30/nfkc_cf.txt
@@ -1,5 +1,5 @@
# Unicode Character Database
-# Copyright (c) 1991-2014 Unicode, Inc.
+# Copyright (c) 1991-2016 Unicode, Inc.
# For terms of use, see http://www.unicode.org/terms_of_use.html
# For documentation, see http://www.unicode.org/reports/tr44/
#
@@ -12,7 +12,7 @@
# and reformatted into syntax for the gennorm2 Normalizer2 data generator tool.
# Use this file as the third gennorm2 input file after nfc.txt and nfkc.txt.
-* Unicode 7.0.0
+* Unicode 9.0.0
0041>0061
0042>0062
@@ -632,8 +632,22 @@
10CD>2D2D
10FC>10DC
115F..1160>
+13F8>13F0
+13F9>13F1
+13FA>13F2
+13FB>13F3
+13FC>13F4
+13FD>13F5
17B4..17B5>
180B..180E>
+1C80>0432
+1C81>0434
+1C82>043E
+1C83>0441
+1C84..1C85>0442
+1C86>044A
+1C87>0463
+1C88>A64B
1D2C>0061
1D2D>00E6
1D2E>0062
@@ -2382,14 +2396,99 @@ A7AA>0266
A7AB>025C
A7AC>0261
A7AD>026C
+A7AE>026A
A7B0>029E
A7B1>0287
+A7B2>029D
+A7B3>AB53
+A7B4>A7B5
+A7B6>A7B7
A7F8>0127
A7F9>0153
AB5C>A727
AB5D>AB37
AB5E>026B
AB5F>AB52
+AB70>13A0
+AB71>13A1
+AB72>13A2
+AB73>13A3
+AB74>13A4
+AB75>13A5
+AB76>13A6
+AB77>13A7
+AB78>13A8
+AB79>13A9
+AB7A>13AA
+AB7B>13AB
+AB7C>13AC
+AB7D>13AD
+AB7E>13AE
+AB7F>13AF
+AB80>13B0
+AB81>13B1
+AB82>13B2
+AB83>13B3
+AB84>13B4
+AB85>13B5
+AB86>13B6
+AB87>13B7
+AB88>13B8
+AB89>13B9
+AB8A>13BA
+AB8B>13BB
+AB8C>13BC
+AB8D>13BD
+AB8E>13BE
+AB8F>13BF
+AB90>13C0
+AB91>13C1
+AB92>13C2
+AB93>13C3
+AB94>13C4
+AB95>13C5
+AB96>13C6
+AB97>13C7
+AB98>13C8
+AB99>13C9
+AB9A>13CA
+AB9B>13CB
+AB9C>13CC
+AB9D>13CD
+AB9E>13CE
+AB9F>13CF
+ABA0>13D0
+ABA1>13D1
+ABA2>13D2
+ABA3>13D3
+ABA4>13D4
+ABA5>13D5
+ABA6>13D6
+ABA7>13D7
+ABA8>13D8
+ABA9>13D9
+ABAA>13DA
+ABAB>13DB
+ABAC>13DC
+ABAD>13DD
+ABAE>13DE
+ABAF>13DF
+ABB0>13E0
+ABB1>13E1
+ABB2>13E2
+ABB3>13E3
+ABB4>13E4
+ABB5>13E5
+ABB6>13E6
+ABB7>13E7
+ABB8>13E8
+ABB9>13E9
+ABBA>13EA
+ABBB>13EB
+ABBC>13EC
+ABBD>13ED
+ABBE>13EE
+ABBF>13EF
F900>8C48
F901>66F4
F902>8ECA
@@ -3766,6 +3865,93 @@ FFF0..FFF8>
10425>1044D
10426>1044E
10427>1044F
+104B0>104D8
+104B1>104D9
+104B2>104DA
+104B3>104DB
+104B4>104DC
+104B5>104DD
+104B6>104DE
+104B7>104DF
+104B8>104E0
+104B9>104E1
+104BA>104E2
+104BB>104E3
+104BC>104E4
+104BD>104E5
+104BE>104E6
+104BF>104E7
+104C0>104E8
+104C1>104E9
+104C2>104EA
+104C3>104EB
+104C4>104EC
+104C5>104ED
+104C6>104EE
+104C7>104EF
+104C8>104F0
+104C9>104F1
+104CA>104F2
+104CB>104F3
+104CC>104F4
+104CD>104F5
+104CE>104F6
+104CF>104F7
+104D0>104F8
+104D1>104F9
+104D2>104FA
+104D3>104FB
+10C80>10CC0
+10C81>10CC1
+10C82>10CC2
+10C83>10CC3
+10C84>10CC4
+10C85>10CC5
+10C86>10CC6
+10C87>10CC7
+10C88>10CC8
+10C89>10CC9
+10C8A>10CCA
+10C8B>10CCB
+10C8C>10CCC
+10C8D>10CCD
+10C8E>10CCE
+10C8F>10CCF
+10C90>10CD0
+10C91>10CD1
+10C92>10CD2
+10C93>10CD3
+10C94>10CD4
+10C95>10CD5
+10C96>10CD6
+10C97>10CD7
+10C98>10CD8
+10C99>10CD9
+10C9A>10CDA
+10C9B>10CDB
+10C9C>10CDC
+10C9D>10CDD
+10C9E>10CDE
+10C9F>10CDF
+10CA0>10CE0
+10CA1>10CE1
+10CA2>10CE2
+10CA3>10CE3
+10CA4>10CE4
+10CA5>10CE5
+10CA6>10CE6
+10CA7>10CE7
+10CA8>10CE8
+10CA9>10CE9
+10CAA>10CEA
+10CAB>10CEB
+10CAC>10CEC
+10CAD>10CED
+10CAE>10CEE
+10CAF>10CEF
+10CB0>10CF0
+10CB1>10CF1
+10CB2>10CF2
118A0>118C0
118A1>118C1
118A2>118C2
@@ -4803,6 +4989,40 @@ FFF0..FFF8>
1D7FD>0037
1D7FE>0038
1D7FF>0039
+1E900>1E922
+1E901>1E923
+1E902>1E924
+1E903>1E925
+1E904>1E926
+1E905>1E927
+1E906>1E928
+1E907>1E929
+1E908>1E92A
+1E909>1E92B
+1E90A>1E92C
+1E90B>1E92D
+1E90C>1E92E
+1E90D>1E92F
+1E90E>1E930
+1E90F>1E931
+1E910>1E932
+1E911>1E933
+1E912>1E934
+1E913>1E935
+1E914>1E936
+1E915>1E937
+1E916>1E938
+1E917>1E939
+1E918>1E93A
+1E919>1E93B
+1E91A>1E93C
+1E91B>1E93D
+1E91C>1E93E
+1E91D>1E93F
+1E91E>1E940
+1E91F>1E941
+1E920>1E942
+1E921>1E943
1EE00>0627
1EE01>0628
1EE02>062C
@@ -5067,6 +5287,7 @@ FFF0..FFF8>
1F238>7533
1F239>5272
1F23A>55B6
+1F23B>914D
1F240>3014 672C 3015
1F241>3014 4E09 3015
1F242>3014 4E8C 3015
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cd567b98/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/Default.brk
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/Default.brk b/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/Default.brk
index 5b84797..c94a023 100644
Binary files a/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/Default.brk and b/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/Default.brk differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cd567b98/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/MyanmarSyllable.brk
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/MyanmarSyllable.brk b/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/MyanmarSyllable.brk
index 41b977b..c3357ef 100644
Binary files a/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/MyanmarSyllable.brk and b/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/MyanmarSyllable.brk differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cd567b98/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/utr30.nrm
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/utr30.nrm b/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/utr30.nrm
index 2680264..1a16f3e 100644
Binary files a/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/utr30.nrm and b/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/utr30.nrm differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cd567b98/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateUTR30DataFiles.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateUTR30DataFiles.java b/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateUTR30DataFiles.java
index 035a3a0..0f2bffe 100644
--- a/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateUTR30DataFiles.java
+++ b/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateUTR30DataFiles.java
@@ -63,7 +63,7 @@ import java.util.regex.Pattern;
public class GenerateUTR30DataFiles {
private static final String ICU_SVN_TAG_URL
= "http://source.icu-project.org/repos/icu/icu/tags";
- private static final String ICU_RELEASE_TAG = "release-54-1";
+ private static final String ICU_RELEASE_TAG = "release-58-1";
private static final String ICU_DATA_NORM2_PATH = "source/data/unidata/norm2";
private static final String NFC_TXT = "nfc.txt";
private static final String NFKC_TXT = "nfkc.txt";
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cd567b98/lucene/ivy-versions.properties
----------------------------------------------------------------------
diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties
index 7e0e7c7..3c45b2c 100644
--- a/lucene/ivy-versions.properties
+++ b/lucene/ivy-versions.properties
@@ -29,7 +29,7 @@ com.fasterxml.jackson.core.version = 2.5.4
/com.googlecode.juniversalchardet/juniversalchardet = 1.0.3
/com.googlecode.mp4parser/isoparser = 1.1.18
/com.healthmarketscience.jackcess/jackcess = 2.1.3
-/com.ibm.icu/icu4j = 56.1
+/com.ibm.icu/icu4j = 59.1
/com.pff/java-libpst = 0.8.1
com.sun.jersey.version = 1.9
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cd567b98/lucene/licenses/icu4j-56.1.jar.sha1
----------------------------------------------------------------------
diff --git a/lucene/licenses/icu4j-56.1.jar.sha1 b/lucene/licenses/icu4j-56.1.jar.sha1
deleted file mode 100644
index 5f8e046..0000000
--- a/lucene/licenses/icu4j-56.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8dd6671f52165a0419e6de5e1016400875a90fa9
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cd567b98/lucene/licenses/icu4j-59.1.jar.sha1
----------------------------------------------------------------------
diff --git a/lucene/licenses/icu4j-59.1.jar.sha1 b/lucene/licenses/icu4j-59.1.jar.sha1
new file mode 100644
index 0000000..f3f0018
--- /dev/null
+++ b/lucene/licenses/icu4j-59.1.jar.sha1
@@ -0,0 +1 @@
+6f06e820cf4c8968bbbaae66ae0b33f6a256b57f
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cd567b98/solr/licenses/icu4j-56.1.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/icu4j-56.1.jar.sha1 b/solr/licenses/icu4j-56.1.jar.sha1
deleted file mode 100644
index 5f8e046..0000000
--- a/solr/licenses/icu4j-56.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8dd6671f52165a0419e6de5e1016400875a90fa9
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cd567b98/solr/licenses/icu4j-59.1.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/icu4j-59.1.jar.sha1 b/solr/licenses/icu4j-59.1.jar.sha1
new file mode 100644
index 0000000..f3f0018
--- /dev/null
+++ b/solr/licenses/icu4j-59.1.jar.sha1
@@ -0,0 +1 @@
+6f06e820cf4c8968bbbaae66ae0b33f6a256b57f
[19/44] lucene-solr:jira/solr-8668: SOLR-10634: calc metrics in first
phase if limit=-1 and no subfacets
Posted by cp...@apache.org.
SOLR-10634: calc metrics in first phase if limit=-1 and no subfacets
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d60c72f3
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d60c72f3
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d60c72f3
Branch: refs/heads/jira/solr-8668
Commit: d60c72f34ca9c63ac6075e00dac844c6f052d0a8
Parents: ea9adc0
Author: yonik <yo...@apache.org>
Authored: Tue May 23 20:52:46 2017 -0400
Committer: yonik <yo...@apache.org>
Committed: Tue May 23 20:52:46 2017 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 8 +
.../solr/search/facet/FacetFieldProcessor.java | 81 +++++++++-
.../solr/collection1/conf/solrconfig-tlog.xml | 8 +
.../org/apache/solr/search/facet/DebugAgg.java | 146 +++++++++++++++++++
.../solr/search/facet/TestJsonFacets.java | 134 +++++++++++++++--
5 files changed, 366 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d60c72f3/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 928a190..5b92e3c 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -231,6 +231,14 @@ Bug Fixes
resulting in exceptions when using a hashing faceting method and sorting by hll(numeric_field).
(yonik)
+Optimizations
+----------------------
+* SOLR-10634: JSON Facet API: When a field/terms facet will retrieve all buckets (i.e. limit:-1)
+ and there are no nested facets, aggregations are computed in the first collection phase
+ so that the second phase which would normally involve calculating the domain for the bucket
+ can be skipped entirely, leading to large performance improvements. (yonik)
+
+
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d60c72f3/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java
index 143d1fc..7b08e14 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java
@@ -129,7 +129,36 @@ abstract class FacetFieldProcessor extends FacetProcessor<FacetField> {
}
sortAcc = indexOrderAcc;
deferredAggs = freq.getFacetStats();
- } else {
+ }
+
+ // If we are going to return all buckets and if there are no subfacets (that would need a domain), then don't defer
+ // any aggregation calculations to a second phase. This way we can avoid calculating domains for each bucket, which
+ // can be expensive.
+ if (freq.limit == -1 && freq.subFacets.size() == 0) {
+ accs = new SlotAcc[ freq.getFacetStats().size() ];
+ int otherAccIdx = 0;
+ for (Map.Entry<String,AggValueSource> entry : freq.getFacetStats().entrySet()) {
+ AggValueSource agg = entry.getValue();
+ SlotAcc acc = agg.createSlotAcc(fcontext, numDocs, numSlots);
+ acc.key = entry.getKey();
+ accMap.put(acc.key, acc);
+ accs[otherAccIdx++] = acc;
+ }
+ if (accs.length == 1) {
+ collectAcc = accs[0];
+ } else {
+ collectAcc = new MultiAcc(fcontext, accs);
+ }
+
+ if (sortAcc == null) {
+ sortAcc = accMap.get(freq.sortVariable);
+ assert sortAcc != null;
+ }
+
+ deferredAggs = null;
+ }
+
+ if (sortAcc == null) {
AggValueSource sortAgg = freq.getFacetStats().get(freq.sortVariable);
if (sortAgg != null) {
collectAcc = sortAgg.createSlotAcc(fcontext, numDocs, numSlots);
@@ -140,7 +169,7 @@ abstract class FacetFieldProcessor extends FacetProcessor<FacetField> {
deferredAggs.remove(freq.sortVariable);
}
- if (deferredAggs.size() == 0) {
+ if (deferredAggs == null || deferredAggs.size() == 0) {
deferredAggs = null;
}
@@ -438,6 +467,54 @@ abstract class FacetFieldProcessor extends FacetProcessor<FacetField> {
}
}
+ static class MultiAcc extends SlotAcc {
+ final SlotAcc[] subAccs;
+
+ MultiAcc(FacetContext fcontext, SlotAcc[] subAccs) {
+ super(fcontext);
+ this.subAccs = subAccs;
+ }
+
+ @Override
+ public void collect(int doc, int slot) throws IOException {
+ for (SlotAcc acc : subAccs) {
+ acc.collect(doc, slot);
+ }
+ }
+
+ @Override
+ public int compare(int slotA, int slotB) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object getValue(int slotNum) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void reset() throws IOException {
+ for (SlotAcc acc : subAccs) {
+ acc.reset();
+ }
+ }
+
+ @Override
+ public void resize(Resizer resizer) {
+ for (SlotAcc acc : subAccs) {
+ acc.resize(resizer);
+ }
+ }
+
+ @Override
+ public void setValues(SimpleOrderedMap<Object> bucket, int slotNum) throws IOException {
+ for (SlotAcc acc : subAccs) {
+ acc.setValues(bucket, slotNum);
+ }
+ }
+ }
+
+
static class SpecialSlotAcc extends SlotAcc {
SlotAcc collectAcc;
SlotAcc[] otherAccs;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d60c72f3/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
index b08af83..cf12c4a 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
@@ -164,4 +164,12 @@
</lst>
</initParams>
+
+ <valueSourceParser name="nvl" class="org.apache.solr.search.function.NvlValueSourceParser">
+ <float name="nvlFloatValue">0.0</float>
+ </valueSourceParser>
+ <valueSourceParser name="agg_debug" class="org.apache.solr.search.facet.DebugAgg$Parser">
+ </valueSourceParser>
+
+
</config>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d60c72f3/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java b/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java
new file mode 100644
index 0000000..ad4fabf
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search.facet;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.queries.function.ValueSource;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.search.DocSet;
+import org.apache.solr.search.FunctionQParser;
+import org.apache.solr.search.SyntaxError;
+import org.apache.solr.search.ValueSourceParser;
+
+
+public class DebugAgg extends AggValueSource {
+
+ public static class Parser extends ValueSourceParser {
+ @Override
+ public ValueSource parse(FunctionQParser fp) throws SyntaxError {
+ return new DebugAgg();
+ }
+
+ @Override
+ public void init(NamedList args) {
+ }
+ }
+
+
+ public DebugAgg() {
+ super("debug");
+ }
+
+ public DebugAgg(String name) {
+ super(name);
+ }
+
+ @Override
+ public SlotAcc createSlotAcc(FacetContext fcontext, int numDocs, int numSlots) {
+ return new Acc(fcontext, numDocs, numSlots);
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ @Override
+ public String description() {
+ return "debug()";
+ }
+
+ public static class Acc extends SlotAcc {
+ public static AtomicLong creates = new AtomicLong(0);
+ public static AtomicLong resets = new AtomicLong(0);
+ public static AtomicLong resizes = new AtomicLong(0);
+ public static Acc last;
+
+ public CountSlotAcc sub;
+ public int numDocs;
+ public int numSlots;
+
+ public Acc(FacetContext fcontext, int numDocs, int numSlots) {
+ super(fcontext);
+ this.last = this;
+ this.numDocs = numDocs;
+ this.numSlots = numSlots;
+ creates.addAndGet(1);
+ sub = new CountSlotArrAcc(fcontext, numSlots);
+
+ new RuntimeException("DEBUG Acc numSlots=" + numSlots).printStackTrace();
+ }
+
+ @Override
+ public void collect(int doc, int slot) throws IOException {
+ sub.collect(doc, slot);
+ }
+
+ @Override
+ public int compare(int slotA, int slotB) {
+ return sub.compare(slotA, slotB);
+ }
+
+ @Override
+ public Object getValue(int slotNum) throws IOException {
+ return sub.getValue(slotNum);
+ }
+
+ @Override
+ public void reset() throws IOException {
+ resets.addAndGet(1);
+ sub.reset();
+ }
+
+ @Override
+ public void resize(Resizer resizer) {
+ resizes.addAndGet(1);
+ this.numSlots = resizer.getNewSize();
+ sub.resize(resizer);
+ }
+
+ @Override
+ public void setNextReader(LeafReaderContext readerContext) throws IOException {
+ sub.setNextReader(readerContext);
+ }
+
+ @Override
+ public int collect(DocSet docs, int slot) throws IOException {
+ return sub.collect(docs, slot);
+ }
+
+ @Override
+ public void setValues(SimpleOrderedMap<Object> bucket, int slotNum) throws IOException {
+ sub.key = this.key; // TODO: Blech... this should be fixed
+ sub.setValues(bucket, slotNum);
+ }
+
+ @Override
+ public void close() throws IOException {
+ sub.close();
+ }
+ }
+
+ @Override
+ public FacetMerger createFacetMerger(Object prototype) {
+ return new FacetLongMerger();
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d60c72f3/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
index 3590426..03cc480 100644
--- a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
+++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
@@ -471,7 +471,7 @@ public class TestJsonFacets extends SolrTestCaseHS {
doStatsTemplated(client, params(p, "facet","true", "rows","0", "noexist","noexist_ss", "cat_s","cat_ss", "where_s","where_ss", "num_d","num_f", "num_i","num_l", "num_is","num_ls", "num_fs", "num_ds", "super_s","super_ss", "val_b","val_b", "date","date_dt", "sparse_s","sparse_ss", "multi_ss","multi_ss") );
// multi-valued strings, method=dv for terms facets
- doStatsTemplated(client, params(p, "terms", "method:dv,", "rows", "0", "noexist", "noexist_ss", "cat_s", "cat_ss", "where_s", "where_ss", "num_d", "num_f", "num_i", "num_l", "super_s", "super_ss", "val_b", "val_b", "date", "date_dt", "sparse_s", "sparse_ss", "multi_ss", "multi_ss"));
+ doStatsTemplated(client, params(p, "terms_method", "method:dv,", "rows", "0", "noexist", "noexist_ss", "cat_s", "cat_ss", "where_s", "where_ss", "num_d", "num_f", "num_i", "num_l", "super_s", "super_ss", "val_b", "val_b", "date", "date_dt", "sparse_s", "sparse_ss", "multi_ss", "multi_ss"));
// single valued docvalues for strings, and single valued numeric doc values for numeric fields
doStatsTemplated(client, params(p, "rows","0", "noexist","noexist_sd", "cat_s","cat_sd", "where_s","where_sd", "num_d","num_dd", "num_i","num_id", "num_is","num_lds", "num_fs","num_dds", "super_s","super_sd", "val_b","val_b", "date","date_dtd", "sparse_s","sparse_sd" ,"multi_ss","multi_sds") );
@@ -491,6 +491,26 @@ public class TestJsonFacets extends SolrTestCaseHS {
if (p.get("num_is") == null) p.add("num_is","num_is");
if (p.get("num_fs") == null) p.add("num_fs","num_fs");
+ String terms = p.get("terms");
+ if (terms == null) terms="";
+ int limit=0;
+ switch (random().nextInt(4)) {
+ case 0: limit=-1;
+ case 1: limit=1000000;
+ case 2: // fallthrough
+ case 3: // fallthrough
+ }
+ if (limit != 0) {
+ terms=terms+"limit:"+limit+",";
+ }
+ String terms_method = p.get("terms_method");
+ if (terms_method != null) {
+ terms=terms+terms_method;
+ }
+ p.set("terms", terms);
+ // "${terms}" should be put at the beginning of generic terms facets.
+ // It may specify "method=..." or "limit:-1", so should not be used if the facet explicitly specifies.
+
MacroExpander m = new MacroExpander( p.getMap() );
String cat_s = m.expand("${cat_s}");
@@ -862,7 +882,7 @@ public class TestJsonFacets extends SolrTestCaseHS {
// test numBuckets
client.testJQ(params(p, "q", "*:*", "rows", "0", "facet", "true"
- , "json.facet", "{f1:{terms:{${terms} field:${cat_s}, numBuckets:true, limit:1}}}" // TODO: limit:0 produced an error
+ , "json.facet", "{f1:{terms:{${terms_method} field:${cat_s}, numBuckets:true, limit:1}}}" // TODO: limit:0 produced an error
)
, "facets=={ 'count':6, " +
"'f1':{ numBuckets:2, buckets:[{val:B, count:3}]} } "
@@ -1048,10 +1068,10 @@ public class TestJsonFacets extends SolrTestCaseHS {
// also test limit:0
client.testJQ(params(p, "q", "*:*"
, "json.facet", "{" +
- " f0:{${terms} type:terms, field:${multi_ss}, allBuckets:true, limit:0} " +
- ",f1:{${terms} type:terms, field:${multi_ss}, allBuckets:true, limit:0, offset:1} " + // offset with 0 limit
- ",f2:{${terms} type:terms, field:${multi_ss}, allBuckets:true, limit:0, facet:{x:'sum(${num_d})'}, sort:'x desc' } " +
- ",f3:{${terms} type:terms, field:${multi_ss}, allBuckets:true, limit:0, missing:true, facet:{x:'sum(${num_d})', y:'avg(${num_d})'}, sort:'x desc' } " +
+ " f0:{${terms_method} type:terms, field:${multi_ss}, allBuckets:true, limit:0} " +
+ ",f1:{${terms_method} type:terms, field:${multi_ss}, allBuckets:true, limit:0, offset:1} " + // offset with 0 limit
+ ",f2:{${terms_method} type:terms, field:${multi_ss}, allBuckets:true, limit:0, facet:{x:'sum(${num_d})'}, sort:'x desc' } " +
+ ",f3:{${terms_method} type:terms, field:${multi_ss}, allBuckets:true, limit:0, missing:true, facet:{x:'sum(${num_d})', y:'avg(${num_d})'}, sort:'x desc' } " +
"}"
)
, "facets=={ 'count':6, " +
@@ -1066,9 +1086,9 @@ public class TestJsonFacets extends SolrTestCaseHS {
// also test limit:0
client.testJQ(params(p, "q", "*:*"
, "json.facet", "{" +
- " f0:{${terms} type:terms, field:${num_i}, allBuckets:true, limit:0} " +
- ",f1:{${terms} type:terms, field:${num_i}, allBuckets:true, limit:0, offset:1} " + // offset with 0 limit
- ",f2:{${terms} type:terms, field:${num_i}, allBuckets:true, limit:0, facet:{x:'sum(${num_d})'}, sort:'x desc' } " +
+ " f0:{${terms_method} type:terms, field:${num_i}, allBuckets:true, limit:0} " +
+ ",f1:{${terms_method} type:terms, field:${num_i}, allBuckets:true, limit:0, offset:1} " + // offset with 0 limit
+ ",f2:{${terms_method} type:terms, field:${num_i}, allBuckets:true, limit:0, facet:{x:'sum(${num_d})'}, sort:'x desc' } " +
"}"
)
, "facets=={ 'count':6, " +
@@ -1398,6 +1418,102 @@ public class TestJsonFacets extends SolrTestCaseHS {
}
+
+ ////////////////////////////////////////////////////////////////
+ // test which phase stats are calculated in
+ ////////////////////////////////////////////////////////////////
+ if (client.local()) {
+ long creates, resets;
+ // NOTE: these test the current implementation and may need to be adjusted to match future optimizations (such as calculating N buckets in parallel in the second phase)
+
+ creates = DebugAgg.Acc.creates.get();
+ resets = DebugAgg.Acc.resets.get();
+ client.testJQ(params(p, "q", "*:*"
+ , "json.facet", "{f1:{terms:{${terms_method} field:${super_s}, limit:1, facet:{x:'debug()'} }}}" // x should be deferred to 2nd phase
+ )
+ , "facets=={ 'count':6, " +
+ "f1:{ buckets:[{ val:batman, count:1, x:1}]} } "
+ );
+
+ assertEquals(1, DebugAgg.Acc.creates.get() - creates);
+ assertTrue( DebugAgg.Acc.resets.get() - resets <= 1);
+ assertTrue( DebugAgg.Acc.last.numSlots <= 2 ); // probably "1", but may be special slot for something. As long as it's not cardinality of the field
+
+
+ creates = DebugAgg.Acc.creates.get();
+ resets = DebugAgg.Acc.resets.get();
+ client.testJQ(params(p, "q", "*:*"
+ , "json.facet", "{f1:{terms:{${terms_method} field:${super_s}, limit:1, facet:{ x:'debug()'} , sort:'x asc' }}}" // sorting by x... must be done all at once in first phase
+ )
+ , "facets=={ 'count':6, " +
+ "f1:{ buckets:[{ val:batman, count:1, x:1}]}" +
+ " } "
+ );
+
+ assertEquals(1, DebugAgg.Acc.creates.get() - creates);
+ assertTrue( DebugAgg.Acc.resets.get() - resets == 0);
+ assertTrue( DebugAgg.Acc.last.numSlots >= 5 ); // all slots should be done in a single shot. there may be more than 5 due to special slots or hashing.
+
+
+ // When limit:-1, we should do most stats in first phase (SOLR-10634)
+ creates = DebugAgg.Acc.creates.get();
+ resets = DebugAgg.Acc.resets.get();
+ client.testJQ(params(p, "q", "*:*"
+ , "json.facet", "{f1:{terms:{${terms_method} field:${super_s}, limit:-1, facet:{x:'debug()'} }}}"
+ )
+ , "facets=="
+ );
+
+ assertEquals(1, DebugAgg.Acc.creates.get() - creates);
+ assertTrue( DebugAgg.Acc.resets.get() - resets == 0);
+ assertTrue( DebugAgg.Acc.last.numSlots >= 5 ); // all slots should be done in a single shot. there may be more than 5 due to special slots or hashing.
+
+ // Now for a numeric field
+ // When limit:-1, we should do most stats in first phase (SOLR-10634)
+ creates = DebugAgg.Acc.creates.get();
+ resets = DebugAgg.Acc.resets.get();
+ client.testJQ(params(p, "q", "*:*"
+ , "json.facet", "{f1:{terms:{${terms_method} field:${num_d}, limit:-1, facet:{x:'debug()'} }}}"
+ )
+ , "facets=="
+ );
+
+ assertEquals(1, DebugAgg.Acc.creates.get() - creates);
+ assertTrue( DebugAgg.Acc.resets.get() - resets == 0);
+ assertTrue( DebugAgg.Acc.last.numSlots >= 5 ); // all slots should be done in a single shot. there may be more than 5 due to special slots or hashing.
+
+
+ // But if we need to calculate domains anyway, it probably makes sense to calculate most stats in the 2nd phase (along with sub-facets)
+ creates = DebugAgg.Acc.creates.get();
+ resets = DebugAgg.Acc.resets.get();
+ client.testJQ(params(p, "q", "*:*"
+ , "json.facet", "{f1:{terms:{${terms_method} field:${super_s}, limit:-1, facet:{ x:'debug()' , y:{terms:${where_s}} } }}}"
+ )
+ , "facets=="
+ );
+
+ assertEquals(1, DebugAgg.Acc.creates.get() - creates);
+ assertTrue( DebugAgg.Acc.resets.get() - resets >=4);
+ assertTrue( DebugAgg.Acc.last.numSlots <= 2 ); // probably 1, but could be higher
+
+ // Now with a numeric field
+ // But if we need to calculate domains anyway, it probably makes sense to calculate most stats in the 2nd phase (along with sub-facets)
+ creates = DebugAgg.Acc.creates.get();
+ resets = DebugAgg.Acc.resets.get();
+ client.testJQ(params(p, "q", "*:*"
+ , "json.facet", "{f1:{terms:{${terms_method} field:${num_d}, limit:-1, facet:{ x:'debug()' , y:{terms:${where_s}} } }}}"
+ )
+ , "facets=="
+ );
+
+ assertEquals(1, DebugAgg.Acc.creates.get() - creates);
+ assertTrue( DebugAgg.Acc.resets.get() - resets >=4);
+ assertTrue( DebugAgg.Acc.last.numSlots <= 2 ); // probably 1, but could be higher
+ }
+ //////////////////////////////////////////////////////////////// end phase testing
+
+
+
}
@Test
[28/44] lucene-solr:jira/solr-8668: LUCENE-7849:
GeoWideLongitudeSlice can fail to construct
Posted by cp...@apache.org.
LUCENE-7849: GeoWideLongitudeSlice can fail to construct
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/fed7343d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/fed7343d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/fed7343d
Branch: refs/heads/jira/solr-8668
Commit: fed7343d2ea8136126084517ade408faaff279f2
Parents: 851ab0a
Author: Karl Wright <Da...@gmail.com>
Authored: Wed May 24 09:46:15 2017 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Wed May 24 09:46:15 2017 -0400
----------------------------------------------------------------------
.../apache/lucene/spatial3d/geom/GeoWideLongitudeSlice.java | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fed7343d/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideLongitudeSlice.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideLongitudeSlice.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideLongitudeSlice.java
index f208266..705d961 100755
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideLongitudeSlice.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoWideLongitudeSlice.java
@@ -75,7 +75,13 @@ class GeoWideLongitudeSlice extends GeoBaseBBox {
while (leftLon > rightLon) {
rightLon += Math.PI * 2.0;
}
- final double middleLon = (leftLon + rightLon) * 0.5;
+ double middleLon = (leftLon + rightLon) * 0.5;
+ while (middleLon > Math.PI) {
+ middleLon -= Math.PI * 2.0;
+ }
+ while (middleLon < -Math.PI) {
+ middleLon += Math.PI * 2.0;
+ }
this.centerPoint = new GeoPoint(planetModel, 0.0, middleLon);
this.leftPlane = new SidedPlane(centerPoint, cosLeftLon, sinLeftLon);
[04/44] lucene-solr:jira/solr-8668: SOLR-10700: Convert
PostingsSolrHighlighter to extend UnifiedSolrHighlighter
Posted by cp...@apache.org.
SOLR-10700: Convert PostingsSolrHighlighter to extend UnifiedSolrHighlighter
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2218ded2
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2218ded2
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2218ded2
Branch: refs/heads/jira/solr-8668
Commit: 2218ded2afd0231005d81bd16c6b0c114ef32546
Parents: 715d8b0
Author: David Smiley <ds...@apache.org>
Authored: Tue May 23 11:02:52 2017 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Tue May 23 11:02:52 2017 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 6 +
.../solr/highlight/PostingsSolrHighlighter.java | 326 +++----------------
.../apispec/core.SchemaEdit.addField.json | 8 +-
solr/solr-ref-guide/src/highlighting.adoc | 32 +-
4 files changed, 55 insertions(+), 317 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2218ded2/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 7a604f4..e29952b 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -76,6 +76,9 @@ Upgrading from Solr 6.x
* Setting <defaultSearchField> in schema is no longer allowed and will cause an exception.
Please use "df" parameter on the request instead. For more details, see SOLR-10585.
+* The PostingsSolrHighlighter is deprecated. Furthermore, it now internally works via a re-configuration
+ of the UnifiedSolrHighlighter.
+
New Features
----------------------
* SOLR-9857, SOLR-9858: Collect aggregated metrics from nodes and shard leaders in overseer. (ab)
@@ -185,6 +188,9 @@ Other Changes
* SOLR-10378: Clicking Solr logo on AdminUI shows blank page (Takumi Yoshida via janhoy)
+* SOLR-10700: Deprecated and converted the PostingsSolrHighlighter to extend UnifiedSolrHighlighter and thus no
+ longer use the PostingsHighlighter. It should behave mostly the same. (David Smiley)
+
================== 6.7.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2218ded2/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java b/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java
index 5ea3db1..2e1c483 100644
--- a/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java
+++ b/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java
@@ -16,306 +16,56 @@
*/
package org.apache.solr.highlight;
-import java.io.IOException;
-import java.text.BreakIterator;
-import java.util.Collections;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
+import java.lang.invoke.MethodHandles;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.postingshighlight.CustomSeparatorBreakIterator;
-import org.apache.lucene.search.postingshighlight.DefaultPassageFormatter;
-import org.apache.lucene.search.postingshighlight.Passage;
-import org.apache.lucene.search.postingshighlight.PassageFormatter;
-import org.apache.lucene.search.postingshighlight.PassageScorer;
-import org.apache.lucene.search.postingshighlight.PostingsHighlighter;
-import org.apache.lucene.search.postingshighlight.WholeBreakIterator;
-import org.apache.solr.common.SolrException;
+import org.apache.lucene.search.uhighlight.UnifiedHighlighter;
import org.apache.solr.common.params.HighlightParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.PluginInfo;
+import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.schema.IndexSchema;
-import org.apache.solr.schema.SchemaField;
-import org.apache.solr.search.DocIterator;
-import org.apache.solr.search.DocList;
-import org.apache.solr.search.SolrIndexSearcher;
-import org.apache.solr.util.plugin.PluginInfoInitialized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
- * Highlighter impl that uses {@link PostingsHighlighter}
- * <p>
- * Example configuration:
- * <pre class="prettyprint">
- * <requestHandler name="/select" class="solr.SearchHandler">
- * <lst name="defaults">
- * <str name="hl.method">postings</str>
- * <int name="hl.snippets">1</int>
- * <str name="hl.tag.pre">&lt;em&gt;</str>
- * <str name="hl.tag.post">&lt;/em&gt;</str>
- * <str name="hl.tag.ellipsis">... </str>
- * <bool name="hl.defaultSummary">true</bool>
- * <str name="hl.encoder">simple</str>
- * <float name="hl.score.k1">1.2</float>
- * <float name="hl.score.b">0.75</float>
- * <float name="hl.score.pivot">87</float>
- * <str name="hl.bs.language"></str>
- * <str name="hl.bs.country"></str>
- * <str name="hl.bs.variant"></str>
- * <str name="hl.bs.type">SENTENCE</str>
- * <int name="hl.maxAnalyzedChars">51200</int>
- * <str name="hl.multiValuedSeparatorChar"> </str>
- * <bool name="hl.highlightMultiTerm">false</bool>
- * </lst>
- * </requestHandler>
- * </pre>
- * <p>
- * Notes:
- * <ul>
- * <li>fields to highlight must be configured with storeOffsetsWithPositions="true"
- * <li>hl.q (string) can specify the query
- * <li>hl.fl (string) specifies the field list.
- * <li>hl.snippets (int) specifies how many underlying passages form the resulting snippet.
- * <li>hl.tag.pre (string) specifies text which appears before a highlighted term.
- * <li>hl.tag.post (string) specifies text which appears after a highlighted term.
- * <li>hl.tag.ellipsis (string) specifies text which joins non-adjacent passages.
- * <li>hl.defaultSummary (bool) specifies if a field should have a default summary.
- * <li>hl.encoder (string) can be 'html' (html escapes content) or 'simple' (no escaping).
- * <li>hl.score.k1 (float) specifies bm25 scoring parameter 'k1'
- * <li>hl.score.b (float) specifies bm25 scoring parameter 'b'
- * <li>hl.score.pivot (float) specifies bm25 scoring parameter 'avgdl'
- * <li>hl.bs.type (string) specifies how to divide text into passages: [SENTENCE, LINE, WORD, CHAR, WHOLE]
- * <li>hl.bs.language (string) specifies language code for BreakIterator. default is empty string (root locale)
- * <li>hl.bs.country (string) specifies country code for BreakIterator. default is empty string (root locale)
- * <li>hl.bs.variant (string) specifies country code for BreakIterator. default is empty string (root locale)
- * <li>hl.maxAnalyzedChars specifies how many characters at most will be processed in a document.
- * <li>hl.multiValuedSeparatorChar specifies the logical separator between values for multi-valued fields.
- * <li>hl.highlightMultiTerm enables highlighting for range/wildcard/fuzzy/prefix queries.
- * NOTE: currently hl.maxAnalyzedChars cannot yet be specified per-field
- * </ul>
- *
- * @lucene.experimental
+ * Highlighter impl that uses {@link UnifiedHighlighter} configured to operate as it's ancestor/predecessor, the
+ * {code PostingsHighlighter}.
+ *
+ * @deprecated Use {@link UnifiedSolrHighlighter} instead
*/
-public class PostingsSolrHighlighter extends SolrHighlighter implements PluginInfoInitialized {
-
- @Override
- public void init(PluginInfo info) {}
+@Deprecated
+public class PostingsSolrHighlighter extends UnifiedSolrHighlighter {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@Override
- public NamedList<Object> doHighlighting(DocList docs, Query query, SolrQueryRequest req, String[] defaultFields) throws IOException {
- final SolrParams params = req.getParams();
-
- // if highlighting isnt enabled, then why call doHighlighting?
- if (!isHighlightingEnabled(params))
- return null;
-
- SolrIndexSearcher searcher = req.getSearcher();
- int[] docIDs = toDocIDs(docs);
-
- // fetch the unique keys
- String[] keys = getUniqueKeys(searcher, docIDs);
-
- // query-time parameters
- String[] fieldNames = getHighlightFields(query, req, defaultFields);
-
- int maxPassages[] = new int[fieldNames.length];
- for (int i = 0; i < fieldNames.length; i++) {
- maxPassages[i] = params.getFieldInt(fieldNames[i], HighlightParams.SNIPPETS, 1);
- }
-
- PostingsHighlighter highlighter = getHighlighter(req);
- Map<String,String[]> snippets = highlighter.highlightFields(fieldNames, query, searcher, docIDs, maxPassages);
- return encodeSnippets(keys, fieldNames, snippets);
- }
-
- /** Creates an instance of the Lucene PostingsHighlighter. Provided for subclass extension so that
- * a subclass can return a subclass of {@link PostingsSolrHighlighter.SolrExtendedPostingsHighlighter}. */
- protected PostingsHighlighter getHighlighter(SolrQueryRequest req) {
- return new SolrExtendedPostingsHighlighter(req);
+ public void init(PluginInfo info) {
+ log.warn("The PostingsSolrHighlighter is deprecated; use the UnifiedSolrHighlighter instead.");
+ super.init(info);
}
- /**
- * Encodes the resulting snippets into a namedlist
- * @param keys the document unique keys
- * @param fieldNames field names to highlight in the order
- * @param snippets map from field name to snippet array for the docs
- * @return encoded namedlist of summaries
- */
- protected NamedList<Object> encodeSnippets(String[] keys, String[] fieldNames, Map<String,String[]> snippets) {
- NamedList<Object> list = new SimpleOrderedMap<>();
- for (int i = 0; i < keys.length; i++) {
- NamedList<Object> summary = new SimpleOrderedMap<>();
- for (String field : fieldNames) {
- String snippet = snippets.get(field)[i];
- // box in an array to match the format of existing highlighters,
- // even though it's always one element.
- if (snippet == null) {
- summary.add(field, new String[0]);
- } else {
- summary.add(field, new String[] { snippet });
- }
- }
- list.add(keys[i], summary);
- }
- return list;
- }
-
- /** Converts solr's DocList to the int[] docIDs */
- protected int[] toDocIDs(DocList docs) {
- int[] docIDs = new int[docs.size()];
- DocIterator iterator = docs.iterator();
- for (int i = 0; i < docIDs.length; i++) {
- if (!iterator.hasNext()) {
- throw new AssertionError();
- }
- docIDs[i] = iterator.nextDoc();
- }
- if (iterator.hasNext()) {
- throw new AssertionError();
- }
- return docIDs;
- }
-
- /** Retrieves the unique keys for the topdocs to key the results */
- protected String[] getUniqueKeys(SolrIndexSearcher searcher, int[] docIDs) throws IOException {
- IndexSchema schema = searcher.getSchema();
- SchemaField keyField = schema.getUniqueKeyField();
- if (keyField != null) {
- Set<String> selector = Collections.singleton(keyField.getName());
- String uniqueKeys[] = new String[docIDs.length];
- for (int i = 0; i < docIDs.length; i++) {
- int docid = docIDs[i];
- Document doc = searcher.doc(docid, selector);
- String id = schema.printableUniqueKey(doc);
- uniqueKeys[i] = id;
- }
- return uniqueKeys;
- } else {
- return new String[docIDs.length];
- }
- }
-
- /** From {@link #getHighlighter(org.apache.solr.request.SolrQueryRequest)}. */
- public class SolrExtendedPostingsHighlighter extends PostingsHighlighter {
- protected final SolrParams params;
- protected final IndexSchema schema;
-
- public SolrExtendedPostingsHighlighter(SolrQueryRequest req) {
- super(req.getParams().getInt(HighlightParams.MAX_CHARS, DEFAULT_MAX_CHARS));
- this.params = req.getParams();
- this.schema = req.getSchema();
- }
-
- @Override
- protected Passage[] getEmptyHighlight(String fieldName, BreakIterator bi, int maxPassages) {
- boolean defaultSummary = params.getFieldBool(fieldName, HighlightParams.DEFAULT_SUMMARY, true);
- if (defaultSummary) {
- return super.getEmptyHighlight(fieldName, bi, maxPassages);
- } else {
- //TODO reuse logic of DefaultSolrHighlighter.alternateField
- return new Passage[0];
- }
- }
-
- @Override
- protected PassageFormatter getFormatter(String fieldName) {
- String preTag = params.getFieldParam(fieldName, HighlightParams.TAG_PRE, "<em>");
- String postTag = params.getFieldParam(fieldName, HighlightParams.TAG_POST, "</em>");
- String ellipsis = params.getFieldParam(fieldName, HighlightParams.TAG_ELLIPSIS, "... ");
- String encoder = params.getFieldParam(fieldName, HighlightParams.ENCODER, "simple");
- return new DefaultPassageFormatter(preTag, postTag, ellipsis, "html".equals(encoder));
- }
-
- @Override
- protected PassageScorer getScorer(String fieldName) {
- float k1 = params.getFieldFloat(fieldName, HighlightParams.SCORE_K1, 1.2f);
- float b = params.getFieldFloat(fieldName, HighlightParams.SCORE_B, 0.75f);
- float pivot = params.getFieldFloat(fieldName, HighlightParams.SCORE_PIVOT, 87f);
- return new PassageScorer(k1, b, pivot);
- }
-
- @Override
- protected BreakIterator getBreakIterator(String field) {
- String type = params.getFieldParam(field, HighlightParams.BS_TYPE);
- if ("WHOLE".equals(type)) {
- return new WholeBreakIterator();
- } else if ("SEPARATOR".equals(type)) {
- char customSep = parseBiSepChar(params.getFieldParam(field, HighlightParams.BS_SEP));
- return new CustomSeparatorBreakIterator(customSep);
- } else {
- String language = params.getFieldParam(field, HighlightParams.BS_LANGUAGE);
- String country = params.getFieldParam(field, HighlightParams.BS_COUNTRY);
- String variant = params.getFieldParam(field, HighlightParams.BS_VARIANT);
- Locale locale = parseLocale(language, country, variant);
- return parseBreakIterator(type, locale);
- }
- }
-
- /**
- * parse custom separator char for {@link CustomSeparatorBreakIterator}
- */
- protected char parseBiSepChar(String sepChar) {
- if (sepChar == null) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, HighlightParams.BS_SEP + " not passed");
- }
- if (sepChar.length() != 1) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, HighlightParams.BS_SEP +
- " must be a single char but got: '" + sepChar + "'");
- }
- return sepChar.charAt(0);
- }
-
- @Override
- protected char getMultiValuedSeparator(String field) {
- String sep = params.getFieldParam(field, HighlightParams.MULTI_VALUED_SEPARATOR, " ");
- if (sep.length() != 1) {
- throw new IllegalArgumentException(HighlightParams.MULTI_VALUED_SEPARATOR + " must be exactly one character.");
- }
- return sep.charAt(0);
- }
-
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- if (params.getFieldBool(field, HighlightParams.HIGHLIGHT_MULTI_TERM, false)) {
- return schema.getIndexAnalyzer();
- } else {
- return null;
- }
- }
- }
-
- /** parse a break iterator type for the specified locale */
- protected BreakIterator parseBreakIterator(String type, Locale locale) {
- if (type == null || "SENTENCE".equals(type)) {
- return BreakIterator.getSentenceInstance(locale);
- } else if ("LINE".equals(type)) {
- return BreakIterator.getLineInstance(locale);
- } else if ("WORD".equals(type)) {
- return BreakIterator.getWordInstance(locale);
- } else if ("CHARACTER".equals(type)) {
- return BreakIterator.getCharacterInstance(locale);
- } else {
- throw new IllegalArgumentException("Unknown " + HighlightParams.BS_TYPE + ": " + type);
- }
- }
-
- /** parse a locale from a language+country+variant spec */
- protected Locale parseLocale(String language, String country, String variant) {
- if (language == null && country == null && variant == null) {
- return Locale.ROOT;
- } else if (language != null && country == null && variant != null) {
- throw new IllegalArgumentException("To specify variant, country is required");
- } else if (language != null && country != null && variant != null) {
- return new Locale(language, country, variant);
- } else if (language != null && country != null) {
- return new Locale(language, country);
- } else {
- return new Locale(language);
+ @Override
+ protected UnifiedHighlighter getHighlighter(SolrQueryRequest req) {
+ // Adjust the highlight parameters to match what the old PostingsHighlighter had.
+ ModifiableSolrParams invariants = new ModifiableSolrParams();
+ invariants.set(HighlightParams.OFFSET_SOURCE, "POSTINGS");
+ invariants.set(HighlightParams.FIELD_MATCH, true);
+ invariants.set(HighlightParams.USE_PHRASE_HIGHLIGHTER, false);
+ invariants.set(HighlightParams.FRAGSIZE, -1);
+
+ ModifiableSolrParams defaults = new ModifiableSolrParams();
+ defaults.set(HighlightParams.DEFAULT_SUMMARY, true);
+ defaults.set(HighlightParams.TAG_ELLIPSIS, "... ");
+
+ SolrParams newParams = SolrParams.wrapDefaults(
+ invariants,// this takes precedence
+ SolrParams.wrapDefaults(
+ req.getParams(), // then this (original)
+ defaults // finally our defaults
+ )
+ );
+ try (LocalSolrQueryRequest fakeReq = new LocalSolrQueryRequest(req.getCore(), newParams)) {
+ return super.getHighlighter(fakeReq);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2218ded2/solr/core/src/resources/apispec/core.SchemaEdit.addField.json
----------------------------------------------------------------------
diff --git a/solr/core/src/resources/apispec/core.SchemaEdit.addField.json b/solr/core/src/resources/apispec/core.SchemaEdit.addField.json
index d4a6996..b8cb40b 100644
--- a/solr/core/src/resources/apispec/core.SchemaEdit.addField.json
+++ b/solr/core/src/resources/apispec/core.SchemaEdit.addField.json
@@ -34,17 +34,17 @@
},
"termVectors": {
"type": "boolean",
- "description": "If true, term vectors will be stored to be able to compute similarity between two documents. This is required to use More Like This. If this is not defined, it will inherit the value from the fieldType. If the fieldType does not define a value, it will default to false. Do not enable this if using the PostingsHighlighter.",
+ "description": "If true, term vectors will be stored which can be used to optimize More Like This and optimizing highlighting wildcard queries. If this is not defined, it will inherit the value from the fieldType. If the fieldType does not define a value, it will default to false.",
"default": "false"
},
"termPositions": {
"type": "boolean",
- "description": "If true, term positions will be stored for use with highlighting. If this is not defined, it will inherit the value from the fieldType. If the fieldType does not define a value, it will default to false. Do not enable this if using the PostingsHighlighter.",
+ "description": "If true, term vectors will include positions. If this is not defined, it will inherit the value from the fieldType. If the fieldType does not define a value, it will default to false.",
"default": "false"
},
"termOffsets": {
"type": "boolean",
- "description": "If true, term offsets will be stored for use with highlighting. If this is not defined, it will inherit the value from the fieldType. If the fieldType does not define a value, it will default to false. Do not enable this if using the PostingsHighlighter.",
+ "description": "If true, term vectors will include offsets. If this is not defined, it will inherit the value from the fieldType. If the fieldType does not define a value, it will default to false.",
"default": "false"
},
"multiValued": {
@@ -73,7 +73,7 @@
},
"storeOffsetsWithPositions": {
"type": "boolean",
- "description": "If true, term offsets will be stored with positions in the postings list in the index. This is required if using the PostingsHighlighter. If this is not defined, it will inherit the value from the fieldType. If the fieldType does not define a value, it will default to false.",
+ "description": "If true, term offsets will be stored with positions in the postings list in the index. This optimizes highlighting with the UnifiedHighlighter. If this is not defined, it will inherit the value from the fieldType. If the fieldType does not define a value, it will default to false.",
"default": "false"
},
"docValues": {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2218ded2/solr/solr-ref-guide/src/highlighting.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/highlighting.adoc b/solr/solr-ref-guide/src/highlighting.adoc
index 5a55fd2..7207ce9 100644
--- a/solr/solr-ref-guide/src/highlighting.adoc
+++ b/solr/solr-ref-guide/src/highlighting.adoc
@@ -137,11 +137,14 @@ This highlighter's query-representation is less advanced than the Original or Un
+
Note that both the FastVector and Original Highlighters can be used in conjunction in a search request to highlight some fields with one and some the other. In contrast, the other highlighters can only be chosen exclusively.
-<<Highlighting-ThePostingsHighlighter,Postings Highlighter>>:: (`hl.method=postings`)
+The Postings Highlighter:: (`hl.method=postings`)
+
-The Postings Highlighter is the ancestor of the Unified Highlighter, supporting a subset of its options and none of its index configuration flexibility - it _requires_ `storeOffsetsWithPositions` on all fields to highlight. This option is here for backwards compatibility; if you find you need it, please share your experience with the Solr community.
+The Postings Highlighter is the ancestor of the Unified Highlighter, supporting a subset of its options and none of its index configuration flexibility - it _requires_ `storeOffsetsWithPositions` on all fields to highlight.
+This option is here for backwards compatibility; it's deprecated.
+In 7.0, it is internally implemented as a reconfiguration of the Unified Highlighter.
+See older reference guide editions for its options.
-The Unified Highlighter and Postings Highlighter from which it derives, are exclusively configured via search parameters. In contrast, some settings for the Original and FastVector Highlighters are set in `solrconfig.xml`. There's a robust example of the latter in the "```techproducts```" configset.
+The Unified Highlighter is exclusively configured via search parameters. In contrast, some settings for the Original and FastVector Highlighters are set in `solrconfig.xml`. There's a robust example of the latter in the "```techproducts```" configset.
In addition to further information below, more information can be found in the {solr-javadocs}/solr-core/org/apache/solr/highlight/package-summary.html[Solr javadocs].
@@ -157,7 +160,7 @@ The benefit of this approach is that your index won't grow larger with any extra
The down side is that highlighting speed is roughly linear with the amount of text to process, with a large factor being the complexity of your analysis chain.
+
For "short" text, this is a good choice. Or maybe it's not short but you're prioritizing a smaller index and indexing speed over highlighting performance.
-* *Postings*: Supported by the Unified and Postings Highlighters. Set `storeOffsetsWithPositions` to `true`. This adds a moderate amount of extra data to the index but it speeds up highlighting tremendously, especially compared to analysis with longer text fields.
+* *Postings*: Supported by the Unified Highlighter. Set `storeOffsetsWithPositions` to `true`. This adds a moderate amount of extra data to the index but it speeds up highlighting tremendously, especially compared to analysis with longer text fields.
+
However, wildcard queries will fall back to analysis unless "light" term vectors are added.
@@ -193,27 +196,6 @@ The Unified Highlighter supports these following additional parameters to the on
|hl.bs.separator |_(blank)_ |Indicates which character to break the text on. Requires `hl.bs.type=SEPARATOR`. This is useful when the text has already been manipulated in advance to have a special delineation character at desired highlight passage boundaries. This character will still appear in the text as the last character of a passage.
|===
-[[Highlighting-ThePostingsHighlighter]]
-=== The Postings Highlighter
-
-The Postings Highlighter is the ancestor of the Unified Highlighter, supporting a subset of it's options and sometimes with different default settings for some common parameters.
-
-Viewed from the perspective of the Unified Highlighter, these settings are effectively non-settings and fixed as-such:
-
-* `hl.offsetSource=POSTINGS`
-* `hl.requireFieldMatch=true`
-* `hl.usePhraseHighlighter=false`
-* `hl.fragsize=-1` (none).
-
-It has these different default settings:
-
-* `hl.defaultSummary=true`
-* `hl.tag.ellipsis="... "`.
-
-In addition, it has a setting `hl.multiValuedSeparatorChar=" "` (space).
-
-This highlighter never returns separate snippets as separate values; they are always joined by `hl.tag.ellipsis`.
-
[[Highlighting-TheOriginalHighlighter]]
== The Original Highlighter
[44/44] lucene-solr:jira/solr-8668: Merge branch 'master' into
jira/solr-8668
Posted by cp...@apache.org.
Merge branch 'master' into jira/solr-8668
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7524bd02
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7524bd02
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7524bd02
Branch: refs/heads/jira/solr-8668
Commit: 7524bd020fd9cbcc1a9aad2940f3d93d94bf3bd1
Parents: ab9aad2 1e4d205
Author: Christine Poerschke <cp...@apache.org>
Authored: Thu May 25 18:40:00 2017 +0100
Committer: Christine Poerschke <cp...@apache.org>
Committed: Thu May 25 18:40:00 2017 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 18 +-
lucene/NOTICE.txt | 10 +-
.../lucene/analysis/util/UnicodeProps.java | 6 +-
.../icu/src/data/utr30/DiacriticFolding.txt | 5 +
.../icu/src/data/utr30/NativeDigitFolding.txt | 30 +
lucene/analysis/icu/src/data/utr30/nfc.txt | 22 +-
lucene/analysis/icu/src/data/utr30/nfkc.txt | 5 +-
lucene/analysis/icu/src/data/utr30/nfkc_cf.txt | 225 +++-
.../analysis/icu/segmentation/Default.brk | Bin 35264 -> 36768 bytes
.../icu/segmentation/MyanmarSyllable.brk | Bin 19776 -> 20744 bytes
.../org/apache/lucene/analysis/icu/utr30.nrm | Bin 53840 -> 55184 bytes
.../analysis/icu/GenerateUTR30DataFiles.java | 2 +-
.../uk/UkrainianMorfologikAnalyzer.java | 2 +
.../analysis/uk/TestUkrainianAnalyzer.java | 9 +-
lucene/benchmark/conf/highlighters-postings.alg | 4 +-
.../tasks/SearchTravRetHighlightTask.java | 30 -
.../apache/lucene/document/RangeFieldQuery.java | 48 +-
.../search/TestDoubleRangeFieldQueries.java | 15 +-
.../search/TestFloatRangeFieldQueries.java | 15 +-
.../lucene/search/TestIntRangeFieldQueries.java | 23 +-
.../search/TestLongRangeFieldQueries.java | 23 +-
.../CustomSeparatorBreakIterator.java | 150 ---
.../DefaultPassageFormatter.java | 137 --
.../MultiTermHighlighting.java | 282 -----
.../search/postingshighlight/Passage.java | 159 ---
.../postingshighlight/PassageFormatter.java | 40 -
.../search/postingshighlight/PassageScorer.java | 104 --
.../postingshighlight/PostingsHighlighter.java | 820 ------------
.../postingshighlight/WholeBreakIterator.java | 116 --
.../search/postingshighlight/package-info.java | 21 -
.../CustomSeparatorBreakIterator.java | 150 +++
.../search/uhighlight/WholeBreakIterator.java | 116 ++
.../search/postingshighlight/CambridgeMA.utf8 | 1 -
.../TestCustomSeparatorBreakIterator.java | 114 --
.../TestMultiTermHighlighting.java | 884 -------------
.../TestPostingsHighlighter.java | 1185 ------------------
.../TestPostingsHighlighterRanking.java | 324 -----
.../TestWholeBreakIterator.java | 134 --
.../uhighlight/LengthGoalBreakIteratorTest.java | 1 -
.../TestCustomSeparatorBreakIterator.java | 114 ++
.../uhighlight/TestUnifiedHighlighter.java | 1 -
.../uhighlight/TestWholeBreakIterator.java | 134 ++
lucene/ivy-versions.properties | 4 +-
.../lucene/search/join/GlobalOrdinalsQuery.java | 6 +-
.../join/GlobalOrdinalsWithScoreQuery.java | 18 +-
.../org/apache/lucene/search/join/JoinUtil.java | 23 +-
.../join/PointInSetIncludingScoreQuery.java | 10 +-
.../search/join/TermsIncludingScoreQuery.java | 91 +-
.../apache/lucene/search/join/TermsQuery.java | 39 +-
.../apache/lucene/search/join/TestJoinUtil.java | 246 ++++
lucene/licenses/icu4j-56.1.jar.sha1 | 1 -
lucene/licenses/icu4j-59.1.jar.sha1 | 1 +
.../morfologik-ukrainian-search-3.7.5.jar.sha1 | 1 -
.../morfologik-ukrainian-search-3.7.6.jar.sha1 | 1 +
.../search/TestInetAddressRangeQueries.java | 18 +-
.../spatial3d/geom/GeoWideLongitudeSlice.java | 8 +-
.../search/BaseRangeFieldQueryTestCase.java | 4 +-
solr/CHANGES.txt | 45 +
.../solr/ltr/feature/FieldLengthFeature.java | 3 +-
.../apache/solr/ltr/TestLTRQParserPlugin.java | 11 +-
.../solr/ltr/TestParallelWeightCreation.java | 5 +-
.../solr/ltr/TestSelectiveWeightCreation.java | 44 +-
.../org/apache/solr/core/CoreContainer.java | 17 +-
.../apache/solr/handler/AnalyzeEvaluator.java | 1 +
.../org/apache/solr/handler/StreamHandler.java | 3 +
.../component/HttpShardHandlerFactory.java | 28 +-
.../solr/highlight/PostingsSolrHighlighter.java | 326 +----
.../solr/highlight/UnifiedSolrHighlighter.java | 4 +-
.../apache/solr/request/SolrQueryRequest.java | 42 +-
.../org/apache/solr/schema/GeoHashField.java | 3 +
.../java/org/apache/solr/schema/LatLonType.java | 3 +
.../schema/SpatialPointVectorFieldType.java | 3 +-
.../SpatialTermQueryPrefixTreeFieldType.java | 3 +-
.../apache/solr/search/SolrIndexSearcher.java | 3 +
.../solr/search/facet/FacetFieldProcessor.java | 81 +-
.../SearchGroupShardResponseProcessor.java | 4 +-
.../java/org/apache/solr/update/PeerSync.java | 6 +-
.../apache/solr/update/SolrCmdDistributor.java | 4 +-
.../java/org/apache/solr/update/UpdateLog.java | 11 +-
.../apispec/core.SchemaEdit.addField.json | 10 +-
.../solr/collection1/conf/schema-point.xml | 47 +-
.../solr/collection1/conf/solrconfig-tlog.xml | 8 +
.../solr-shardhandler-loadBalancerRequests.xml | 23 +
...aosMonkeySafeLeaderWithPullReplicasTest.java | 3 +-
.../org/apache/solr/core/TestCoreDiscovery.java | 20 +-
.../component/TestHttpShardHandlerFactory.java | 102 ++
.../org/apache/solr/search/facet/DebugAgg.java | 146 +++
.../solr/search/facet/TestJsonFacets.java | 134 +-
.../search/join/TestScoreJoinQPNoScore.java | 9 +
.../solr/atom/conf/atom-data-config.xml | 2 +-
solr/licenses/icu4j-56.1.jar.sha1 | 1 -
solr/licenses/icu4j-59.1.jar.sha1 | 1 +
solr/solr-ref-guide/src/highlighting.adoc | 31 +-
.../src/update-request-processors.adoc | 20 +-
...store-data-with-the-data-import-handler.adoc | 950 +++++++-------
.../client/solrj/impl/LBHttpSolrClient.java | 50 +-
.../org/apache/solr/client/solrj/io/Tuple.java | 4 +-
.../solrj/io/eval/AbsoluteValueEvaluator.java | 3 -
.../solr/client/solrj/io/eval/AddEvaluator.java | 3 -
.../solr/client/solrj/io/eval/AndEvaluator.java | 3 -
.../solrj/io/eval/ArcCosineEvaluator.java | 3 -
.../client/solrj/io/eval/ArcSineEvaluator.java | 3 -
.../solrj/io/eval/ArcTangentEvaluator.java | 3 -
.../client/solrj/io/eval/BooleanEvaluator.java | 3 -
.../client/solrj/io/eval/CeilingEvaluator.java | 3 -
.../client/solrj/io/eval/CoalesceEvaluator.java | 3 -
.../client/solrj/io/eval/ColumnEvaluator.java | 76 ++
.../solrj/io/eval/ConditionalEvaluator.java | 3 -
.../solrj/io/eval/ConvolutionEvaluator.java | 80 ++
.../client/solrj/io/eval/CopyOfEvaluator.java | 82 ++
.../solrj/io/eval/CopyOfRangeEvaluator.java | 83 ++
.../solrj/io/eval/CorrelationEvaluator.java | 75 ++
.../client/solrj/io/eval/CosineEvaluator.java | 3 -
.../solrj/io/eval/CovarianceEvaluator.java | 75 ++
.../solrj/io/eval/CubedRootEvaluator.java | 3 -
.../client/solrj/io/eval/DescribeEvaluator.java | 90 ++
.../client/solrj/io/eval/DistanceEvaluator.java | 75 ++
.../client/solrj/io/eval/DivideEvaluator.java | 3 -
.../io/eval/EmpiricalDistributionEvaluator.java | 127 ++
.../client/solrj/io/eval/EqualsEvaluator.java | 3 -
.../solrj/io/eval/ExclusiveOrEvaluator.java | 3 -
.../client/solrj/io/eval/FieldEvaluator.java | 3 -
.../solrj/io/eval/FindDelayEvaluator.java | 90 ++
.../client/solrj/io/eval/FloorEvaluator.java | 3 -
.../io/eval/GreaterThanEqualToEvaluator.java | 3 -
.../solrj/io/eval/GreaterThanEvaluator.java | 3 -
.../io/eval/HyperbolicCosineEvaluator.java | 3 -
.../solrj/io/eval/HyperbolicSineEvaluator.java | 3 -
.../io/eval/HyperbolicTangentEvaluator.java | 3 -
.../solrj/io/eval/IfThenElseEvaluator.java | 3 -
.../client/solrj/io/eval/LengthEvaluator.java | 58 +
.../solrj/io/eval/LessThanEqualToEvaluator.java | 3 -
.../client/solrj/io/eval/LessThanEvaluator.java | 3 -
.../client/solrj/io/eval/ModuloEvaluator.java | 3 -
.../client/solrj/io/eval/MultiplyEvaluator.java | 3 -
.../solrj/io/eval/NaturalLogEvaluator.java | 3 -
.../solrj/io/eval/NormalizeEvaluator.java | 74 ++
.../solr/client/solrj/io/eval/NotEvaluator.java | 3 -
.../client/solrj/io/eval/NumberEvaluator.java | 3 -
.../solr/client/solrj/io/eval/OrEvaluator.java | 3 -
.../solrj/io/eval/PercentileEvaluator.java | 66 +
.../client/solrj/io/eval/PowerEvaluator.java | 3 -
.../client/solrj/io/eval/PredictEvaluator.java | 66 +
.../client/solrj/io/eval/RankEvaluator.java | 73 ++
.../client/solrj/io/eval/RawValueEvaluator.java | 3 -
.../solrj/io/eval/RegressionEvaluator.java | 110 ++
.../client/solrj/io/eval/ReverseEvaluator.java | 65 +
.../client/solrj/io/eval/RoundEvaluator.java | 3 -
.../client/solrj/io/eval/ScaleEvaluator.java | 76 ++
.../client/solrj/io/eval/SequenceEvaluator.java | 72 ++
.../client/solrj/io/eval/SimpleEvaluator.java | 3 -
.../client/solrj/io/eval/SineEvaluator.java | 3 -
.../solrj/io/eval/SquareRootEvaluator.java | 3 -
.../client/solrj/io/eval/StreamEvaluator.java | 3 -
.../client/solrj/io/eval/SubtractEvaluator.java | 3 -
.../client/solrj/io/eval/TangentEvaluator.java | 3 -
.../client/solrj/io/stream/ColumnEvaluator.java | 77 --
.../solrj/io/stream/ConvolutionEvaluator.java | 82 --
.../client/solrj/io/stream/CopyOfEvaluator.java | 84 --
.../solrj/io/stream/CopyOfRangeEvaluator.java | 85 --
.../solrj/io/stream/CorrelationEvaluator.java | 77 --
.../solrj/io/stream/CovarianceEvaluator.java | 77 --
.../solrj/io/stream/DescribeEvaluator.java | 92 --
.../solrj/io/stream/DistanceEvaluator.java | 77 --
.../stream/EmpiricalDistributionEvaluator.java | 129 --
.../solr/client/solrj/io/stream/KnnStream.java | 256 ++++
.../client/solrj/io/stream/LengthEvaluator.java | 60 -
.../solrj/io/stream/NormalizeEvaluator.java | 76 --
.../solrj/io/stream/PercentileEvaluator.java | 68 -
.../solrj/io/stream/PredictEvaluator.java | 68 -
.../client/solrj/io/stream/RankEvaluator.java | 75 --
.../solrj/io/stream/RegressionEvaluator.java | 112 --
.../solrj/io/stream/ReverseEvaluator.java | 67 -
.../client/solrj/io/stream/ScaleEvaluator.java | 78 --
.../solr/common/params/HighlightParams.java | 31 +-
.../solrj/io/stream/StreamExpressionTest.java | 199 +++
.../apache/solr/cloud/SolrCloudTestCase.java | 6 +-
177 files changed, 4618 insertions(+), 7052 deletions(-)
----------------------------------------------------------------------
[18/44] lucene-solr:jira/solr-8668: SOLR-10438: Assign explicit
useDocValuesAsStored values to all points field types in
schema-point.xml/TestPointFields.
Posted by cp...@apache.org.
SOLR-10438: Assign explicit useDocValuesAsStored values to all points field types in schema-point.xml/TestPointFields.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ea9adc05
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ea9adc05
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ea9adc05
Branch: refs/heads/jira/solr-8668
Commit: ea9adc055bd4b88ff08b4b1ba20eb08b346b6847
Parents: 85c3ae2
Author: Steve Rowe <sa...@apache.org>
Authored: Tue May 23 20:04:43 2017 -0400
Committer: Steve Rowe <sa...@apache.org>
Committed: Tue May 23 20:04:43 2017 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 3 ++
.../solr/collection1/conf/schema-point.xml | 47 ++++++++------------
2 files changed, 22 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ea9adc05/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 6b91c4d..928a190 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -239,6 +239,9 @@ Other Changes
* SOLR-10400: Replace (instanceof TrieFooField || instanceof FooPointField) constructs with
FieldType.getNumberType() or SchemaField.getSortField() where appropriate. (hossman, Steve Rowe)
+
+* SOLR-10438: Assign explicit useDocValuesAsStored values to all points field types in
+ schema-point.xml/TestPointFields. (hossman, Steve Rowe)
================== 6.6.0 ==================
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ea9adc05/solr/core/src/test-files/solr/collection1/conf/schema-point.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-point.xml b/solr/core/src/test-files/solr/collection1/conf/schema-point.xml
index ae6a11e..8bf545a 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema-point.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-point.xml
@@ -16,15 +16,16 @@
limitations under the License.
-->
-<schema name="example" version="1.6">
+<schema name="schema-point" version="1.6">
<types>
<fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
- <fieldType name="pint" class="solr.IntPointField"/>
- <fieldType name="plong" class="solr.LongPointField"/>
- <fieldType name="pdouble" class="solr.DoublePointField"/>
- <fieldType name="pfloat" class="solr.FloatPointField"/>
- <fieldType name="pdate" class="solr.DatePointField"/>
+ <!-- Override useDocValuesAsStored="true" default for schema version=1.6 -->
+ <fieldType name="pint" class="solr.IntPointField" useDocValuesAsStored="false"/>
+ <fieldType name="plong" class="solr.LongPointField" useDocValuesAsStored="false"/>
+ <fieldType name="pdouble" class="solr.DoublePointField" useDocValuesAsStored="false"/>
+ <fieldType name="pfloat" class="solr.FloatPointField" useDocValuesAsStored="false"/>
+ <fieldType name="pdate" class="solr.DatePointField" useDocValuesAsStored="false"/>
<fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="float" class="solr.TrieFloatField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
@@ -33,12 +34,12 @@
<fieldType name="date" class="solr.TrieDateField" sortMissingLast="true" omitNorms="true"/>
</types>
-
<fields>
<field name="id" type="string"/>
<field name="text" type="string"/>
<field name="_version_" type="long" indexed="true" stored="true" multiValued="false" />
<field name="signatureField" type="string" indexed="true" stored="false"/>
+
<dynamicField name="*_s" type="string" indexed="true" stored="true"/>
<dynamicField name="*_sS" type="string" indexed="false" stored="true"/>
<dynamicField name="*_i" type="int" indexed="true" stored="true"/>
@@ -53,8 +54,8 @@
<dynamicField name="*_p_i_mv_dv" type="pint" indexed="true" stored="true" docValues="true" multiValued="true"/>
<dynamicField name="*_p_i_ni" type="pint" indexed="false" stored="true" docValues="false"/>
<dynamicField name="*_p_i_ni_dv" type="pint" indexed="false" stored="true" docValues="true"/>
- <dynamicField name="*_p_i_ni_dv_ns" type="pint" indexed="false" stored="false" docValues="true" useDocValuesAsStored="false"/>
- <dynamicField name="*_p_i_ni_dv_ns_mv" type="pint" indexed="false" stored="false" docValues="true" useDocValuesAsStored="false" multiValued="true"/>
+ <dynamicField name="*_p_i_ni_dv_ns" type="pint" indexed="false" stored="false" docValues="true"/>
+ <dynamicField name="*_p_i_ni_dv_ns_mv" type="pint" indexed="false" stored="false" docValues="true" multiValued="true"/>
<dynamicField name="*_p_i_ni_mv" type="pint" indexed="false" stored="true" docValues="false" multiValued="true"/>
<dynamicField name="*_p_i_ni_mv_dv" type="pint" indexed="false" stored="true" docValues="true" multiValued="true"/>
<dynamicField name="*_p_i_ni_ns" type="pint" indexed="false" stored="false" docValues="false" />
@@ -66,8 +67,8 @@
<dynamicField name="*_p_l_mv_dv" type="plong" indexed="true" stored="true" docValues="true" multiValued="true"/>
<dynamicField name="*_p_l_ni" type="plong" indexed="false" stored="true" docValues="false"/>
<dynamicField name="*_p_l_ni_dv" type="plong" indexed="false" stored="true" docValues="true"/>
- <dynamicField name="*_p_l_ni_dv_ns" type="plong" indexed="false" stored="false" docValues="true" useDocValuesAsStored="false"/>
- <dynamicField name="*_p_l_ni_dv_ns_mv" type="plong" indexed="false" stored="false" docValues="true" useDocValuesAsStored="false" multiValued="true"/>
+ <dynamicField name="*_p_l_ni_dv_ns" type="plong" indexed="false" stored="false" docValues="true"/>
+ <dynamicField name="*_p_l_ni_dv_ns_mv" type="plong" indexed="false" stored="false" docValues="true" multiValued="true"/>
<dynamicField name="*_p_l_ni_mv" type="plong" indexed="false" stored="true" docValues="false" multiValued="true"/>
<dynamicField name="*_p_l_ni_mv_dv" type="plong" indexed="false" stored="true" docValues="true" multiValued="true"/>
<dynamicField name="*_p_l_ni_ns" type="plong" indexed="false" stored="false" docValues="false" />
@@ -79,8 +80,8 @@
<dynamicField name="*_p_d_mv_dv" type="pdouble" indexed="true" stored="true" docValues="true" multiValued="true"/>
<dynamicField name="*_p_d_ni" type="pdouble" indexed="false" stored="true" docValues="false"/>
<dynamicField name="*_p_d_ni_dv" type="pdouble" indexed="false" stored="true" docValues="true"/>
- <dynamicField name="*_p_d_ni_dv_ns" type="pdouble" indexed="false" stored="false" docValues="true" useDocValuesAsStored="false"/>
- <dynamicField name="*_p_d_ni_dv_ns_mv" type="pdouble" indexed="false" stored="false" docValues="true" useDocValuesAsStored="false" multiValued="true"/>
+ <dynamicField name="*_p_d_ni_dv_ns" type="pdouble" indexed="false" stored="false" docValues="true"/>
+ <dynamicField name="*_p_d_ni_dv_ns_mv" type="pdouble" indexed="false" stored="false" docValues="true" multiValued="true"/>
<dynamicField name="*_p_d_ni_mv" type="pdouble" indexed="false" stored="true" docValues="false" multiValued="true"/>
<dynamicField name="*_p_d_ni_mv_dv" type="pdouble" indexed="false" stored="true" docValues="true" multiValued="true"/>
<dynamicField name="*_p_d_ni_ns" type="pdouble" indexed="false" stored="false" docValues="false"/>
@@ -92,8 +93,8 @@
<dynamicField name="*_p_f_mv_dv" type="pfloat" indexed="true" stored="true" docValues="true" multiValued="true"/>
<dynamicField name="*_p_f_ni" type="pfloat" indexed="false" stored="true" docValues="false"/>
<dynamicField name="*_p_f_ni_dv" type="pfloat" indexed="false" stored="true" docValues="true"/>
- <dynamicField name="*_p_f_ni_dv_ns" type="pfloat" indexed="false" stored="false" docValues="true" useDocValuesAsStored="false"/>
- <dynamicField name="*_p_f_ni_dv_ns_mv" type="pfloat" indexed="false" stored="false" docValues="true" useDocValuesAsStored="false" multiValued="true"/>
+ <dynamicField name="*_p_f_ni_dv_ns" type="pfloat" indexed="false" stored="false" docValues="true"/>
+ <dynamicField name="*_p_f_ni_dv_ns_mv" type="pfloat" indexed="false" stored="false" docValues="true" multiValued="true"/>
<dynamicField name="*_p_f_ni_mv" type="pfloat" indexed="false" stored="true" docValues="false" multiValued="true"/>
<dynamicField name="*_p_f_ni_mv_dv" type="pfloat" indexed="false" stored="true" docValues="true" multiValued="true"/>
<dynamicField name="*_p_f_ni_ns" type="pfloat" indexed="false" stored="false" docValues="false"/>
@@ -105,21 +106,14 @@
<dynamicField name="*_p_dt_mv_dv" type="pdate" indexed="true" stored="true" docValues="true" multiValued="true"/>
<dynamicField name="*_p_dt_ni" type="pdate" indexed="false" stored="true" docValues="false"/>
<dynamicField name="*_p_dt_ni_dv" type="pdate" indexed="false" stored="true" docValues="true"/>
- <dynamicField name="*_p_dt_ni_dv_ns" type="pdate" indexed="false" stored="false" docValues="true" useDocValuesAsStored="false"/>
- <dynamicField name="*_p_dt_ni_dv_ns_mv" type="pdate" indexed="false" stored="false" docValues="true" useDocValuesAsStored="false" multiValued="true"/>
+ <dynamicField name="*_p_dt_ni_dv_ns" type="pdate" indexed="false" stored="false" docValues="true"/>
+ <dynamicField name="*_p_dt_ni_dv_ns_mv" type="pdate" indexed="false" stored="false" docValues="true" multiValued="true"/>
<dynamicField name="*_p_dt_ni_mv" type="pdate" indexed="false" stored="true" docValues="false" multiValued="true"/>
<dynamicField name="*_p_dt_ni_mv_dv" type="pdate" indexed="false" stored="true" docValues="true" multiValued="true"/>
<dynamicField name="*_p_dt_ni_ns" type="pdate" indexed="false" stored="false" docValues="false"/>
<dynamicField name="*_p_dt_ni_ns_mv" type="pdate" indexed="false" stored="false" docValues="false" multiValued="true"/>
-
- <!-- NOTE: https://issues.apache.org/jira/browse/SOLR-10438
-
- NOTE: because schema version=1.6, *all* DV fields default to useDocValuesAsStored="true"
-
- NOTE: we need to audit if this is breaking any assumptions elsewhere in the test code
- -->
- <!-- return DV fields as -->
+ <!-- return DV fields as stored -->
<dynamicField name="*_p_i_dv_ns" type="pint" indexed="true" stored="false" docValues="true" useDocValuesAsStored="true"/>
<dynamicField name="*_p_l_dv_ns" type="plong" indexed="true" stored="false" docValues="true" useDocValuesAsStored="true"/>
<dynamicField name="*_p_d_dv_ns" type="pdouble" indexed="true" stored="false" docValues="true" useDocValuesAsStored="true"/>
@@ -136,11 +130,8 @@
<dynamicField name="*_p_l_dv_ns_mv" type="plong" indexed="true" stored="false" docValues="true" useDocValuesAsStored="true" multiValued="true"/>
<dynamicField name="*_p_f_dv_ns_mv" type="pfloat" indexed="true" stored="false" docValues="true" useDocValuesAsStored="true" multiValued="true"/>
<dynamicField name="*_p_dt_dv_ns_mv" type="pdate" indexed="true" stored="false" docValues="true" useDocValuesAsStored="true" multiValued="true"/>
-
-
</fields>
<uniqueKey>id</uniqueKey>
-
</schema>
[39/44] lucene-solr:jira/solr-8668: SOLR-10743: Add sequence
StreamEvaluator
Posted by cp...@apache.org.
SOLR-10743: Add sequence StreamEvaluator
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0b47126e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0b47126e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0b47126e
Branch: refs/heads/jira/solr-8668
Commit: 0b47126e5c10cb6a092c4f49e4b1835bd15aa878
Parents: 7fef7e3
Author: Joel Bernstein <jb...@apache.org>
Authored: Thu May 25 11:20:15 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Thu May 25 11:20:15 2017 -0400
----------------------------------------------------------------------
.../org/apache/solr/handler/StreamHandler.java | 1 +
.../client/solrj/io/eval/SequenceEvaluator.java | 72 ++++++++++++++++++++
.../solrj/io/stream/StreamExpressionTest.java | 53 ++++++++++++++
3 files changed, 126 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0b47126e/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
index 9880512..b219a2c 100644
--- a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
@@ -186,6 +186,7 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
.withFunctionName("empiricalDistribution", EmpiricalDistributionEvaluator.class)
.withFunctionName("describe", DescribeEvaluator.class)
.withFunctionName("finddelay", FindDelayEvaluator.class)
+ .withFunctionName("sequence", SequenceEvaluator.class)
// metrics
.withFunctionName("min", MinMetric.class)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0b47126e/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SequenceEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SequenceEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SequenceEvaluator.java
new file mode 100644
index 0000000..a88d695
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SequenceEvaluator.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.math3.util.MathArrays;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation;
+import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
+import org.apache.solr.client.solrj.io.stream.expr.Expressible;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class SequenceEvaluator extends ComplexEvaluator implements Expressible {
+
+ private static final long serialVersionUID = 1;
+
+ public SequenceEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ public List<Number> evaluate(Tuple tuple) throws IOException {
+ StreamEvaluator sizeEval = subEvaluators.get(0);
+ StreamEvaluator startEval = subEvaluators.get(1);
+ StreamEvaluator strideEval = subEvaluators.get(2);
+
+ Number sizeNum = (Number)sizeEval.evaluate(tuple);
+ Number startNum = (Number)startEval.evaluate(tuple);
+ Number strideNum = (Number)strideEval.evaluate(tuple);
+
+ int[] sequence = MathArrays.sequence(sizeNum.intValue(), startNum.intValue(), strideNum.intValue());
+ List<Number> numbers = new ArrayList(sequence.length);
+ for(int i : sequence) {
+ numbers.add(i);
+ }
+
+ return numbers;
+ }
+
+ @Override
+ public StreamExpressionParameter toExpression(StreamFactory factory) throws IOException {
+ StreamExpression expression = new StreamExpression(factory.getFunctionName(getClass()));
+ return expression;
+ }
+
+ @Override
+ public Explanation toExplanation(StreamFactory factory) throws IOException {
+ return new Explanation(nodeId.toString())
+ .withExpressionType(ExpressionType.EVALUATOR)
+ .withFunctionName(factory.getFunctionName(getClass()))
+ .withImplementingClass(getClass().getName())
+ .withExpression(toExpression(factory).toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0b47126e/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index ab792f9..0dff6fc 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -5207,6 +5207,59 @@ public class StreamExpressionTest extends SolrCloudTestCase {
@Test
+ public void testSequence() throws Exception {
+ String expr = "tuple(seq=sequence(20, 0, 1))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", expr);
+ paramsLoc.set("qt", "/stream");
+
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<Number> sequence = (List<Number>)tuples.get(0).get("seq");
+ assertTrue(sequence.size() == 20);
+ for(int i=0; i<sequence.size(); i++) {
+ assertTrue(sequence.get(i).intValue() == i);
+ }
+
+ //Change the size, stride
+ expr = "tuple(seq=sequence(100, 0, 4))";
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", expr);
+ paramsLoc.set("qt", "/stream");
+
+ solrStream = new SolrStream(url, paramsLoc);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ sequence = (List<Number>)tuples.get(0).get("seq");
+ assertTrue(sequence.size() == 100);
+ for(int i=0; i<sequence.size(); i++) {
+ assertTrue(sequence.get(i).intValue() == (i*4));
+ }
+
+ //Change the start
+ expr = "tuple(seq=sequence(100, 10, 1))";
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", expr);
+ paramsLoc.set("qt", "/stream");
+
+ solrStream = new SolrStream(url, paramsLoc);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ sequence = (List<Number>)tuples.get(0).get("seq");
+ assertTrue(sequence.size() == 100);
+ for(int i=0; i<sequence.size(); i++) {
+ assertTrue(sequence.get(i).intValue() == (i+10));
+ }
+ }
+
+
+
+ @Test
public void testEvalStream() throws Exception {
UpdateRequest updateRequest = new UpdateRequest();
updateRequest.add(id, "hello", "test_t", "l b c d c");