You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ds...@apache.org on 2017/05/23 18:39:59 UTC
[1/4] lucene-solr:master: LUCENE-7815: Removed the PostingsHighlighter
Repository: lucene-solr
Updated Branches:
refs/heads/master 14320a584 -> 0d3c73eaa
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java b/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java
deleted file mode 100644
index 4b7e66b..0000000
--- a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java
+++ /dev/null
@@ -1,1185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.postingshighlight;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.StoredField;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.queries.CustomScoreQuery;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.PhraseQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.LuceneTestCase;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
-import java.text.BreakIterator;
-import java.util.Arrays;
-import java.util.Map;
-
-public class TestPostingsHighlighter extends LuceneTestCase {
-
- public void testBasics() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");
- iw.addDocument(doc);
- body.setStringValue("Highlighting the first term. Hope it works.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- Query query = new TermQuery(new Term("body", "highlighting"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("Just a test <b>highlighting</b> from postings. ", snippets[0]);
- assertEquals("<b>Highlighting</b> the first term. ", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testFormatWithMatchExceedingContentLength2() throws Exception {
-
- String bodyText = "123 TEST 01234 TEST";
-
- String[] snippets = formatWithMatchExceedingContentLength(bodyText);
-
- assertEquals(1, snippets.length);
- assertEquals("123 <b>TEST</b> 01234 TE", snippets[0]);
- }
-
- public void testFormatWithMatchExceedingContentLength3() throws Exception {
-
- String bodyText = "123 5678 01234 TEST TEST";
-
- String[] snippets = formatWithMatchExceedingContentLength(bodyText);
-
- assertEquals(1, snippets.length);
- assertEquals("123 5678 01234 TE", snippets[0]);
- }
-
- public void testFormatWithMatchExceedingContentLength() throws Exception {
-
- String bodyText = "123 5678 01234 TEST";
-
- String[] snippets = formatWithMatchExceedingContentLength(bodyText);
-
- assertEquals(1, snippets.length);
- // LUCENE-5166: no snippet
- assertEquals("123 5678 01234 TE", snippets[0]);
- }
-
- private String[] formatWithMatchExceedingContentLength(String bodyText) throws IOException {
-
- int maxLength = 17;
-
- final Analyzer analyzer = new MockAnalyzer(random());
-
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- final FieldType fieldType = new FieldType(TextField.TYPE_STORED);
- fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- final Field body = new Field("body", bodyText, fieldType);
-
- Document doc = new Document();
- doc.add(body);
-
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
-
- Query query = new TermQuery(new Term("body", "test"));
-
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(1, topDocs.totalHits);
-
- PostingsHighlighter highlighter = new PostingsHighlighter(maxLength);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
-
-
- ir.close();
- dir.close();
- return snippets;
- }
-
- // simple test highlighting last word.
- public void testHighlightLastWord() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- Query query = new TermQuery(new Term("body", "test"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(1, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(1, snippets.length);
- assertEquals("This is a <b>test</b>", snippets[0]);
-
- ir.close();
- dir.close();
- }
-
- // simple test with one sentence documents.
- public void testOneSentence() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- Query query = new TermQuery(new Term("body", "test"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- // simple test with multiple values that make a result longer than maxLength.
- public void testMaxLengthWithMultivalue() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Document doc = new Document();
-
- for(int i = 0; i < 3 ; i++) {
- Field body = new Field("body", "", offsetsType);
- body.setStringValue("This is a multivalued field");
- doc.add(body);
- }
-
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter(40);
- Query query = new TermQuery(new Term("body", "field"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(1, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(1, snippets.length);
- assertTrue("Snippet should have maximum 40 characters plus the pre and post tags",
- snippets[0].length() == (40 + "<b></b>".length()));
-
- ir.close();
- dir.close();
- }
-
- public void testMultipleFields() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Field title = new Field("title", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
- doc.add(title);
-
- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");
- title.setStringValue("I am hoping for the best.");
- iw.addDocument(doc);
- body.setStringValue("Highlighting the first term. Hope it works.");
- title.setStringValue("But best may not be good enough.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- BooleanQuery.Builder query = new BooleanQuery.Builder();
- query.add(new TermQuery(new Term("body", "highlighting")), BooleanClause.Occur.SHOULD);
- query.add(new TermQuery(new Term("title", "best")), BooleanClause.Occur.SHOULD);
- TopDocs topDocs = searcher.search(query.build(), 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- Map<String,String[]> snippets = highlighter.highlightFields(new String [] { "body", "title" }, query.build(), searcher, topDocs);
- assertEquals(2, snippets.size());
- assertEquals("Just a test <b>highlighting</b> from postings. ", snippets.get("body")[0]);
- assertEquals("<b>Highlighting</b> the first term. ", snippets.get("body")[1]);
- assertEquals("I am hoping for the <b>best</b>.", snippets.get("title")[0]);
- assertEquals("But <b>best</b> may not be good enough.", snippets.get("title")[1]);
- ir.close();
- dir.close();
- }
-
- public void testMultipleTerms() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");
- iw.addDocument(doc);
- body.setStringValue("Highlighting the first term. Hope it works.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- BooleanQuery.Builder query = new BooleanQuery.Builder();
- query.add(new TermQuery(new Term("body", "highlighting")), BooleanClause.Occur.SHOULD);
- query.add(new TermQuery(new Term("body", "just")), BooleanClause.Occur.SHOULD);
- query.add(new TermQuery(new Term("body", "first")), BooleanClause.Occur.SHOULD);
- TopDocs topDocs = searcher.search(query.build(), 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query.build(), searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("<b>Just</b> a test <b>highlighting</b> from postings. ", snippets[0]);
- assertEquals("<b>Highlighting</b> the <b>first</b> term. ", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testMultiplePassages() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");
- iw.addDocument(doc);
- body.setStringValue("This test is another test. Not a good sentence. Test test test test.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- Query query = new TermQuery(new Term("body", "test"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs, 2);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>. Just a <b>test</b> highlighting from postings. ", snippets[0]);
- assertEquals("This <b>test</b> is another <b>test</b>. ... <b>Test</b> <b>test</b> <b>test</b> <b>test</b>.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testUserFailedToIndexOffsets() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType positionsType = new FieldType(TextField.TYPE_STORED);
- positionsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
- Field body = new Field("body", "", positionsType);
- Field title = new StringField("title", "", Field.Store.YES);
- Document doc = new Document();
- doc.add(body);
- doc.add(title);
-
- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");
- title.setStringValue("test");
- iw.addDocument(doc);
- body.setStringValue("This test is another test. Not a good sentence. Test test test test.");
- title.setStringValue("test");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- Query query = new TermQuery(new Term("body", "test"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- expectThrows(IllegalArgumentException.class, () -> {
- highlighter.highlight("body", query, searcher, topDocs, 2);
- });
-
- expectThrows(IllegalArgumentException.class, () -> {
- highlighter.highlight("title", new TermQuery(new Term("title", "test")), searcher, topDocs, 2);
- fail("did not hit expected exception");
- });
-
- ir.close();
- dir.close();
- }
-
- public void testBuddhism() throws Exception {
- String text = "This eight-volume set brings together seminal papers in Buddhist studies from a vast " +
- "range of academic disciplines published over the last forty years. With a new introduction " +
- "by the editor, this collection is a unique and unrivalled research resource for both " +
- "student and scholar. Coverage includes: - Buddhist origins; early history of Buddhism in " +
- "South and Southeast Asia - early Buddhist Schools and Doctrinal History; Theravada Doctrine " +
- "- the Origins and nature of Mahayana Buddhism; some Mahayana religious topics - Abhidharma " +
- "and Madhyamaka - Yogacara, the Epistemological tradition, and Tathagatagarbha - Tantric " +
- "Buddhism (Including China and Japan); Buddhism in Nepal and Tibet - Buddhism in South and " +
- "Southeast Asia, and - Buddhism in China, East Asia, and Japan.";
- Directory dir = newDirectory();
- Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, analyzer);
-
- FieldType positionsType = new FieldType(TextField.TYPE_STORED);
- positionsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", text, positionsType);
- Document document = new Document();
- document.add(body);
- iw.addDocument(document);
- IndexReader ir = iw.getReader();
- iw.close();
- IndexSearcher searcher = newSearcher(ir);
- PhraseQuery query = new PhraseQuery("body", "buddhist", "origins");
- TopDocs topDocs = searcher.search(query, 10);
- assertEquals(1, topDocs.totalHits);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs, 2);
- assertEquals(1, snippets.length);
- assertTrue(snippets[0].contains("<b>Buddhist</b> <b>origins</b>"));
- ir.close();
- dir.close();
- }
-
- public void testCuriousGeorge() throws Exception {
- String text = "It’s the formula for success for preschoolers—Curious George and fire trucks! " +
- "Curious George and the Firefighters is a story based on H. A. and Margret Rey’s " +
- "popular primate and painted in the original watercolor and charcoal style. " +
- "Firefighters are a famously brave lot, but can they withstand a visit from one curious monkey?";
- Directory dir = newDirectory();
- Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, analyzer);
- FieldType positionsType = new FieldType(TextField.TYPE_STORED);
- positionsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", text, positionsType);
- Document document = new Document();
- document.add(body);
- iw.addDocument(document);
- IndexReader ir = iw.getReader();
- iw.close();
- IndexSearcher searcher = newSearcher(ir);
- PhraseQuery query = new PhraseQuery("body", "curious", "george");
- TopDocs topDocs = searcher.search(query, 10);
- assertEquals(1, topDocs.totalHits);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs, 2);
- assertEquals(1, snippets.length);
- assertFalse(snippets[0].contains("<b>Curious</b>Curious"));
- ir.close();
- dir.close();
- }
-
- public void testCambridgeMA() throws Exception {
- BufferedReader r = new BufferedReader(new InputStreamReader(
- this.getClass().getResourceAsStream("CambridgeMA.utf8"), StandardCharsets.UTF_8));
- String text = r.readLine();
- r.close();
- Directory dir = newDirectory();
- Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, analyzer);
- FieldType positionsType = new FieldType(TextField.TYPE_STORED);
- positionsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", text, positionsType);
- Document document = new Document();
- document.add(body);
- iw.addDocument(document);
- IndexReader ir = iw.getReader();
- iw.close();
- IndexSearcher searcher = newSearcher(ir);
- BooleanQuery.Builder query = new BooleanQuery.Builder();
- query.add(new TermQuery(new Term("body", "porter")), BooleanClause.Occur.SHOULD);
- query.add(new TermQuery(new Term("body", "square")), BooleanClause.Occur.SHOULD);
- query.add(new TermQuery(new Term("body", "massachusetts")), BooleanClause.Occur.SHOULD);
- TopDocs topDocs = searcher.search(query.build(), 10);
- assertEquals(1, topDocs.totalHits);
- PostingsHighlighter highlighter = new PostingsHighlighter(Integer.MAX_VALUE-1);
- String snippets[] = highlighter.highlight("body", query.build(), searcher, topDocs, 2);
- assertEquals(1, snippets.length);
- assertTrue(snippets[0].contains("<b>Square</b>"));
- assertTrue(snippets[0].contains("<b>Porter</b>"));
- ir.close();
- dir.close();
- }
-
- public void testPassageRanking() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test. Just highlighting from postings. This is also a much sillier test. Feel free to test test test test test test test.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- Query query = new TermQuery(new Term("body", "test"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(1, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs, 2);
- assertEquals(1, snippets.length);
- assertEquals("This is a <b>test</b>. ... Feel free to <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b>.", snippets[0]);
-
- ir.close();
- dir.close();
- }
-
- public void testBooleanMustNot() throws Exception {
- Directory dir = newDirectory();
- Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, analyzer);
- FieldType positionsType = new FieldType(TextField.TYPE_STORED);
- positionsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "This sentence has both terms. This sentence has only terms.", positionsType);
- Document document = new Document();
- document.add(body);
- iw.addDocument(document);
- IndexReader ir = iw.getReader();
- iw.close();
- IndexSearcher searcher = newSearcher(ir);
- BooleanQuery.Builder query = new BooleanQuery.Builder();
- query.add(new TermQuery(new Term("body", "terms")), BooleanClause.Occur.SHOULD);
- BooleanQuery.Builder query2 = new BooleanQuery.Builder();
- query.add(query2.build(), BooleanClause.Occur.SHOULD);
- query2.add(new TermQuery(new Term("body", "both")), BooleanClause.Occur.MUST_NOT);
- TopDocs topDocs = searcher.search(query.build(), 10);
- assertEquals(1, topDocs.totalHits);
- PostingsHighlighter highlighter = new PostingsHighlighter(Integer.MAX_VALUE-1);
- String snippets[] = highlighter.highlight("body", query.build(), searcher, topDocs, 2);
- assertEquals(1, snippets.length);
- assertFalse(snippets[0].contains("<b>both</b>"));
- ir.close();
- dir.close();
- }
-
- public void testHighlightAllText() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test. Just highlighting from postings. This is also a much sillier test. Feel free to test test test test test test test.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter(10000) {
- @Override
- protected BreakIterator getBreakIterator(String field) {
- return new WholeBreakIterator();
- }
- };
- Query query = new TermQuery(new Term("body", "test"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(1, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs, 2);
- assertEquals(1, snippets.length);
- assertEquals("This is a <b>test</b>. Just highlighting from postings. This is also a much sillier <b>test</b>. Feel free to <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b>.", snippets[0]);
-
- ir.close();
- dir.close();
- }
-
- public void testSpecificDocIDs() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");
- iw.addDocument(doc);
- body.setStringValue("Highlighting the first term. Hope it works.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- Query query = new TermQuery(new Term("body", "highlighting"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- ScoreDoc[] hits = topDocs.scoreDocs;
- int[] docIDs = new int[2];
- docIDs[0] = hits[0].doc;
- docIDs[1] = hits[1].doc;
- String snippets[] = highlighter.highlightFields(new String[] {"body"}, query, searcher, docIDs, new int[] { 1 }).get("body");
- assertEquals(2, snippets.length);
- assertEquals("Just a test <b>highlighting</b> from postings. ", snippets[0]);
- assertEquals("<b>Highlighting</b> the first term. ", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testCustomFieldValueSource() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- Document doc = new Document();
-
- FieldType offsetsType = new FieldType(TextField.TYPE_NOT_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- final String text = "This is a test. Just highlighting from postings. This is also a much sillier test. Feel free to test test test test test test test.";
- Field body = new Field("body", text, offsetsType);
- doc.add(body);
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
-
- PostingsHighlighter highlighter = new PostingsHighlighter(10000) {
- @Override
- protected String[][] loadFieldValues(IndexSearcher searcher, String[] fields, int[] docids, int maxLength) throws IOException {
- assert fields.length == 1;
- assert docids.length == 1;
- String[][] contents = new String[1][1];
- contents[0][0] = text;
- return contents;
- }
-
- @Override
- protected BreakIterator getBreakIterator(String field) {
- return new WholeBreakIterator();
- }
- };
-
- Query query = new TermQuery(new Term("body", "test"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(1, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs, 2);
- assertEquals(1, snippets.length);
- assertEquals("This is a <b>test</b>. Just highlighting from postings. This is also a much sillier <b>test</b>. Feel free to <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b>.", snippets[0]);
-
- ir.close();
- dir.close();
- }
-
- /** Make sure highlighter returns first N sentences if
- * there were no hits. */
- public void testEmptyHighlights() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Document doc = new Document();
-
- Field body = new Field("body", "test this is. another sentence this test has. far away is that planet.", offsetsType);
- doc.add(body);
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- Query query = new TermQuery(new Term("body", "highlighting"));
- int[] docIDs = new int[] {0};
- String snippets[] = highlighter.highlightFields(new String[] {"body"}, query, searcher, docIDs, new int[] { 2 }).get("body");
- assertEquals(1, snippets.length);
- assertEquals("test this is. another sentence this test has. ", snippets[0]);
-
- ir.close();
- dir.close();
- }
-
- /** Make sure highlighter we can customize how emtpy
- * highlight is returned. */
- public void testCustomEmptyHighlights() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Document doc = new Document();
-
- Field body = new Field("body", "test this is. another sentence this test has. far away is that planet.", offsetsType);
- doc.add(body);
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- public Passage[] getEmptyHighlight(String fieldName, BreakIterator bi, int maxPassages) {
- return new Passage[0];
- }
- };
- Query query = new TermQuery(new Term("body", "highlighting"));
- int[] docIDs = new int[] {0};
- String snippets[] = highlighter.highlightFields(new String[] {"body"}, query, searcher, docIDs, new int[] { 2 }).get("body");
- assertEquals(1, snippets.length);
- assertNull(snippets[0]);
-
- ir.close();
- dir.close();
- }
-
- /** Make sure highlighter returns whole text when there
- * are no hits and BreakIterator is null. */
- public void testEmptyHighlightsWhole() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Document doc = new Document();
-
- Field body = new Field("body", "test this is. another sentence this test has. far away is that planet.", offsetsType);
- doc.add(body);
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter(10000) {
- @Override
- protected BreakIterator getBreakIterator(String field) {
- return new WholeBreakIterator();
- }
- };
- Query query = new TermQuery(new Term("body", "highlighting"));
- int[] docIDs = new int[] {0};
- String snippets[] = highlighter.highlightFields(new String[] {"body"}, query, searcher, docIDs, new int[] { 2 }).get("body");
- assertEquals(1, snippets.length);
- assertEquals("test this is. another sentence this test has. far away is that planet.", snippets[0]);
-
- ir.close();
- dir.close();
- }
-
- /** Make sure highlighter is OK with entirely missing
- * field. */
- public void testFieldIsMissing() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Document doc = new Document();
-
- Field body = new Field("body", "test this is. another sentence this test has. far away is that planet.", offsetsType);
- doc.add(body);
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- Query query = new TermQuery(new Term("bogus", "highlighting"));
- int[] docIDs = new int[] {0};
- String snippets[] = highlighter.highlightFields(new String[] {"bogus"}, query, searcher, docIDs, new int[] { 2 }).get("bogus");
- assertEquals(1, snippets.length);
- assertNull(snippets[0]);
-
- ir.close();
- dir.close();
- }
-
- public void testFieldIsJustSpace() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
-
- Document doc = new Document();
- doc.add(new Field("body", " ", offsetsType));
- doc.add(new Field("id", "id", offsetsType));
- iw.addDocument(doc);
-
- doc = new Document();
- doc.add(new Field("body", "something", offsetsType));
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- int docID = searcher.search(new TermQuery(new Term("id", "id")), 1).scoreDocs[0].doc;
-
- Query query = new TermQuery(new Term("body", "highlighting"));
- int[] docIDs = new int[1];
- docIDs[0] = docID;
- String snippets[] = highlighter.highlightFields(new String[] {"body"}, query, searcher, docIDs, new int[] { 2 }).get("body");
- assertEquals(1, snippets.length);
- assertEquals(" ", snippets[0]);
-
- ir.close();
- dir.close();
- }
-
- public void testFieldIsEmptyString() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
-
- Document doc = new Document();
- doc.add(new Field("body", "", offsetsType));
- doc.add(new Field("id", "id", offsetsType));
- iw.addDocument(doc);
-
- doc = new Document();
- doc.add(new Field("body", "something", offsetsType));
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- int docID = searcher.search(new TermQuery(new Term("id", "id")), 1).scoreDocs[0].doc;
-
- Query query = new TermQuery(new Term("body", "highlighting"));
- int[] docIDs = new int[1];
- docIDs[0] = docID;
- String snippets[] = highlighter.highlightFields(new String[] {"body"}, query, searcher, docIDs, new int[] { 2 }).get("body");
- assertEquals(1, snippets.length);
- assertNull(snippets[0]);
-
- ir.close();
- dir.close();
- }
-
- public void testMultipleDocs() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
-
- int numDocs = atLeast(100);
- for(int i=0;i<numDocs;i++) {
- Document doc = new Document();
- String content = "the answer is " + i;
- if ((i & 1) == 0) {
- content += " some more terms";
- }
- doc.add(new Field("body", content, offsetsType));
- doc.add(newStringField("id", ""+i, Field.Store.YES));
- iw.addDocument(doc);
-
- if (random().nextInt(10) == 2) {
- iw.commit();
- }
- }
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- Query query = new TermQuery(new Term("body", "answer"));
- TopDocs hits = searcher.search(query, numDocs);
- assertEquals(numDocs, hits.totalHits);
-
- String snippets[] = highlighter.highlight("body", query, searcher, hits);
- assertEquals(numDocs, snippets.length);
- for(int hit=0;hit<numDocs;hit++) {
- Document doc = searcher.doc(hits.scoreDocs[hit].doc);
- int id = Integer.parseInt(doc.get("id"));
- String expected = "the <b>answer</b> is " + id;
- if ((id & 1) == 0) {
- expected += " some more terms";
- }
- assertEquals(expected, snippets[hit]);
- }
-
- ir.close();
- dir.close();
- }
-
- public void testMultipleSnippetSizes() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Field title = new Field("title", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
- doc.add(title);
-
- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");
- title.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- BooleanQuery.Builder query = new BooleanQuery.Builder();
- query.add(new TermQuery(new Term("body", "test")), BooleanClause.Occur.SHOULD);
- query.add(new TermQuery(new Term("title", "test")), BooleanClause.Occur.SHOULD);
- Map<String,String[]> snippets = highlighter.highlightFields(new String[] { "title", "body" }, query.build(), searcher, new int[] { 0 }, new int[] { 1, 2 });
- String titleHighlight = snippets.get("title")[0];
- String bodyHighlight = snippets.get("body")[0];
- assertEquals("This is a <b>test</b>. ", titleHighlight);
- assertEquals("This is a <b>test</b>. Just a <b>test</b> highlighting from postings. ", bodyHighlight);
- ir.close();
- dir.close();
- }
-
- public void testEncode() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test. Just a test highlighting from <i>postings</i>. Feel free to ignore.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected PassageFormatter getFormatter(String field) {
- return new DefaultPassageFormatter("<b>", "</b>", "... ", true);
- }
- };
- Query query = new TermQuery(new Term("body", "highlighting"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(1, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(1, snippets.length);
- assertEquals("Just a test <b>highlighting</b> from <i>postings</i>. ", snippets[0]);
-
- ir.close();
- dir.close();
- }
-
- /** customizing the gap separator to force a sentence break */
- public void testGapSeparator() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Document doc = new Document();
-
- Field body1 = new Field("body", "", offsetsType);
- body1.setStringValue("This is a multivalued field");
- doc.add(body1);
-
- Field body2 = new Field("body", "", offsetsType);
- body2.setStringValue("This is something different");
- doc.add(body2);
-
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected char getMultiValuedSeparator(String field) {
- assert field.equals("body");
- return '\u2029';
- }
- };
- Query query = new TermQuery(new Term("body", "field"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(1, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(1, snippets.length);
- assertEquals("This is a multivalued <b>field</b>\u2029", snippets[0]);
-
- ir.close();
- dir.close();
- }
-
- // LUCENE-4906
- public void testObjectFormatter() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected PassageFormatter getFormatter(String field) {
- return new PassageFormatter() {
- PassageFormatter defaultFormatter = new DefaultPassageFormatter();
-
- @Override
- public String[] format(Passage passages[], String content) {
- // Just turns the String snippet into a length 2
- // array of String
- return new String[] {"blah blah", defaultFormatter.format(passages, content).toString()};
- }
- };
- }
- };
-
- Query query = new TermQuery(new Term("body", "highlighting"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(1, topDocs.totalHits);
- int[] docIDs = new int[1];
- docIDs[0] = topDocs.scoreDocs[0].doc;
- Map<String,Object[]> snippets = highlighter.highlightFieldsAsObjects(new String[]{"body"}, query, searcher, docIDs, new int[] {1});
- Object[] bodySnippets = snippets.get("body");
- assertEquals(1, bodySnippets.length);
- assertTrue(Arrays.equals(new String[] {"blah blah", "Just a test <b>highlighting</b> from postings. "}, (String[]) bodySnippets[0]));
-
- ir.close();
- dir.close();
- }
-
- public void testFieldSometimesMissingFromSegment() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "foo", offsetsType);
- Document doc = new Document();
- doc.add(body);
- iw.addDocument(doc);
-
- // Make a 2nd segment where body is only stored:
- iw.commit();
- doc = new Document();
- doc.add(new StoredField("body", "foo"));
- iw.addDocument(doc);
-
- IndexReader ir = DirectoryReader.open(iw.w);
- iw.close();
-
- IndexSearcher searcher = new IndexSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
- Query query = new MatchAllDocsQuery();
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("foo", snippets[0]);
- assertNull(snippets[1]);
- ir.close();
- dir.close();
- }
-
- public void testCustomScoreQueryHighlight() throws Exception{
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This piece of text refers to Kennedy at the beginning then has a longer piece of text that is very long in the middle and finally ends with another reference to Kennedy");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- TermQuery termQuery = new TermQuery(new Term("body", "very"));
- PostingsHighlighter highlighter = new PostingsHighlighter();
- CustomScoreQuery query = new CustomScoreQuery(termQuery);
-
- IndexSearcher searcher = newSearcher(ir);
- TopDocs hits = searcher.search(query, 10);
- assertEquals(1, hits.totalHits);
-
- String snippets[] = highlighter.highlight("body", query, searcher, hits);
- assertEquals(1, snippets.length);
- assertEquals("This piece of text refers to Kennedy at the beginning then has a longer piece of text that is <b>very</b> long in the middle and finally ends with another reference to Kennedy",
- snippets[0]);
-
- ir.close();
- dir.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java b/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java
deleted file mode 100644
index 7a693d9..0000000
--- a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java
+++ /dev/null
@@ -1,324 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.postingshighlight;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Random;
-
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.TestUtil;
-
-public class TestPostingsHighlighterRanking extends LuceneTestCase {
- /**
- * indexes a bunch of gibberish, and then highlights top(n).
- * asserts that top(n) highlights is a subset of top(n+1) up to some max N
- */
- // TODO: this only tests single-valued fields. we should also index multiple values per field!
- public void testRanking() throws Exception {
- // number of documents: we will check each one
- final int numDocs = atLeast(100);
- // number of top-N snippets, we will check 1 .. N
- final int maxTopN = 5;
- // maximum number of elements to put in a sentence.
- final int maxSentenceLength = 10;
- // maximum number of sentences in a document
- final int maxNumSentences = 20;
-
- Directory dir = newDirectory();
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- Document document = new Document();
- Field id = new StringField("id", "", Field.Store.NO);
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- document.add(id);
- document.add(body);
-
- for (int i = 0; i < numDocs; i++) {
- StringBuilder bodyText = new StringBuilder();
- int numSentences = TestUtil.nextInt(random(), 1, maxNumSentences);
- for (int j = 0; j < numSentences; j++) {
- bodyText.append(newSentence(random(), maxSentenceLength));
- }
- body.setStringValue(bodyText.toString());
- id.setStringValue(Integer.toString(i));
- iw.addDocument(document);
- }
-
- IndexReader ir = iw.getReader();
- IndexSearcher searcher = newSearcher(ir);
- for (int i = 0; i < numDocs; i++) {
- checkDocument(searcher, i, maxTopN);
- }
- iw.close();
- ir.close();
- dir.close();
- }
-
- private void checkDocument(IndexSearcher is, int doc, int maxTopN) throws IOException {
- for (int ch = 'a'; ch <= 'z'; ch++) {
- Term term = new Term("body", "" + (char)ch);
- // check a simple term query
- checkQuery(is, new TermQuery(term), doc, maxTopN);
- // check a boolean query
- BooleanQuery.Builder bq = new BooleanQuery.Builder();
- bq.add(new TermQuery(term), BooleanClause.Occur.SHOULD);
- Term nextTerm = new Term("body", "" + (char)(ch+1));
- bq.add(new TermQuery(nextTerm), BooleanClause.Occur.SHOULD);
- checkQuery(is, bq.build(), doc, maxTopN);
- }
- }
-
- private void checkQuery(IndexSearcher is, Query query, int doc, int maxTopN) throws IOException {
- for (int n = 1; n < maxTopN; n++) {
- final FakePassageFormatter f1 = new FakePassageFormatter();
- PostingsHighlighter p1 = new PostingsHighlighter(Integer.MAX_VALUE-1) {
- @Override
- protected PassageFormatter getFormatter(String field) {
- assertEquals("body", field);
- return f1;
- }
- };
-
- final FakePassageFormatter f2 = new FakePassageFormatter();
- PostingsHighlighter p2 = new PostingsHighlighter(Integer.MAX_VALUE-1) {
- @Override
- protected PassageFormatter getFormatter(String field) {
- assertEquals("body", field);
- return f2;
- }
- };
-
- BooleanQuery.Builder bq = new BooleanQuery.Builder();
- bq.add(query, BooleanClause.Occur.MUST);
- bq.add(new TermQuery(new Term("id", Integer.toString(doc))), BooleanClause.Occur.MUST);
- TopDocs td = is.search(bq.build(), 1);
- p1.highlight("body", bq.build(), is, td, n);
- p2.highlight("body", bq.build(), is, td, n+1);
- assertTrue(f2.seen.containsAll(f1.seen));
- }
- }
-
- /**
- * returns a new random sentence, up to maxSentenceLength "words" in length.
- * each word is a single character (a-z). The first one is capitalized.
- */
- private String newSentence(Random r, int maxSentenceLength) {
- StringBuilder sb = new StringBuilder();
- int numElements = TestUtil.nextInt(r, 1, maxSentenceLength);
- for (int i = 0; i < numElements; i++) {
- if (sb.length() > 0) {
- sb.append(' ');
- sb.append((char) TestUtil.nextInt(r, 'a', 'z'));
- } else {
- // capitalize the first word to help breakiterator
- sb.append((char) TestUtil.nextInt(r, 'A', 'Z'));
- }
- }
- sb.append(". "); // finalize sentence
- return sb.toString();
- }
-
- /**
- * a fake formatter that doesn't actually format passages.
- * instead it just collects them for asserts!
- */
- static class FakePassageFormatter extends PassageFormatter {
- HashSet<Pair> seen = new HashSet<>();
-
- @Override
- public String format(Passage passages[], String content) {
- for (Passage p : passages) {
- // verify some basics about the passage
- assertTrue(p.getScore() >= 0);
- assertTrue(p.getNumMatches() > 0);
- assertTrue(p.getStartOffset() >= 0);
- assertTrue(p.getStartOffset() <= content.length());
- assertTrue(p.getEndOffset() >= p.getStartOffset());
- assertTrue(p.getEndOffset() <= content.length());
- // we use a very simple analyzer. so we can assert the matches are correct
- int lastMatchStart = -1;
- for (int i = 0; i < p.getNumMatches(); i++) {
- BytesRef term = p.getMatchTerms()[i];
- int matchStart = p.getMatchStarts()[i];
- assertTrue(matchStart >= 0);
- // must at least start within the passage
- assertTrue(matchStart < p.getEndOffset());
- int matchEnd = p.getMatchEnds()[i];
- assertTrue(matchEnd >= 0);
- // always moving forward
- assertTrue(matchStart >= lastMatchStart);
- lastMatchStart = matchStart;
- // single character terms
- assertEquals(matchStart+1, matchEnd);
- // and the offsets must be correct...
- assertEquals(1, term.length);
- assertEquals((char)term.bytes[term.offset], Character.toLowerCase(content.charAt(matchStart)));
- }
- // record just the start/end offset for simplicity
- seen.add(new Pair(p.getStartOffset(), p.getEndOffset()));
- }
- return "bogus!!!!!!";
- }
- }
-
- static class Pair {
- final int start;
- final int end;
-
- Pair(int start, int end) {
- this.start = start;
- this.end = end;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + end;
- result = prime * result + start;
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- Pair other = (Pair) obj;
- if (end != other.end) {
- return false;
- }
- if (start != other.start) {
- return false;
- }
- return true;
- }
-
- @Override
- public String toString() {
- return "Pair [start=" + start + ", end=" + end + "]";
- }
- }
-
- /** sets b=0 to disable passage length normalization */
- public void testCustomB() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test. This test is a better test but the sentence is excruiatingly long, " +
- "you have no idea how painful it was for me to type this long sentence into my IDE.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter(10000) {
- @Override
- protected PassageScorer getScorer(String field) {
- return new PassageScorer(1.2f, 0, 87);
- }
- };
- Query query = new TermQuery(new Term("body", "test"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(1, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs, 1);
- assertEquals(1, snippets.length);
- assertTrue(snippets[0].startsWith("This <b>test</b> is a better <b>test</b>"));
-
- ir.close();
- dir.close();
- }
-
- /** sets k1=0 for simple coordinate-level match (# of query terms present) */
- public void testCustomK1() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This has only foo foo. " +
- "On the other hand this sentence contains both foo and bar. " +
- "This has only bar bar bar bar bar bar bar bar bar bar bar bar.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter(10000) {
- @Override
- protected PassageScorer getScorer(String field) {
- return new PassageScorer(0, 0.75f, 87);
- }
- };
- BooleanQuery.Builder query = new BooleanQuery.Builder();
- query.add(new TermQuery(new Term("body", "foo")), BooleanClause.Occur.SHOULD);
- query.add(new TermQuery(new Term("body", "bar")), BooleanClause.Occur.SHOULD);
- TopDocs topDocs = searcher.search(query.build(), 10, Sort.INDEXORDER);
- assertEquals(1, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query.build(), searcher, topDocs, 1);
- assertEquals(1, snippets.length);
- assertTrue(snippets[0].startsWith("On the other hand"));
-
- ir.close();
- dir.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/LengthGoalBreakIteratorTest.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/LengthGoalBreakIteratorTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/LengthGoalBreakIteratorTest.java
index 4dd30e2..2434f52 100644
--- a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/LengthGoalBreakIteratorTest.java
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/LengthGoalBreakIteratorTest.java
@@ -23,7 +23,6 @@ import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.postingshighlight.CustomSeparatorBreakIterator;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.QueryBuilder;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestCustomSeparatorBreakIterator.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestCustomSeparatorBreakIterator.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestCustomSeparatorBreakIterator.java
new file mode 100644
index 0000000..cacb9cb
--- /dev/null
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestCustomSeparatorBreakIterator.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search.uhighlight;
+
+import java.text.BreakIterator;
+import java.util.Locale;
+
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import org.apache.lucene.util.LuceneTestCase;
+
+import static org.apache.lucene.search.uhighlight.TestWholeBreakIterator.assertSameBreaks;
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class TestCustomSeparatorBreakIterator extends LuceneTestCase {
+
+ private static final Character[] SEPARATORS = new Character[]{' ', '\u0000', 8233};
+
+ public void testBreakOnCustomSeparator() throws Exception {
+ Character separator = randomSeparator();
+ BreakIterator bi = new CustomSeparatorBreakIterator(separator);
+ String source = "this" + separator + "is" + separator + "the" + separator + "first" + separator + "sentence";
+ bi.setText(source);
+ assertThat(bi.current(), equalTo(0));
+ assertThat(bi.first(), equalTo(0));
+ assertThat(source.substring(bi.current(), bi.next()), equalTo("this" + separator));
+ assertThat(source.substring(bi.current(), bi.next()), equalTo("is" + separator));
+ assertThat(source.substring(bi.current(), bi.next()), equalTo("the" + separator));
+ assertThat(source.substring(bi.current(), bi.next()), equalTo("first" + separator));
+ assertThat(source.substring(bi.current(), bi.next()), equalTo("sentence"));
+ assertThat(bi.next(), equalTo(BreakIterator.DONE));
+
+ assertThat(bi.last(), equalTo(source.length()));
+ int current = bi.current();
+ assertThat(source.substring(bi.previous(), current), equalTo("sentence"));
+ current = bi.current();
+ assertThat(source.substring(bi.previous(), current), equalTo("first" + separator));
+ current = bi.current();
+ assertThat(source.substring(bi.previous(), current), equalTo("the" + separator));
+ current = bi.current();
+ assertThat(source.substring(bi.previous(), current), equalTo("is" + separator));
+ current = bi.current();
+ assertThat(source.substring(bi.previous(), current), equalTo("this" + separator));
+ assertThat(bi.previous(), equalTo(BreakIterator.DONE));
+ assertThat(bi.current(), equalTo(0));
+
+ assertThat(source.substring(0, bi.following(9)), equalTo("this" + separator + "is" + separator + "the" + separator));
+
+ assertThat(source.substring(0, bi.preceding(9)), equalTo("this" + separator + "is" + separator));
+
+ assertThat(bi.first(), equalTo(0));
+ assertThat(source.substring(0, bi.next(3)), equalTo("this" + separator + "is" + separator + "the" + separator));
+ }
+
+ public void testSingleSentences() throws Exception {
+ BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
+ BreakIterator actual = new CustomSeparatorBreakIterator(randomSeparator());
+ assertSameBreaks("a", expected, actual);
+ assertSameBreaks("ab", expected, actual);
+ assertSameBreaks("abc", expected, actual);
+ assertSameBreaks("", expected, actual);
+ }
+
+ public void testSliceEnd() throws Exception {
+ BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
+ BreakIterator actual = new CustomSeparatorBreakIterator(randomSeparator());
+ assertSameBreaks("a000", 0, 1, expected, actual);
+ assertSameBreaks("ab000", 0, 1, expected, actual);
+ assertSameBreaks("abc000", 0, 1, expected, actual);
+ assertSameBreaks("000", 0, 0, expected, actual);
+ }
+
+ public void testSliceStart() throws Exception {
+ BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
+ BreakIterator actual = new CustomSeparatorBreakIterator(randomSeparator());
+ assertSameBreaks("000a", 3, 1, expected, actual);
+ assertSameBreaks("000ab", 3, 2, expected, actual);
+ assertSameBreaks("000abc", 3, 3, expected, actual);
+ assertSameBreaks("000", 3, 0, expected, actual);
+ }
+
+ public void testSliceMiddle() throws Exception {
+ BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
+ BreakIterator actual = new CustomSeparatorBreakIterator(randomSeparator());
+ assertSameBreaks("000a000", 3, 1, expected, actual);
+ assertSameBreaks("000ab000", 3, 2, expected, actual);
+ assertSameBreaks("000abc000", 3, 3, expected, actual);
+ assertSameBreaks("000000", 3, 0, expected, actual);
+ }
+
+ /** the current position must be ignored, initial position is always first() */
+ public void testFirstPosition() throws Exception {
+ BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
+ BreakIterator actual = new CustomSeparatorBreakIterator(randomSeparator());
+ assertSameBreaks("000ab000", 3, 2, 4, expected, actual);
+ }
+
+ private static char randomSeparator() {
+ return RandomPicks.randomFrom(random(), SEPARATORS);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.java
index ddf8a92..3ffe95a 100644
--- a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.java
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.java
@@ -49,7 +49,6 @@ import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.postingshighlight.WholeBreakIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestWholeBreakIterator.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestWholeBreakIterator.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestWholeBreakIterator.java
new file mode 100644
index 0000000..8e5b2ff
--- /dev/null
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestWholeBreakIterator.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search.uhighlight;
+
+import java.text.BreakIterator;
+import java.text.CharacterIterator;
+import java.text.StringCharacterIterator;
+import java.util.Locale;
+
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestWholeBreakIterator extends LuceneTestCase {
+
+ /** For single sentences, we know WholeBreakIterator should break the same as a sentence iterator */
+ public void testSingleSentences() throws Exception {
+ BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
+ BreakIterator actual = new WholeBreakIterator();
+ assertSameBreaks("a", expected, actual);
+ assertSameBreaks("ab", expected, actual);
+ assertSameBreaks("abc", expected, actual);
+ assertSameBreaks("", expected, actual);
+ }
+
+ public void testSliceEnd() throws Exception {
+ BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
+ BreakIterator actual = new WholeBreakIterator();
+ assertSameBreaks("a000", 0, 1, expected, actual);
+ assertSameBreaks("ab000", 0, 1, expected, actual);
+ assertSameBreaks("abc000", 0, 1, expected, actual);
+ assertSameBreaks("000", 0, 0, expected, actual);
+ }
+
+ public void testSliceStart() throws Exception {
+ BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
+ BreakIterator actual = new WholeBreakIterator();
+ assertSameBreaks("000a", 3, 1, expected, actual);
+ assertSameBreaks("000ab", 3, 2, expected, actual);
+ assertSameBreaks("000abc", 3, 3, expected, actual);
+ assertSameBreaks("000", 3, 0, expected, actual);
+ }
+
+ public void testSliceMiddle() throws Exception {
+ BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
+ BreakIterator actual = new WholeBreakIterator();
+ assertSameBreaks("000a000", 3, 1, expected, actual);
+ assertSameBreaks("000ab000", 3, 2, expected, actual);
+ assertSameBreaks("000abc000", 3, 3, expected, actual);
+ assertSameBreaks("000000", 3, 0, expected, actual);
+ }
+
+ /** the current position must be ignored, initial position is always first() */
+ public void testFirstPosition() throws Exception {
+ BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);
+ BreakIterator actual = new WholeBreakIterator();
+ assertSameBreaks("000ab000", 3, 2, 4, expected, actual);
+ }
+
+ public static void assertSameBreaks(String text, BreakIterator expected, BreakIterator actual) {
+ assertSameBreaks(new StringCharacterIterator(text),
+ new StringCharacterIterator(text),
+ expected,
+ actual);
+ }
+
+ public static void assertSameBreaks(String text, int offset, int length, BreakIterator expected, BreakIterator actual) {
+ assertSameBreaks(text, offset, length, offset, expected, actual);
+ }
+
+ public static void assertSameBreaks(String text, int offset, int length, int current, BreakIterator expected, BreakIterator actual) {
+ assertSameBreaks(new StringCharacterIterator(text, offset, offset+length, current),
+ new StringCharacterIterator(text, offset, offset+length, current),
+ expected,
+ actual);
+ }
+
+ /** Asserts that two breakiterators break the text the same way */
+ public static void assertSameBreaks(CharacterIterator one, CharacterIterator two, BreakIterator expected, BreakIterator actual) {
+ expected.setText(one);
+ actual.setText(two);
+
+ assertEquals(expected.current(), actual.current());
+
+ // next()
+ int v = expected.current();
+ while (v != BreakIterator.DONE) {
+ assertEquals(v = expected.next(), actual.next());
+ assertEquals(expected.current(), actual.current());
+ }
+
+ // first()
+ assertEquals(expected.first(), actual.first());
+ assertEquals(expected.current(), actual.current());
+ // last()
+ assertEquals(expected.last(), actual.last());
+ assertEquals(expected.current(), actual.current());
+
+ // previous()
+ v = expected.current();
+ while (v != BreakIterator.DONE) {
+ assertEquals(v = expected.previous(), actual.previous());
+ assertEquals(expected.current(), actual.current());
+ }
+
+ // following()
+ for (int i = one.getBeginIndex(); i <= one.getEndIndex(); i++) {
+ expected.first();
+ actual.first();
+ assertEquals(expected.following(i), actual.following(i));
+ assertEquals(expected.current(), actual.current());
+ }
+
+ // preceding()
+ for (int i = one.getBeginIndex(); i <= one.getEndIndex(); i++) {
+ expected.last();
+ actual.last();
+ assertEquals(expected.preceding(i), actual.preceding(i));
+ assertEquals(expected.current(), actual.current());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java b/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java
index e9c842c..bd32fc1 100644
--- a/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java
+++ b/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java
@@ -28,13 +28,13 @@ import java.util.function.Predicate;
import org.apache.lucene.document.Document;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.postingshighlight.CustomSeparatorBreakIterator;
-import org.apache.lucene.search.postingshighlight.WholeBreakIterator;
+import org.apache.lucene.search.uhighlight.CustomSeparatorBreakIterator;
import org.apache.lucene.search.uhighlight.DefaultPassageFormatter;
import org.apache.lucene.search.uhighlight.LengthGoalBreakIterator;
import org.apache.lucene.search.uhighlight.PassageFormatter;
import org.apache.lucene.search.uhighlight.PassageScorer;
import org.apache.lucene.search.uhighlight.UnifiedHighlighter;
+import org.apache.lucene.search.uhighlight.WholeBreakIterator;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.HighlightParams;
import org.apache.solr.common.params.SolrParams;
[2/4] lucene-solr:master: LUCENE-7815: Removed the PostingsHighlighter
Posted by ds...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java b/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java
deleted file mode 100644
index 0620bd6..0000000
--- a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java
+++ /dev/null
@@ -1,884 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.postingshighlight;
-
-import java.util.Collections;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.ConstantScoreQuery;
-import org.apache.lucene.search.DisjunctionMaxQuery;
-import org.apache.lucene.search.FuzzyQuery;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.PrefixQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.RegexpQuery;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TermRangeQuery;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.WildcardQuery;
-import org.apache.lucene.search.spans.SpanFirstQuery;
-import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper;
-import org.apache.lucene.search.spans.SpanNearQuery;
-import org.apache.lucene.search.spans.SpanNotQuery;
-import org.apache.lucene.search.spans.SpanOrQuery;
-import org.apache.lucene.search.spans.SpanQuery;
-import org.apache.lucene.search.spans.SpanTermQuery;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.LuceneTestCase;
-
-/**
- * Some tests that override {@link PostingsHighlighter#getIndexAnalyzer} to
- * highlight wilcard, fuzzy, etc queries.
- */
-public class TestMultiTermHighlighting extends LuceneTestCase {
-
- public void testWildcards() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- Query query = new WildcardQuery(new Term("body", "te*"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- // wrong field
- BooleanQuery.Builder bq = new BooleanQuery.Builder();
- bq.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
- bq.add(new WildcardQuery(new Term("bogus", "te*")), BooleanClause.Occur.SHOULD);
- topDocs = searcher.search(bq.build(), 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", bq.build(), searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a test.", snippets[0]);
- assertEquals("Test a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testOnePrefix() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- Query query = new PrefixQuery(new Term("body", "te"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- // wrong field
- BooleanQuery.Builder bq = new BooleanQuery.Builder();
- bq.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
- bq.add(new PrefixQuery(new Term("bogus", "te")), BooleanClause.Occur.SHOULD);
- topDocs = searcher.search(bq.build(), 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", bq.build(), searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a test.", snippets[0]);
- assertEquals("Test a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testOneRegexp() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- Query query = new RegexpQuery(new Term("body", "te.*"));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- // wrong field
- BooleanQuery.Builder bq = new BooleanQuery.Builder();
- bq.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
- bq.add(new RegexpQuery(new Term("bogus", "te.*")), BooleanClause.Occur.SHOULD);
- topDocs = searcher.search(bq.build(), 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", bq.build(), searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a test.", snippets[0]);
- assertEquals("Test a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testOneFuzzy() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- Query query = new FuzzyQuery(new Term("body", "tets"), 1);
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- // with prefix
- query = new FuzzyQuery(new Term("body", "tets"), 1, 2);
- topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- // wrong field
- BooleanQuery.Builder bq = new BooleanQuery.Builder();
- bq.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
- bq.add(new FuzzyQuery(new Term("bogus", "tets"), 1), BooleanClause.Occur.SHOULD);
- topDocs = searcher.search(bq.build(), 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", bq.build(), searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a test.", snippets[0]);
- assertEquals("Test a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testRanges() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- Query query = TermRangeQuery.newStringRange("body", "ta", "tf", true, true);
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- // null start
- query = TermRangeQuery.newStringRange("body", null, "tf", true, true);
- topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This <b>is</b> <b>a</b> <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> <b>a</b> <b>one</b> <b>sentence</b> <b>document</b>.", snippets[1]);
-
- // null end
- query = TermRangeQuery.newStringRange("body", "ta", null, true, true);
- topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("<b>This</b> is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- // exact start inclusive
- query = TermRangeQuery.newStringRange("body", "test", "tf", true, true);
- topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- // exact end inclusive
- query = TermRangeQuery.newStringRange("body", "ta", "test", true, true);
- topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- // exact start exclusive
- BooleanQuery.Builder bq = new BooleanQuery.Builder();
- bq.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
- bq.add(TermRangeQuery.newStringRange("body", "test", "tf", false, true), BooleanClause.Occur.SHOULD);
- topDocs = searcher.search(bq.build(), 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", bq.build(), searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a test.", snippets[0]);
- assertEquals("Test a one sentence document.", snippets[1]);
-
- // exact end exclusive
- bq = new BooleanQuery.Builder();
- bq.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
- bq.add(TermRangeQuery.newStringRange("body", "ta", "test", true, false), BooleanClause.Occur.SHOULD);
- topDocs = searcher.search(bq.build(), 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", bq.build(), searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a test.", snippets[0]);
- assertEquals("Test a one sentence document.", snippets[1]);
-
- // wrong field
- bq = new BooleanQuery.Builder();
- bq.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
- bq.add(TermRangeQuery.newStringRange("bogus", "ta", "tf", true, true), BooleanClause.Occur.SHOULD);
- topDocs = searcher.search(bq.build(), 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", bq.build(), searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a test.", snippets[0]);
- assertEquals("Test a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testWildcardInBoolean() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- BooleanQuery.Builder query = new BooleanQuery.Builder();
- query.add(new WildcardQuery(new Term("body", "te*")), BooleanClause.Occur.SHOULD);
- TopDocs topDocs = searcher.search(query.build(), 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query.build(), searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- // must not
- query = new BooleanQuery.Builder();
- query.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
- query.add(new WildcardQuery(new Term("bogus", "te*")), BooleanClause.Occur.MUST_NOT);
- topDocs = searcher.search(query.build(), 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- snippets = highlighter.highlight("body", query.build(), searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a test.", snippets[0]);
- assertEquals("Test a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testWildcardInFiltered() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- Query query = new BooleanQuery.Builder()
- .add(new WildcardQuery(new Term("body", "te*")), Occur.MUST)
- .add(new TermQuery(new Term("body", "test")), Occur.FILTER)
- .build();
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testWildcardInConstantScore() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- ConstantScoreQuery query = new ConstantScoreQuery(new WildcardQuery(new Term("body", "te*")));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testWildcardInDisjunctionMax() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- DisjunctionMaxQuery query = new DisjunctionMaxQuery(
- Collections.singleton(new WildcardQuery(new Term("body", "te*"))), 0);
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testSpanWildcard() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- Query query = new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*")));
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testSpanOr() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- SpanQuery childQuery = new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*")));
- Query query = new SpanOrQuery(new SpanQuery[] { childQuery });
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testSpanNear() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- SpanQuery childQuery = new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*")));
- Query query = new SpanNearQuery(new SpanQuery[] { childQuery, childQuery }, 0, false);
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testSpanNot() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- SpanQuery include = new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*")));
- SpanQuery exclude = new SpanTermQuery(new Term("body", "bogus"));
- Query query = new SpanNotQuery(include, exclude);
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- public void testSpanPositionCheck() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("This is a test.");
- iw.addDocument(doc);
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- SpanQuery childQuery = new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*")));
- Query query = new SpanFirstQuery(childQuery, 1000000);
- TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
- assertEquals(2, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
- assertEquals(2, snippets.length);
- assertEquals("This is a <b>test</b>.", snippets[0]);
- assertEquals("<b>Test</b> a one sentence document.", snippets[1]);
-
- ir.close();
- dir.close();
- }
-
- /** Runs a query with two MTQs and confirms the formatter
- * can tell which query matched which hit. */
- public void testWhichMTQMatched() throws Exception {
- Directory dir = newDirectory();
- // use simpleanalyzer for more natural tokenization (else "test." is a token)
- final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-
- FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field body = new Field("body", "", offsetsType);
- Document doc = new Document();
- doc.add(body);
-
- body.setStringValue("Test a one sentence document.");
- iw.addDocument(doc);
-
- IndexReader ir = iw.getReader();
- iw.close();
-
- IndexSearcher searcher = newSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
- };
- BooleanQuery.Builder query = new BooleanQuery.Builder();
- query.add(new WildcardQuery(new Term("body", "te*")), BooleanClause.Occur.SHOULD);
- query.add(new WildcardQuery(new Term("body", "one")), BooleanClause.Occur.SHOULD);
- query.add(new WildcardQuery(new Term("body", "se*")), BooleanClause.Occur.SHOULD);
- TopDocs topDocs = searcher.search(query.build(), 10, Sort.INDEXORDER);
- assertEquals(1, topDocs.totalHits);
- String snippets[] = highlighter.highlight("body", query.build(), searcher, topDocs);
- assertEquals(1, snippets.length);
-
- // Default formatter just bolds each hit:
- assertEquals("<b>Test</b> a <b>one</b> <b>sentence</b> document.", snippets[0]);
-
- // Now use our own formatter, that also stuffs the
- // matching term's text into the result:
- highlighter = new PostingsHighlighter() {
- @Override
- protected Analyzer getIndexAnalyzer(String field) {
- return analyzer;
- }
-
- @Override
- protected PassageFormatter getFormatter(String field) {
- return new PassageFormatter() {
-
- @Override
- public Object format(Passage passages[], String content) {
- // Copied from DefaultPassageFormatter, but
- // tweaked to include the matched term:
- StringBuilder sb = new StringBuilder();
- int pos = 0;
- for (Passage passage : passages) {
- // don't add ellipsis if it's the first one, or if it's connected.
- if (passage.startOffset > pos && pos > 0) {
- sb.append("... ");
- }
- pos = passage.startOffset;
- for (int i = 0; i < passage.numMatches; i++) {
- int start = passage.matchStarts[i];
- int end = passage.matchEnds[i];
- // it's possible to have overlapping terms
- if (start > pos) {
- sb.append(content, pos, start);
- }
- if (end > pos) {
- sb.append("<b>");
- sb.append(content, Math.max(pos, start), end);
- sb.append('(');
- sb.append(passage.getMatchTerms()[i].utf8ToString());
- sb.append(')');
- sb.append("</b>");
- pos = end;
- }
- }
- // it's possible a "term" from the analyzer could span a sentence boundary.
- sb.append(content, pos, Math.max(pos, passage.endOffset));
- pos = passage.endOffset;
- }
- return sb.toString();
- }
- };
- }
- };
-
- assertEquals(1, topDocs.totalHits);
- snippets = highlighter.highlight("body", query.build(), searcher, topDocs);
- assertEquals(1, snippets.length);
-
- // Default formatter bolds each hit:
- assertEquals("<b>Test(body:te*)</b> a <b>one(body:one)</b> <b>sentence(body:se*)</b> document.", snippets[0]);
-
- ir.close();
- dir.close();
- }
-}
[3/4] lucene-solr:master: LUCENE-7815: Removed the PostingsHighlighter
Posted by ds...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/CambridgeMA.utf8
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/CambridgeMA.utf8 b/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/CambridgeMA.utf8
deleted file mode 100644
index d60b6fa..0000000
--- a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/CambridgeMA.utf8
+++ /dev/null
@@ -1 +0,0 @@
-{{Distinguish|Cambridge, England}} {{primary sources|date=June 2012}} {{Use mdy dates|date=January 2011}} {{Infobox settlement |official_name = Cambridge, Massachusetts |nickname = |motto = "Boston's Left Bank"<ref>{{cite web|url= http://www.epodunk.com/cgi-bin/genInfo.php?locIndex=2894|title=Profile for Cambridge, Massachusetts, MA|publisher= ePodunk |accessdate= November 1, 2012}}</ref> |image_skyline = CambridgeMACityHall2.jpg |imagesize = 175px |image_caption = Cambridge City Hall |image_seal = |image_flag = |image_map = Cambridge ma highlight.png |mapsize = 250px |map_caption = Location in Middlesex County in Massachusetts |image_map1 = |mapsize1 = |map_caption1 = |coordinates_region = US-MA |subdivision_type = Country |subdivision_name = United States |subdivision_type1 = State |subdivision_name1 = [[Massachusetts]] |subdivision_type2 = [[List of counties in Massachusetts|County]] |subdivision_name2 = [[Middlesex County, Massachusetts|Middlesex]] |established_title = Settled |
established_date = 1630 |established_title2 = Incorporated |established_date2 = 1636 |established_title3 = |established_date3 = |government_type = [[Council-manager government|Council-City Manager]] |leader_title = Mayor |leader_name = Henrietta Davis |leader_title1 = [[City manager|City Manager]] |leader_name1 = [[Robert W. Healy]] |area_magnitude = |area_total_km2 = 18.47 |area_total_sq_mi = 7.13 |area_land_km2 = 16.65 |area_land_sq_mi = 6.43 |area_water_km2 = 1.81 |area_water_sq_mi = 0.70 |population_as_of = 2010 |population_blank2_title = [[Demonym]] |population_blank2 = [[Cantabrigian]] |settlement_type = City |population_total = 105,162 |population_density_km2 = 6,341.98 |population_density_sq_mi = 16,422.08 |elevation_m = 12 |elevation_ft = 40 |timezone = [[Eastern Time Zone|Eastern]] |utc_offset = -5 |timezone_DST = [[Eastern Time Zone|Eastern]] |utc_offset_DST = -4 |coordinates_display = display=inline,title |latd = 42 |latm = 22 |lats = 25 |latNS = N |longd = 71 |longm = 0
6 |longs = 38 |longEW = W |website = [http://www.cambridgema.gov/ www.cambridgema.gov] |postal_code_type = ZIP code |postal_code = 02138, 02139, 02140, 02141, 02142 |area_code = [[Area code 617|617]] / [[Area code 857|857]] |blank_name = [[Federal Information Processing Standard|FIPS code]] |blank_info = 25-11000 |blank1_name = [[Geographic Names Information System|GNIS]] feature ID |blank1_info = 0617365 |footnotes = }} '''Cambridge''' is a city in [[Middlesex County, Massachusetts|Middlesex County]], [[Massachusetts]], [[United States]], in the [[Greater Boston]] area. It was named in honor of the [[University of Cambridge]] in [[England]], an important center of the [[Puritan]] theology embraced by the town's founders.<ref>{{cite book|last=Degler|first=Carl Neumann|title=Out of Our Pasts: The Forces That Shaped Modern America|publisher=HarperCollins|location=New York|year=1984|url=http://books.google.com/books?id=NebLe1ueuGQC&pg=PA18&lpg=PA18&dq=cambridge+university+puritans+newt
owne#v=onepage&q=&f=false|accessdate=September 9, 2009 | isbn=978-0-06-131985-3}}</ref> Cambridge is home to two of the world's most prominent universities, [[Harvard University]] and the [[Massachusetts Institute of Technology]]. According to the [[2010 United States Census]], the city's population was 105,162.<ref name="2010.census.gov">{{cite web|url=http://2010.census.gov/news/releases/operations/cb11-cn104.html |title=Census 2010 News | U.S. Census Bureau Delivers Massachusetts' 2010 Census Population Totals, Including First Look at Race and Hispanic Origin Data for Legislative Redistricting |publisher=2010.census.gov |date=2011-03-22 |accessdate=2012-04-28}}</ref> It is the fifth most populous city in the state, behind [[Boston]], [[Worcester, MA|Worcester]], [[Springfield, MA|Springfield]], and [[Lowell, Massachusetts|Lowell]].<ref name="2010.census.gov"/> Cambridge was one of the two [[county seat]]s of Middlesex County prior to the abolition of county government in 199
7; [[Lowell, Massachusetts|Lowell]] was the other. ==History== {{See also|Timeline of Cambridge, Massachusetts history}} [[File:Formation of Massachusetts towns.svg|thumb|A map showing the original boundaries of Cambridge]] The site for what would become Cambridge was chosen in December 1630, because it was located safely upriver from Boston Harbor, which made it easily defensible from attacks by enemy ships. Also, the water from the local spring was so good that the local Native Americans believed it had medicinal properties.{{Citation needed|date=November 2009}} [[Thomas Dudley]], his daughter [[Anne Bradstreet]] and her husband Simon were among the first settlers of the town. The first houses were built in the spring of 1631. The settlement was initially referred to as "the newe towne".<ref name=drake>{{cite book|last=Drake|first=Samuel Adams|title=History of Middlesex County, Massachusetts|publisher=Estes and Lauriat|location=Boston|year=1880|volume=1|pages=305–16|url=http://b
ooks.google.com/books?id=QGolOAyd9RMC&pg=PA316&lpg=PA305&dq=newetowne&ct=result#PPA305,M1|accessdate=December 26, 2008}}</ref> Official Massachusetts records show the name capitalized as '''Newe Towne''' by 1632.<ref name=public>{{cite book|title=Report on the Custody and Condition of the Public Records of Parishes|publisher=Massachusetts Secretary of the Commonwealth|url=http://books.google.com/books?id=IyYWAAAAYAAJ&pg=RA1-PA298&lpg=RA1-PA298&dq=%22Ordered+That+Newtowne+shall+henceforward+be+called%22|location=Boston|year=1889|page=298|accessdate=December 24, 2008}}</ref> Located at the first convenient [[Charles River]] crossing west of [[Boston]], Newe Towne was one of a number of towns (including Boston, [[Dorchester, Massachusetts|Dorchester]], [[Watertown, Massachusetts|Watertown]], and [[Weymouth, Massachusetts|Weymouth]]) founded by the 700 original [[Puritan]] colonists of the [[Massachusetts Bay Colony]] under governor [[John Winthrop]]. The original village site is in the
heart of today's [[Harvard Square]]. The marketplace where farmers brought in crops from surrounding towns to sell survives today as the small park at the corner of John F. Kennedy (J.F.K.) and Winthrop Streets, then at the edge of a salt marsh, since filled. The town included a much larger area than the present city, with various outlying parts becoming independent towns over the years: [[Newton, Massachusetts|Newton (originally Cambridge Village, then Newtown)]] in 1688,<ref>{{cite book |last= Ritter |first= Priscilla R. |coauthors= Thelma Fleishman |title= Newton, Massachusetts 1679–1779: A Biographical Directory |year= 1982 |publisher= New England Historic Genealogical Society }}</ref> [[Lexington, Massachusetts|Lexington (Cambridge Farms)]] in 1712, and both [[Arlington, Massachusetts|West Cambridge (originally Menotomy)]] and [[Brighton, Massachusetts|Brighton (Little Cambridge)]] in 1807.<ref>{{cite web |url=http://www.brightonbot.com/history.php |title=A Short History of
Allston-Brighton |first=Marchione |last=William P. |author= |authorlink= |coauthors= |date= |month= |year=2011 |work=Brighton-Allston Historical Society |publisher=Brighton Board of Trade |location= |page= |pages= |at= |language= |trans_title= |arxiv= |asin= |bibcode= |doi= |doibroken= |isbn= |issn= |jfm= |jstor= |lccn= |mr= |oclc= |ol= |osti= |pmc = |pmid= |rfc= |ssrn= |zbl= |id= |archiveurl= |archivedate= |deadurl= |accessdate=December 21, 2011 |quote= |ref= |separator= |postscript=}}</ref> Part of West Cambridge joined the new town of [[Belmont, Massachusetts|Belmont]] in 1859, and the rest of West Cambridge was renamed Arlington in 1867; Brighton was annexed by Boston in 1874. In the late 19th century, various schemes for annexing Cambridge itself to the City of Boston were pursued and rejected.<ref>{{cite news |title=ANNEXATION AND ITS FRUITS |author=Staff writer |first= |last= |authorlink= |url=http://query.nytimes.com/gst/abstract.html?res=9901E4DC173BEF34BC4D52DFB766838F669F
DE |agency= |newspaper=[[The New York Times]] |publisher= |isbn= |issn= |pmid= |pmd= |bibcode= |doi= |date=January 15, 1874, Wednesday |page= 4 |pages= |accessdate=|archiveurl=http://query.nytimes.com/mem/archive-free/pdf?res=9901E4DC173BEF34BC4D52DFB766838F669FDE |archivedate=January 15, 1874 |ref= }}</ref><ref>{{cite news |title=BOSTON'S ANNEXATION SCHEMES.; PROPOSAL TO ABSORB CAMBRIDGE AND OTHER NEAR-BY TOWNS |author=Staff writer |first= |last= |authorlink= |url=http://query.nytimes.com/gst/abstract.html?res=9C05E1DC1F39E233A25754C2A9659C94639ED7CF |agency= |newspaper=[[The New York Times]] |publisher= |isbn= |issn= |pmid= |pmd= |bibcode= |doi= |date=March 26, 1892, Wednesday |page= 11 |pages= |accessdate=August 21, 2010|archiveurl=http://query.nytimes.com/mem/archive-free/pdf?res=9C05E1DC1F39E233A25754C2A9659C94639ED7CF |archivedate=March 27, 1892 |ref= }}</ref> In 1636, [[Harvard College]] was founded by the colony to train [[minister (religion)|ministers]] and the new town was
chosen for its site by [[Thomas Dudley]]. By 1638, the name "Newe Towne" had "compacted by usage into 'Newtowne'."<ref name=drake /> In May 1638<ref>{{cite book|title=The Cambridge of Eighteen Hundred and Ninety-six|editor=Arthur Gilman, ed.|publisher=Committee on the Memorial Volume|location=Cambridge|year=1896|page=8}}</ref><ref>{{cite web|author=Harvard News Office |url=http://news.harvard.edu/gazette/2002/05.02/02-history.html |title=''Harvard Gazette'' historical calendar giving May 12, 1638 as date of name change; certain other sources say May 2, 1638 or late 1637 |publisher=News.harvard.edu |date=2002-05-02 |accessdate=2012-04-28}}</ref> the name was changed to '''Cambridge''' in honor of the [[University of Cambridge|university]] in [[Cambridge, England]].<ref>{{cite book |last= Hannah Winthrop Chapter, D.A.R. |title= Historic Guide to Cambridge |edition= Second |year= 1907 |publisher= Hannah Winthrop Chapter, D.A.R. |location= Cambridge, Mass. |pages= 20–21 |quot
e= On October 15, 1637, the Great and General Court passed a vote that: "The college is ordered to bee at Newetowne." In this same year the name of Newetowne was changed to Cambridge, ("It is ordered that Newetowne shall henceforward be called Cambridge") in honor of the university in Cambridge, England, where many of the early settlers were educated. }}</ref> The first president ([[Henry Dunster]]), the first benefactor ([[John Harvard (clergyman)|John Harvard]]), and the first schoolmaster ([[Nathaniel Eaton]]) of Harvard were all Cambridge University alumni, as was the then ruling (and first) governor of the [[Massachusetts Bay Colony]], John Winthrop. In 1629, Winthrop had led the signing of the founding document of the city of Boston, which was known as the [[Cambridge Agreement]], after the university.<ref>{{cite web|url=http://www.winthropsociety.org/doc_cambr.php|publisher=The Winthrop Society|title=Descendants of the Great Migration|accessdate=September 8, 2008}}</ref>
It was Governor Thomas Dudley who, in 1650, signed the charter creating the corporation which still governs Harvard College.<ref>{{cite web|url=http://hul.harvard.edu/huarc/charter.html |title=Harvard Charter of 1650, Harvard University Archives, Harvard University, harvard.edu |publisher=Hul.harvard.edu |date= |accessdate=2012-04-28}}</ref><ref>{{cite book |last1= |first1= |authorlink1= |editor1-first= |editor1-last= |editor1-link= |others= |title=Constitution of the Commonwealth of Massachusetts|url=http://www.mass.gov/legis/const.htm |accessdate=December 13, 2009 |edition= |series= |volume= |date=September 1, 1779 |publisher=The General Court of Massachusetts |location= |isbn= |oclc= |doi= |page= |pages=|chapter=Chapter V: The University at Cambridge, and encouragement of literature, etc. |chapterurl= |ref= |bibcode= }}</ref> [[Image:Washington taking command of the American Army at Cambridge, 1775 - NARA - 532874.tif|thumb|right|George Washington in Cambridge, 1775]] Cambridge
grew slowly as an agricultural village eight miles (13 km) by road from Boston, the capital of the colony. By the [[American Revolution]], most residents lived near the [[Cambridge Common|Common]] and Harvard College, with farms and estates comprising most of the town. Most of the inhabitants were descendants of the original Puritan colonists, but there was also a small elite of [[Anglicans|Anglican]] "worthies" who were not involved in village life, who made their livings from estates, investments, and trade, and lived in mansions along "the Road to Watertown" (today's [[Brattle Street (Cambridge, Massachusetts)|Brattle Street]], still known as [[Tory Row]]). In 1775, [[George Washington]] came up from [[Virginia]] to take command of fledgling volunteer American soldiers camped on the [[Cambridge Common]]—today called the birthplace of the [[U.S. Army]]. (The name of today's nearby Sheraton Commander Hotel refers to that event.) Most of the Tory estates were confiscated afte
r the Revolution. On January 24, 1776, [[Henry Knox]] arrived with artillery captured from [[Fort Ticonderoga]], which enabled Washington to drive the British army out of Boston. [[File:Cambridge 1873 WardMap.jpg|thumb|300px|left|A map of Cambridge from 1873]] Between 1790 and 1840, Cambridge began to grow rapidly, with the construction of the [[West Boston Bridge]] in 1792, that connected Cambridge directly to Boston, making it no longer necessary to travel eight miles (13 km) through the [[Boston Neck]], [[Roxbury, Massachusetts|Roxbury]], and [[Brookline, Massachusetts|Brookline]] to cross the [[Charles River]]. A second bridge, the Canal Bridge, opened in 1809 alongside the new [[Middlesex Canal]]. The new bridges and roads made what were formerly estates and [[marsh]]land into prime industrial and residential districts. In the mid-19th century, Cambridge was the center of a literary revolution when it gave the country a new identity through poetry and literature. Cambridge
was home to the famous Fireside Poets—so called because their poems would often be read aloud by families in front of their evening fires. In their day, the [[Fireside Poets]]—[[Henry Wadsworth Longfellow]], [[James Russell Lowell]], and [[Oliver Wendell Holmes, Sr.|Oliver Wendell Holmes]]—were as popular and influential as rock stars are today.{{Citation needed|date=November 2009}} Soon after, [[Toll road|turnpikes]] were built: the [[Cambridge and Concord Turnpike]] (today's Broadway and Concord Ave.), the [[Middlesex Turnpike (Massachusetts)|Middlesex Turnpike]] (Hampshire St. and [[Massachusetts Avenue (Boston)|Massachusetts Ave.]] northwest of [[Porter Square]]), and what are today's Cambridge, Main, and Harvard Streets were roads to connect various areas of Cambridge to the bridges. In addition, railroads crisscrossed the town during the same era, leading to the development of Porter Square as well as the creation of neighboring town [[Somerville, Massachusetts|Somervil
le]] from the formerly rural parts of [[Charlestown, Massachusetts|Charlestown]]. [[File:Middlesex Canal (Massachusetts) map, 1852.jpg|thumb|1852 Map of Boston area showing Cambridge and rail lines.]] Cambridge was incorporated as a city in 1846. This was despite noticeable tensions between East Cambridge, Cambridgeport, and Old Cambridge that stemmed from differences in in each area's culture, sources of income, and the national origins of the residents.<ref>Cambridge Considered: A Very Brief History of Cambridge, 1800-1900, Part I. http://cambridgeconsidered.blogspot.com/2011/01/very-brief-history-of-cambridge-1800.html</ref> The city's commercial center began to shift from Harvard Square to Central Square, which became the downtown of the city around this time. Between 1850 and 1900, Cambridge took on much of its present character—[[streetcar suburb]]an development along the turnpikes, with working-class and industrial neighborhoods focused on East Cambridge, comfortable middle
-class housing being built on old estates in Cambridgeport and Mid-Cambridge, and upper-class enclaves near Harvard University and on the minor hills of the city. The coming of the railroad to North Cambridge and Northwest Cambridge then led to three major changes in the city: the development of massive brickyards and brickworks between Massachusetts Ave., Concord Ave. and [[Alewife Brook]]; the ice-cutting industry launched by [[Frederic Tudor]] on [[Fresh Pond, Cambridge, Massachusetts|Fresh Pond]]; and the carving up of the last estates into residential subdivisions to provide housing to the thousands of immigrants that arrived to work in the new industries. For many years, the city's largest employer was the [[New England Glass Company]], founded in 1818. By the middle of the 19th century it was the largest and most modern glassworks in the world. In 1888, all production was moved, by [[Edward Libbey|Edward Drummond Libbey]], to [[Toledo, Ohio]], where it continues today under t
he name Owens Illinois. Flint glassware with heavy lead content, produced by that company, is prized by antique glass collectors. There is none on public display in Cambridge, but there is a large collection in the [[Toledo Museum of Art]]. Among the largest businesses located in Cambridge was the firm of [[Carter's Ink Company]], whose neon sign long adorned the [[Charles River]] and which was for many years the largest manufacturer of ink in the world. By 1920, Cambridge was one of the main industrial cities of [[New England]], with nearly 120,000 residents. As industry in New England began to decline during the [[Great Depression]] and after World War II, Cambridge lost much of its industrial base. It also began the transition to being an intellectual, rather than an industrial, center. Harvard University had always been important in the city (both as a landowner and as an institution), but it began to play a more dominant role in the city's life and culture. Also, the move of th
e [[Massachusetts Institute of Technology]] from Boston in 1916 ensured Cambridge's status as an intellectual center of the United States. After the 1950s, the city's population began to decline slowly, as families tended to be replaced by single people and young couples. The 1980s brought a wave of high-technology startups, creating software such as [[Visicalc]] and [[Lotus 1-2-3]], and advanced computers, but many of these companies fell into decline with the fall of the minicomputer and [[DOS]]-based systems. However, the city continues to be home to many startups as well as a thriving biotech industry. By the end of the 20th century, Cambridge had one of the most expensive housing markets in the Northeastern United States. While maintaining much diversity in class, race, and age, it became harder and harder for those who grew up in the city to be able to afford to stay. The end of [[rent control]] in 1994 prompted many Cambridge renters to move to housing that was more affordabl
e, in Somerville and other communities. In 2005, a reassessment of residential property values resulted in a disproportionate number of houses owned by non-affluent people jumping in value relative to other houses, with hundreds having their property tax increased by over 100%; this forced many homeowners in Cambridge to move elsewhere.<ref>Cambridge Chronicle, October 6, 13, 20, 27, 2005</ref> As of 2012, Cambridge's mix of amenities and proximity to Boston has kept housing prices relatively stable. ==Geography== [[File:Charles River Cambridge USA.jpg|thumb|upright|A view from Boston of Harvard's [[Weld Boathouse]] and Cambridge in winter. The [[Charles River]] is in the foreground.]] According to the [[United States Census Bureau]], Cambridge has a total area of {{convert|7.1|sqmi|km2}}, of which {{convert|6.4|sqmi|km2}} of it is land and {{convert|0.7|sqmi|km2}} of it (9.82%) is water. ===Adjacent municipalities=== Cambridge is located in eastern Massachusetts, bordered by: *the
city of [[Boston]] to the south (across the [[Charles River]]) and east *the city of [[Somerville, Massachusetts|Somerville]] to the north *the town of [[Arlington, Massachusetts|Arlington]] to the northwest *the town of [[Belmont, Massachusetts|Belmont]] and *the city of [[Watertown, Massachusetts|Watertown]] to the west The border between Cambridge and the neighboring city of [[Somerville, Massachusetts|Somerville]] passes through densely populated neighborhoods which are connected by the [[Red Line (MBTA)|MBTA Red Line]]. Some of the main squares, [[Inman Square|Inman]], [[Porter Square|Porter]], and to a lesser extent, [[Harvard Square|Harvard]], are very close to the city line, as are Somerville's [[Union Square (Somerville)|Union]] and [[Davis Square]]s. ===Neighborhoods=== ====Squares==== [[File:Centralsquarecambridgemass.jpg|thumb|[[Central Square (Cambridge)|Central Square]]]] [[File:Harvard square 2009j.JPG|thumb|[[Harvard Square]]]] [[File:Cambridge MA Inman Square.jpg|th
umb|[[Inman Square]]]] Cambridge has been called the "City of Squares" by some,<ref>{{cite web|author=No Writer Attributed |url=http://www.thecrimson.com/article/1969/9/18/cambridge-a-city-of-squares-pcambridge/ |title="Cambridge: A City of Squares" Harvard Crimson, Sept. 18, 1969 |publisher=Thecrimson.com |date=1969-09-18 |accessdate=2012-04-28}}</ref><ref>{{cite web|url=http://www.travelwritersmagazine.com/RonBernthal/Cambridge.html |title=Cambridge Journal: Massachusetts City No Longer in Boston's Shadow |publisher=Travelwritersmagazine.com |date= |accessdate=2012-04-28}}</ref> as most of its commercial districts are major street intersections known as [[Town square|squares]]. Each of the squares acts as a neighborhood center. These include: * [[Kendall Square]], formed by the junction of Broadway, Main Street, and Third Street, is also known as '''Technology Square''', a name shared with an office and laboratory building cluster in the neighborhood. Just over the [[Longfellow Br
idge]] from Boston, at the eastern end of the [[Massachusetts Institute of Technology|MIT]] campus, it is served by the [[Kendall (MBTA station)|Kendall/MIT]] station on the [[Massachusetts Bay Transportation Authority|MBTA]] [[Red Line (MBTA)|Red Line]] subway. Most of Cambridge's large office towers are located here, giving the area somewhat of an office park feel. A flourishing [[biotech]] industry has grown up around this area. The "One Kendall Square" complex is nearby, but—confusingly—not actually in Kendall Square. Also, the "Cambridge Center" office complex is located here, and not at the actual center of Cambridge. * [[Central Square (Cambridge)|Central Square]], formed by the junction of Massachusetts Avenue, Prospect Street, and Western Avenue, is well known for its wide variety of ethnic restaurants. As recently as the late 1990s it was rather run-down; it underwent a controversial [[gentrification]] in recent years (in conjunction with the development of the nearby
[[University Park at MIT]]), and continues to grow more expensive. It is served by the [[Central (MBTA station)|Central Station]] stop on the MBTA Red Line subway. '''Lafayette Square''', formed by the junction of Massachusetts Avenue, Columbia Street, Sidney Street, and Main Street, is considered part of the Central Square area. [[Cambridgeport]] is south of Central Square along Magazine Street and Brookline Street. * [[Harvard Square]], formed by the junction of Massachusetts Avenue, Brattle Street, and JFK Street. This is the primary site of [[Harvard University]], and is a major Cambridge shopping area. It is served by a [[Harvard (MBTA station)|Red Line station]]. Harvard Square was originally the northwestern terminus of the Red Line and a major transfer point to streetcars that also operated in a short [[Harvard Bus Tunnel|tunnel]]—which is still a major bus terminal, although the area under the Square was reconfigured dramatically in the 1980s when the Red Line was extende
d. The Harvard Square area includes '''Brattle Square''' and '''Eliot Square'''. A short distance away from the square lies the [[Cambridge Common]], while the neighborhood north of Harvard and east of Massachusetts Avenue is known as Agassiz in honor of the famed scientist [[Louis Agassiz]]. * [[Porter Square]], about a mile north on Massachusetts Avenue from Harvard Square, is formed by the junction of Massachusetts and Somerville Avenues, and includes part of the city of [[Somerville, Massachusetts|Somerville]]. It is served by the [[Porter (MBTA station)|Porter Square Station]], a complex housing a [[Red Line (MBTA)|Red Line]] stop and a [[Fitchburg Line]] [[MBTA commuter rail|commuter rail]] stop. [[Lesley University]]'s University Hall and Porter campus are located at Porter Square. * [[Inman Square]], at the junction of Cambridge and Hampshire streets in Mid-Cambridge. Inman Square is home to many diverse restaurants, bars, music venues and boutiques. The funky street scene s
till holds some urban flair, but was dressed up recently with Victorian streetlights, benches and bus stops. A new community park was installed and is a favorite place to enjoy some takeout food from the nearby restaurants and ice cream parlor. * [[Lechmere Square]], at the junction of Cambridge and First streets, adjacent to the CambridgeSide Galleria shopping mall. Perhaps best known as the northern terminus of the [[Massachusetts Bay Transportation Authority|MBTA]] [[Green Line (MBTA)|Green Line]] subway, at [[Lechmere (MBTA station)|Lechmere Station]]. ====Other neighborhoods==== The residential neighborhoods ([http://www.cambridgema.gov/CPD/publications/neighborhoods.cfm map]) in Cambridge border, but are not defined by the squares. These include: * [[East Cambridge, Massachusetts|East Cambridge]] (Area 1) is bordered on the north by the [[Somerville, Massachusetts|Somerville]] border, on the east by the Charles River, on the south by Broadway and Main Street, and on the west b
y the [[Grand Junction Railroad]] tracks. It includes the [[NorthPoint (Cambridge, Massachusetts)|NorthPoint]] development. * [[Massachusetts Institute of Technology|MIT]] Campus ([[MIT Campus (Area 2), Cambridge|Area 2]]) is bordered on the north by Broadway, on the south and east by the Charles River, and on the west by the Grand Junction Railroad tracks. * [[Wellington-Harrington]] (Area 3) is bordered on the north by the [[Somerville, Massachusetts|Somerville]] border, on the south and west by Hampshire Street, and on the east by the Grand Junction Railroad tracks. Referred to as "Mid-Block".{{clarify|What is? By whom? A full sentence would help.|date=September 2011}} * [[Area 4, Cambridge|Area 4]] is bordered on the north by Hampshire Street, on the south by Massachusetts Avenue, on the west by Prospect Street, and on the east by the Grand Junction Railroad tracks. Residents of Area 4 often refer to their neighborhood simply as "The Port", and refer to the area of Cambridgeport
and Riverside as "The Coast". * [[Cambridgeport]] (Area 5) is bordered on the north by Massachusetts Avenue, on the south by the Charles River, on the west by River Street, and on the east by the Grand Junction Railroad tracks. * [[Mid-Cambridge]] (Area 6) is bordered on the north by Kirkland and Hampshire Streets and the [[Somerville, Massachusetts|Somerville]] border, on the south by Massachusetts Avenue, on the west by Peabody Street, and on the east by Prospect Street. * [[Riverside, Cambridge|Riverside]] (Area 7), an area sometimes referred to as "The Coast," is bordered on the north by Massachusetts Avenue, on the south by the Charles River, on the west by JFK Street, and on the east by River Street. * [[Agassiz, Cambridge, Massachusetts|Agassiz (Harvard North)]] (Area 8) is bordered on the north by the [[Somerville, Massachusetts|Somerville]] border, on the south and east by Kirkland Street, and on the west by Massachusetts Avenue. * [[Peabody, Cambridge, Massachusetts|Peabo
dy]] (Area 9) is bordered on the north by railroad tracks, on the south by Concord Avenue, on the west by railroad tracks, and on the east by Massachusetts Avenue. The Avon Hill sub-neighborhood consists of the higher elevations bounded by Upland Road, Raymond Street, Linnaean Street and Massachusetts Avenue. * Brattle area/[[West Cambridge (neighborhood)|West Cambridge]] (Area 10) is bordered on the north by Concord Avenue and Garden Street, on the south by the Charles River and the [[Watertown, Massachusetts|Watertown]] border, on the west by Fresh Pond and the Collins Branch Library, and on the east by JFK Street. It includes the sub-neighborhoods of Brattle Street (formerly known as [[Tory Row]]) and Huron Village. * [[North Cambridge, Massachusetts|North Cambridge]] (Area 11) is bordered on the north by the [[Arlington, Massachusetts|Arlington]] and [[Somerville, Massachusetts|Somerville]] borders, on the south by railroad tracks, on the west by the [[Belmont, Massachusetts|Bel
mont]] border, and on the east by the [[Somerville, Massachusetts|Somerville]] border. * [[Cambridge Highlands]] (Area 12) is bordered on the north and east by railroad tracks, on the south by Fresh Pond, and on the west by the [[Belmont, Massachusetts|Belmont]] border. * [[Strawberry Hill, Cambridge|Strawberry Hill]] (Area 13) is bordered on the north by Fresh Pond, on the south by the [[Watertown, Massachusetts|Watertown]] border, on the west by the [[Belmont, Massachusetts|Belmont]] border, and on the east by railroad tracks. ===Parks and outdoors=== [[File:Alewife Brook Reservation.jpg|thumb|Alewife Brook Reservation]] Consisting largely of densely built residential space, Cambridge lacks significant tracts of public parkland. This is partly compensated for, however, by the presence of easily accessible open space on the university campuses, including [[Harvard Yard]] and MIT's Great Lawn, as well as the considerable open space of [[Mount Auburn Cemetery]]. At the western edge o
f Cambridge, the cemetery is well known as the first garden cemetery, for its distinguished inhabitants, for its superb landscaping (the oldest planned landscape in the country), and as a first-rate [[arboretum]]. Although known as a Cambridge landmark, much of the cemetery lies within the bounds of Watertown.<ref>http://www2.cambridgema.gov/CityOfCambridge_Content/documents/CambridgeStreetMap18x24_032007.pdf</ref> It is also a significant [[Important Bird Area]] (IBA) in the Greater Boston area. Public parkland includes the esplanade along the Charles River, which mirrors its [[Charles River Esplanade|Boston counterpart]], [[Cambridge Common]], a busy and historic public park immediately adjacent to the Harvard campus, and the [[Alewife Brook Reservation]] and [[Fresh Pond, Cambridge, Massachusetts|Fresh Pond]] in the western part of the city. ==Demographics== {{Historical populations | type=USA | align=right | 1790|2115 | 1800|2453 | 1810|2323 | 1820|3295 | 1830|6072 | 1840|8409 |
1850|15215 | 1860|26060 | 1870|39634 | 1880|52669 | 1890|70028 | 1900|91886 | 1910|104839 | 1920|109694 | 1930|113643 | 1940|110879 | 1950|120740 | 1960|107716 | 1970|100361 | 1980|95322 | 1990|95802 | 2000|101355 | 2010|105162 | footnote= {{Historical populations/Massachusetts municipalities references}}<ref name="1950_Census_Urban_populations_since_1790">{{cite journal | title=1950 Census of Population | volume=1: Number of Inhabitants | at=Section 6, Pages 21-7 through 21-09, Massachusetts Table 4. Population of Urban Places of 10,000 or more from Earliest Census to 1920 | publisher=Bureau of the Census | accessdate=July 12, 2011 | year=1952 | url=http://www2.census.gov/prod2/decennial/documents/23761117v1ch06.pdf}}</ref> }} As of the census{{GR|2}} of 2010, there were 105,162 people, 44,032 households, and 17,420 families residing in the city. The population density was 16,422.08 people per square mile (6,341.98/km²), making Cambridge the fifth most densely populated city in t
he US<ref name=CountyCityDataBook>County and City Data Book: 2000. Washington, DC: US Department of Commerce, Bureau of the Census. Table C-1.</ref> and the second most densely populated city in [[Massachusetts]] behind neighboring [[Somerville, Massachusetts|Somerville]].<ref>[http://www.boston.com/realestate/news/articles/2008/07/13/highest_population_density/ Highest Population Density, The Boston Globe]</ref> There were 47,291 housing units at an average density of 7,354.7 per square mile (2,840.3/km²). The racial makeup of the city was 66.60% [[White (U.S. Census)|White]], 11.70% [[Black (people)|Black]] or [[Race (United States Census)|African American]], 0.20% [[Native American (U.S. Census)|Native American]], 15.10% [[Asian (U.S. Census)|Asian]], 0.01% [[Pacific Islander (U.S. Census)|Pacific Islander]], 2.10% from [[Race (United States Census)|other races]], and 4.30% from two or more races. 7.60% of the population were [[Hispanics in the United States|Hispanic]] or [[Lati
no (U.S. Census)|Latino]] of any race. [[Non-Hispanic Whites]] were 62.1% of the population in 2010,<ref>{{cite web |url=http://quickfacts.census.gov/qfd/states/25/2511000.html |title=Cambridge (city), Massachusetts |work=State & County QuickFacts |publisher=U.S. Census Bureau}}</ref> down from 89.7% in 1970.<ref>{{cite web|title=Massachusetts - Race and Hispanic Origin for Selected Cities and Other Places: Earliest Census to 1990|publisher=U.S. Census Bureau|url=http://www.census.gov/population/www/documentation/twps0076/twps0076.html}}</ref> This rather closely parallels the average [[racial demographics of the United States]] as a whole, although Cambridge has significantly more Asians than the average, and fewer Hispanics and Caucasians. 11.0% were of [[irish people|Irish]], 7.2% English, 6.9% [[italians|Italian]], 5.5% [[West Indian]] and 5.3% [[germans|German]] ancestry according to [[Census 2000]]. 69.4% spoke English, 6.9% Spanish, 3.2% [[Standard Mandarin|Chinese]] or [[Sta
ndard Mandarin|Mandarin]], 3.0% [[portuguese language|Portuguese]], 2.9% [[French-based creole languages|French Creole]], 2.3% French, 1.5% [[korean language|Korean]], and 1.0% [[italian language|Italian]] as their first language. There were 44,032 households out of which 16.9% had children under the age of 18 living with them, 28.9% were married couples living together, 8.4% had a female householder with no husband present, and 60.4% were non-families. 40.7% of all households were made up of individuals and 9.6% had someone living alone who was 65 years of age or older. The average household size was 2.00 and the average family size was 2.76. In the city the population was spread out with 13.3% under the age of 18, 21.2% from 18 to 24, 38.6% from 25 to 44, 17.8% from 45 to 64, and 9.2% who were 65 years of age or older. The median age was 30.5 years. For every 100 females, there were 96.1 males. For every 100 females age 18 and over, there were 94.7 males. The median income for a h
ousehold in the city was $47,979, and the median income for a family was $59,423 (these figures had risen to $58,457 and $79,533 respectively {{as of|2007|alt=as of a 2007 estimate}}<ref>{{cite web|url=http://factfinder.census.gov/servlet/ACSSAFFFacts?_event=Search&geo_id=16000US2418750&_geoContext=01000US%7C04000US24%7C16000US2418750&_street=&_county=cambridge&_cityTown=cambridge&_state=04000US25&_zip=&_lang=en&_sse=on&ActiveGeoDiv=geoSelect&_useEV=&pctxt=fph&pgsl=160&_submenuId=factsheet_1&ds_name=ACS_2007_3YR_SAFF&_ci_nbr=null&qr_name=null®=null%3Anull&_keyword=&_industry= |title=U.S. Census, 2000 |publisher=Factfinder.census.gov |date= |accessdate=2012-04-28}}</ref>). Males had a median income of $43,825 versus $38,489 for females. The per capita income for the city was $31,156. About 8.7% of families and 12.9% of the population were below the poverty line, including 15.1% of those under age 18 and 12.9% of those age 65 or over. Cambridge was ranked as one of the most liberal
cities in America.<ref>{{cite web|author=Aug 16, 2005 12:00 AM |url=http://www.govpro.com/News/Article/31439/ |title=Study Ranks America’s Most Liberal and Conservative Cities |publisher=Govpro.com |date=2005-08-16 |accessdate=2012-04-28}}</ref> Locals living in and near the city jokingly refer to it as "The People's Republic of Cambridge."<ref>[http://www.universalhub.com/glossary/peoples_republic_the.html Wicked Good Guide to Boston English] Accessed February 2, 2009</ref> For 2012, the residential property tax rate in Cambridge is $8.48 per $1,000.<ref>{{cite web|url=http://www.cambridgema.gov/finance/propertytaxinformation/fy12propertytaxinformation.aspx |title=FY12 Property Tax Information - City of Cambridge, Massachusetts |publisher=Cambridgema.gov |date= |accessdate=2012-04-28}}</ref> Cambridge enjoys the highest possible [[bond credit rating]], AAA, with all three Wall Street rating agencies.<ref>http://www.cambridgema.gov/CityOfCambridge_Content/documents/Understanding_
Your_Taxes_2007.pdf</ref> Cambridge is noted for its diverse population, both racially and economically. Residents, known as ''Cantabrigians'', include affluent [[MIT]] and Harvard professors. The first legal applications in America for same-sex marriage licenses were issued at Cambridge's City Hall.<ref>{{cite web|url=http://www.boston.com/news/local/articles/2004/05/17/free_to_marry/ |title=Free to Marry |work=[[The Boston Globe]] |date=2004-05-17 |accessdate=2012-07-18}}</ref> Cambridge is also the birthplace of [[Thailand|Thai]] king [[Bhumibol Adulyadej|Bhumibol Adulyadej (Rama IX)]], who is the world's longest reigning monarch at age 82 (2010), as well as the longest reigning monarch in Thai history. He is also the first king of a foreign country to be born in the United States. ==Government== ===Federal and state representation=== {| class=wikitable ! colspan = 6 | Voter registration and party enrollment {{as of|lc=y|df=US|2008|10|15}}<ref>{{cite web|title = 2008 State Party
Election Party Enrollment Statistics | publisher = Massachusetts Elections Division | format = PDF | accessdate = July 7, 2010 | url = http://www.sec.state.ma.us/ele/elepdf/st_county_town_enroll_breakdown_08.pdf}}</ref> |- ! colspan = 2 | Party ! Number of voters ! Percentage {{American politics/party colors/Democratic/row}} | [[Democratic Party (United States)|Democratic]] | style="text-align:center;"| 37,822 | style="text-align:center;"| 58.43% {{American politics/party colors/Republican/row}} | [[Republican Party (United States)|Republican]] | style="text-align:center;"| 3,280 | style="text-align:center;"| 5.07% {{American politics/party colors/Independent/row}} | Unaffiliated | style="text-align:center;"| 22,935 | style="text-align:center;"| 35.43% {{American politics/party colors/Libertarian/row}} | Minor Parties | style="text-align:center;"| 690 | style="text-align:center;"| 1.07% |- ! colspan = 2 | Total ! style="text-align:center;"| 64,727 ! style="text-align:center;"| 100%
|} Cambridge is part of [[Massachusetts's 8th congressional district]], represented by Democrat [[Mike Capuano]], elected in 1998. The state's senior member of the [[United States Senate]] is Democrat [[John Kerry]], elected in 1984. The state's junior member is Republican [[Scott Brown]], [[United States Senate special election in Massachusetts, 2010|elected in 2010]] to fill the vacancy caused by the death of long-time Democratic Senator [[Ted Kennedy]]. The Governor of Massachusetts is Democrat [[Deval Patrick]], elected in 2006 and re-elected in 2010. On the state level, Cambridge is represented in six districts in the [[Massachusetts House of Representatives]]: the 24th Middlesex (which includes parts of Belmont and Arlington), the 25th and 26th Middlesex (the latter which includes a portion of Somerville), the 29th Middlesex (which includes a small part of Watertown), and the Eighth and Ninth Suffolk (both including parts of the City of Boston). The city is represented in the
[[Massachusetts Senate]] as a part of the "First Suffolk and Middlesex" district (this contains parts of Boston, Revere and Winthrop each in Suffolk County); the "Middlesex, Suffolk and Essex" district, which includes Everett and Somerville, with Boston, Chelsea, and Revere of Suffolk, and Saugus in Essex; and the "Second Suffolk and Middlesex" district, containing parts of the City of Boston in Suffolk county, and Cambridge, Belmont and Watertown in Middlesex county.<ref>{{cite web|url=http://www.malegislature.gov/ |title=Index of Legislative Representation by City and Town, from |publisher=Mass.gov |date= |accessdate=2012-04-28}}</ref> In addition to the [[Cambridge Police Department (Massachusetts)|Cambridge Police Department]], the city is patrolled by the Fifth (Brighton) Barracks of Troop H of the [[Massachusetts State Police]].<ref>[http://www.mass.gov/?pageID=eopsterminal&L=5&L0=Home&L1=Law+Enforcement+%26+Criminal+Justice&L2=Law+Enforcement&L3=State+Police+Troops&L4=Troop+H
&sid=Eeops&b=terminalcontent&f=msp_divisions_field_services_troops_troop_h_msp_field_troop_h_station_h5&csid=Eeops Station H-5, SP Brighton]{{dead link|date=April 2012}}</ref> Due, however, to close proximity, the city also practices functional cooperation with the Fourth (Boston) Barracks of Troop H, as well.<ref>[http://www.mass.gov/?pageID=eopsterminal&L=5&L0=Home&L1=Law+Enforcement+%26+Criminal+Justice&L2=Law+Enforcement&L3=State+Police+Troops&L4=Troop+H&sid=Eeops&b=terminalcontent&f=msp_divisions_field_services_troops_troop_h_msp_field_troop_h_station_h4&csid=Eeops Station H-4, SP Boston]{{dead link|date=April 2012}}</ref> ===City government=== [[File:CambridgeMACityHall1.jpg|thumb|right|[[Cambridge, Massachusetts City Hall|Cambridge City Hall]] in the 1980s]] Cambridge has a city government led by a [[List of mayors of Cambridge, Massachusetts|Mayor]] and nine-member City Council. There is also a six-member School Committee which functions alongside the Superintendent of publi
c schools. The councilors and school committee members are elected every two years using the [[single transferable vote]] (STV) system.<ref>{{cite web|url=http://www.cambridgema.gov/election/Proportional_Representation.cfm |title=Proportional Representation Voting in Cambridge |publisher=Cambridgema.gov |date= |accessdate=2012-04-28}}</ref> Once a laborious process that took several days to complete by hand, ballot sorting and calculations to determine the outcome of elections are now quickly performed by computer, after the ballots have been [[Optical scan voting system|optically scanned]]. The mayor is elected by the city councilors from amongst themselves, and serves as the chair of City Council meetings. The mayor also sits on the School Committee. However, the Mayor is not the Chief Executive of the City. Rather, the City Manager, who is appointed by the City Council, serves in that capacity. Under the City's Plan E form of government the city council does not have the power to
appoint or remove city officials who are under direction of the city manager. The city council and its individual members are also forbidden from giving orders to any subordinate of the city manager.<ref>http://www.cambridgema.gov/CityOfCambridge_Content/documents/planE.pdf</ref> [[Robert W. Healy]] is the City Manager; he has served in the position since 1981. In recent history, the media has highlighted the salary of the City Manager as being one of the highest in the State of Massachusetts.<ref>{{cite news |title=Cambridge city manager's salary almost as much as Obama's pay |url=http://www.wickedlocal.com/cambridge/features/x1837730973/Cambridge-city-managers-salary-almost-as-much-as-Obamas |agency= |newspaper=Wicked Local: Cambridge |publisher= |date=August 11, 2011 |accessdate=December 30, 2011 |quote= |archiveurl= |archivedate= |deadurl= |ref=}}</ref> The city council consists of:<ref>{{cite web|url=http://www.cambridgema.gov/ccouncil/citycouncilmembers.aspx |title=City of Ca
mbridge – City Council Members |publisher=Cambridgema.gov |date= |accessdate=2012-04-28}}</ref>{{Refbegin|3}} *[[Leland Cheung]] (Jan. 2010–present) *Henrietta Davis (Jan. 1996–present)* *Marjorie C. Decker (Jan. 2000–present)<ref>{{cite web |url= http://www.wickedlocal.com/cambridge/news/x738245499/Marjorie-Decker-announces-she-will-run-for-Alice-Wolfs-Cambridge-State-Representative-seat |title= Marjorie Decker announces she will run for Alice Wolf's Cambridge State Representative seat |date= 22 March 2012 |work= Wicked Local Cambridge |publisher= GateHouse Media, Inc. |accessdate= 4 April 2012 }}</ref> *Craig A. Kelley (Jan. 2006–present) *David Maher (Jan. 2000-Jan. 2006, Sept. 2007–present<ref>{{cite web|author=By ewelin, on September 5th, 2007 |url=http://www.cambridgehighlands.com/2007/09/david-p-maher-elected-to-fill-michael-sullivans-vacated-city-council-seat |title=David P. Maher Elected to fill Michael Sullivan’s Vacated City Council Seat • Cambridge Highla
nds Neighborhood Association |publisher=Cambridgehighlands.com |date=2007-09-05 |accessdate=2012-04-28}}</ref>)** *[[Kenneth Reeves]] (Jan. 1990–present)** *[[E. Denise Simmons]] (Jan. 2002–present)** *[[Timothy J. Toomey, Jr.]] (Jan. 1990–present) *Minka vanBeuzekom (Jan. 2012–present){{Refend}} ''* = Current Mayor''<br> ''** = former Mayor'' ===Fire Department=== The city of Cambridge is protected full-time by the 274 professional firefighters of the Cambridge Fire Department. The current Chief of Department is Gerald R. Reardon. The Cambridge Fire Department operates out of eight fire stations, located throughout the city, under the command of two divisions. The CFD also maintains and operates a front-line fire apparatus fleet of eight engines, four ladders, two Non-Transport Paramedic EMS units, a Haz-Mat unit, a Tactical Rescue unit, a Dive Rescue unit, two Marine units, and numerous special, support, and reserve units. John J. Gelinas, Chief of Operations, is in charge
of day to day operation of the department.<ref>{{cite web|url=http://www2.cambridgema.gov/cfd/ |title=City of Cambridge Fire Department |publisher=.cambridgema.gov |date=2005-03-13 |accessdate=2012-06-26}}</ref> The CFD is rated as a Class 1 fire department by the [[Insurance Services Office]] (ISO), and is one of only 32 fire departments so rated, out of 37,000 departments in the United States. The other class 1 departments in New England are in [[Hartford, Connecticut]] and [[Milford, Connecticut]]. Class 1 signifies the highest level of fire protection according to various criteria.<ref>{{cite web|url=http://www2.cambridgema.gov/CFD/Class1FD.cfm |title=Class 1 Fire Department |publisher=.cambridgema.gov |date=1999-07-01 |accessdate=2012-06-26}}</ref> The CFD responds to approximately 15,000 emergency calls annually. {| class=wikitable |- valign=bottom ! Engine Company ! Ladder Company ! Special Unit ! Division ! Address ! Neighborhood |- | Engine 1 || Ladder 1 || || || 491 Broad
way || Harvard Square |- | Engine 2 || Ladder 3 || Squad 2 || || 378 Massachusetts Ave. || Lafayette Square |- | Engine 3 || Ladder 2 || || || 175 Cambridge St. || East Cambridge |- | Engine 4 || || Squad 4 || || 2029 Massachusetts Ave. || Porter Square |- | Engine 5 || || || Division 1 || 1384 Cambridge St. || Inman Square |- | Engine 6 || || || || 176 River St. || Cambridgeport |- | Engine 8 || Ladder 4 || || Division 2 || 113 Garden St. || Taylor Square |- | Engine 9 || || || || 167 Lexington Ave || West Cambridge |- | Maintenance Facility || || || || 100 Smith Pl. || |} ===Water Department=== Cambridge is unusual among cities inside Route 128 in having a non-[[MWRA]] water supply. City water is obtained from [[Hobbs Brook]] (in [[Lincoln, Massachusetts|Lincoln]] and [[Waltham, Massachusetts|Waltham]]), [[Stony Brook (Boston)|Stony Brook]] (Waltham and [[Weston, Massachusetts|Weston]]), and [[Fresh Pond (Cambridge, Massachusetts)|Fresh Pond]] (Cambridge). The city owns over 1200
acres of land in other towns that includes these reservoirs and portions of their watershed.<ref>{{cite web|url=http://www2.cambridgema.gov/CWD/wat_lands.cfm |title=Cambridge Watershed Lands & Facilities |publisher=.cambridgema.gov |date= |accessdate=2012-04-28}}</ref> Water is treated at Fresh Pond, then pumped uphill to an elevation of {{convert|176|ft|m}} [[above sea level]] at the Payson Park Reservoir ([[Belmont, Massachusetts|Belmont]]); From there, the water is redistributed downhill via gravity to individual users in the city.<ref>{{cite web|url=http://www.cambridgema.gov/CityOfCambridge_Content/documents/CWD_March_2010.pdf |title=Water supply system |format=PDF |date= |accessdate=2012-04-28}}</ref><ref>[http://www.cambridgema.gov/CWD/fpfaqs.cfm Is Fresh Pond really used for drinking water?], Cambridge Water Department</ref> ===County government=== Cambridge is a [[county seat]] of [[Middlesex County, Massachusetts]], along with [[Lowell, Massachusetts|Lowell]]. Though the c
ounty government was abolished in 1997, the county still exists as a geographical and political region. The employees of Middlesex County courts, jails, registries, and other county agencies now work directly for the state. At present, the county's registrars of [[Deed]]s and Probate remain in Cambridge; however, the Superior Court and District Attorney have had their base of operations transferred to [[Woburn, Massachusetts|Woburn]]. Third District court has shifted operations to [[Medford, Massachusetts|Medford]], and the Sheriff's office for the county is still awaiting a near-term relocation.<ref>{{cite news | url=http://www.boston.com/news/local/massachusetts/articles/2008/02/14/court_move_a_hassle_for_commuters/ |title=Court move a hassle for commuters |accessdate=July 25, 2009 |first=Eric |last=Moskowitz |authorlink= |coauthors= |date=February 14, 2008 |work=[[Boston Globe|The Boston Globe]] |pages= |archiveurl= |archivedate= |quote=In a little more than a month, Middlesex Su
perior Court will open in Woburn after nearly four decades at the Edward J. Sullivan Courthouse in Cambridge. With it, the court will bring the roughly 500 people who pass through its doors each day – the clerical staff, lawyers, judges, jurors, plaintiffs, defendants, and others who use or work in the system.}}</ref><ref>{{cite news | url=http://www.wickedlocal.com/cambridge/homepage/x135741754/Cambridges-Middlesex-Jail-courts-may-be-shuttered-for-good |title=Cambridge's Middlesex Jail, courts may be shuttered for good |accessdate=July 25, 2009 |first=Charlie |last=Breitrose |authorlink= |coauthors= |date=July 7, 2009 |work=Wicked Local News: Cambridge |pages= |archiveurl= |archivedate= |quote=The courts moved out of the building to allow workers to remove asbestos. Superior Court moved to Woburn in March 2008, and in February, the Third District Court moved to Medford.}}</ref> ==Education== [[File:MIT Main Campus Aerial.jpg|thumb|Aerial view of part of [[MIT]]'s main campus]] [[
File:Dunster House.jpg|thumb|[[Dunster House]], Harvard]] ===Higher education=== Cambridge is perhaps best known as an academic and intellectual center, owing to its colleges and universities, which include: *[[Cambridge College]] *[[Cambridge School of Culinary Arts]] *[[Episcopal Divinity School]] *[[Harvard University]] *[[Hult International Business School]] *[[Lesley University]] *[[Longy School of Music]] *[[Massachusetts Institute of Technology]] *[[Le Cordon Bleu College of Culinary Arts in Boston]] [[Nobel laureates by university affiliation|At least 129]] of the world's total 780 [[Nobel Prize]] winners have been, at some point in their careers, affiliated with universities in Cambridge. The [[American Academy of Arts and Sciences]] is also based in Cambridge. ===Primary and secondary public education=== The Cambridge Public School District encompasses 12 elementary schools that follow a variety of different educational systems and philosophies. All but one of the elementa
ry schools extend up to the [[middle school]] grades as well. The 12 elementary schools are: *[[Amigos School]] *Baldwin School *Cambridgeport School *Fletcher-Maynard Academy *Graham and Parks Alternative School *Haggerty School *Kennedy-Longfellow School *King Open School *Martin Luther King, Jr. School *Morse School (a [[Core Knowledge Foundation|Core Knowledge]] school) *Peabody School *Tobin School (a [[Montessori school]]) There are three public high schools serving Cambridge students, including the [[Cambridge Rindge and Latin School]].<ref>{{cite web|url=http://www.cpsd.us/Web/PubInfo/SchoolsAtAGlance06-07.pdf|title=Cambridge Public Schools at a Glance|format=PDF}}{{dead link|date=June 2012}}</ref> and Community Charter School of Cambridge (www.ccscambridge.org) In 2003, the CRLS, also known as Rindge, came close to losing its educational accreditation when it was placed on probation by the [[New England Association of Schools and Colleges]].<ref name="Crimson MCAS">{{cite w
eb|url=http://www.thecrimson.com/article.aspx?ref=512061|title=School Fights Achievement Gap|publisher=The Harvard Crimson|accessdate=May 14, 2009}}</ref> The school has improved under Principal Chris Saheed, graduation rates hover around 98%, and 70% of students gain college admission. Community Charter School of Cambridge serves 350 students, primarily from Boston and Cambridge, and is a tuition free public charter school with a college preparatory curriculum. All students from the class of 2009 and 2010 gained admission to college. Outside of the main public schools are public charter schools including: [[Benjamin Banneker Charter School]], which serves students in grades K-6,<ref>{{cite web|url=http://www.banneker.org/ |title=The Benjamin Banneker Charter Public School |publisher=Banneker.org |date=2012-03-01 |accessdate=2012-04-28}}</ref> [[Community Charter School of Cambridge]],<ref>{{cite web|url=http://www.ccscambridge.org/ |title=Community Charter School of Cambridge |publ
isher=Ccscambridge.org |date= |accessdate=2012-04-28}}</ref> which is located in Kendall Square and serves students in grades 7–12, and [[Prospect Hill Academy]], a [[charter school]] whose upper school is in [[Central Square (Cambridge)|Central Square]], though it is not a part of the Cambridge Public School District. ===Primary and secondary private education=== [[File:Cambridge Public Library, Cambridge, Massachusetts.JPG|thumb|right|[[Cambridge Public Library]] original building, part of an expanded facility]] There are also many private schools in the city including: <!-- please keep alphabetical --> *[[Boston Archdiocesan Choir School]] (BACS) *[[Buckingham Browne & Nichols]] (BB&N) *[[Cambridge montessori school|Cambridge Montessori School]] (CMS) *Cambridge [[Religious Society of Friends|Friends]] School. Thomas Waring served as founding headmaster of the school. *Fayerweather Street School (FSS)[http://www.fayerweather.org/ ] *[[International School of Boston]] (ISB, form
erly École Bilingue) *[[Matignon High School]] *[[North Cambridge Catholic High School]] (re-branded as Cristo Rey Boston and relocated to Dorchester, MA in 2010) *[[Shady Hill School]] *St. Peter School ==Economy== [[File:Cambridge Skyline.jpg|thumb|Buildings of [[Kendall Square]], center of Cambridge's [[biotech]] economy, seen from the [[Charles River]]]] Manufacturing was an important part of the economy in the late 19th and early 20th century, but educational institutions are the city's biggest employers today. Harvard and [[Massachusetts Institute of Technology|MIT]] together employ about 20,000.<ref name="2008top25">[http://www2.cambridgema.gov/cdd/data/labor/top25/top25_2008.html Top 25 Cambridge Employers: 2008], City of Cambridge</ref> As a cradle of technological innovation, Cambridge was home to technology firms [[Analog Devices]], [[Akamai Technologies|Akamai]], [[BBN Technologies|Bolt, Beranek, and Newman (BBN Technologies)]] (now part of Raytheon), [[General Radio|Ge
neral Radio (later GenRad)]], [[Lotus Development Corporation]] (now part of [[IBM]]), [[Polaroid Corporation|Polaroid]], [[Symbolics]], and [[Thinking Machines]]. In 1996, [[Polaroid Corporation|Polaroid]], [[Arthur D. Little]], and [[Lotus Development Corporation|Lotus]] were top employers with over 1,000 employees in Cambridge, but faded out a few years later. Health care and biotechnology firms such as [[Genzyme]], [[Biogen Idec]], [[Millennium Pharmaceuticals]], [[Sanofi]], [[Pfizer]] and [[Novartis]]<ref>{{cite news |title=Novartis doubles plan for Cambridge |author=Casey Ross and Robert Weisman |first= |last= |authorlink= |authorlink2= |url=http://articles.boston.com/2010-10-27/business/29323650_1_french-drug-maker-astrazeneca-plc-research-operations |agency= |newspaper=[[The Boston Globe]] |publisher= |isbn= |issn= |pmid= |pmd= |bibcode= |doi= |date=October 27, 2010 |page= |pages= |accessdate=April 12, 2011|quote=Already Cambridge’s largest corporate employer, the Swiss fi
rm expects to hire an additional 200 to 300 employees over the next five years, bringing its total workforce in the city to around 2,300. Novartis’s global research operations are headquartered in Cambridge, across Massachusetts Avenue from the site of the new four-acre campus. |archiveurl= |archivedate= |ref=}}</ref> have significant presences in the city. Though headquartered in Switzerland, Novartis continues to expand its operations in Cambridge. Other major biotech and pharmaceutical firms expanding their presence in Cambridge include [[GlaxoSmithKline]], [[AstraZeneca]], [[Shire plc|Shire]], and [[Pfizer]].<ref>{{cite news|title=Novartis Doubles Plan for Cambridge|url=http://www.boston.com/business/healthcare/articles/2010/10/27/novartis_doubles_plan_for_cambridge/|accessdate=23 February 2012 | work=The Boston Globe|first1=Casey|last1=Ross|first2=Robert|last2=Weisman|date=October 27, 2010}}</ref> Most Biotech firms in Cambridge are located around [[Kendall Square]] and [[Eas
t Cambridge, Massachusetts|East Cambridge]], which decades ago were the city's center of manufacturing. A number of biotechnology companies are also located in [[University Park at MIT]], a new development in another former manufacturing area. None of the high technology firms that once dominated the economy was among the 25 largest employers in 2005, but by 2008 high tech companies [[Akamai Technologies|Akamai]] and [[ITA Software]] had grown to be among the largest 25 employers.<ref name="2008top25" /> [[Google]],<ref>{{cite web|url=http://www.google.com/corporate/address.html |title=Google Offices |publisher=Google.com |date= |accessdate=2012-07-18}}</ref> [[IBM Research]], and [[Microsoft Research]] maintain offices in Cambridge. In late January 2012—less than a year after acquiring [[Billerica, Massachusetts|Billerica]]-based analytic database management company, [[Vertica]]—[[Hewlett-Packard]] announced it would also be opening its first offices in Cambridge.<ref>{{cite we
b|last=Huang|first=Gregory|title=Hewlett-Packard Expands to Cambridge via Vertica’s "Big Data" Center|url=http://www.xconomy.com/boston/2012/01/23/hewlett-packard-expands-to-cambridge-via-verticas-big-data-center/?single_page=true}}</ref> Around this same time, e-commerce giants [[Staples Inc.|Staples]]<ref>{{cite web|title=Staples to bring e-commerce office to Cambridge's Kendall Square Read more: Staples to bring e-commerce office to Cambridge's Kendall Square - Cambridge, Massachusetts - Cambridge Chronicle http://www.wickedlocal.com/cambridge/news/x690035936/Staples-to-bring-E-commerce-office-to-Cambridges-Kendall-Square#ixzz1nDY39Who|url=http://www.wickedlocal.com/cambridge/news/x690035936/Staples-to-bring-E-commerce-office-to-Cambridges-Kendall-Square#axzz1kg3no7Zg}}</ref> and [[Amazon.com]]<ref>{{cite web|title=Amazon Seeks Brick-And-Mortar Presence In Boston Area|url=http://www.wbur.org/2011/12/22/amazon-boston}}</ref> said they would be opening research and innovation cen
ters in Kendall Square. Video game developer [[Harmonix Music Systems]] is based in [[Central Square (Cambridge)|Central Square]]. The proximity of Cambridge's universities has also made the city a center for nonprofit groups and think tanks, including the [[National Bureau of Economic Research]], the [[Smithsonian Astrophysical Observatory]], the [[Lincoln Institute of Land Policy]], [[Cultural Survival]], and [[One Laptop per Child]]. In September 2011, an initiative by the City of Cambridge called the "[[Entrepreneur Walk of Fame]]" was launched. It seeks to highlight individuals who have made contributions to innovation in the global business community.<ref>{{cite news |title=Stars of invention |author= |first=Kathleen |last=Pierce |url=http://articles.boston.com/2011-09-16/business/30165912_1_gates-and-jobs-microsoft-granite-stars |agency= |newspaper=The Boston Globe|date=September 16, 2011 |page= |pages= |at= |accessdate=October 1, 2011}}</ref> ===Top employers=== The top ten
employers in the city are:<ref>{{cite web|url=http://cambridgema.gov/citynewsandpublications/news/2012/01/fy11comprehensiveannualfinancialreportnowavailable.aspx |title=City of Cambridge, Massachusetts Comprehensive Annual Financial Report July 1, 2010—June 30, 2011 |publisher=Cambridgema.gov |date=2011-06-30 |accessdate=2012-04-28}}</ref> {| class="wikitable" |- ! # ! Employer ! # of employees |- | 1 |[[Harvard University]] |10,718 |- |2 |[[Massachusetts Institute of Technology]] |7,604 |- |3 |City of Cambridge |2,922 |- |4 |[[Novartis]] Institutes for BioMedical Research |2,095 |- |5 |[[Mount Auburn Hospital]] |1,665 |- |6 |[[Vertex Pharmaceuticals]] |1,600 |- |7 |[[Genzyme]] |1,504 |- |8 |[[Biogen Idec]] |1,350 |- |9 |[[Federal government of the United States|Federal Government]] |1,316 |- |10 |[[Pfizer]] |1,300 |} ==Transportation== {{See also|Boston transportation}} ===Road=== [[File:Harvard Square at Peabody Street and Mass Avenue.jpg|thumb|[[Massachusetts Avenue (Boston)|Ma
ssachusetts Avenue]] in [[Harvard Square]]]] Several major roads lead to Cambridge, including [[Massachusetts State Highway 2|Route 2]], [[Massachusetts State Highway 16|Route 16]] and the [[Massachusetts State Highway 28|McGrath Highway (Route 28)]]. The [[Massachusetts Turnpike]] does not pass through Cambridge, but provides access by an exit in nearby [[Allston, Massachusetts|Allston]]. Both [[U.S. Route 1]] and [[I-93 (MA)]] also provide additional access on the eastern end of Cambridge at Leverett Circle in [[Boston]]. [[Massachusetts State Highway 2A|Route 2A]] runs the length of the city, chiefly along Massachusetts Avenue. The Charles River forms the southern border of Cambridge and is crossed by 11 bridges connecting Cambridge to Boston, including the [[Longfellow Bridge]] and the [[Harvard Bridge]], eight of which are open to motorized road traffic. Cambridge has an irregular street network because many of the roads date from the colonial era. Contrary to popular belief, t
he road system did not evolve from longstanding cow-paths. Roads connected various village settlements with each other and nearby towns, and were shaped by geographic features, most notably streams, hills, and swampy areas. Today, the major "squares" are typically connected by long, mostly straight roads, such as Massachusetts Avenue between [[Harvard Square]] and [[Central Square (Cambridge)|Central Square]], or Hampshire Street between [[Kendall Square]] and [[Inman Square]]. ===Mass transit=== [[File:Central MBTA station.jpg|thumb|[[Central (MBTA)|Central station on the MBTA Red Line]]]] Cambridge is well served by the [[MBTA]], including the [[Porter (MBTA station)|Porter Square stop]] on the regional [[MBTA Commuter Rail|Commuter Rail]], the [[Lechmere (MBTA station)|Lechmere stop]] on the [[Green Line (MBTA)|Green Line]], and five stops on the [[Red Line (MBTA)|Red Line]] ([[Alewife Station (MBTA)|Alewife]], [[Porter (MBTA)|Porter Square]], [[Harvard (MBTA station)|Harvard Squ
are]], [[Central (MBTA station)|Central Square]], and [[Kendall/MIT (MBTA station)|Kendall Square/MIT]]). Alewife Station, the current terminus of the Red Line, has a large multi-story parking garage (at a rate of $7 per day {{as of|lc=y|2009}}).<ref>{{cite web|url=http://www.mbta.com/schedules_and_maps/subway/lines/stations/?stopId=10029 |title=> Schedules & Maps > Subway > Alewife Station |publisher=MBTA |date= |accessdate=2012-04-28}}</ref> The [[Harvard Bus Tunnel]], under Harvard Square, reduces traffic congestion on the surface, and connects to the Red Line underground. This tunnel was originally opened for streetcars in 1912, and served trackless trolleys and buses as the routes were converted. The tunnel was partially reconfigured when the Red Line was extended to Alewife in the early 1980s. Outside of the state-owned transit agency, the city is also served by the Charles River Transportation Management Agency (CRTMA) shuttles which are supported by some of the largest compa
nies operating in city, in addition to the municipal government itself.<ref>{{cite web |url=http://www.charlesrivertma.org/members.htm |title=Charles River TMA Members |author=Staff writer |date=(As of) January 1, 2013 |work=CRTMA |publisher= |language= |trans_title= |type= |archiveurl= |archivedate= |deadurl= |accessdate=January 1, 2013 |quote= |ref= |separator= |postscript=}} </ref> ===Cycling=== Cambridge has several [[bike path]]s, including one along the Charles River,<ref>{{cite web|url=http://www.mass.gov/dcr/parks/metroboston/maps/bikepaths_dudley.gif |title=Dr. Paul Dudley White Bikepath |date= |accessdate=2012-04-28}}</ref> and the [[Cambridge Linear Park|Linear Park]] connecting the [[Minuteman Bikeway]] at Alewife with the [[Somerville Community Path]]. Bike parking is common and there are bike lanes on many streets, although concerns have been expressed regarding the suitability of many of the lanes. On several central MIT streets, bike lanes transfer onto the sidewalk.
Cambridge bans cycling on certain sections of sidewalk where pedestrian traffic is heavy.<ref>{{cite web|url=http://www.cambridgema.gov/cdd/et/bike/bike_ban.html |title=Sidewalk Bicycling Banned Areas – Cambridge Massachusetts |publisher=Cambridgema.gov |date= |accessdate=2012-04-28}}</ref><ref>{{cite web|url=http://www.cambridgema.gov/cdd/et/bike/bike_reg.html |title=Traffic Regulations for Cyclists – Cambridge Massachusetts |publisher=Cambridgema.gov |date=1997-05-01 |accessdate=2012-04-28}}</ref> While ''[[Bicycling Magazine]]'' has rated Boston as one of the worst cities in the nation for bicycling (In their words, for "lousy roads, scarce and unconnected bike lanes and bike-friendly gestures from City Hall that go nowhere—such as hiring a bike coordinator in 2001, only to cut the position two years later"),<ref>[http://www.bicycling.com/article/1,6610,s1-2-16-14593-11,00.html Urban Treasures – bicycling.com]{{dead link|date=April 2012}}</ref> it has listed Cambridge as
an honorable mention as one of the best<ref>[http://www.bicycling.com/article/1,6610,s1-2-16-14593-9,00.html Urban Treasures – bicycling.com]{{dead link|date=April 2012}}</ref> and was called by the magazine "Boston's Great Hope." Cambridge has an active, official bicycle committee. ===Walking=== [[File:Weeks Footbridge Cambridge, MA.jpg|thumb|The [[John W. Weeks Bridge|Weeks Bridge]] provides a pedestrian-only connection between Boston's Allston-Brighton neighborhood and Cambridge over the Charles River]] Walking is a popular activity in Cambridge. Per year 2000 data, of the communities in the U.S. with more than 100,000 residents, Cambridge has the highest percentage of commuters who walk to work.<ref>{{cite web|url=http://www.bikesatwork.com/carfree/census-lookup.php?state_select=ALL_STATES&lower_pop=100000&upper_pop=99999999&sort_num=2&show_rows=25&first_row=0 |title=The Carfree Census Database: Result of search for communities in any state with population over 100,000, sorte
d in descending order by % Pedestrian Commuters |publisher=Bikesatwork.com |date= |accessdate=2012-04-28}}</ref> Cambridge receives a "Walk Score" of 100 out of 100 possible points.<ref>[http://www.walkscore.com/get-score.php?street=cambridge%2C+ma&go=Go Walk Score site] Accessed July 28, 2009</ref> Cambridge's major historic squares have been recently changed into a modern walking landscape, which has sparked a traffic calming program based on the needs of pedestrians rather than of motorists. ===Intercity=== The Boston intercity bus and train stations at [[South Station]], Boston, and [[Logan International Airport]] in [[East Boston]], are accessible by [[Red Line (MBTA)|subway]]. The [[Fitchburg Line]] rail service from [[Porter (MBTA station)|Porter Square]] connects to some western suburbs. Since October 2010, there has also been intercity bus service between [[Alewife (MBTA station)|Alewife Station]] (Cambridge) and [[New York City]].<ref>{{cite web|last=Thomas |first=Sarah |u
rl=http://www.boston.com/yourtown/news/cambridge/2010/10/warren_mbta_welcome_world_wide.html |title=NYC-bound buses will roll from Newton, Cambridge |publisher=Boston.com |date=2010-10-19 |accessdate=2012-04-28}}</ref> ==Media== ===Newspapers=== Cambridge is served by several weekly newspapers. The most prominent is the ''[[Cambridge Chronicle]]'', which is also the oldest surviving weekly paper in the United States. ===Radio=== Cambridge is home to the following commercially licensed and student-run radio stations: {| class=wikitable |- ! [[Callsign]] !! Frequency !! City/town !! Licensee !! Format |- | [[WHRB]] || align=right | 95.3 FM || Cambridge (Harvard) || Harvard Radio Broadcasting Co., Inc. || [[Variety (US radio)|Musical variety]] |- | [[WJIB]] || align=right | 740 AM || Cambridge || Bob Bittner Broadcasting || [[Adult Standards]]/Pop |- | [[WMBR]] || align=right | 88.1 FM || Cambridge (MIT) || Technology Broadcasting Corporation || [[College radio]] |} ===Television=
== Cambridge Community Television (CCTV) has served the Cambridge community since its inception in 1988. CCTV operates Cambridge's public access television facility and programs three television channels, 8, 9, and 96 on the Cambridge cable system (Comcast). ===Social media=== As of 2011, a growing number of social media efforts provide means for participatory engagement with the locality of Cambridge, such as Localocracy<ref>"Localocracy is an online town common where registered voters using real names can weigh in on local issues." [http://cambridge.localocracy.com/ Localocracy Cambridge, Massachusetts]. Accessed 2011-10-01</ref> and [[foursquare (website)|Foursquare]]. ==Culture, art and architecture== [[File:Fogg.jpg|thumb|[[Fogg Museum]], Harvard]] ===Museums=== * [[Harvard Art Museum]], including the [[Busch-Reisinger Museum]], a collection of Germanic art the [[Fogg Art Museum]], a comprehensive collection of Western art, and the [[Arthur M. Sackler Museum]], a collection of
Middle East and Asian art * [[Harvard Museum of Natural History]], including the [[Glass Flowers]] collection * [[Peabody Museum of Archaeology and Ethnology]], Harvard *[[Semitic Museum]], Harvard * [[MIT Museum]] * [[List Visual Arts Center]], MIT ===Public art=== Cambridge has a large and varied collection of permanent public art, both on city property (managed by the Cambridge Arts Council),<ref>{{cite web|url=http://www.cambridgema.gov/CAC/Public/overview.cfm |title=CAC Public Art Program |publisher=Cambridgema.gov |date=2007-03-13 |accessdate=2012-04-28}}</ref> and on the campuses of Harvard<ref>{{cite web|url=http://ofa.fas.harvard.edu/visualarts/pubart.php |title=Office for the Arts at Harvard: Public Art |publisher=Ofa.fas.harvard.edu |date= |accessdate=2012-04-28}}</ref> and MIT.<ref>{{cite web|url=http://listart.mit.edu/map |title=MIT Public Art Collection Map |publisher=Listart.mit.edu |date= |accessdate=2012-04-28}}</ref> Temporary public artworks are displayed as part
of the annual Cambridge River Festival on the banks of the Charles River, during winter celebrations in Harvard and Central Squares, and at university campus sites. Experimental forms of public artistic and cultural expression include the Central Square World's Fair, the Somerville-based annual Honk! Festival,<ref>{{cite web|url=http://honkfest.org/ |title= Honk Fest}}</ref> and [[If This House Could Talk]],<ref>{{cite web|url=http://cambridgehistory.org/discover/ifthishousecouldtalk/index.html |title=The Cambridge Historical Society}}</ref> a neighborhood art and history event. {{or|date=April 2012}} {{Citation needed|date=April 2012}} An active tradition of street musicians and other performers in Harvard Square entertains an audience of tourists and local residents during the warmer months of the year. The performances are coordinated through a public process that has been developed collaboratively by the performers,<ref>{{cite web|url=http://www.buskersadvocates.org/ | title= St
reet Arts & Buskers Advocates}}</ref> city administrators, private organizations and business groups.<ref>{{cite web|url=http://harvardsquare.com/Home/Arts-and-Entertainment/Street-Arts-and-Buskers-Advocates.aspx |title=Street Arts and Buskers Advocates |publisher=Harvardsquare.com |date= |accessdate=2012-04-28}}</ref> [[File:Longfellow National Historic Site, Cambridge, Massachusetts.JPG|thumb|right|The [[Longfellow National Historic Site]]]] [[File:Wfm stata center.jpg|thumb|[[Stata Center]], MIT]] [[File:Simmons Hall, MIT, Cambridge, Massachusetts.JPG|thumb|[[List of MIT undergraduate dormitories|Simmons Hall]], MIT]] ===Architecture=== Despite intensive urbanization during the late 19th century and 20th century, Cambridge has preserved an unusual number of historic buildings, including some dating to the 17th century. The city also contains an abundance of innovative contemporary architecture, largely built by Harvard and MIT. ;Notable historic buildings in the city include: * T
he [[Asa Gray House]] (1810) * [[Austin Hall, Harvard University]] (1882–84) * [[Cambridge, Massachusetts City Hall|Cambridge City Hall]] (1888–89) * [[Cambridge Public Library]] (1888) * [[Christ Church, Cambridge]] (1761) * [[Cooper-Frost-Austin House]] (1689–1817) * [[Elmwood (Cambridge, Massachusetts)|Elmwood House]] (1767), residence of the [[President of Harvard University]] * [[First Church of Christ, Scientist (Cambridge, Massachusetts)|First Church of Christ, Scientist]] (1924–30) * [[The First Parish in Cambridge]] (1833) * [[Harvard-Epworth United Methodist Church]] (1891–93) * [[Harvard Lampoon Building]] (1909) * The [[Hooper-Lee-Nichols House]] (1685–1850) * [[Longfellow National Historic Site]] (1759), former home of poet [[Henry Wadsworth Longfellow]] * [[The Memorial Church of Harvard University]] (1932) * [[Memorial Hall, Harvard University]] (1870–77) * [[Middlesex County Courthouse (Massachusetts)|Middlesex County Courthouse]] (1814–48) * [[Urban
Rowhouse (40-48 Pearl Street, Cambridge, Massachusetts)|Urban Rowhouse]] (1875) * [[spite house|O'Reilly Spite House]] (1908), built to spite a neighbor who would not sell his adjacent land<ref name="existing">Bloom, Jonathan. (February 2, 2003) [[Boston Globe]] ''[http://nl.newsbank.com/nl-search/we/Archives?p_product=BG&p_theme=bg&p_action=search&p_maxdocs=200&p_topdoc=1&p_text_direct-0=0F907F2342522B5D&p_field_direct-0=document_id&p_perpage=10&p_sort=YMD_date:D Existing by the Thinnest of Margins. A Concord Avenue Landmark Gives New Meaning to Cozy.]'' Section: City Weekly; Page 11. Location: 260 Concord Ave, Cambridge, MA 02138.</ref> {{See also|List of Registered Historic Places in Cambridge, Massachusetts}} ;Contemporary architecture: * [[List of MIT undergraduate dormitories#Baker House|Baker House]] dormitory, MIT, by Finnish architect [[Alvar Aalto]], one of only two buildings by Aalto in the US * Harvard Graduate Center/Harkness Commons, by [[The Architects Collaborative]]
(TAC, with [[Walter Gropius]]) * [[Carpenter Center for the Visual Arts]], Harvard, the only building in North America by [[Le Corbusier]] * [[Kresge Auditorium]], MIT, by [[Eero Saarinen]] * [[MIT Chapel]], by [[Eero Saarinen]] * [[Design Research Building]], by [[Benjamin Thompson and Associates]] * [[American Academy of Arts and Sciences]], by [[Kallmann McKinnell and Wood]], also architects of Boston City Hall * [[Arthur M. Sackler Museum]], Harvard, one of the few buildings in the U.S. by [[James Stirling (architect)|James Stirling]], winner of the [[Pritzker Prize]] * [[Stata Center]], MIT, by [[Frank Gehry]] * [[List of MIT undergraduate dormitories#Simmons Hall|Simmons Hall]], MIT, by [[Steven Holl]] ===Music=== <!-- make section generic. NEEDS MORE WORK. remove marketing fluff for Ryles. --> The city has an active music scene from classical performances to the latest popular bands. ==Sister cities== Cambridge has 8 active, official [[Twin towns and sister cities|sister cit
ies]], and an unofficial relationship with [[Cambridge]], England:<ref name="peacom">"A message from the Peace Commission" [http://www.cambridgema.gov/peace/newsandpublications/news/detail.aspx?path=%2fsitecore%2fcontent%2fhome%2fpeace%2fnewsandpublications%2fnews%2f2008%2f02%2finformationoncambridgessistercities].</ref> *{{Flagicon|PRT}} [[Coimbra]], [[Portugal]] *{{Flagicon|CUB}} [[Cienfuegos]], [[Cuba]] *{{Flagicon|ITA}} [[Gaeta]], [[Italy]] *{{Flagicon|IRL}} [[Galway]], [[Republic of Ireland|Ireland]] *{{Flagicon|ARM}} [[Yerevan]], [[Armenia]]<ref>{{cite web|url=http://www.cysca.org/ |title=Cambridge-Yerevan Sister City Association |publisher=Cysca.org |date= |accessdate=2012-04-28}}</ref> *{{Flagicon|SLV}} [[San José Las Flores, Chalatenango|San José Las Flores]], [[El Salvador]] *{{Flagicon|JPN}} [[Tsukuba, Ibaraki|Tsukuba Science City]], Japan *{{Flagicon|POL}} [[Kraków]], [[Poland]] *{{Flagicon|CHN}} [[Haidian District]], [[China]] Ten other official sister city relations
hips are inactive: [[Dublin]], Ireland; [[Ischia]], [[Catania]], and [[Florence]], Italy; [[Kraków]], Poland; [[Santo Domingo Oeste]], Dominican Republic; [[Southwark]], London, England; [[Yuseong]], Daejeon, Korea; and [[Haidian District|Haidian]], Beijing, China.<ref name="peacom"/> There has also been an unofficial relationship with: *{{Flagicon|GBR}} [[Cambridge]], England, UK<ref>{{cite web|url=http://www.cambridgema.gov/peace/newsandpublications/news/detail.aspx?path=%2fsitecore%2fcontent%2fhome%2fpeace%2fnewsandpublications%2fnews%2f2008%2f02%2finformationoncambridgessistercities |title="Sister Cities", Cambridge Peace Commission |publisher=Cambridgema.gov |date=2008-02-15 |accessdate=2012-07-18}}</ref> ==Zip codes== *02138—Harvard Square/West Cambridge *02139—Central Square/Inman Square/MIT *02140—Porter Square/North Cambridge *02141—East Cambridge *02142—Kendall Square ==References== {{reflist|30em}} ==General references== * ''History of Middlesex County, Massach
usetts'', [http://books.google.com/books?id=QGolOAyd9RMC&dq=intitle:History+intitle:of+intitle:Middlesex+intitle:County+intitle:Massachusetts&lr=&num=50&as_brr=0&source=gbs_other_versions_sidebar_s&cad=5 Volume 1 (A-H)], [http://books.google.com/books?id=hNaAnwRMedUC&pg=PA506&dq=intitle:History+intitle:of+intitle:Middlesex+intitle:County+intitle:Massachusetts&lr=&num=50&as_brr=0#PPA3,M1 Volume 2 (L-W)] compiled by Samuel Adams Drake, published 1879–1880. ** [http://books.google.com/books?id=QGolOAyd9RMC&printsec=titlepage#PPA305,M1 Cambridge article] by Rev. Edward Abbott in volume 1, pages 305–358. *Eliot, Samuel Atkins. ''A History of Cambridge, Massachusetts: 1630–1913''. Cambridge: The Cambridge Tribune, 1913. *Hiestand, Emily. "Watershed: An Excursion in Four Parts" The Georgia Review Spring 1998 pages 7–28 *[[Lucius Robinson Paige|Paige, Lucius]]. ''History of Cambridge, Massachusetts: 1630–1877''. Cambridge: The Riverside Press, 1877. *Survey of Architectural Histor
y in Cambridge: Mid Cambridge, 1967, Cambridge Historical Commission, Cambridge, Mass.{{ISBN missing}} *Survey of Architectural History in Cambridge: Cambridgeport, 1971 ISBN 0-262-53013-9, Cambridge Historical Commission, Cambridge, Mass. *Survey of Architectural History in Cambridge: Old Cambridge, 1973 ISBN 0-262-53014-7, Cambridge Historical Commission, Cambridge, Mass. *Survey of Architectural History in Cambridge: Northwest Cambridge, 1977 ISBN 0-262-53032-5, Cambridge Historical Commission, Cambridge, Mass. *Survey of Architectural History in Cambridge: East Cambridge, 1988 (revised) ISBN 0-262-53078-3, Cambridge Historical Commission, Cambridge, Mass. *{{cite book|last=Sinclair|first=Jill|title=Fresh Pond: The History of a Cambridge Landscape|publisher=MIT Press|location=Cambridge, Mass.|date=April 2009|isbn=978-0-262-19591-1 }} *{{cite book|last=Seaburg|first=Alan|title=Cambridge on the Charles|url=http://books.google.com/books?id=c7_oCS782-8C|publisher=Anne Miniver Press|l
ocation=Billerica, Mass.|year=2001|author=Seaburg, A. and Dahill, T. and Rose, C.H.|isbn=978-0-9625794-9-3}} ==External links== {{Commons category}} <!-- for current and future use if material is uploaded --> {{Wikivoyage|Cambridge (Massachusetts)}} {{Portal|Boston}} {{Commons category|Cambridge, Massachusetts}} *{{Official website|http://www.cambridgema.gov/}} *[http://www.cambridge-usa.org/ Cambridge Office for Tourism] *[http://www.city-data.com/city/Cambridge-Massachusetts.html City-Data.com] *[http://www.epodunk.com/cgi-bin/genInfo.php?locIndex=2894 ePodunk: Profile for Cambridge, Massachusetts] *{{dmoz|Regional/North_America/United_States/Massachusetts/Localities/C/Cambridge}} <br/><!--this break is to put visual space between the last information and the following template if needed--> ===Maps=== *[http://www.cambridgema.gov/GIS/FindMapAtlas.cfm Cambridge Maps] *[http://www.cambridgema.gov/GIS City of Cambridge Geographic Information System (GIS)] *[http://www.salemdeeds.com/
atlases_results.asp?ImageType=index&atlastype=MassWorld&atlastown=&atlas=MASSACHUSETTS+1871&atlas_desc=MASSACHUSETTS+1871 ''1871 Atlas of Massachusetts''.] by Wall & Gray. [http://www.salemdeeds.com/atlases_pages.asp?ImageName=PAGE_0010_0011.jpg&atlastype=MassWorld&atlastown=&atlas=MASSACHUSETTS+1871&atlas_desc=MASSACHUSETTS+1871&pageprefix= Map of Massachusetts.] [http://www.salemdeeds.com/atlases_pages.asp?ImageName=PAGE_0044_0045.jpg&atlastype=MassWorld&atlastown=&atlas=MASSACHUSETTS+1871&atlas_desc=MASSACHUSETTS+1871&pageprefix= Map of Middlesex County.] *Dutton, E.P. [http://maps.bpl.org/details_10717/?srch_query=Dutton%2C+E.P.&srch_fields=all&srch_author=on&srch_style=exact&srch_fa=save&srch_ok=Go+Search Chart of Boston Harbor and Massachusetts Bay with Map of Adjacent Country.] Published 1867. A good map of roads and rail lines around Cambridge. *[http://www.citymap.com/cambridge/index.htm Cambridge Citymap – Community, Business, and Visitor Map.] *[http://docs.unh.edu/town
s/CambridgeMassachusettsMapList.htm Old USGS maps of Cambridge area.] {{Greater Boston}} {{Middlesex County, Massachusetts}} {{Massachusetts}} {{New England}} {{Massachusetts cities and mayors of 100,000 population}} [[Category:Cambridge, Massachusetts| ]] [[Category:University towns in the United States]] [[Category:County seats in Massachusetts]] [[Category:Populated places established in 1630]] [[Category:Charles River]] [[Category:Place names of English origin in the United States]] [[af:Cambridge, Massachusetts]] [[ar:كامبريدج، ماساتشوستس]] [[zh-min-nan:Cambridge, Massachusetts]] [[be:Горад Кембрыдж, Масачусетс]] [[be-x-old:Кембрыдж (Масачусэтс)]] [[bg:Кеймбридж (Масачузетс)]] [[br:Cambridge (Massachusetts)]] [[ca:Cambridge (Massachusetts)]] [[cs:Cambridge (Massachusetts)]] [[cy:Cambridge, Massachusetts]] [[da:Cambridge (Massachusetts)]] [[de:Cambridge (Massachusetts)]] [[et:Cambridge (Massachusetts)
]] [[es:Cambridge (Massachusetts)]] [[eo:Kembriĝo (Masaĉuseco)]] [[eu:Cambridge (Massachusetts)]] [[fa:کمبریج (ماساچوست)]] [[fr:Cambridge (Massachusetts)]] [[gd:Cambridge (MA)]] [[ko:케임브리지 (매사추세츠 주)]] [[hy:Քեմբրիջ (Մասաչուսեթս)]] [[id:Cambridge, Massachusetts]] [[it:Cambridge (Massachusetts)]] [[he:קיימברידג' (מסצ'וסטס)]] [[jv:Cambridge, Massachusetts]] [[kk:Кэмбридж (Массачусетс)]] [[kw:Cambridge, Massachusetts]] [[sw:Cambridge, Massachusetts]] [[ht:Cambridge, Massachusetts]] [[la:Cantabrigia (Massachusetta)]] [[lv:Keimbridža]] [[lb:Cambridge (Massachusetts)]] [[hu:Cambridge (Massachusetts)]] [[mr:केंब्रिज, मॅसेच्युसेट्स]] [[ms:Cambridge, Massachusetts]] [[nl:Cambridge (Massachusetts)]] [[ja:ケンブリッジ (マサチューセッツ州)]] [[no:Cambridge (Massachusetts)]] [[pl:Cambridge (Massachusetts)]] [[pt:Cambridge (Massachusetts)]] [[ro:Cambri
dge, Massachusetts]] [[ru:Кембридж (Массачусетс)]] [[scn:Cambridge (Massachusetts), USA]] [[simple:Cambridge, Massachusetts]] [[sk:Cambridge (Massachusetts)]] [[sl:Cambridge, Massachusetts]] [[sr:Кембриџ (Масачусетс)]] [[fi:Cambridge (Massachusetts)]] [[sv:Cambridge, Massachusetts]] [[tl:Cambridge, Massachusetts]] [[ta:கேம்பிரிஜ், மாசசூசெட்ஸ்]] [[th:เคมบริดจ์ (รัฐแมสซาชูเซตส์)]] [[tg:Кембриҷ (Массачусетс)]] [[tr:Cambridge, Massachusetts]] [[uk:Кембридж (Массачусетс)]] [[vi:Cambridge, Massachusetts]] [[vo:Cambridge (Massachusetts)]] [[war:Cambridge, Massachusetts]] [[yi:קעמברידזש, מאסאטשוסעטס]] [[zh:剑桥 (马萨诸塞州)]]
\ No newline at end of file
[4/4] lucene-solr:master: LUCENE-7815: Removed the PostingsHighlighter
Posted by ds...@apache.org.
LUCENE-7815: Removed the PostingsHighlighter
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0d3c73ea
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0d3c73ea
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0d3c73ea
Branch: refs/heads/master
Commit: 0d3c73eaa2dd26af73461fd6ec3494bc12edbe8a
Parents: 14320a5
Author: David Smiley <ds...@apache.org>
Authored: Tue May 23 14:39:51 2017 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Tue May 23 14:39:51 2017 -0400
----------------------------------------------------------------------
lucene/CHANGES.txt | 4 +
lucene/benchmark/conf/highlighters-postings.alg | 4 +-
.../tasks/SearchTravRetHighlightTask.java | 30 -
.../DefaultPassageFormatter.java | 137 --
.../MultiTermHighlighting.java | 282 -----
.../search/postingshighlight/Passage.java | 159 ---
.../postingshighlight/PassageFormatter.java | 40 -
.../search/postingshighlight/PassageScorer.java | 104 --
.../postingshighlight/PostingsHighlighter.java | 820 ------------
.../search/postingshighlight/package-info.java | 21 -
.../CustomSeparatorBreakIterator.java | 150 +++
.../search/uhighlight/WholeBreakIterator.java | 116 ++
.../search/postingshighlight/CambridgeMA.utf8 | 1 -
.../TestMultiTermHighlighting.java | 884 -------------
.../TestPostingsHighlighter.java | 1185 ------------------
.../TestPostingsHighlighterRanking.java | 324 -----
.../uhighlight/LengthGoalBreakIteratorTest.java | 1 -
.../TestCustomSeparatorBreakIterator.java | 114 ++
.../uhighlight/TestUnifiedHighlighter.java | 1 -
.../uhighlight/TestWholeBreakIterator.java | 134 ++
.../solr/highlight/UnifiedSolrHighlighter.java | 4 +-
21 files changed, 522 insertions(+), 3993 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 3951cea..dadba8b 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -53,6 +53,10 @@ API Changes
* LUCENE-7741: DoubleValuesSource now has an explain() method (Alan Woodward,
Adrien Grand)
+* LUCENE-7815: Removed the PostingsHighlighter; you should use the UnifiedHighlighter
+ instead, which derived from the UH. WholeBreakIterator and
+ CustomSeparatorBreakIterator were moved to UH's package. (David Smiley)
+
Bug Fixes
* LUCENE-7626: IndexWriter will no longer accept broken token offsets
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/benchmark/conf/highlighters-postings.alg
----------------------------------------------------------------------
diff --git a/lucene/benchmark/conf/highlighters-postings.alg b/lucene/benchmark/conf/highlighters-postings.alg
index 610908f..2560dad 100644
--- a/lucene/benchmark/conf/highlighters-postings.alg
+++ b/lucene/benchmark/conf/highlighters-postings.alg
@@ -38,7 +38,7 @@ file.query.maker.file=conf/query-terms.txt
log.queries=false
log.step.SearchTravRetHighlight=-1
-highlighter=HlImpl:NONE:SH_A:UH_A:PH_P:UH_P:UH_PV
+highlighter=HlImpl:NONE:SH_A:UH_A:UH_P:UH_PV
{ "Populate"
CreateIndex
@@ -60,6 +60,6 @@ highlighter=HlImpl:NONE:SH_A:UH_A:PH_P:UH_P:UH_PV
CloseReader
NewRound
-} : 6
+} : 5
RepSumByPrefRound HL
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetHighlightTask.java
----------------------------------------------------------------------
diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetHighlightTask.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetHighlightTask.java
index f36854d..d90d3a7 100644
--- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetHighlightTask.java
+++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetHighlightTask.java
@@ -42,7 +42,6 @@ import org.apache.lucene.search.highlight.Highlighter;
import org.apache.lucene.search.highlight.QueryScorer;
import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
import org.apache.lucene.search.highlight.TokenSources;
-import org.apache.lucene.search.postingshighlight.PostingsHighlighter;
import org.apache.lucene.search.uhighlight.UnifiedHighlighter;
import org.apache.lucene.search.vectorhighlight.BoundaryScanner;
import org.apache.lucene.search.vectorhighlight.BreakIteratorBoundaryScanner;
@@ -133,8 +132,6 @@ public class SearchTravRetHighlightTask extends SearchTravTask {
case "UH_P": hlImpl = new UnifiedHLImpl(UnifiedHighlighter.OffsetSource.POSTINGS); break;
case "UH_PV": hlImpl = new UnifiedHLImpl(UnifiedHighlighter.OffsetSource.POSTINGS_WITH_TERM_VECTORS); break;
- case "PH_P": hlImpl = new PostingsHLImpl(); break;
-
default: throw new Exception("unrecognized highlighter type: " + type + " (try 'UH')");
}
}
@@ -224,33 +221,6 @@ public class SearchTravRetHighlightTask extends SearchTravTask {
return clone;
}
- private class PostingsHLImpl implements HLImpl {
- PostingsHighlighter highlighter;
- String[] fields = hlFields.toArray(new String[hlFields.size()]);
- int[] maxPassages;
- PostingsHLImpl() {
- highlighter = new PostingsHighlighter(maxDocCharsToAnalyze) {
- @Override
- protected Analyzer getIndexAnalyzer(String field) { // thus support wildcards
- return analyzer;
- }
-
- @Override
- protected BreakIterator getBreakIterator(String field) {
- return BreakIterator.getSentenceInstance(Locale.ENGLISH);
- }
- };
- maxPassages = new int[hlFields.size()];
- Arrays.fill(maxPassages, maxFrags);
- }
-
- @Override
- public void withTopDocs(IndexSearcher searcher, Query q, TopDocs hits) throws Exception {
- Map<String, String[]> result = highlighter.highlightFields(fields, q, searcher, hits, maxPassages);
- preventOptimizeAway = result.size();
- }
- }
-
private class UnifiedHLImpl implements HLImpl {
UnifiedHighlighter highlighter;
IndexSearcher lastSearcher;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/DefaultPassageFormatter.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/DefaultPassageFormatter.java b/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/DefaultPassageFormatter.java
deleted file mode 100644
index 73822c8..0000000
--- a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/DefaultPassageFormatter.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.postingshighlight;
-
-/**
- * Creates a formatted snippet from the top passages.
- * <p>
- * The default implementation marks the query terms as bold, and places
- * ellipses between unconnected passages.
- */
-public class DefaultPassageFormatter extends PassageFormatter {
- /** text that will appear before highlighted terms */
- protected final String preTag;
- /** text that will appear after highlighted terms */
- protected final String postTag;
- /** text that will appear between two unconnected passages */
- protected final String ellipsis;
- /** true if we should escape for html */
- protected final boolean escape;
-
- /**
- * Creates a new DefaultPassageFormatter with the default tags.
- */
- public DefaultPassageFormatter() {
- this("<b>", "</b>", "... ", false);
- }
-
- /**
- * Creates a new DefaultPassageFormatter with custom tags.
- * @param preTag text which should appear before a highlighted term.
- * @param postTag text which should appear after a highlighted term.
- * @param ellipsis text which should be used to connect two unconnected passages.
- * @param escape true if text should be html-escaped
- */
- public DefaultPassageFormatter(String preTag, String postTag, String ellipsis, boolean escape) {
- if (preTag == null || postTag == null || ellipsis == null) {
- throw new NullPointerException();
- }
- this.preTag = preTag;
- this.postTag = postTag;
- this.ellipsis = ellipsis;
- this.escape = escape;
- }
-
- @Override
- public String format(Passage passages[], String content) {
- StringBuilder sb = new StringBuilder();
- int pos = 0;
- for (Passage passage : passages) {
- // don't add ellipsis if it's the first one, or if it's connected.
- if (passage.startOffset > pos && pos > 0) {
- sb.append(ellipsis);
- }
- pos = passage.startOffset;
- for (int i = 0; i < passage.numMatches; i++) {
- int start = passage.matchStarts[i];
- int end = passage.matchEnds[i];
- // it's possible to have overlapping terms
- if (start > pos) {
- append(sb, content, pos, start);
- }
- if (end > pos) {
- sb.append(preTag);
- append(sb, content, Math.max(pos, start), end);
- sb.append(postTag);
- pos = end;
- }
- }
- // it's possible a "term" from the analyzer could span a sentence boundary.
- append(sb, content, pos, Math.max(pos, passage.endOffset));
- pos = passage.endOffset;
- }
- return sb.toString();
- }
-
- /**
- * Appends original text to the response.
- * @param dest resulting text, possibly transformed or encoded
- * @param content original text content
- * @param start index of the first character in content
- * @param end index of the character following the last character in content
- */
- protected void append(StringBuilder dest, String content, int start, int end) {
- if (escape) {
- // note: these are the rules from owasp.org
- for (int i = start; i < end; i++) {
- char ch = content.charAt(i);
- switch(ch) {
- case '&':
- dest.append("&");
- break;
- case '<':
- dest.append("<");
- break;
- case '>':
- dest.append(">");
- break;
- case '"':
- dest.append(""");
- break;
- case '\'':
- dest.append("'");
- break;
- case '/':
- dest.append("/");
- break;
- default:
- if (ch >= 0x30 && ch <= 0x39 || ch >= 0x41 && ch <= 0x5A || ch >= 0x61 && ch <= 0x7A) {
- dest.append(ch);
- } else if (ch < 0xff) {
- dest.append("&#");
- dest.append((int)ch);
- dest.append(";");
- } else {
- dest.append(ch);
- }
- }
- }
- } else {
- dest.append(content, start, end);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/MultiTermHighlighting.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/MultiTermHighlighting.java b/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/MultiTermHighlighting.java
deleted file mode 100644
index c9733d3..0000000
--- a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/MultiTermHighlighting.java
+++ /dev/null
@@ -1,282 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.postingshighlight;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.List;
-
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.AutomatonQuery;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.ConstantScoreQuery;
-import org.apache.lucene.search.DisjunctionMaxQuery;
-import org.apache.lucene.search.FuzzyQuery;
-import org.apache.lucene.search.PrefixQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermRangeQuery;
-import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper;
-import org.apache.lucene.search.spans.SpanNearQuery;
-import org.apache.lucene.search.spans.SpanNotQuery;
-import org.apache.lucene.search.spans.SpanOrQuery;
-import org.apache.lucene.search.spans.SpanPositionCheckQuery;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.CharsRef;
-import org.apache.lucene.util.UnicodeUtil;
-import org.apache.lucene.util.automaton.Automata;
-import org.apache.lucene.util.automaton.Automaton;
-import org.apache.lucene.util.automaton.CharacterRunAutomaton;
-import org.apache.lucene.util.automaton.LevenshteinAutomata;
-import org.apache.lucene.util.automaton.Operations;
-
-/**
- * Support for highlighting multiterm queries in PostingsHighlighter.
- */
-class MultiTermHighlighting {
-
- /**
- * Extracts all MultiTermQueries for {@code field}, and returns equivalent
- * automata that will match terms.
- */
- static CharacterRunAutomaton[] extractAutomata(Query query, String field) {
- List<CharacterRunAutomaton> list = new ArrayList<>();
- if (query instanceof BooleanQuery) {
- for (BooleanClause clause : (BooleanQuery) query) {
- if (!clause.isProhibited()) {
- list.addAll(Arrays.asList(extractAutomata(clause.getQuery(), field)));
- }
- }
- } else if (query instanceof ConstantScoreQuery) {
- list.addAll(Arrays.asList(extractAutomata(((ConstantScoreQuery) query).getQuery(), field)));
- } else if (query instanceof DisjunctionMaxQuery) {
- for (Query sub : ((DisjunctionMaxQuery) query).getDisjuncts()) {
- list.addAll(Arrays.asList(extractAutomata(sub, field)));
- }
- } else if (query instanceof SpanOrQuery) {
- for (Query sub : ((SpanOrQuery) query).getClauses()) {
- list.addAll(Arrays.asList(extractAutomata(sub, field)));
- }
- } else if (query instanceof SpanNearQuery) {
- for (Query sub : ((SpanNearQuery) query).getClauses()) {
- list.addAll(Arrays.asList(extractAutomata(sub, field)));
- }
- } else if (query instanceof SpanNotQuery) {
- list.addAll(Arrays.asList(extractAutomata(((SpanNotQuery) query).getInclude(), field)));
- } else if (query instanceof SpanPositionCheckQuery) {
- list.addAll(Arrays.asList(extractAutomata(((SpanPositionCheckQuery) query).getMatch(), field)));
- } else if (query instanceof SpanMultiTermQueryWrapper) {
- list.addAll(Arrays.asList(extractAutomata(((SpanMultiTermQueryWrapper<?>) query).getWrappedQuery(), field)));
- } else if (query instanceof PrefixQuery) {
- final PrefixQuery pq = (PrefixQuery) query;
- Term prefix = pq.getPrefix();
- if (prefix.field().equals(field)) {
- list.add(new CharacterRunAutomaton(Operations.concatenate(Automata.makeString(prefix.text()),
- Automata.makeAnyString())) {
- @Override
- public String toString() {
- return pq.toString();
- }
- });
- }
- } else if (query instanceof FuzzyQuery) {
- final FuzzyQuery fq = (FuzzyQuery) query;
- if (fq.getField().equals(field)) {
- String utf16 = fq.getTerm().text();
- int termText[] = new int[utf16.codePointCount(0, utf16.length())];
- for (int cp, i = 0, j = 0; i < utf16.length(); i += Character.charCount(cp)) {
- termText[j++] = cp = utf16.codePointAt(i);
- }
- int termLength = termText.length;
- int prefixLength = Math.min(fq.getPrefixLength(), termLength);
- String suffix = UnicodeUtil.newString(termText, prefixLength, termText.length - prefixLength);
- LevenshteinAutomata builder = new LevenshteinAutomata(suffix, fq.getTranspositions());
- String prefix = UnicodeUtil.newString(termText, 0, prefixLength);
- Automaton automaton = builder.toAutomaton(fq.getMaxEdits(), prefix);
- list.add(new CharacterRunAutomaton(automaton) {
- @Override
- public String toString() {
- return fq.toString();
- }
- });
- }
- } else if (query instanceof TermRangeQuery) {
- final TermRangeQuery tq = (TermRangeQuery) query;
- if (tq.getField().equals(field)) {
- final CharsRef lowerBound;
- if (tq.getLowerTerm() == null) {
- lowerBound = null;
- } else {
- lowerBound = new CharsRef(tq.getLowerTerm().utf8ToString());
- }
-
- final CharsRef upperBound;
- if (tq.getUpperTerm() == null) {
- upperBound = null;
- } else {
- upperBound = new CharsRef(tq.getUpperTerm().utf8ToString());
- }
-
- final boolean includeLower = tq.includesLower();
- final boolean includeUpper = tq.includesUpper();
- final CharsRef scratch = new CharsRef();
- final Comparator<CharsRef> comparator = CharsRef.getUTF16SortedAsUTF8Comparator();
-
- // this is *not* an automaton, but it's very simple
- list.add(new CharacterRunAutomaton(Automata.makeEmpty()) {
- @Override
- public boolean run(char[] s, int offset, int length) {
- scratch.chars = s;
- scratch.offset = offset;
- scratch.length = length;
-
- if (lowerBound != null) {
- int cmp = comparator.compare(scratch, lowerBound);
- if (cmp < 0 || (!includeLower && cmp == 0)) {
- return false;
- }
- }
-
- if (upperBound != null) {
- int cmp = comparator.compare(scratch, upperBound);
- if (cmp > 0 || (!includeUpper && cmp == 0)) {
- return false;
- }
- }
- return true;
- }
-
- @Override
- public String toString() {
- return tq.toString();
- }
- });
- }
- } else if (query instanceof AutomatonQuery) {
- final AutomatonQuery aq = (AutomatonQuery) query;
- if (aq.getField().equals(field)) {
- list.add(new CharacterRunAutomaton(aq.getAutomaton()) {
- @Override
- public String toString() {
- return aq.toString();
- }
- });
- }
- }
- return list.toArray(new CharacterRunAutomaton[list.size()]);
- }
-
- /**
- * Returns a "fake" DocsAndPositionsEnum over the tokenstream, returning offsets where {@code matchers}
- * matches tokens.
- * <p>
- * This is solely used internally by PostingsHighlighter: <b>DO NOT USE THIS METHOD!</b>
- */
- static PostingsEnum getDocsEnum(final TokenStream ts, final CharacterRunAutomaton[] matchers) throws IOException {
- final CharTermAttribute charTermAtt = ts.addAttribute(CharTermAttribute.class);
- final OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
- ts.reset();
-
- // TODO: we could use CachingWrapperFilter, (or consume twice) to allow us to have a true freq()
- // but this would have a performance cost for likely little gain in the user experience, it
- // would only serve to make this method less bogus.
- // instead, we always return freq() = Integer.MAX_VALUE and let PH terminate based on offset...
-
- return new PostingsEnum() {
- int currentDoc = -1;
- int currentMatch = -1;
- int currentStartOffset = -1;
- int currentEndOffset = -1;
- TokenStream stream = ts;
-
- final BytesRef matchDescriptions[] = new BytesRef[matchers.length];
-
- @Override
- public int nextPosition() throws IOException {
- if (stream != null) {
- while (stream.incrementToken()) {
- for (int i = 0; i < matchers.length; i++) {
- if (matchers[i].run(charTermAtt.buffer(), 0, charTermAtt.length())) {
- currentStartOffset = offsetAtt.startOffset();
- currentEndOffset = offsetAtt.endOffset();
- currentMatch = i;
- return 0;
- }
- }
- }
- stream.end();
- stream.close();
- stream = null;
- }
- // exhausted
- currentStartOffset = currentEndOffset = Integer.MAX_VALUE;
- return Integer.MAX_VALUE;
- }
-
- @Override
- public int freq() throws IOException {
- return Integer.MAX_VALUE; // lie
- }
-
- @Override
- public int startOffset() throws IOException {
- assert currentStartOffset >= 0;
- return currentStartOffset;
- }
-
- @Override
- public int endOffset() throws IOException {
- assert currentEndOffset >= 0;
- return currentEndOffset;
- }
-
- @Override
- public BytesRef getPayload() throws IOException {
- if (matchDescriptions[currentMatch] == null) {
- matchDescriptions[currentMatch] = new BytesRef(matchers[currentMatch].toString());
- }
- return matchDescriptions[currentMatch];
- }
-
- @Override
- public int docID() {
- return currentDoc;
- }
-
- @Override
- public int nextDoc() throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public int advance(int target) throws IOException {
- return currentDoc = target;
- }
-
- @Override
- public long cost() {
- return 0;
- }
- };
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/Passage.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/Passage.java b/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/Passage.java
deleted file mode 100644
index 50aebea..0000000
--- a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/Passage.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.postingshighlight;
-
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.InPlaceMergeSorter;
-import org.apache.lucene.util.RamUsageEstimator;
-
-/**
- * Represents a passage (typically a sentence of the document).
- * <p>
- * A passage contains {@link #getNumMatches} highlights from the query,
- * and the offsets and query terms that correspond with each match.
- * @lucene.experimental
- */
-public final class Passage {
- int startOffset = -1;
- int endOffset = -1;
- float score = 0.0f;
-
- int matchStarts[] = new int[8];
- int matchEnds[] = new int[8];
- BytesRef matchTerms[] = new BytesRef[8];
- int numMatches = 0;
-
- void addMatch(int startOffset, int endOffset, BytesRef term) {
- assert startOffset >= this.startOffset && startOffset <= this.endOffset;
- if (numMatches == matchStarts.length) {
- int newLength = ArrayUtil.oversize(numMatches+1, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
- int newMatchStarts[] = new int[newLength];
- int newMatchEnds[] = new int[newLength];
- BytesRef newMatchTerms[] = new BytesRef[newLength];
- System.arraycopy(matchStarts, 0, newMatchStarts, 0, numMatches);
- System.arraycopy(matchEnds, 0, newMatchEnds, 0, numMatches);
- System.arraycopy(matchTerms, 0, newMatchTerms, 0, numMatches);
- matchStarts = newMatchStarts;
- matchEnds = newMatchEnds;
- matchTerms = newMatchTerms;
- }
- assert matchStarts.length == matchEnds.length && matchEnds.length == matchTerms.length;
- matchStarts[numMatches] = startOffset;
- matchEnds[numMatches] = endOffset;
- matchTerms[numMatches] = term;
- numMatches++;
- }
-
- void sort() {
- final int starts[] = matchStarts;
- final int ends[] = matchEnds;
- final BytesRef terms[] = matchTerms;
- new InPlaceMergeSorter() {
- @Override
- protected void swap(int i, int j) {
- int temp = starts[i];
- starts[i] = starts[j];
- starts[j] = temp;
-
- temp = ends[i];
- ends[i] = ends[j];
- ends[j] = temp;
-
- BytesRef tempTerm = terms[i];
- terms[i] = terms[j];
- terms[j] = tempTerm;
- }
-
- @Override
- protected int compare(int i, int j) {
- return Integer.compare(starts[i], starts[j]);
- }
-
- }.sort(0, numMatches);
- }
-
- void reset() {
- startOffset = endOffset = -1;
- score = 0.0f;
- numMatches = 0;
- }
-
- /**
- * Start offset of this passage.
- * @return start index (inclusive) of the passage in the
- * original content: always >= 0.
- */
- public int getStartOffset() {
- return startOffset;
- }
-
- /**
- * End offset of this passage.
- * @return end index (exclusive) of the passage in the
- * original content: always >= {@link #getStartOffset()}
- */
- public int getEndOffset() {
- return endOffset;
- }
-
- /**
- * Passage's score.
- */
- public float getScore() {
- return score;
- }
-
- /**
- * Number of term matches available in
- * {@link #getMatchStarts}, {@link #getMatchEnds},
- * {@link #getMatchTerms}
- */
- public int getNumMatches() {
- return numMatches;
- }
-
- /**
- * Start offsets of the term matches, in increasing order.
- * <p>
- * Only {@link #getNumMatches} are valid. Note that these
- * offsets are absolute (not relative to {@link #getStartOffset()}).
- */
- public int[] getMatchStarts() {
- return matchStarts;
- }
-
- /**
- * End offsets of the term matches, corresponding with {@link #getMatchStarts}.
- * <p>
- * Only {@link #getNumMatches} are valid. Note that it's possible that an end offset
- * could exceed beyond the bounds of the passage ({@link #getEndOffset()}), if the
- * Analyzer produced a term which spans a passage boundary.
- */
- public int[] getMatchEnds() {
- return matchEnds;
- }
-
- /**
- * BytesRef (term text) of the matches, corresponding with {@link #getMatchStarts()}.
- * <p>
- * Only {@link #getNumMatches()} are valid.
- */
- public BytesRef[] getMatchTerms() {
- return matchTerms;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PassageFormatter.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PassageFormatter.java b/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PassageFormatter.java
deleted file mode 100644
index f1596c1..0000000
--- a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PassageFormatter.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.postingshighlight;
-
-/**
- * Creates a formatted snippet from the top passages.
- *
- * @lucene.experimental
- */
-public abstract class PassageFormatter {
-
- /**
- * Formats the top <code>passages</code> from <code>content</code>
- * into a human-readable text snippet.
- *
- * @param passages top-N passages for the field. Note these are sorted in
- * the order that they appear in the document for convenience.
- * @param content content for the field.
- * @return formatted highlight. Note that for the
- * non-expert APIs in {@link PostingsHighlighter} that
- * return String, the toString method on the Object
- * returned by this method is used to compute the string.
- */
- public abstract Object format(Passage passages[], String content);
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PassageScorer.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PassageScorer.java b/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PassageScorer.java
deleted file mode 100644
index 1f74f7a..0000000
--- a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PassageScorer.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.postingshighlight;
-
-/**
- * Ranks passages found by {@link PostingsHighlighter}.
- * <p>
- * Each passage is scored as a miniature document within the document.
- * The final score is computed as {@link #norm} * ∑ ({@link #weight} * {@link #tf}).
- * The default implementation is {@link #norm} * BM25.
- * @lucene.experimental
- */
-public class PassageScorer {
-
- // TODO: this formula is completely made up. It might not provide relevant snippets!
-
- /** BM25 k1 parameter, controls term frequency normalization */
- final float k1;
- /** BM25 b parameter, controls length normalization. */
- final float b;
- /** A pivot used for length normalization. */
- final float pivot;
-
- /**
- * Creates PassageScorer with these default values:
- * <ul>
- * <li>{@code k1 = 1.2},
- * <li>{@code b = 0.75}.
- * <li>{@code pivot = 87}
- * </ul>
- */
- public PassageScorer() {
- // 1.2 and 0.75 are well-known bm25 defaults (but maybe not the best here) ?
- // 87 is typical average english sentence length.
- this(1.2f, 0.75f, 87f);
- }
-
- /**
- * Creates PassageScorer with specified scoring parameters
- * @param k1 Controls non-linear term frequency normalization (saturation).
- * @param b Controls to what degree passage length normalizes tf values.
- * @param pivot Pivot value for length normalization (some rough idea of average sentence length in characters).
- */
- public PassageScorer(float k1, float b, float pivot) {
- this.k1 = k1;
- this.b = b;
- this.pivot = pivot;
- }
-
- /**
- * Computes term importance, given its in-document statistics.
- *
- * @param contentLength length of document in characters
- * @param totalTermFreq number of time term occurs in document
- * @return term importance
- */
- public float weight(int contentLength, int totalTermFreq) {
- // approximate #docs from content length
- float numDocs = 1 + contentLength / pivot;
- // numDocs not numDocs - docFreq (ala DFR), since we approximate numDocs
- return (k1 + 1) * (float) Math.log(1 + (numDocs + 0.5D)/(totalTermFreq + 0.5D));
- }
-
- /**
- * Computes term weight, given the frequency within the passage
- * and the passage's length.
- *
- * @param freq number of occurrences of within this passage
- * @param passageLen length of the passage in characters.
- * @return term weight
- */
- public float tf(int freq, int passageLen) {
- float norm = k1 * ((1 - b) + b * (passageLen / pivot));
- return freq / (freq + norm);
- }
-
- /**
- * Normalize a passage according to its position in the document.
- * <p>
- * Typically passages towards the beginning of the document are
- * more useful for summarizing the contents.
- * <p>
- * The default implementation is <code>1 + 1/log(pivot + passageStart)</code>
- * @param passageStart start offset of the passage
- * @return a boost value multiplied into the passage's core.
- */
- public float norm(int passageStart) {
- return 1 + 1/(float)Math.log(pivot + passageStart);
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java b/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java
deleted file mode 100644
index e4d3667..0000000
--- a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java
+++ /dev/null
@@ -1,820 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.postingshighlight;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.text.BreakIterator;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.PriorityQueue;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexReaderContext;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.MultiReader;
-import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.ReaderUtil;
-import org.apache.lucene.index.StoredFieldVisitor;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.InPlaceMergeSorter;
-import org.apache.lucene.util.UnicodeUtil;
-import org.apache.lucene.util.automaton.CharacterRunAutomaton;
-
-/**
- * Simple highlighter that does not analyze fields nor use
- * term vectors. Instead it requires
- * {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}.
- * <p>
- * PostingsHighlighter treats the single original document as the whole corpus, and then scores individual
- * passages as if they were documents in this corpus. It uses a {@link BreakIterator} to find
- * passages in the text; by default it breaks using {@link BreakIterator#getSentenceInstance(Locale)
- * getSentenceInstance(Locale.ROOT)}. It then iterates in parallel (merge sorting by offset) through
- * the positions of all terms from the query, coalescing those hits that occur in a single passage
- * into a {@link Passage}, and then scores each Passage using a separate {@link PassageScorer}.
- * Passages are finally formatted into highlighted snippets with a {@link PassageFormatter}.
- * <p>
- * You can customize the behavior by subclassing this highlighter, some important hooks:
- * <ul>
- * <li>{@link #getBreakIterator(String)}: Customize how the text is divided into passages.
- * <li>{@link #getScorer(String)}: Customize how passages are ranked.
- * <li>{@link #getFormatter(String)}: Customize how snippets are formatted.
- * <li>{@link #getIndexAnalyzer(String)}: Enable highlighting of MultiTermQuerys such as {@code WildcardQuery}.
- * </ul>
- * <p>
- * <b>WARNING</b>: The code is very new and probably still has some exciting bugs!
- * <p>
- * Example usage:
- * <pre class="prettyprint">
- * // configure field with offsets at index time
- * FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
- * offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- * Field body = new Field("body", "foobar", offsetsType);
- *
- * // retrieve highlights at query time
- * PostingsHighlighter highlighter = new PostingsHighlighter();
- * Query query = new TermQuery(new Term("body", "highlighting"));
- * TopDocs topDocs = searcher.search(query, n);
- * String highlights[] = highlighter.highlight("body", query, searcher, topDocs);
- * </pre>
- * <p>
- * This is thread-safe, and can be used across different readers.
- * @lucene.experimental
- */
-public class PostingsHighlighter {
-
- // TODO: maybe allow re-analysis for tiny fields? currently we require offsets,
- // but if the analyzer is really fast and the field is tiny, this might really be
- // unnecessary.
-
- /** for rewriting: we don't want slow processing from MTQs */
- private static final IndexSearcher EMPTY_INDEXSEARCHER;
- static {
- try {
- IndexReader emptyReader = new MultiReader();
- EMPTY_INDEXSEARCHER = new IndexSearcher(emptyReader);
- EMPTY_INDEXSEARCHER.setQueryCache(null);
- } catch (IOException bogus) {
- throw new RuntimeException(bogus);
- }
- }
-
- /** Default maximum content size to process. Typically snippets
- * closer to the beginning of the document better summarize its content */
- public static final int DEFAULT_MAX_LENGTH = 10000;
-
- private final int maxLength;
-
- /** Set the first time {@link #getFormatter} is called,
- * and then reused. */
- private PassageFormatter defaultFormatter;
-
- /** Set the first time {@link #getScorer} is called,
- * and then reused. */
- private PassageScorer defaultScorer;
-
- /**
- * Creates a new highlighter with {@link #DEFAULT_MAX_LENGTH}.
- */
- public PostingsHighlighter() {
- this(DEFAULT_MAX_LENGTH);
- }
-
- /**
- * Creates a new highlighter, specifying maximum content length.
- * @param maxLength maximum content size to process.
- * @throws IllegalArgumentException if <code>maxLength</code> is negative or <code>Integer.MAX_VALUE</code>
- */
- public PostingsHighlighter(int maxLength) {
- if (maxLength < 0 || maxLength == Integer.MAX_VALUE) {
- // two reasons: no overflow problems in BreakIterator.preceding(offset+1),
- // our sentinel in the offsets queue uses this value to terminate.
- throw new IllegalArgumentException("maxLength must be < Integer.MAX_VALUE");
- }
- this.maxLength = maxLength;
- }
-
- /** Returns the {@link BreakIterator} to use for
- * dividing text into passages. This returns
- * {@link BreakIterator#getSentenceInstance(Locale)} by default;
- * subclasses can override to customize. */
- protected BreakIterator getBreakIterator(String field) {
- return BreakIterator.getSentenceInstance(Locale.ROOT);
- }
-
- /** Returns the {@link PassageFormatter} to use for
- * formatting passages into highlighted snippets. This
- * returns a new {@code PassageFormatter} by default;
- * subclasses can override to customize. */
- protected PassageFormatter getFormatter(String field) {
- if (defaultFormatter == null) {
- defaultFormatter = new DefaultPassageFormatter();
- }
- return defaultFormatter;
- }
-
- /** Returns the {@link PassageScorer} to use for
- * ranking passages. This
- * returns a new {@code PassageScorer} by default;
- * subclasses can override to customize. */
- protected PassageScorer getScorer(String field) {
- if (defaultScorer == null) {
- defaultScorer = new PassageScorer();
- }
- return defaultScorer;
- }
-
- /**
- * Highlights the top passages from a single field.
- *
- * @param field field name to highlight.
- * Must have a stored string value and also be indexed with offsets.
- * @param query query to highlight.
- * @param searcher searcher that was previously used to execute the query.
- * @param topDocs TopDocs containing the summary result documents to highlight.
- * @return Array of formatted snippets corresponding to the documents in <code>topDocs</code>.
- * If no highlights were found for a document, the
- * first sentence for the field will be returned.
- * @throws IOException if an I/O error occurred during processing
- * @throws IllegalArgumentException if <code>field</code> was indexed without
- * {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}
- */
- public String[] highlight(String field, Query query, IndexSearcher searcher, TopDocs topDocs) throws IOException {
- return highlight(field, query, searcher, topDocs, 1);
- }
-
- /**
- * Highlights the top-N passages from a single field.
- *
- * @param field field name to highlight.
- * Must have a stored string value and also be indexed with offsets.
- * @param query query to highlight.
- * @param searcher searcher that was previously used to execute the query.
- * @param topDocs TopDocs containing the summary result documents to highlight.
- * @param maxPassages The maximum number of top-N ranked passages used to
- * form the highlighted snippets.
- * @return Array of formatted snippets corresponding to the documents in <code>topDocs</code>.
- * If no highlights were found for a document, the
- * first {@code maxPassages} sentences from the
- * field will be returned.
- * @throws IOException if an I/O error occurred during processing
- * @throws IllegalArgumentException if <code>field</code> was indexed without
- * {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}
- */
- public String[] highlight(String field, Query query, IndexSearcher searcher, TopDocs topDocs, int maxPassages) throws IOException {
- Map<String,String[]> res = highlightFields(new String[] { field }, query, searcher, topDocs, new int[] { maxPassages });
- return res.get(field);
- }
-
- /**
- * Highlights the top passages from multiple fields.
- * <p>
- * Conceptually, this behaves as a more efficient form of:
- * <pre class="prettyprint">
- * Map m = new HashMap();
- * for (String field : fields) {
- * m.put(field, highlight(field, query, searcher, topDocs));
- * }
- * return m;
- * </pre>
- *
- * @param fields field names to highlight.
- * Must have a stored string value and also be indexed with offsets.
- * @param query query to highlight.
- * @param searcher searcher that was previously used to execute the query.
- * @param topDocs TopDocs containing the summary result documents to highlight.
- * @return Map keyed on field name, containing the array of formatted snippets
- * corresponding to the documents in <code>topDocs</code>.
- * If no highlights were found for a document, the
- * first sentence from the field will be returned.
- * @throws IOException if an I/O error occurred during processing
- * @throws IllegalArgumentException if <code>field</code> was indexed without
- * {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}
- */
- public Map<String,String[]> highlightFields(String fields[], Query query, IndexSearcher searcher, TopDocs topDocs) throws IOException {
- int maxPassages[] = new int[fields.length];
- Arrays.fill(maxPassages, 1);
- return highlightFields(fields, query, searcher, topDocs, maxPassages);
- }
-
- /**
- * Highlights the top-N passages from multiple fields.
- * <p>
- * Conceptually, this behaves as a more efficient form of:
- * <pre class="prettyprint">
- * Map m = new HashMap();
- * for (String field : fields) {
- * m.put(field, highlight(field, query, searcher, topDocs, maxPassages));
- * }
- * return m;
- * </pre>
- *
- * @param fields field names to highlight.
- * Must have a stored string value and also be indexed with offsets.
- * @param query query to highlight.
- * @param searcher searcher that was previously used to execute the query.
- * @param topDocs TopDocs containing the summary result documents to highlight.
- * @param maxPassages The maximum number of top-N ranked passages per-field used to
- * form the highlighted snippets.
- * @return Map keyed on field name, containing the array of formatted snippets
- * corresponding to the documents in <code>topDocs</code>.
- * If no highlights were found for a document, the
- * first {@code maxPassages} sentences from the
- * field will be returned.
- * @throws IOException if an I/O error occurred during processing
- * @throws IllegalArgumentException if <code>field</code> was indexed without
- * {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}
- */
- public Map<String,String[]> highlightFields(String fields[], Query query, IndexSearcher searcher, TopDocs topDocs, int maxPassages[]) throws IOException {
- final ScoreDoc scoreDocs[] = topDocs.scoreDocs;
- int docids[] = new int[scoreDocs.length];
- for (int i = 0; i < docids.length; i++) {
- docids[i] = scoreDocs[i].doc;
- }
-
- return highlightFields(fields, query, searcher, docids, maxPassages);
- }
-
- /**
- * Highlights the top-N passages from multiple fields,
- * for the provided int[] docids.
- *
- * @param fieldsIn field names to highlight.
- * Must have a stored string value and also be indexed with offsets.
- * @param query query to highlight.
- * @param searcher searcher that was previously used to execute the query.
- * @param docidsIn containing the document IDs to highlight.
- * @param maxPassagesIn The maximum number of top-N ranked passages per-field used to
- * form the highlighted snippets.
- * @return Map keyed on field name, containing the array of formatted snippets
- * corresponding to the documents in <code>docidsIn</code>.
- * If no highlights were found for a document, the
- * first {@code maxPassages} from the field will
- * be returned.
- * @throws IOException if an I/O error occurred during processing
- * @throws IllegalArgumentException if <code>field</code> was indexed without
- * {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}
- */
- public Map<String,String[]> highlightFields(String fieldsIn[], Query query, IndexSearcher searcher, int[] docidsIn, int maxPassagesIn[]) throws IOException {
- Map<String,String[]> snippets = new HashMap<>();
- for(Map.Entry<String,Object[]> ent : highlightFieldsAsObjects(fieldsIn, query, searcher, docidsIn, maxPassagesIn).entrySet()) {
- Object[] snippetObjects = ent.getValue();
- String[] snippetStrings = new String[snippetObjects.length];
- snippets.put(ent.getKey(), snippetStrings);
- for(int i=0;i<snippetObjects.length;i++) {
- Object snippet = snippetObjects[i];
- if (snippet != null) {
- snippetStrings[i] = snippet.toString();
- }
- }
- }
-
- return snippets;
- }
-
- /**
- * Expert: highlights the top-N passages from multiple fields,
- * for the provided int[] docids, to custom Object as
- * returned by the {@link PassageFormatter}. Use
- * this API to render to something other than String.
- *
- * @param fieldsIn field names to highlight.
- * Must have a stored string value and also be indexed with offsets.
- * @param query query to highlight.
- * @param searcher searcher that was previously used to execute the query.
- * @param docidsIn containing the document IDs to highlight.
- * @param maxPassagesIn The maximum number of top-N ranked passages per-field used to
- * form the highlighted snippets.
- * @return Map keyed on field name, containing the array of formatted snippets
- * corresponding to the documents in <code>docidsIn</code>.
- * If no highlights were found for a document, the
- * first {@code maxPassages} from the field will
- * be returned.
- * @throws IOException if an I/O error occurred during processing
- * @throws IllegalArgumentException if <code>field</code> was indexed without
- * {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}
- */
- protected Map<String,Object[]> highlightFieldsAsObjects(String fieldsIn[], Query query, IndexSearcher searcher, int[] docidsIn, int maxPassagesIn[]) throws IOException {
- if (fieldsIn.length < 1) {
- throw new IllegalArgumentException("fieldsIn must not be empty");
- }
- if (fieldsIn.length != maxPassagesIn.length) {
- throw new IllegalArgumentException("invalid number of maxPassagesIn");
- }
- SortedSet<Term> queryTerms = new TreeSet<>();
- EMPTY_INDEXSEARCHER.createNormalizedWeight(query, false).extractTerms(queryTerms);
-
- IndexReaderContext readerContext = searcher.getIndexReader().getContext();
- List<LeafReaderContext> leaves = readerContext.leaves();
-
- // Make our own copies because we sort in-place:
- int[] docids = new int[docidsIn.length];
- System.arraycopy(docidsIn, 0, docids, 0, docidsIn.length);
- final String fields[] = new String[fieldsIn.length];
- System.arraycopy(fieldsIn, 0, fields, 0, fieldsIn.length);
- final int maxPassages[] = new int[maxPassagesIn.length];
- System.arraycopy(maxPassagesIn, 0, maxPassages, 0, maxPassagesIn.length);
-
- // sort for sequential io
- Arrays.sort(docids);
- new InPlaceMergeSorter() {
-
- @Override
- protected void swap(int i, int j) {
- String tmp = fields[i];
- fields[i] = fields[j];
- fields[j] = tmp;
- int tmp2 = maxPassages[i];
- maxPassages[i] = maxPassages[j];
- maxPassages[j] = tmp2;
- }
-
- @Override
- protected int compare(int i, int j) {
- return fields[i].compareTo(fields[j]);
- }
-
- }.sort(0, fields.length);
-
- // pull stored data:
- String[][] contents = loadFieldValues(searcher, fields, docids, maxLength);
-
- Map<String,Object[]> highlights = new HashMap<>();
- for (int i = 0; i < fields.length; i++) {
- String field = fields[i];
- int numPassages = maxPassages[i];
- Term floor = new Term(field, "");
- Term ceiling = new Term(field, UnicodeUtil.BIG_TERM);
- SortedSet<Term> fieldTerms = queryTerms.subSet(floor, ceiling);
- // TODO: should we have some reasonable defaults for term pruning? (e.g. stopwords)
-
- // Strip off the redundant field:
- BytesRef terms[] = new BytesRef[fieldTerms.size()];
- int termUpto = 0;
- for(Term term : fieldTerms) {
- terms[termUpto++] = term.bytes();
- }
- Map<Integer,Object> fieldHighlights = highlightField(field, contents[i], getBreakIterator(field), terms, docids, leaves, numPassages, query);
-
- Object[] result = new Object[docids.length];
- for (int j = 0; j < docidsIn.length; j++) {
- result[j] = fieldHighlights.get(docidsIn[j]);
- }
- highlights.put(field, result);
- }
- return highlights;
- }
-
- /** Loads the String values for each field X docID to be
- * highlighted. By default this loads from stored
- * fields, but a subclass can change the source. This
- * method should allocate the String[fields.length][docids.length]
- * and fill all values. The returned Strings must be
- * identical to what was indexed. */
- protected String[][] loadFieldValues(IndexSearcher searcher, String[] fields, int[] docids, int maxLength) throws IOException {
- String contents[][] = new String[fields.length][docids.length];
- char valueSeparators[] = new char[fields.length];
- for (int i = 0; i < fields.length; i++) {
- valueSeparators[i] = getMultiValuedSeparator(fields[i]);
- }
- LimitedStoredFieldVisitor visitor = new LimitedStoredFieldVisitor(fields, valueSeparators, maxLength);
- for (int i = 0; i < docids.length; i++) {
- searcher.doc(docids[i], visitor);
- for (int j = 0; j < fields.length; j++) {
- contents[j][i] = visitor.getValue(j).toString();
- }
- visitor.reset();
- }
- return contents;
- }
-
- /**
- * Returns the logical separator between values for multi-valued fields.
- * The default value is a space character, which means passages can span across values,
- * but a subclass can override, for example with {@code U+2029 PARAGRAPH SEPARATOR (PS)}
- * if each value holds a discrete passage for highlighting.
- */
- protected char getMultiValuedSeparator(String field) {
- return ' ';
- }
-
- /**
- * Returns the analyzer originally used to index the content for {@code field}.
- * <p>
- * This is used to highlight some MultiTermQueries.
- * @return Analyzer or null (the default, meaning no special multi-term processing)
- */
- protected Analyzer getIndexAnalyzer(String field) {
- return null;
- }
-
- private Map<Integer,Object> highlightField(String field, String contents[], BreakIterator bi, BytesRef terms[], int[] docids, List<LeafReaderContext> leaves, int maxPassages, Query query) throws IOException {
- Map<Integer,Object> highlights = new HashMap<>();
-
- PassageFormatter fieldFormatter = getFormatter(field);
- if (fieldFormatter == null) {
- throw new NullPointerException("PassageFormatter must not be null");
- }
-
- // check if we should do any multiterm processing
- Analyzer analyzer = getIndexAnalyzer(field);
- CharacterRunAutomaton automata[] = new CharacterRunAutomaton[0];
- if (analyzer != null) {
- automata = MultiTermHighlighting.extractAutomata(query, field);
- }
-
- // resize 'terms', where the last term is the multiterm matcher
- if (automata.length > 0) {
- BytesRef newTerms[] = new BytesRef[terms.length + 1];
- System.arraycopy(terms, 0, newTerms, 0, terms.length);
- terms = newTerms;
- }
-
- // we are processing in increasing docid order, so we only need to reinitialize stuff on segment changes
- // otherwise, we will just advance() existing enums to the new document in the same segment.
- PostingsEnum postings[] = null;
- TermsEnum termsEnum = null;
- int lastLeaf = -1;
-
- for (int i = 0; i < docids.length; i++) {
- String content = contents[i];
- if (content.length() == 0) {
- continue; // nothing to do
- }
- bi.setText(content);
- int doc = docids[i];
- int leaf = ReaderUtil.subIndex(doc, leaves);
- LeafReaderContext subContext = leaves.get(leaf);
- LeafReader r = subContext.reader();
-
- assert leaf >= lastLeaf; // increasing order
-
- // if the segment has changed, we must initialize new enums.
- if (leaf != lastLeaf) {
- Terms t = r.terms(field);
- if (t != null) {
- if (!t.hasOffsets()) {
- // no offsets available
- throw new IllegalArgumentException("field '" + field + "' was indexed without offsets, cannot highlight");
- }
- termsEnum = t.iterator();
- postings = new PostingsEnum[terms.length];
- } else {
- termsEnum = null;
- }
- }
- if (termsEnum == null) {
- continue; // no terms for this field, nothing to do
- }
-
- // if there are multi-term matches, we have to initialize the "fake" enum for each document
- if (automata.length > 0) {
- PostingsEnum dp = MultiTermHighlighting.getDocsEnum(analyzer.tokenStream(field, content), automata);
- dp.advance(doc - subContext.docBase);
- postings[terms.length-1] = dp; // last term is the multiterm matcher
- }
-
- Passage passages[] = highlightDoc(field, terms, content.length(), bi, doc - subContext.docBase, termsEnum, postings, maxPassages);
-
- if (passages.length == 0) {
- // no passages were returned, so ask for a default summary
- passages = getEmptyHighlight(field, bi, maxPassages);
- }
-
- if (passages.length > 0) {
- highlights.put(doc, fieldFormatter.format(passages, content));
- }
-
- lastLeaf = leaf;
- }
-
- return highlights;
- }
-
- // algorithm: treat sentence snippets as miniature documents
- // we can intersect these with the postings lists via BreakIterator.preceding(offset),s
- // score each sentence as norm(sentenceStartOffset) * sum(weight * tf(freq))
- private Passage[] highlightDoc(String field, BytesRef terms[], int contentLength, BreakIterator bi, int doc,
- TermsEnum termsEnum, PostingsEnum[] postings, int n) throws IOException {
- PassageScorer scorer = getScorer(field);
- if (scorer == null) {
- throw new NullPointerException("PassageScorer must not be null");
- }
- PriorityQueue<OffsetsEnum> pq = new PriorityQueue<>();
- float weights[] = new float[terms.length];
- // initialize postings
- for (int i = 0; i < terms.length; i++) {
- PostingsEnum de = postings[i];
- int pDoc;
- if (de == EMPTY) {
- continue;
- } else if (de == null) {
- postings[i] = EMPTY; // initially
- if (!termsEnum.seekExact(terms[i])) {
- continue; // term not found
- }
- de = postings[i] = termsEnum.postings(null, PostingsEnum.OFFSETS);
- assert de != null;
- pDoc = de.advance(doc);
- } else {
- pDoc = de.docID();
- if (pDoc < doc) {
- pDoc = de.advance(doc);
- }
- }
-
- if (doc == pDoc) {
- weights[i] = scorer.weight(contentLength, de.freq());
- de.nextPosition();
- pq.add(new OffsetsEnum(de, i));
- }
- }
-
- pq.add(new OffsetsEnum(EMPTY, Integer.MAX_VALUE)); // a sentinel for termination
-
- PriorityQueue<Passage> passageQueue = new PriorityQueue<>(n, new Comparator<Passage>() {
- @Override
- public int compare(Passage left, Passage right) {
- if (left.score < right.score) {
- return -1;
- } else if (left.score > right.score) {
- return 1;
- } else {
- return left.startOffset - right.startOffset;
- }
- }
- });
- Passage current = new Passage();
-
- OffsetsEnum off;
- while ((off = pq.poll()) != null) {
- final PostingsEnum dp = off.dp;
- int start = dp.startOffset();
- assert start >= 0;
- int end = dp.endOffset();
- // LUCENE-5166: this hit would span the content limit... however more valid
- // hits may exist (they are sorted by start). so we pretend like we never
- // saw this term, it won't cause a passage to be added to passageQueue or anything.
- assert EMPTY.startOffset() == Integer.MAX_VALUE;
- if (start < contentLength && end > contentLength) {
- continue;
- }
- if (start >= current.endOffset) {
- if (current.startOffset >= 0) {
- // finalize current
- current.score *= scorer.norm(current.startOffset);
- // new sentence: first add 'current' to queue
- if (passageQueue.size() == n && current.score < passageQueue.peek().score) {
- current.reset(); // can't compete, just reset it
- } else {
- passageQueue.offer(current);
- if (passageQueue.size() > n) {
- current = passageQueue.poll();
- current.reset();
- } else {
- current = new Passage();
- }
- }
- }
- // if we exceed limit, we are done
- if (start >= contentLength) {
- Passage passages[] = new Passage[passageQueue.size()];
- passageQueue.toArray(passages);
- for (Passage p : passages) {
- p.sort();
- }
- // sort in ascending order
- Arrays.sort(passages, new Comparator<Passage>() {
- @Override
- public int compare(Passage left, Passage right) {
- return left.startOffset - right.startOffset;
- }
- });
- return passages;
- }
- // advance breakiterator
- assert BreakIterator.DONE < 0;
- current.startOffset = Math.max(bi.preceding(start+1), 0);
- current.endOffset = Math.min(bi.next(), contentLength);
- }
- int tf = 0;
- while (true) {
- tf++;
- BytesRef term = terms[off.id];
- if (term == null) {
- // multitermquery match, pull from payload
- term = off.dp.getPayload();
- assert term != null;
- }
- current.addMatch(start, end, term);
- if (off.pos == dp.freq()) {
- break; // removed from pq
- } else {
- off.pos++;
- dp.nextPosition();
- start = dp.startOffset();
- end = dp.endOffset();
- }
- if (start >= current.endOffset || end > contentLength) {
- pq.offer(off);
- break;
- }
- }
- current.score += weights[off.id] * scorer.tf(tf, current.endOffset - current.startOffset);
- }
-
- // Dead code but compiler disagrees:
- assert false;
- return null;
- }
-
- /** Called to summarize a document when no hits were
- * found. By default this just returns the first
- * {@code maxPassages} sentences; subclasses can override
- * to customize. */
- protected Passage[] getEmptyHighlight(String fieldName, BreakIterator bi, int maxPassages) {
- // BreakIterator should be un-next'd:
- List<Passage> passages = new ArrayList<>();
- int pos = bi.current();
- assert pos == 0;
- while (passages.size() < maxPassages) {
- int next = bi.next();
- if (next == BreakIterator.DONE) {
- break;
- }
- Passage passage = new Passage();
- passage.score = Float.NaN;
- passage.startOffset = pos;
- passage.endOffset = next;
- passages.add(passage);
- pos = next;
- }
-
- return passages.toArray(new Passage[passages.size()]);
- }
-
- private static class OffsetsEnum implements Comparable<OffsetsEnum> {
- PostingsEnum dp;
- int pos;
- int id;
-
- OffsetsEnum(PostingsEnum dp, int id) throws IOException {
- this.dp = dp;
- this.id = id;
- this.pos = 1;
- }
-
- @Override
- public int compareTo(OffsetsEnum other) {
- try {
- int off = dp.startOffset();
- int otherOff = other.dp.startOffset();
- if (off == otherOff) {
- return id - other.id;
- } else {
- return Integer.compare(off, otherOff);
- }
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- private static final PostingsEnum EMPTY = new PostingsEnum() {
-
- @Override
- public int nextPosition() throws IOException { return -1; }
-
- @Override
- public int startOffset() throws IOException { return Integer.MAX_VALUE; }
-
- @Override
- public int endOffset() throws IOException { return Integer.MAX_VALUE; }
-
- @Override
- public BytesRef getPayload() throws IOException { return null; }
-
- @Override
- public int freq() throws IOException { return 0; }
-
- @Override
- public int docID() { return NO_MORE_DOCS; }
-
- @Override
- public int nextDoc() throws IOException { return NO_MORE_DOCS; }
-
- @Override
- public int advance(int target) throws IOException { return NO_MORE_DOCS; }
-
- @Override
- public long cost() { return 0; }
- };
-
- private static class LimitedStoredFieldVisitor extends StoredFieldVisitor {
- private final String fields[];
- private final char valueSeparators[];
- private final int maxLength;
- private final StringBuilder builders[];
- private int currentField = -1;
-
- public LimitedStoredFieldVisitor(String fields[], char valueSeparators[], int maxLength) {
- assert fields.length == valueSeparators.length;
- this.fields = fields;
- this.valueSeparators = valueSeparators;
- this.maxLength = maxLength;
- builders = new StringBuilder[fields.length];
- for (int i = 0; i < builders.length; i++) {
- builders[i] = new StringBuilder();
- }
- }
-
- @Override
- public void stringField(FieldInfo fieldInfo, byte[] bytes) throws IOException {
- String value = new String(bytes, StandardCharsets.UTF_8);
- assert currentField >= 0;
- StringBuilder builder = builders[currentField];
- if (builder.length() > 0 && builder.length() < maxLength) {
- builder.append(valueSeparators[currentField]);
- }
- if (builder.length() + value.length() > maxLength) {
- builder.append(value, 0, maxLength - builder.length());
- } else {
- builder.append(value);
- }
- }
-
- @Override
- public Status needsField(FieldInfo fieldInfo) throws IOException {
- currentField = Arrays.binarySearch(fields, fieldInfo.name);
- if (currentField < 0) {
- return Status.NO;
- } else if (builders[currentField].length() > maxLength) {
- return fields.length == 1 ? Status.STOP : Status.NO;
- }
- return Status.YES;
- }
-
- String getValue(int i) {
- return builders[i].toString();
- }
-
- void reset() {
- currentField = -1;
- for (int i = 0; i < fields.length; i++) {
- builders[i].setLength(0);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/package-info.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/package-info.java b/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/package-info.java
deleted file mode 100644
index 10013c2..0000000
--- a/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Highlighter implementation that uses offsets from postings lists.
- */
-package org.apache.lucene.search.postingshighlight;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/CustomSeparatorBreakIterator.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/CustomSeparatorBreakIterator.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/CustomSeparatorBreakIterator.java
new file mode 100644
index 0000000..7119aed
--- /dev/null
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/CustomSeparatorBreakIterator.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search.uhighlight;
+
+import java.text.BreakIterator;
+import java.text.CharacterIterator;
+
+/**
+ * A {@link BreakIterator} that breaks the text whenever a certain separator, provided as a constructor argument, is found.
+ */
+public final class CustomSeparatorBreakIterator extends BreakIterator {
+
+ private final char separator;
+ private CharacterIterator text;
+ private int current;
+
+ public CustomSeparatorBreakIterator(char separator) {
+ this.separator = separator;
+ }
+
+ @Override
+ public int current() {
+ return current;
+ }
+
+ @Override
+ public int first() {
+ text.setIndex(text.getBeginIndex());
+ return current = text.getIndex();
+ }
+
+ @Override
+ public int last() {
+ text.setIndex(text.getEndIndex());
+ return current = text.getIndex();
+ }
+
+ @Override
+ public int next() {
+ if (text.getIndex() == text.getEndIndex()) {
+ return DONE;
+ } else {
+ return advanceForward();
+ }
+ }
+
+ private int advanceForward() {
+ char c;
+ while ((c = text.next()) != CharacterIterator.DONE) {
+ if (c == separator) {
+ return current = text.getIndex() + 1;
+ }
+ }
+ assert text.getIndex() == text.getEndIndex();
+ return current = text.getIndex();
+ }
+
+ @Override
+ public int following(int pos) {
+ if (pos < text.getBeginIndex() || pos > text.getEndIndex()) {
+ throw new IllegalArgumentException("offset out of bounds");
+ } else if (pos == text.getEndIndex()) {
+ // this conflicts with the javadocs, but matches actual behavior (Oracle has a bug in something)
+ // https://bugs.openjdk.java.net/browse/JDK-8015110
+ text.setIndex(text.getEndIndex());
+ current = text.getIndex();
+ return DONE;
+ } else {
+ text.setIndex(pos);
+ current = text.getIndex();
+ return advanceForward();
+ }
+ }
+
+ @Override
+ public int previous() {
+ if (text.getIndex() == text.getBeginIndex()) {
+ return DONE;
+ } else {
+ return advanceBackward();
+ }
+ }
+
+ private int advanceBackward() {
+ char c;
+ while ((c = text.previous()) != CharacterIterator.DONE) {
+ if (c == separator) {
+ return current = text.getIndex() + 1;
+ }
+ }
+ assert text.getIndex() == text.getBeginIndex();
+ return current = text.getIndex();
+ }
+
+ @Override
+ public int preceding(int pos) {
+ if (pos < text.getBeginIndex() || pos > text.getEndIndex()) {
+ throw new IllegalArgumentException("offset out of bounds");
+ } else if (pos == text.getBeginIndex()) {
+ // this conflicts with the javadocs, but matches actual behavior (Oracle has a bug in something)
+ // https://bugs.openjdk.java.net/browse/JDK-8015110
+ text.setIndex(text.getBeginIndex());
+ current = text.getIndex();
+ return DONE;
+ } else {
+ text.setIndex(pos);
+ current = text.getIndex();
+ return advanceBackward();
+ }
+ }
+
+ @Override
+ public int next(int n) {
+ if (n < 0) {
+ for (int i = 0; i < -n; i++) {
+ previous();
+ }
+ } else {
+ for (int i = 0; i < n; i++) {
+ next();
+ }
+ }
+ return current();
+ }
+
+ @Override
+ public CharacterIterator getText() {
+ return text;
+ }
+
+ @Override
+ public void setText(CharacterIterator newText) {
+ text = newText;
+ current = text.getBeginIndex();
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0d3c73ea/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/WholeBreakIterator.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/WholeBreakIterator.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/WholeBreakIterator.java
new file mode 100644
index 0000000..37f48aa
--- /dev/null
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/WholeBreakIterator.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search.uhighlight;
+
+import java.text.BreakIterator;
+import java.text.CharacterIterator;
+
+/** Just produces one single fragment for the entire text */
+public final class WholeBreakIterator extends BreakIterator {
+ private CharacterIterator text;
+ private int start;
+ private int end;
+ private int current;
+
+ @Override
+ public int current() {
+ return current;
+ }
+
+ @Override
+ public int first() {
+ return (current = start);
+ }
+
+ @Override
+ public int following(int pos) {
+ if (pos < start || pos > end) {
+ throw new IllegalArgumentException("offset out of bounds");
+ } else if (pos == end) {
+ // this conflicts with the javadocs, but matches actual behavior (Oracle has a bug in something)
+ // https://bugs.openjdk.java.net/browse/JDK-8015110
+ current = end;
+ return DONE;
+ } else {
+ return last();
+ }
+ }
+
+ @Override
+ public CharacterIterator getText() {
+ return text;
+ }
+
+ @Override
+ public int last() {
+ return (current = end);
+ }
+
+ @Override
+ public int next() {
+ if (current == end) {
+ return DONE;
+ } else {
+ return last();
+ }
+ }
+
+ @Override
+ public int next(int n) {
+ if (n < 0) {
+ for (int i = 0; i < -n; i++) {
+ previous();
+ }
+ } else {
+ for (int i = 0; i < n; i++) {
+ next();
+ }
+ }
+ return current();
+ }
+
+ @Override
+ public int preceding(int pos) {
+ if (pos < start || pos > end) {
+ throw new IllegalArgumentException("offset out of bounds");
+ } else if (pos == start) {
+ // this conflicts with the javadocs, but matches actual behavior (Oracle has a bug in something)
+ // https://bugs.openjdk.java.net/browse/JDK-8015110
+ current = start;
+ return DONE;
+ } else {
+ return first();
+ }
+ }
+
+ @Override
+ public int previous() {
+ if (current == start) {
+ return DONE;
+ } else {
+ return first();
+ }
+ }
+
+ @Override
+ public void setText(CharacterIterator newText) {
+ start = newText.getBeginIndex();
+ end = newText.getEndIndex();
+ text = newText;
+ current = start;
+ }
+}