You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by pr...@apache.org on 2016/05/11 06:34:14 UTC
incubator-zeppelin git commit: [ZEPPELIN-599]notebook search should
search paragraph title
Repository: incubator-zeppelin
Updated Branches:
refs/heads/master 81b47c039 -> a87d45ec0
[ZEPPELIN-599]notebook search should search paragraph title
### What is this PR for?
Allow notebook search to search paragraph title too.
### What type of PR is it?
[Bug Fix]
### Todos
### What is the Jira issue?
[ZEPPELIN-599](https://issues.apache.org/jira/browse/ZEPPELIN-599?jql=project%20%3D%20ZEPPELIN%20AND%20status%20%3D%20Open%20AND%20text%20~%20%22Title%22)
### How should this be tested?
You should be able to search the the note by searching queryterm in the paragraph title.
### Screenshots (if appropriate)
Before:
After:
### Questions:
* Does the licenses files need update?NO
* Is there breaking changes for older versions?NO
* Does this needs documentation?NO
Author: Ravi Ranjan <ra...@gmail.com>
Closes #859 from ravicodder/searchTitle and squashes the following commits:
59b5a76 [Ravi Ranjan] Make code consistent
c8a9eaf [Ravi Ranjan] Merge branch 'master' of https://github.com/apache/incubator-zeppelin into searchTitle
cbded8d [Ravi Ranjan] Add test in LuceneSearchTest.java
b3e66bb [Ravi Ranjan] Add test
f6d64fd [Ravi Ranjan] Fix spacing
5830c8f [Ravi Ranjan] revert indent
ed69177 [Ravi Ranjan] Search Title of notebook
Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/a87d45ec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/a87d45ec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/a87d45ec
Branch: refs/heads/master
Commit: a87d45ec0460c64d709f8bc67e847bf17dc8f9d3
Parents: 81b47c0
Author: Ravi Ranjan <ra...@gmail.com>
Authored: Thu May 5 10:53:15 2016 +0530
Committer: Prabhjyot Singh <pr...@gmail.com>
Committed: Wed May 11 12:04:06 2016 +0530
----------------------------------------------------------------------
.../zeppelin/rest/ZeppelinRestApiTest.java | 26 ++++++++
.../src/app/search/result-list.controller.js | 36 ++++++++--
zeppelin-web/src/app/search/search.css | 5 ++
.../apache/zeppelin/search/LuceneSearch.java | 52 ++++++++++-----
.../zeppelin/search/LuceneSearchTest.java | 69 +++++++++++++++-----
5 files changed, 149 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/a87d45ec/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java
index 2f2a36b..36c95af 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java
@@ -756,5 +756,31 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
ZeppelinServer.notebook.removeNote(note2.getId());
}
+ @Test
+ public void testTitleSearch() throws IOException {
+ Note note = ZeppelinServer.notebook.createNote();
+ String jsonRequest = "{\"title\": \"testTitleSearchOfParagraph\", \"text\": \"ThisIsToTestSearchMethodWithTitle \"}";
+ PostMethod postNotebookText = httpPost("/notebook/" + note.getId() + "/paragraph", jsonRequest);
+ postNotebookText.releaseConnection();
+
+ GetMethod searchNotebook = httpGet("/notebook/search?q='testTitleSearchOfParagraph'");
+ searchNotebook.addRequestHeader("Origin", "http://localhost");
+ Map<String, Object> respSearchResult = gson.fromJson(searchNotebook.getResponseBodyAsString(),
+ new TypeToken<Map<String, Object>>() {
+ }.getType());
+ ArrayList searchBody = (ArrayList) respSearchResult.get("body");
+
+ int numberOfTitleHits = 0;
+ for (int i = 0; i < searchBody.size(); i++) {
+ Map<String, String> searchResult = (Map<String, String>) searchBody.get(i);
+ if (searchResult.get("header").contains("testTitleSearchOfParagraph")) {
+ numberOfTitleHits++;
+ }
+ }
+ assertEquals("Paragraph title hits must be at-least one", true, numberOfTitleHits >= 1);
+ searchNotebook.releaseConnection();
+ ZeppelinServer.notebook.removeNote(note.getId());
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/a87d45ec/zeppelin-web/src/app/search/result-list.controller.js
----------------------------------------------------------------------
diff --git a/zeppelin-web/src/app/search/result-list.controller.js b/zeppelin-web/src/app/search/result-list.controller.js
index 0d55442..949e01f 100644
--- a/zeppelin-web/src/app/search/result-list.controller.js
+++ b/zeppelin-web/src/app/search/result-list.controller.js
@@ -74,12 +74,20 @@ angular
};
}
- var lines = note.snippet
+ var result = '';
+ if (note.header !== '') {
+ result = note.header + '\n\n' + note.snippet;
+ } else {
+ result = note.snippet;
+ }
+
+ var lines = result
.split('\n')
.map(function(line, row) {
+
var match = line.match(/<B>(.+?)<\/B>/);
- // return early if nothing to highlight
+ // return early if nothing to highlight
if (!match) {
return line;
}
@@ -93,15 +101,31 @@ angular
indeces.forEach(function(start) {
var end = start + term.length;
- _editor
- .getSession()
- .addMarker(
+ if (note.header !== '' && row === 0) {
+ _editor
+ .getSession()
+ .addMarker(
+ new Range(row, 0, row, line.length),
+ 'search-results-highlight-header',
+ 'background'
+ );
+ _editor
+ .getSession()
+ .addMarker(
new Range(row, start, row, end),
'search-results-highlight',
'line'
);
+ } else {
+ _editor
+ .getSession()
+ .addMarker(
+ new Range(row, start, row, end),
+ 'search-results-highlight',
+ 'line'
+ );
+ }
});
-
return __line;
});
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/a87d45ec/zeppelin-web/src/app/search/search.css
----------------------------------------------------------------------
diff --git a/zeppelin-web/src/app/search/search.css b/zeppelin-web/src/app/search/search.css
index e89c765..b06b4a9 100644
--- a/zeppelin-web/src/app/search/search.css
+++ b/zeppelin-web/src/app/search/search.css
@@ -31,6 +31,11 @@
position: absolute;
}
+.search-results-highlight-header {
+ background-color: #e6f2ff;
+ position: absolute;
+}
+
/* remove error highlighting */
.search-results .ace_invalid {
background: none !important;
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/a87d45ec/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java
index 7f9cbbd..b43c453 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java
@@ -37,6 +37,7 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
+import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.IndexSearcher;
@@ -69,7 +70,8 @@ import com.google.common.collect.Lists;
public class LuceneSearch implements SearchService {
private static final Logger LOG = LoggerFactory.getLogger(LuceneSearch.class);
- private static final String SEARCH_FIELD = "contents";
+ private static final String SEARCH_FIELD_TEXT = "contents";
+ private static final String SEARCH_FIELD_TITLE = "header";
static final String PARAGRAPH = "paragraph";
static final String ID_FIELD = "id";
@@ -85,7 +87,7 @@ public class LuceneSearch implements SearchService {
try {
writer = new IndexWriter(ramDirectory, iwc);
} catch (IOException e) {
- LOG.error("Failed to reate new IndexWriter", e);
+ LOG.error("Failed to create new IndexWriter", e);
}
}
@@ -102,10 +104,12 @@ public class LuceneSearch implements SearchService {
try (IndexReader indexReader = DirectoryReader.open(ramDirectory)) {
IndexSearcher indexSearcher = new IndexSearcher(indexReader);
Analyzer analyzer = new StandardAnalyzer();
- QueryParser parser = new QueryParser(SEARCH_FIELD, analyzer);
+ MultiFieldQueryParser parser = new MultiFieldQueryParser(
+ new String[] {SEARCH_FIELD_TEXT, SEARCH_FIELD_TITLE},
+ analyzer);
Query query = parser.parse(queryStr);
- LOG.debug("Searching for: " + query.toString(SEARCH_FIELD));
+ LOG.debug("Searching for: " + query.toString(SEARCH_FIELD_TEXT));
SimpleHTMLFormatter htmlFormatter = new SimpleHTMLFormatter();
Highlighter highlighter = new Highlighter(htmlFormatter, new QueryScorer(query));
@@ -139,20 +143,33 @@ public class LuceneSearch implements SearchService {
LOG.debug(" Title: {}", doc.get("title"));
}
- String text = doc.get(SEARCH_FIELD);
- TokenStream tokenStream = TokenSources.getTokenStream(searcher.getIndexReader(), id,
- SEARCH_FIELD, analyzer);
- TextFragment[] frag = highlighter.getBestTextFragments(tokenStream, text, true, 3);
- LOG.debug(" {} fragments found for query '{}'", frag.length, query);
- for (int j = 0; j < frag.length; j++) {
- if ((frag[j] != null) && (frag[j].getScore() > 0)) {
- LOG.debug(" Fragment: {}", frag[j].toString());
+ String text = doc.get(SEARCH_FIELD_TEXT);
+ String header = doc.get(SEARCH_FIELD_TITLE);
+ String fragment = "";
+
+ if (text != null) {
+ TokenStream tokenStream = TokenSources.getTokenStream(searcher.getIndexReader(), id,
+ SEARCH_FIELD_TEXT, analyzer);
+ TextFragment[] frag = highlighter.getBestTextFragments(tokenStream, text, true, 3);
+ LOG.debug(" {} fragments found for query '{}'", frag.length, query);
+ for (int j = 0; j < frag.length; j++) {
+ if ((frag[j] != null) && (frag[j].getScore() > 0)) {
+ LOG.debug(" Fragment: {}", frag[j].toString());
+ }
}
+ fragment = (frag != null && frag.length > 0) ? frag[0].toString() : "";
}
- String fragment = (frag != null && frag.length > 0) ? frag[0].toString() : "";
+ if (header != null) {
+ TokenStream tokenTitle = TokenSources.getTokenStream(searcher.getIndexReader(), id,
+ SEARCH_FIELD_TITLE, analyzer);
+ TextFragment[] frgTitle = highlighter.getBestTextFragments(tokenTitle, header, true, 3);
+ header = (frgTitle != null && frgTitle.length > 0) ? frgTitle[0].toString() : "";
+ } else {
+ header = "";
+ }
matchingParagraphs.add(ImmutableMap.of("id", path, // <noteId>/paragraph/<paragraphId>
- "name", title, "snippet", fragment, "text", text));
+ "name", title, "snippet", fragment, "text", text, "header", header));
} else {
LOG.info("{}. No {} for this document", i + 1, ID_FIELD);
}
@@ -252,11 +269,14 @@ public class LuceneSearch implements SearchService {
doc.add(new StringField("title", noteName, Field.Store.YES));
if (null != p) {
- doc.add(new TextField(SEARCH_FIELD, p.getText(), Field.Store.YES));
+ doc.add(new TextField(SEARCH_FIELD_TEXT, p.getText(), Field.Store.YES));
+ if (p.getTitle() != null) {
+ doc.add(new TextField(SEARCH_FIELD_TITLE, p.getTitle(), Field.Store.YES));
+ }
Date date = p.getDateStarted() != null ? p.getDateStarted() : p.getDateCreated();
doc.add(new LongField("modified", date.getTime(), Field.Store.NO));
} else {
- doc.add(new TextField(SEARCH_FIELD, noteName, Field.Store.YES));
+ doc.add(new TextField(SEARCH_FIELD_TEXT, noteName, Field.Store.YES));
}
return doc;
}
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/a87d45ec/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java
index f74d95e..c744267 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java
@@ -65,8 +65,8 @@ public class LuceneSearchTest {
@Test public void canIndexNotebook() {
//give
- Note note1 = newNoteWithParapgraph("Notebook1", "test");
- Note note2 = newNoteWithParapgraph("Notebook2", "not test");
+ Note note1 = newNoteWithParagraph("Notebook1", "test");
+ Note note2 = newNoteWithParagraph("Notebook2", "not test");
List<Note> notebook = Arrays.asList(note1, note2);
//when
@@ -75,8 +75,8 @@ public class LuceneSearchTest {
@Test public void canIndexAndQuery() {
//given
- Note note1 = newNoteWithParapgraph("Notebook1", "test");
- Note note2 = newNoteWithParapgraphs("Notebook2", "not test", "not test at all");
+ Note note1 = newNoteWithParagraph("Notebook1", "test");
+ Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all");
notebookIndex.addIndexDocs(Arrays.asList(note1, note2));
//when
@@ -91,8 +91,8 @@ public class LuceneSearchTest {
@Test public void canIndexAndQueryByNotebookName() {
//given
- Note note1 = newNoteWithParapgraph("Notebook1", "test");
- Note note2 = newNoteWithParapgraphs("Notebook2", "not test", "not test at all");
+ Note note1 = newNoteWithParagraph("Notebook1", "test");
+ Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all");
notebookIndex.addIndexDocs(Arrays.asList(note1, note2));
//when
@@ -104,9 +104,31 @@ public class LuceneSearchTest {
assertThat(results.get(0)).containsEntry("id", note1.getId());
}
+ @Test
+ public void canIndexAndQueryByParagraphTitle() {
+ //given
+ Note note1 = newNoteWithParagraph("Notebook1", "test", "testingTitleSearch");
+ Note note2 = newNoteWithParagraph("Notebook2", "not test", "notTestingTitleSearch");
+ notebookIndex.addIndexDocs(Arrays.asList(note1, note2));
+
+ //when
+ List<Map<String, String>> results = notebookIndex.query("testingTitleSearch");
+
+ //then
+ assertThat(results).isNotEmpty();
+ assertThat(results.size()).isAtLeast(1);
+ int TitleHits = 0;
+ for (Map<String, String> res : results) {
+ if (res.get("header").contains("testingTitleSearch")) {
+ TitleHits++;
+ }
+ }
+ assertThat(TitleHits).isAtLeast(1);
+ }
+
@Test public void indexKeyContract() throws IOException {
//give
- Note note1 = newNoteWithParapgraph("Notebook1", "test");
+ Note note1 = newNoteWithParagraph("Notebook1", "test");
//when
notebookIndex.addIndexDoc(note1);
//then
@@ -129,8 +151,8 @@ public class LuceneSearchTest {
@Test public void canIndexAndReIndex() throws IOException {
//given
- Note note1 = newNoteWithParapgraph("Notebook1", "test");
- Note note2 = newNoteWithParapgraphs("Notebook2", "not test", "not test at all");
+ Note note1 = newNoteWithParagraph("Notebook1", "test");
+ Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all");
notebookIndex.addIndexDocs(Arrays.asList(note1, note2));
//when
@@ -155,8 +177,8 @@ public class LuceneSearchTest {
@Test public void canDeleteFromIndex() throws IOException {
//given
- Note note1 = newNoteWithParapgraph("Notebook1", "test");
- Note note2 = newNoteWithParapgraphs("Notebook2", "not test", "not test at all");
+ Note note1 = newNoteWithParagraph("Notebook1", "test");
+ Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all");
notebookIndex.addIndexDocs(Arrays.asList(note1, note2));
assertThat(resultForQuery("Notebook2")).isNotEmpty();
@@ -174,8 +196,8 @@ public class LuceneSearchTest {
@Test public void indexParagraphUpdatedOnNoteSave() throws IOException {
//given: total 2 notebooks, 3 paragraphs
- Note note1 = newNoteWithParapgraph("Notebook1", "test");
- Note note2 = newNoteWithParapgraphs("Notebook2", "not test", "not test at all");
+ Note note1 = newNoteWithParagraph("Notebook1", "test");
+ Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all");
notebookIndex.addIndexDocs(Arrays.asList(note1, note2));
assertThat(resultForQuery("test").size()).isEqualTo(3);
@@ -199,8 +221,8 @@ public class LuceneSearchTest {
@Test public void indexNoteNameUpdatedOnNoteSave() throws IOException {
//given: total 2 notebooks, 3 paragraphs
- Note note1 = newNoteWithParapgraph("Notebook1", "test");
- Note note2 = newNoteWithParapgraphs("Notebook2", "not test", "not test at all");
+ Note note1 = newNoteWithParagraph("Notebook1", "test");
+ Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all");
notebookIndex.addIndexDocs(Arrays.asList(note1, note2));
assertThat(resultForQuery("test").size()).isEqualTo(3);
@@ -226,17 +248,23 @@ public class LuceneSearchTest {
* @param parText text of the paragraph
* @return Note
*/
- private Note newNoteWithParapgraph(String noteName, String parText) {
+ private Note newNoteWithParagraph(String noteName, String parText) {
Note note1 = newNote(noteName);
addParagraphWithText(note1, parText);
return note1;
}
+ private Note newNoteWithParagraph(String noteName, String parText,String title) {
+ Note note = newNote(noteName);
+ addParagraphWithTextAndTitle(note, parText, title);
+ return note;
+ }
+
/**
* Creates a new Note \w given name,
* adds N paragraphs \w given texts
*/
- private Note newNoteWithParapgraphs(String noteName, String... parTexts) {
+ private Note newNoteWithParagraphs(String noteName, String... parTexts) {
Note note1 = newNote(noteName);
for (String parText : parTexts) {
addParagraphWithText(note1, parText);
@@ -250,6 +278,13 @@ public class LuceneSearchTest {
return p;
}
+ private Paragraph addParagraphWithTextAndTitle(Note note, String text, String title) {
+ Paragraph p = note.addParagraph();
+ p.setText(text);
+ p.setTitle(title);
+ return p;
+ }
+
private Note newNote(String name) {
Note note = new Note(notebookRepoMock, replLoaderMock, null, notebookIndex);
note.setName(name);