You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by jo...@apache.org on 2018/07/23 05:23:15 UTC
zeppelin git commit: Revert "[ZEPPELIN-3593] Change LuceneSearch's
directory to file system from memory"
Repository: zeppelin
Updated Branches:
refs/heads/master 716dfd1c5 -> 4467ac01d
Revert "[ZEPPELIN-3593] Change LuceneSearch's directory to file system from memory"
This reverts commit c195b5609bb4dbb9fd037381ea58d1650cee92bf.
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/4467ac01
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/4467ac01
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/4467ac01
Branch: refs/heads/master
Commit: 4467ac01dfe4a39c8d191d2266d7e772ab35a1a9
Parents: 716dfd1
Author: Jongyoul Lee <jo...@gmail.com>
Authored: Mon Jul 23 14:22:49 2018 +0900
Committer: Jongyoul Lee <jo...@gmail.com>
Committed: Mon Jul 23 14:22:49 2018 +0900
----------------------------------------------------------------------
.../zeppelin/conf/ZeppelinConfiguration.java | 24 +--
.../apache/zeppelin/server/ZeppelinServer.java | 2 +-
.../apache/zeppelin/search/LuceneSearch.java | 187 +++++++++----------
.../zeppelin/search/LuceneSearchTest.java | 147 +++++++--------
4 files changed, 165 insertions(+), 195 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4467ac01/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
index 9cb30c1..7245cb9 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -17,12 +17,6 @@
package org.apache.zeppelin.conf;
-import java.io.File;
-import java.net.URL;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.XMLConfiguration;
import org.apache.commons.configuration.tree.ConfigurationNode;
@@ -31,6 +25,13 @@ import org.apache.zeppelin.util.Util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
/**
* Zeppelin configuration.
*
@@ -497,10 +498,10 @@ public class ZeppelinConfiguration extends XMLConfiguration {
}
public String getRelativeDir(String path) {
- if (path != null && path.startsWith(File.separator) || isWindowsPath(path)) {
+ if (path != null && path.startsWith("/") || isWindowsPath(path)) {
return path;
} else {
- return getString(ConfVars.ZEPPELIN_HOME) + File.separator + path;
+ return getString(ConfVars.ZEPPELIN_HOME) + "/" + path;
}
}
@@ -626,10 +627,6 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getString(ConfVars.ZEPPELIN_PROXY_PASSWORD);
}
- public String getZeppelinSearchTempPath() {
- return getRelativeDir(ConfVars.ZEPPELIN_SEARCH_TEMP_PATH);
- }
-
public Map<String, String> dumpConfigurations(ZeppelinConfiguration conf,
ConfigurationKeyPredicate predicate) {
Map<String, String> configurations = new HashMap<>();
@@ -831,8 +828,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ZEPPELIN_NOTEBOOK_CRON_FOLDERS("zeppelin.notebook.cron.folders", null),
ZEPPELIN_PROXY_URL("zeppelin.proxy.url", null),
ZEPPELIN_PROXY_USER("zeppelin.proxy.user", null),
- ZEPPELIN_PROXY_PASSWORD("zeppelin.proxy.password", null),
- ZEPPELIN_SEARCH_TEMP_PATH("zeppelin.search.temp.path", System.getProperty("java.io.tmpdir"));
+ ZEPPELIN_PROXY_PASSWORD("zeppelin.proxy.password", null);
private String varName;
@SuppressWarnings("rawtypes")
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4467ac01/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
index 5c85c23..9e2c5f8 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
@@ -168,7 +168,7 @@ public class ZeppelinServer extends Application {
notebookWsServer, notebookWsServer);
this.replFactory = new InterpreterFactory(interpreterSettingManager);
this.notebookRepo = new NotebookRepoSync(conf);
- this.noteSearchService = new LuceneSearch(conf);
+ this.noteSearchService = new LuceneSearch();
this.notebookAuthorization = NotebookAuthorization.getInstance();
this.credentials = new Credentials(
conf.credentialsPersist(),
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4467ac01/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java
index 49719a6..b43c453 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java
@@ -16,20 +16,14 @@
*/
package org.apache.zeppelin.search;
-import com.google.common.base.Joiner;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
-import org.apache.commons.io.FileUtils;
+
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
@@ -45,6 +39,7 @@ import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
@@ -56,50 +51,43 @@ import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
import org.apache.lucene.search.highlight.TextFragment;
import org.apache.lucene.search.highlight.TokenSources;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.MMapDirectory;
import org.apache.lucene.store.RAMDirectory;
-import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.Paragraph;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+
/**
- * Search (both, indexing and query) the notebooks using Lucene. Query is thread-safe, as creates
- * new IndexReader every time. Index is thread-safe, as re-uses single IndexWriter, which is
- * thread-safe.
+ * Search (both, indexing and query) the notebooks using Lucene.
+ *
+ * Query is thread-safe, as creates new IndexReader every time.
+ * Index is thread-safe, as re-uses single IndexWriter, which is thread-safe.
*/
public class LuceneSearch implements SearchService {
- private static final Logger logger = LoggerFactory.getLogger(LuceneSearch.class);
+ private static final Logger LOG = LoggerFactory.getLogger(LuceneSearch.class);
private static final String SEARCH_FIELD_TEXT = "contents";
private static final String SEARCH_FIELD_TITLE = "header";
- private static final String PARAGRAPH = "paragraph";
- private static final String ID_FIELD = "id";
-
- private Directory directory;
- private Path directoryPath;
- private Analyzer analyzer;
- private IndexWriterConfig indexWriterConfig;
- private IndexWriter indexWriter;
-
- public LuceneSearch(ZeppelinConfiguration zeppelinConfiguration) {
- try {
- this.directoryPath =
- Files.createTempDirectory(
- Paths.get(zeppelinConfiguration.getZeppelinSearchTempPath()), "zeppelin-search-");
- this.directory = new MMapDirectory(directoryPath);
- } catch (IOException e) {
- logger.error(
- "Failed to create temporary directory for search service. Use memory instead", e);
- this.directory = new RAMDirectory();
- }
- this.analyzer = new StandardAnalyzer();
- this.indexWriterConfig = new IndexWriterConfig(analyzer);
+ static final String PARAGRAPH = "paragraph";
+ static final String ID_FIELD = "id";
+
+ Directory ramDirectory;
+ Analyzer analyzer;
+ IndexWriterConfig iwc;
+ IndexWriter writer;
+
+ public LuceneSearch() {
+ ramDirectory = new RAMDirectory();
+ analyzer = new StandardAnalyzer();
+ iwc = new IndexWriterConfig(analyzer);
try {
- this.indexWriter = new IndexWriter(directory, indexWriterConfig);
+ writer = new IndexWriter(ramDirectory, iwc);
} catch (IOException e) {
- logger.error("Failed to create new IndexWriter", e);
+ LOG.error("Failed to create new IndexWriter", e);
}
}
@@ -108,49 +96,51 @@ public class LuceneSearch implements SearchService {
*/
@Override
public List<Map<String, String>> query(String queryStr) {
- if (null == directory) {
+ if (null == ramDirectory) {
throw new IllegalStateException(
"Something went wrong on instance creation time, index dir is null");
}
List<Map<String, String>> result = Collections.emptyList();
- try (IndexReader indexReader = DirectoryReader.open(directory)) {
+ try (IndexReader indexReader = DirectoryReader.open(ramDirectory)) {
IndexSearcher indexSearcher = new IndexSearcher(indexReader);
Analyzer analyzer = new StandardAnalyzer();
- MultiFieldQueryParser parser =
- new MultiFieldQueryParser(new String[] {SEARCH_FIELD_TEXT, SEARCH_FIELD_TITLE}, analyzer);
+ MultiFieldQueryParser parser = new MultiFieldQueryParser(
+ new String[] {SEARCH_FIELD_TEXT, SEARCH_FIELD_TITLE},
+ analyzer);
Query query = parser.parse(queryStr);
- logger.debug("Searching for: " + query.toString(SEARCH_FIELD_TEXT));
+ LOG.debug("Searching for: " + query.toString(SEARCH_FIELD_TEXT));
SimpleHTMLFormatter htmlFormatter = new SimpleHTMLFormatter();
Highlighter highlighter = new Highlighter(htmlFormatter, new QueryScorer(query));
result = doSearch(indexSearcher, query, analyzer, highlighter);
+ indexReader.close();
} catch (IOException e) {
- logger.error("Failed to open index dir {}, make sure indexing finished OK", directory, e);
+ LOG.error("Failed to open index dir {}, make sure indexing finished OK", ramDirectory, e);
} catch (ParseException e) {
- logger.error("Failed to parse query " + queryStr, e);
+ LOG.error("Failed to parse query " + queryStr, e);
}
return result;
}
- private List<Map<String, String>> doSearch(
- IndexSearcher searcher, Query query, Analyzer analyzer, Highlighter highlighter) {
+ private List<Map<String, String>> doSearch(IndexSearcher searcher, Query query,
+ Analyzer analyzer, Highlighter highlighter) {
List<Map<String, String>> matchingParagraphs = Lists.newArrayList();
ScoreDoc[] hits;
try {
hits = searcher.search(query, 20).scoreDocs;
for (int i = 0; i < hits.length; i++) {
- logger.debug("doc={} score={}", hits[i].doc, hits[i].score);
+ LOG.debug("doc={} score={}", hits[i].doc, hits[i].score);
int id = hits[i].doc;
Document doc = searcher.doc(id);
String path = doc.get(ID_FIELD);
if (path != null) {
- logger.debug((i + 1) + ". " + path);
+ LOG.debug((i + 1) + ". " + path);
String title = doc.get("title");
if (title != null) {
- logger.debug(" Title: {}", doc.get("title"));
+ LOG.debug(" Title: {}", doc.get("title"));
}
String text = doc.get(SEARCH_FIELD_TEXT);
@@ -158,38 +148,34 @@ public class LuceneSearch implements SearchService {
String fragment = "";
if (text != null) {
- TokenStream tokenStream =
- TokenSources.getTokenStream(
- searcher.getIndexReader(), id, SEARCH_FIELD_TEXT, analyzer);
+ TokenStream tokenStream = TokenSources.getTokenStream(searcher.getIndexReader(), id,
+ SEARCH_FIELD_TEXT, analyzer);
TextFragment[] frag = highlighter.getBestTextFragments(tokenStream, text, true, 3);
- logger.debug(" {} fragments found for query '{}'", frag.length, query);
+ LOG.debug(" {} fragments found for query '{}'", frag.length, query);
for (int j = 0; j < frag.length; j++) {
if ((frag[j] != null) && (frag[j].getScore() > 0)) {
- logger.debug(" Fragment: {}", frag[j].toString());
+ LOG.debug(" Fragment: {}", frag[j].toString());
}
}
fragment = (frag != null && frag.length > 0) ? frag[0].toString() : "";
}
if (header != null) {
- TokenStream tokenTitle =
- TokenSources.getTokenStream(
- searcher.getIndexReader(), id, SEARCH_FIELD_TITLE, analyzer);
+ TokenStream tokenTitle = TokenSources.getTokenStream(searcher.getIndexReader(), id,
+ SEARCH_FIELD_TITLE, analyzer);
TextFragment[] frgTitle = highlighter.getBestTextFragments(tokenTitle, header, true, 3);
header = (frgTitle != null && frgTitle.length > 0) ? frgTitle[0].toString() : "";
} else {
header = "";
}
- matchingParagraphs.add(
- ImmutableMap.of(
- "id", path, // <noteId>/paragraph/<paragraphId>
- "name", title, "snippet", fragment, "text", text, "header", header));
+ matchingParagraphs.add(ImmutableMap.of("id", path, // <noteId>/paragraph/<paragraphId>
+ "name", title, "snippet", fragment, "text", text, "header", header));
} else {
- logger.info("{}. No {} for this document", i + 1, ID_FIELD);
+ LOG.info("{}. No {} for this document", i + 1, ID_FIELD);
}
}
} catch (IOException | InvalidTokenOffsetsException e) {
- logger.error("Exception on searching for {}", query, e);
+ LOG.error("Exception on searching for {}", query, e);
}
return matchingParagraphs;
}
@@ -200,7 +186,7 @@ public class LuceneSearch implements SearchService {
@Override
public void updateIndexDoc(Note note) throws IOException {
updateIndexNoteName(note);
- for (Paragraph p : note.getParagraphs()) {
+ for (Paragraph p: note.getParagraphs()) {
updateIndexParagraph(note, p);
}
}
@@ -208,9 +194,9 @@ public class LuceneSearch implements SearchService {
private void updateIndexNoteName(Note note) throws IOException {
String noteName = note.getName();
String noteId = note.getId();
- logger.debug("Indexing Notebook {}, '{}'", noteId, noteName);
+ LOG.debug("Indexing Notebook {}, '{}'", noteId, noteName);
if (null == noteName || noteName.isEmpty()) {
- logger.debug("Skipping empty notebook name");
+ LOG.debug("Skipping empty notebook name");
return;
}
updateDoc(noteId, noteName, null);
@@ -218,15 +204,15 @@ public class LuceneSearch implements SearchService {
private void updateIndexParagraph(Note note, Paragraph p) throws IOException {
if (p.getText() == null) {
- logger.debug("Skipping empty paragraph");
+ LOG.debug("Skipping empty paragraph");
return;
}
updateDoc(note.getId(), note.getName(), p);
}
/**
- * Updates index for the given note: either note.name or a paragraph If paragraph is <code>null
- * </code> - updates only for the note.name
+ * Updates index for the given note: either note.name or a paragraph If
+ * paragraph is <code>null</code> - updates only for the note.name
*
* @param noteId
* @param noteName
@@ -237,16 +223,16 @@ public class LuceneSearch implements SearchService {
String id = formatId(noteId, p);
Document doc = newDocument(id, noteName, p);
try {
- indexWriter.updateDocument(new Term(ID_FIELD, id), doc);
- indexWriter.commit();
+ writer.updateDocument(new Term(ID_FIELD, id), doc);
+ writer.commit();
} catch (IOException e) {
- logger.error("Failed to updaet index of notebook {}", noteId, e);
+ LOG.error("Failed to updaet index of notebook {}", noteId, e);
}
}
/**
- * If paragraph is not null, id is <noteId>/paragraphs/<paragraphId>, otherwise it's just
- * <noteId>.
+ * If paragraph is not null, id is <noteId>/paragraphs/<paragraphId>,
+ * otherwise it's just <noteId>.
*/
static String formatId(String noteId, Paragraph p) {
String id = noteId;
@@ -267,7 +253,8 @@ public class LuceneSearch implements SearchService {
}
/**
- * If paragraph is not null, indexes code in the paragraph, otherwise indexes the notebook name.
+ * If paragraph is not null, indexes code in the paragraph, otherwise indexes
+ * the notebook name.
*
* @param id id of the document, different for Note name and paragraph
* @param noteName name of the note
@@ -307,17 +294,15 @@ public class LuceneSearch implements SearchService {
docsIndexed++;
}
} catch (IOException e) {
- logger.error("Failed to index all Notebooks", e);
+ LOG.error("Failed to index all Notebooks", e);
} finally {
try { // save what's been indexed, even if not full collection
- indexWriter.commit();
+ writer.commit();
} catch (IOException e) {
- logger.error("Failed to save index", e);
+ LOG.error("Failed to save index", e);
}
long end = System.nanoTime();
- logger.info(
- "Indexing {} notebooks took {}ms",
- docsIndexed,
+ LOG.info("Indexing {} notebooks took {}ms", docsIndexed,
TimeUnit.NANOSECONDS.toMillis(end - start));
}
}
@@ -329,9 +314,9 @@ public class LuceneSearch implements SearchService {
public void addIndexDoc(Note note) {
try {
addIndexDocAsync(note);
- indexWriter.commit();
+ writer.commit();
} catch (IOException e) {
- logger.error("Failed to add note {} to index", note, e);
+ LOG.error("Failed to add note {} to index", note, e);
}
}
@@ -342,13 +327,13 @@ public class LuceneSearch implements SearchService {
* @throws IOException
*/
private void addIndexDocAsync(Note note) throws IOException {
- indexNoteName(indexWriter, note.getId(), note.getName());
+ indexNoteName(writer, note.getId(), note.getName());
for (Paragraph doc : note.getParagraphs()) {
if (doc.getText() == null) {
- logger.debug("Skipping empty paragraph");
+ LOG.debug("Skipping empty paragraph");
continue;
}
- indexDoc(indexWriter, note.getId(), note.getName(), doc);
+ indexDoc(writer, note.getId(), note.getName(), doc);
}
}
@@ -371,18 +356,18 @@ public class LuceneSearch implements SearchService {
private void deleteDoc(Note note, Paragraph p) {
if (null == note) {
- logger.error("Trying to delete note by reference to NULL");
+ LOG.error("Trying to delete note by reference to NULL");
return;
}
String fullNoteOrJustParagraph = formatDeleteId(note.getId(), p);
- logger.debug("Deleting note {}, out of: {}", note.getId(), indexWriter.numDocs());
+ LOG.debug("Deleting note {}, out of: {}", note.getId(), writer.numDocs());
try {
- indexWriter.deleteDocuments(new WildcardQuery(new Term(ID_FIELD, fullNoteOrJustParagraph)));
- indexWriter.commit();
+ writer.deleteDocuments(new WildcardQuery(new Term(ID_FIELD, fullNoteOrJustParagraph)));
+ writer.commit();
} catch (IOException e) {
- logger.error("Failed to delete {} from index by '{}'", note, fullNoteOrJustParagraph, e);
+ LOG.error("Failed to delete {} from index by '{}'", note, fullNoteOrJustParagraph, e);
}
- logger.debug("Done, index contains {} docs now" + indexWriter.numDocs());
+ LOG.debug("Done, index contains {} docs now" + writer.numDocs());
}
/* (non-Javadoc)
@@ -391,12 +376,9 @@ public class LuceneSearch implements SearchService {
@Override
public void close() {
try {
- indexWriter.close();
- if (null != directoryPath) {
- FileUtils.deleteDirectory(directoryPath.toFile());
- }
+ writer.close();
} catch (IOException e) {
- logger.error("Failed to .close() the notebook index", e);
+ LOG.error("Failed to .close() the notebook index", e);
}
}
@@ -406,19 +388,24 @@ public class LuceneSearch implements SearchService {
* @throws IOException
*/
private void indexNoteName(IndexWriter w, String noteId, String noteName) throws IOException {
- logger.debug("Indexing Notebook {}, '{}'", noteId, noteName);
+ LOG.debug("Indexing Notebook {}, '{}'", noteId, noteName);
if (null == noteName || noteName.isEmpty()) {
- logger.debug("Skipping empty notebook name");
+ LOG.debug("Skipping empty notebook name");
return;
}
indexDoc(w, noteId, noteName, null);
}
- /** Indexes a single document: - code of the paragraph (if non-null) - or just a note name */
+ /**
+ * Indexes a single document:
+ * - code of the paragraph (if non-null)
+ * - or just a note name
+ */
private void indexDoc(IndexWriter w, String noteId, String noteName, Paragraph p)
throws IOException {
String id = formatId(noteId, p);
Document doc = newDocument(id, noteName, p);
w.addDocument(doc);
}
+
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4467ac01/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java
index aeead7e..0ac4697 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java
@@ -17,15 +17,14 @@
package org.apache.zeppelin.search;
import static com.google.common.truth.Truth.assertThat;
+import static org.mockito.Mockito.*;
import static org.apache.zeppelin.search.LuceneSearch.formatId;
-import static org.mockito.Mockito.mock;
-import com.google.common.base.Splitter;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
-import org.apache.zeppelin.conf.ZeppelinConfiguration;
+
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.notebook.Note;
@@ -37,12 +36,14 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+import com.google.common.base.Splitter;
+
public class LuceneSearchTest {
private static NotebookRepo notebookRepoMock;
private static InterpreterFactory interpreterFactory;
private static InterpreterSettingManager interpreterSettingManager;
-
+
private SearchService noteSearchService;
private AuthenticationInfo anonymous;
@@ -52,13 +53,13 @@ public class LuceneSearchTest {
interpreterFactory = mock(InterpreterFactory.class);
interpreterSettingManager = mock(InterpreterSettingManager.class);
- // when(replLoaderMock.getInterpreterSettings())
- // .thenReturn(ImmutableList.<InterpreterSetting>of());
+// when(replLoaderMock.getInterpreterSettings())
+// .thenReturn(ImmutableList.<InterpreterSetting>of());
}
@Before
public void startUp() {
- noteSearchService = new LuceneSearch(ZeppelinConfiguration.create());
+ noteSearchService = new LuceneSearch();
anonymous = new AuthenticationInfo("anonymous");
}
@@ -67,45 +68,42 @@ public class LuceneSearchTest {
noteSearchService.close();
}
- @Test
- public void canIndexNotebook() {
- // give
+ @Test public void canIndexNotebook() {
+ //give
Note note1 = newNoteWithParagraph("Notebook1", "test");
Note note2 = newNoteWithParagraph("Notebook2", "not test");
List<Note> notebook = Arrays.asList(note1, note2);
- // when
+ //when
noteSearchService.addIndexDocs(notebook);
}
- @Test
- public void canIndexAndQuery() {
- // given
+ @Test public void canIndexAndQuery() {
+ //given
Note note1 = newNoteWithParagraph("Notebook1", "test");
Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all");
noteSearchService.addIndexDocs(Arrays.asList(note1, note2));
- // when
+ //when
List<Map<String, String>> results = noteSearchService.query("all");
- // then
+ //then
assertThat(results).isNotEmpty();
assertThat(results.size()).isEqualTo(1);
assertThat(results.get(0))
- .containsEntry("id", formatId(note2.getId(), note2.getLastParagraph()));
+ .containsEntry("id", formatId(note2.getId(), note2.getLastParagraph()));
}
- @Test
- public void canIndexAndQueryByNotebookName() {
- // given
+ @Test public void canIndexAndQueryByNotebookName() {
+ //given
Note note1 = newNoteWithParagraph("Notebook1", "test");
Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all");
noteSearchService.addIndexDocs(Arrays.asList(note1, note2));
- // when
+ //when
List<Map<String, String>> results = noteSearchService.query("Notebook1");
- // then
+ //then
assertThat(results).isNotEmpty();
assertThat(results.size()).isEqualTo(1);
assertThat(results.get(0)).containsEntry("id", note1.getId());
@@ -113,15 +111,15 @@ public class LuceneSearchTest {
@Test
public void canIndexAndQueryByParagraphTitle() {
- // given
+ //given
Note note1 = newNoteWithParagraph("Notebook1", "test", "testingTitleSearch");
Note note2 = newNoteWithParagraph("Notebook2", "not test", "notTestingTitleSearch");
noteSearchService.addIndexDocs(Arrays.asList(note1, note2));
- // when
+ //when
List<Map<String, String>> results = noteSearchService.query("testingTitleSearch");
- // then
+ //then
assertThat(results).isNotEmpty();
assertThat(results.size()).isAtLeast(1);
int TitleHits = 0;
@@ -133,44 +131,41 @@ public class LuceneSearchTest {
assertThat(TitleHits).isAtLeast(1);
}
- @Test
- public void indexKeyContract() throws IOException {
- // give
+ @Test public void indexKeyContract() throws IOException {
+ //give
Note note1 = newNoteWithParagraph("Notebook1", "test");
- // when
+ //when
noteSearchService.addIndexDoc(note1);
- // then
- String id = resultForQuery("test").get(0).get("id"); // LuceneSearch.ID_FIELD
+ //then
+ String id = resultForQuery("test").get(0).get(LuceneSearch.ID_FIELD);
- assertThat(Splitter.on("/").split(id)) // key structure <noteId>/paragraph/<paragraphId>
- .containsAllOf(
- note1.getId(), "paragraph", note1.getLastParagraph().getId()); // LuceneSearch.PARAGRAPH
+ assertThat(Splitter.on("/").split(id)) //key structure <noteId>/paragraph/<paragraphId>
+ .containsAllOf(note1.getId(), LuceneSearch.PARAGRAPH, note1.getLastParagraph().getId());
}
- @Test // (expected=IllegalStateException.class)
+ @Test //(expected=IllegalStateException.class)
public void canNotSearchBeforeIndexing() {
- // given NO noteSearchService.index() was called
- // when
+ //given NO noteSearchService.index() was called
+ //when
List<Map<String, String>> result = noteSearchService.query("anything");
- // then
+ //then
assertThat(result).isEmpty();
- // assert logs were printed
- // "ERROR org.apache.zeppelin.search.SearchService:97 - Failed to open index dir RAMDirectory"
+ //assert logs were printed
+ //"ERROR org.apache.zeppelin.search.SearchService:97 - Failed to open index dir RAMDirectory"
}
- @Test
- public void canIndexAndReIndex() throws IOException {
- // given
+ @Test public void canIndexAndReIndex() throws IOException {
+ //given
Note note1 = newNoteWithParagraph("Notebook1", "test");
Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all");
noteSearchService.addIndexDocs(Arrays.asList(note1, note2));
- // when
+ //when
Paragraph p2 = note2.getLastParagraph();
p2.setText("test indeed");
noteSearchService.updateIndexDoc(note2);
- // then
+ //then
List<Map<String, String>> results = noteSearchService.query("all");
assertThat(results).isEmpty();
@@ -178,26 +173,24 @@ public class LuceneSearchTest {
assertThat(results).isNotEmpty();
}
- @Test
- public void canDeleteNull() throws IOException {
- // give
+ @Test public void canDeleteNull() throws IOException {
+ //give
// looks like a bug in web UI: it tries to delete a note twice (after it has just been deleted)
- // when
+ //when
noteSearchService.deleteIndexDocs(null);
}
- @Test
- public void canDeleteFromIndex() throws IOException {
- // given
+ @Test public void canDeleteFromIndex() throws IOException {
+ //given
Note note1 = newNoteWithParagraph("Notebook1", "test");
Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all");
noteSearchService.addIndexDocs(Arrays.asList(note1, note2));
assertThat(resultForQuery("Notebook2")).isNotEmpty();
- // when
+ //when
noteSearchService.deleteIndexDocs(note2);
- // then
+ //then
assertThat(noteSearchService.query("all")).isEmpty();
assertThat(resultForQuery("Notebook2")).isEmpty();
@@ -206,46 +199,43 @@ public class LuceneSearchTest {
assertThat(results.size()).isEqualTo(1);
}
- @Test
- public void indexParagraphUpdatedOnNoteSave() throws IOException {
- // given: total 2 notebooks, 3 paragraphs
+ @Test public void indexParagraphUpdatedOnNoteSave() throws IOException {
+ //given: total 2 notebooks, 3 paragraphs
Note note1 = newNoteWithParagraph("Notebook1", "test");
Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all");
noteSearchService.addIndexDocs(Arrays.asList(note1, note2));
assertThat(resultForQuery("test").size()).isEqualTo(3);
- // when
+ //when
Paragraph p1 = note1.getLastParagraph();
p1.setText("no no no");
note1.persist(anonymous);
- // then
+ //then
assertThat(resultForQuery("Notebook1").size()).isEqualTo(1);
List<Map<String, String>> results = resultForQuery("test");
assertThat(results).isNotEmpty();
assertThat(results.size()).isEqualTo(2);
- // does not include Notebook1's paragraph any more
- for (Map<String, String> result : results) {
- assertThat(result.get("id").startsWith(note1.getId())).isFalse();
- ;
+ //does not include Notebook1's paragraph any more
+ for (Map<String, String> result: results) {
+ assertThat(result.get("id").startsWith(note1.getId())).isFalse();;
}
}
- @Test
- public void indexNoteNameUpdatedOnNoteSave() throws IOException {
- // given: total 2 notebooks, 3 paragraphs
+ @Test public void indexNoteNameUpdatedOnNoteSave() throws IOException {
+ //given: total 2 notebooks, 3 paragraphs
Note note1 = newNoteWithParagraph("Notebook1", "test");
Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all");
noteSearchService.addIndexDocs(Arrays.asList(note1, note2));
assertThat(resultForQuery("test").size()).isEqualTo(3);
- // when
+ //when
note1.setName("NotebookN");
note1.persist(anonymous);
- // then
+ //then
assertThat(resultForQuery("Notebook1")).isEmpty();
assertThat(resultForQuery("NotebookN")).isNotEmpty();
assertThat(resultForQuery("NotebookN").size()).isEqualTo(1);
@@ -256,7 +246,8 @@ public class LuceneSearchTest {
}
/**
- * Creates a new Note \w given name, adds a new paragraph \w given text
+ * Creates a new Note \w given name,
+ * adds a new paragraph \w given text
*
* @param noteName name of the note
* @param parText text of the paragraph
@@ -268,13 +259,16 @@ public class LuceneSearchTest {
return note1;
}
- private Note newNoteWithParagraph(String noteName, String parText, String title) {
+ private Note newNoteWithParagraph(String noteName, String parText,String title) {
Note note = newNote(noteName);
addParagraphWithTextAndTitle(note, parText, title);
return note;
}
- /** Creates a new Note \w given name, adds N paragraphs \w given texts */
+ /**
+ * Creates a new Note \w given name,
+ * adds N paragraphs \w given texts
+ */
private Note newNoteWithParagraphs(String noteName, String... parTexts) {
Note note1 = newNote(noteName);
for (String parText : parTexts) {
@@ -297,16 +291,9 @@ public class LuceneSearchTest {
}
private Note newNote(String name) {
- Note note =
- new Note(
- notebookRepoMock,
- interpreterFactory,
- interpreterSettingManager,
- null,
- noteSearchService,
- null,
- null);
+ Note note = new Note(notebookRepoMock, interpreterFactory, interpreterSettingManager, null, noteSearchService, null, null);
note.setName(name);
return note;
}
+
}