You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by pd...@apache.org on 2021/07/01 06:41:45 UTC
[zeppelin] branch master updated: [ZEPPELIN-5178] notebook search
not working on JDK11
This is an automated email from the ASF dual-hosted git repository.
pdallig pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new 7288702 [ZEPPELIN-5178] notebook search not working on JDK11
7288702 is described below
commit 7288702a53d9c246bd6e7194e03e00f9b64fe61a
Author: Kalyan Kanuri <kk...@amazon.com>
AuthorDate: Mon Jun 28 13:48:38 2021 +0200
[ZEPPELIN-5178] notebook search not working on JDK11
### What is this PR for?
On JDK11, Notebook search is throwing this error for any query
We couldn't find any notebook matching 'spark'
Logs Indicate this may be issue with using older version of Lucene (5.3.0)
### What type of PR is it?
Upgrades Lucene to the current latest version 8.7.0 and upgrades the deprecated methods
### Todos
[ ] Verify if Lucene handles the index upgrade seamlessly.
### What is the Jira issue?
https://issues.apache.org/jira/browse/ZEPPELIN-5178
### How should this be tested?
* Unit Tests
* Manual verification of search
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update?
* Is there breaking changes for older versions?
* Does this needs documentation?
Author: Kalyan Kanuri <kk...@amazon.com>
Author: Philipp Dallig <ph...@gmail.com>
Closes #4002 from kkalyan/lucene and squashes the following commits:
6eac42b5c [Philipp Dallig] Adding both Apache Lucene licenses for zengine and elasticsearch interpreter
82e8227b6 [Kalyan Kanuri] aws sdk licenese upgrade
21f5ef6b7 [Kalyan Kanuri] license
d8d78ca1e [Kalyan Kanuri] lucene upgrade
---
zeppelin-distribution/src/bin_license/LICENSE | 41 +++++++++++++---------
zeppelin-zengine/pom.xml | 2 +-
.../org/apache/zeppelin/search/LuceneSearch.java | 8 ++---
3 files changed, 30 insertions(+), 21 deletions(-)
diff --git a/zeppelin-distribution/src/bin_license/LICENSE b/zeppelin-distribution/src/bin_license/LICENSE
index bba35ee..4787826 100644
--- a/zeppelin-distribution/src/bin_license/LICENSE
+++ b/zeppelin-distribution/src/bin_license/LICENSE
@@ -2,7 +2,7 @@ The following components are provided under Apache License.
(Apache 2.0) nvd3.js v1.7.1 (http://nvd3.org/) - https://github.com/novus/nvd3/blob/v1.7.1/LICENSE.md
(Apache 2.0) gson v2.2 (com.google.code.gson:gson:jar:2.2 - https://github.com/google/gson) - https://github.com/google/gson/blob/gson-2.2/LICENSE
- (Apache 2.0) Amazon Web Services SDK for Java v1.10.62 (https://aws.amazon.com/sdk-for-java/) - https://raw.githubusercontent.com/aws/aws-sdk-java/1.10.62/LICENSE.txt
+ (Apache 2.0) Amazon Web Services SDK for Java v1.11.736 (https://aws.amazon.com/sdk-for-java/) - https://raw.githubusercontent.com/aws/aws-sdk-java/1.11.736/LICENSE.txt
(Apache 2.0) JavaEWAH v0.7.9 (https://github.com/lemire/javaewah) - https://github.com/lemire/javaewah/blob/master/LICENSE-2.0.txt
(Apache 2.0) Apache Commons Logging (commons-logging:commons-logging:1.1.1 - http://commons.apache.org/proper/commons-logging/)
(Apache 2.0) Apache Commons Codec (commons-codec:commons-codec:1.5 - http://commons.apache.org/proper/commons-codec/)
@@ -79,21 +79,30 @@ The following components are provided under Apache License.
(Apache 2.0) Spatial4J (com.spatial4j:spatial4j:0.4.1 - https://github.com/spatial4j/spatial4j)
(Apache 2.0) T-Digest (com.tdunning:t-digest:3.0 - https://github.com/tdunning/t-digest)
(Apache 2.0) Netty (io.netty:netty:3.10.5.Final - http://netty.io/)
- (Apache 2.0) Lucene Common Analyzers (org.apache.lucene:lucene-analyzers-common:5.3.1 - http://lucene.apache.org/lucene-parent/lucene-analyzers-common)
- (Apache 2.0) Lucene Memory (org.apache.lucene:lucene-backward-codecs:5.3.1 - http://lucene.apache.org/lucene-parent/lucene-backward-codecs)
- (Apache 2.0) Lucene Core (org.apache.lucene:lucene-core:5.3.1 - http://lucene.apache.org/lucene-parent/lucene-core)
- (Apache 2.0) Lucene Grouping (org.apache.lucene:lucene-grouping:5.3.1 - http://lucene.apache.org/lucene-parent/lucene-grouping)
- (Apache 2.0) Lucene Highlighter (org.apache.lucene:lucene-highlighter:5.3.1 - http://lucene.apache.org/lucene-parent/lucene-highlighter)
- (Apache 2.0) Lucene Join (org.apache.lucene:lucene-join:5.3.1 - http://lucene.apache.org/lucene-parent/lucene-join)
- (Apache 2.0) Lucene Memory (org.apache.lucene:lucene-memory:5.3.1 - http://lucene.apache.org/lucene-parent/lucene-memory)
- (Apache 2.0) Lucene Miscellaneous (org.apache.lucene:lucene-misc:5.3.1 - http://lucene.apache.org/lucene-parent/lucene-misc)
- (Apache 2.0) Lucene Queries (org.apache.lucene:lucene-queries:5.3.1 - http://lucene.apache.org/lucene-parent/lucene-queries)
- (Apache 2.0) Lucene QueryParsers (org.apache.lucene:lucene-queryparser:5.3.1 - http://lucene.apache.org/lucene-parent/lucene-queryparser)
- (Apache 2.0) Lucene Sandbox (org.apache.lucene:lucene-sandbox:5.3.1 - http://lucene.apache.org/lucene-parent/lucene-sandbox)
- (Apache 2.0) Lucene Spatial (org.apache.lucene:lucene-spatial:5.3.1 - http://lucene.apache.org/lucene-parent/lucene-spatial)
- (Apache 2.0) Lucene Spatial 3D (org.apache.lucene:lucene-spatial3d:5.3.1 - http://lucene.apache.org/lucene-parent/lucene-spatial3d)
- (Apache 2.0) Lucene Suggest (org.apache.lucene:lucene-suggest:5.3.1 - http://lucene.apache.org/lucene-parent/lucene-suggest)
- (Apache 2.0) Elasticsearch: Core (org.elasticsearch:elasticsearch:2.1.0 - http://nexus.sonatype.org/oss-repository-hosting.html/parent/elasticsearch)
+ Apache Lucene via zengine
+ (Apache 2.0) Lucene Core (org.apache.lucene:lucene-core:8.7.0 https://github.com/apache/lucene)
+ (Apache 2.0) Lucene Common Analyzers (org.apache.lucene:lucene-analyzers-common:8.7.0 - https://github.com/apache/lucene)
+ (Apache 2.0) Lucene QueryParsers (org.apache.lucene:lucene-queryparser:8.7.0 https://github.com/apache/lucene)
+ (Apache 2.0) Lucene Queries (org.apache.lucene:lucene-queries:8.7.0 https://github.com/apache/lucene)
+ (Apache 2.0) Lucene Sandbox (org.apache.lucene:lucene-sandbox:8.7.0 https://github.com/apache/lucene)
+ (Apache 2.0) Lucene Highlighter (org.apache.lucene:lucene-highlighter:8.7.0 https://github.com/apache/lucene)
+ (Apache 2.0) Lucene Memory (org.apache.lucene:lucene-memory:8.7.0 https://github.com/apache/lucene)
+ (Apache 2.0) Elasticsearch: Core (org.elasticsearch:elasticsearch:2.4.3 - https://github.com/elastic/elasticsearch)
+ Apache Lucene via elasticsearch interpreter
+ (Apache 2.0) Lucene Core (org.apache.lucene:lucene-core:5.5.2 https://github.com/apache/lucene)
+ (Apache 2.0) Lucene Backward Codes (org.apache.lucene:lucene-backward-codecs:5.5.2 https://github.com/apache/lucene)
+ (Apache 2.0) Lucene Common Analyzers (org.apache.lucene:lucene-analyzers-common:5.5.2 - https://github.com/apache/lucene)
+ (Apache 2.0) Lucene Queries (org.apache.lucene:lucene-queries:5.5.2 https://github.com/apache/lucene)
+ (Apache 2.0) Lucene Memory (org.apache.lucene:lucene-memory:5.5.2 https://github.com/apache/lucene)
+ (Apache 2.0) Lucene Highlighter (org.apache.lucene:lucene-highlighter:5.5.2 https://github.com/apache/lucene)
+ (Apache 2.0) Lucene Join (org.apache.lucene:lucene-join:5.5.2 https://github.com/apache/lucene)
+ (Apache 2.0) Lucene QueryParsers (org.apache.lucene:lucene-queryparser:5.5.2 https://github.com/apache/lucene)
+ (Apache 2.0) Lucene Sandbox (org.apache.lucene:lucene-sandbox:5.5.2 https://github.com/apache/lucene)
+ (Apache 2.0) Lucene Suggest (org.apache.lucene:lucene-suggest:5.5.2 https://github.com/apache/lucene)
+ (Apache 2.0) Lucene Miscellaneous (org.apache.lucene:lucene-misc:5.5.2 https://github.com/apache/lucene)
+ (Apache 2.0) Lucene Grouping (org.apache.lucene:lucene-queries:5.5.2 https://github.com/apache/lucene)
+ (Apache 2.0) Lucene Spatial (org.apache.lucene:lucene-spatial:5.5.2 https://github.com/apache/lucene)
+ (Apache 2.0) Lucene Spatial 3D (org.apache.lucene:lucene-spatial3d:5.5.2 https://github.com/apache/lucene)
(Apache 2.0) Joda convert (org.joda:joda-convert:1.8.1 - http://joda-convert.sourceforge.net)
(Apache 2.0) Shiro Core (org.apache.shiro:shiro-core:1.7.0 - https://shiro.apache.org)
(Apache 2.0) Shiro Web (org.apache.shiro:shiro-web:1.7.0 - https://shiro.apache.org)
diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml
index b1e2f46..0f78fd0 100644
--- a/zeppelin-zengine/pom.xml
+++ b/zeppelin-zengine/pom.xml
@@ -37,7 +37,7 @@
<properties>
<!--library versions-->
- <lucene.version>5.3.1</lucene.version>
+ <lucene.version>8.7.0</lucene.version>
<org.reflections.version>0.9.8</org.reflections.version>
<xml.apis.version>1.4.01</xml.apis.version>
<frontend.maven.plugin.version>1.3</frontend.maven.plugin.version>
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java
index 3c3645c..586b5d9 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java
@@ -35,7 +35,7 @@ import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.document.LongField;
+import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
@@ -289,7 +289,7 @@ public class LuceneSearch extends SearchService {
doc.add(new TextField(SEARCH_FIELD_TITLE, p.getTitle(), Field.Store.YES));
}
Date date = p.getDateStarted() != null ? p.getDateStarted() : p.getDateCreated();
- doc.add(new LongField("modified", date.getTime(), Field.Store.NO));
+ doc.add(new LongPoint("modified", date.getTime()));
} else {
doc.add(new TextField(SEARCH_FIELD_TEXT, noteName, Field.Store.YES));
}
@@ -358,14 +358,14 @@ public class LuceneSearch extends SearchService {
*/
private void deleteDoc(String noteId, Paragraph p) {
String fullNoteOrJustParagraph = formatDeleteId(noteId, p);
- LOGGER.debug("Deleting note {}, out of: {}", noteId, indexWriter.numDocs());
+ LOGGER.debug("Deleting note {}, out of: {}", noteId, indexWriter.getDocStats().numDocs);
try {
indexWriter.deleteDocuments(new WildcardQuery(new Term(ID_FIELD, fullNoteOrJustParagraph)));
indexWriter.commit();
} catch (IOException e) {
LOGGER.error("Failed to delete {} from index by '{}'", noteId, fullNoteOrJustParagraph, e);
}
- LOGGER.debug("Done, index contains {} docs now", indexWriter.numDocs());
+ LOGGER.debug("Done, index contains {} docs now", indexWriter.getDocStats().numDocs);
}
/* (non-Javadoc)