You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by zj...@apache.org on 2020/11/04 02:07:48 UTC
[zeppelin] branch master updated: [ZEPPELIN-5112]. upgrade-note.sh
throws exception while upgrading notes on hdfs storage
This is an automated email from the ASF dual-hosted git repository.
zjffdu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new c1c0e40 [ZEPPELIN-5112]. upgrade-note.sh throws exception while upgrading notes on hdfs storage
c1c0e40 is described below
commit c1c0e40b56c4f9b2d5f9be8c4b48cff426a34e0c
Author: Jeff Zhang <zj...@apache.org>
AuthorDate: Thu Oct 29 13:24:01 2020 +0800
[ZEPPELIN-5112]. upgrade-note.sh throws exception while upgrading notes on hdfs storage
### What is this PR for?
The root cause is that we didn't put hadoop jars in the classpath in `upgrade-note.sh`. This PR update script `upgrade-note.sh`. Besides that this PR also fix an upgrading issue in `ParagraphRuntimeInfo.java`. The `values` is List<String> in old version of zeppelin, but it is changed to List<Map<String, String>> in new version of zeppelin.
### What type of PR is it?
[Bug Fix]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-5112
### How should this be tested?
* CI pass
https://travis-ci.org/github/zjffdu/zeppelin/builds/739821895
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <zj...@apache.org>
Closes #3958 from zjffdu/ZEPPELIN-5112 and squashes the following commits:
5e74dae1c [Jeff Zhang] [ZEPPELIN-5112]. upgrade-note.sh throws exception while upgrading notes on hdfs storage
---
bin/upgrade-note.sh | 14 ++++++++++++++
.../org/apache/zeppelin/socket/NotebookServerTest.java | 9 +++++----
.../org/apache/zeppelin/notebook/ParagraphRuntimeInfo.java | 4 ++--
.../apache/zeppelin/notebook/repo/NotebookRepoSync.java | 1 +
4 files changed, 22 insertions(+), 6 deletions(-)
diff --git a/bin/upgrade-note.sh b/bin/upgrade-note.sh
index 594eff5..e68ef78 100755
--- a/bin/upgrade-note.sh
+++ b/bin/upgrade-note.sh
@@ -51,4 +51,18 @@ addJarInDir "${ZEPPELIN_HOME}/zeppelin-server/target/lib"
ZEPPELIN_CLASSPATH="$CLASSPATH:$ZEPPELIN_CLASSPATH"
+## Add hadoop jars when env USE_HADOOP is true
+if [[ "${USE_HADOOP}" != "false" ]]; then
+ if [[ -z "${HADOOP_CONF_DIR}" ]]; then
+ echo "Please specify HADOOP_CONF_DIR if USE_HADOOP is true"
+ else
+ ZEPPELIN_CLASSPATH+=":${HADOOP_CONF_DIR}"
+ if ! [ -x "$(command -v hadoop)" ]; then
+ echo 'hadoop command is not in PATH when HADOOP_CONF_DIR is specified.'
+ else
+ ZEPPELIN_CLASSPATH+=":`hadoop classpath`"
+ fi
+ fi
+fi
+
exec $ZEPPELIN_RUNNER $JAVA_OPTS -cp $ZEPPELIN_CLASSPATH_OVERRIDES:${ZEPPELIN_CLASSPATH} $MAIN_CLASS "$@"
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java
index 06816c4..0936cff 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java
@@ -679,11 +679,12 @@ public class NotebookServerTest extends AbstractTestRestApi {
// check RuntimeInfos
assertTrue(paragraph.getRuntimeInfos().containsKey("jobUrl"));
- List<Map<String, String>> list = paragraph.getRuntimeInfos().get("jobUrl").getValue();
+ List<Object> list = paragraph.getRuntimeInfos().get("jobUrl").getValue();
assertEquals(1, list.size());
- assertEquals(2, list.get(0).size());
- assertEquals(list.get(0).get("jobUrl"), "jobUrl_value");
- assertEquals(list.get(0).get("jobLabel"), "jobLabel_value");
+ Map<String, String> map = (Map<String, String>) list.get(0);
+ assertEquals(2, map.size());
+ assertEquals(map.get("jobUrl"), "jobUrl_value");
+ assertEquals(map.get("jobLabel"), "jobLabel_value");
}
@Test
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ParagraphRuntimeInfo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ParagraphRuntimeInfo.java
index 23ce9e8..d5edf56 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ParagraphRuntimeInfo.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ParagraphRuntimeInfo.java
@@ -18,7 +18,7 @@ public class ParagraphRuntimeInfo {
// runtimeInfos job url or dropdown-menu key in
// zeppelin-web/src/app/notebook/paragraph/paragraph-control.html
- private List<Map<String, String>> values; // values for the key-value pair property
+ private List<Object> values; // values for the key-value pair property
private String interpreterSettingId;
public ParagraphRuntimeInfo(String propertyName, String label,
@@ -39,7 +39,7 @@ public class ParagraphRuntimeInfo {
}
@VisibleForTesting
- public List<Map<String, String>> getValue() {
+ public List<Object> getValue() {
return values;
}
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java
index 6f37aa2..805a848 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java
@@ -116,6 +116,7 @@ public class NotebookRepoSync implements NotebookRepoWithVersionControl {
LOGGER.info("Convert old note file to new style, note count: " + oldNotesInfo.size());
LOGGER.info("Delete old note: " + deleteOld);
for (OldNoteInfo oldNoteInfo : oldNotesInfo) {
+ LOGGER.info("Converting note, id: {}", oldNoteInfo.getId());
Note note = oldNotebookRepo.get(oldNoteInfo.getId(), AuthenticationInfo.ANONYMOUS);
note.setPath(note.getName());
note.setVersion(Util.getVersion());