You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mo...@apache.org on 2016/02/21 19:47:02 UTC
incubator-zeppelin git commit: selenium test spark,
pyspark and sparkSql
Repository: incubator-zeppelin
Updated Branches:
refs/heads/master 831f426db -> 9b72c8940
selenium test spark, pyspark and sparkSql
### What is this PR for?
Test functionality of spark, pyspark, sparksql
### What type of PR is it?
Improvement
### Todos
* [x] - Selenium for spark
* [x] - Selenium for pyspark
* [x] - Selenium for sparksql
* [x] - refactor with https://github.com/apache/incubator-zeppelin/pull/619
### Is there a relevant Jira issue?
ZEPPELIN-587
### How should this be tested?
On macOS
PATH=~/Applications/Firefox.app/Contents/MacOS/:$PATH CI="" \
mvn -Dtest=org.apache.zeppelin.integration.TestSparkParagraph -Denforcer.skip=true \
test -pl zeppelin-server
Author: Prabhjyot Singh <pr...@gmail.com>
Closes #654 from prabhjyotsingh/ZEPPELIN-587 and squashes the following commits:
8f24695 [Prabhjyot Singh] use handleException in all other test cases remove test for spark 1.1.1 more meaningful log message
28dfc55 [Prabhjyot Singh] thorwong exception similar to https://github.com/apache/incubator-zeppelin/pull/709
21bcc45 [Prabhjyot Singh] Merge remote-tracking branch 'origin/master' into ZEPPELIN-587
b05b81b [Prabhjyot Singh] have SHIFT_ENTER enum
9a206f4 [Prabhjyot Singh] add missing endToEndTestEnabled check for testSparkInterpreterDependencyLoading
ab03287 [Prabhjyot Singh] have static import of AbstractZeppelinIT.HelperKeys.*, and rg.openqa.selenium.Keys.*
5187a16 [Prabhjyot Singh] check if spark version is less than 1.3 then don't append ".toDF()"
6927f7e [Prabhjyot Singh] CI FIX
9236e3c [Prabhjyot Singh] implemeting @bzz review comments
2c28758 [Prabhjyot Singh] missed refactor with #619
d49344f [Prabhjyot Singh] selenium test spark, pyspark and sparkSql
Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/9b72c894
Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/9b72c894
Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/9b72c894
Branch: refs/heads/master
Commit: 9b72c894017880bd4a2a0c018c6f9d9b42a5d51d
Parents: 831f426
Author: Prabhjyot Singh <pr...@gmail.com>
Authored: Wed Feb 17 11:23:52 2016 +0530
Committer: Lee moon soo <mo...@apache.org>
Committed: Sun Feb 21 10:50:33 2016 -0800
----------------------------------------------------------------------
.../org/apache/zeppelin/AbstractZeppelinIT.java | 46 +++-
.../integration/ParagraphActionsIT.java | 8 +-
.../zeppelin/integration/SparkParagraphIT.java | 228 +++++++++++++++++++
.../apache/zeppelin/integration/ZeppelinIT.java | 8 +-
4 files changed, 276 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/9b72c894/zeppelin-server/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java b/zeppelin-server/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java
index 29569fc..2b68356 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java
@@ -19,8 +19,6 @@ package org.apache.zeppelin;
import com.google.common.base.Function;
-import org.junit.After;
-import org.junit.Before;
import org.openqa.selenium.*;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.FluentWait;
@@ -29,10 +27,14 @@ import org.openqa.selenium.support.ui.WebDriverWait;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.TimeUnit;
+import static org.openqa.selenium.Keys.ENTER;
+import static org.openqa.selenium.Keys.SHIFT;
+
abstract public class AbstractZeppelinIT {
protected WebDriver driver;
@@ -131,4 +133,44 @@ abstract public class AbstractZeppelinIT {
sleep(100, true);
}
+ public enum HelperKeys implements CharSequence {
+ OPEN_PARENTHESIS(Keys.chord(Keys.SHIFT, "9")),
+ EXCLAMATION(Keys.chord(Keys.SHIFT, "1")),
+ PERCENTAGE(Keys.chord(Keys.SHIFT, "5")),
+ SHIFT_ENTER(Keys.chord(SHIFT, ENTER));
+
+ private final CharSequence keyCode;
+
+ HelperKeys(CharSequence keyCode) {
+ this.keyCode = keyCode;
+ }
+
+ public char charAt(int index) {
+ return index == 0 ? keyCode.charAt(index) : '\ue000';
+ }
+
+ public int length() {
+ return 1;
+ }
+
+ public CharSequence subSequence(int start, int end) {
+ if (start == 0 && end == 1) {
+ return String.valueOf(this.keyCode);
+ } else {
+ throw new IndexOutOfBoundsException();
+ }
+ }
+
+ public String toString() {
+ return String.valueOf(this.keyCode);
+ }
+
+ }
+
+ protected void handleException(String message, Exception e) throws Exception {
+ LOG.error(message, e);
+ File scrFile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE);
+ throw e;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/9b72c894/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java
index 0af660c..ee27db1 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java
@@ -227,9 +227,7 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
deleteTestNotebook(driver);
} catch (Exception e) {
- LOG.error("Exception in ParagraphActionsIT while testMoveUpAndDown ", e);
- File scrFile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE);
- throw e;
+ handleException("Exception in ParagraphActionsIT while testMoveUpAndDown ", e);
}
}
@@ -267,9 +265,7 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
deleteTestNotebook(driver);
} catch (Exception e) {
- LOG.error("Exception in ParagraphActionsIT while testDisableParagraphRunButton ", e);
- File scrFile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE);
- throw e;
+ handleException("Exception in ParagraphActionsIT while testDisableParagraphRunButton ", e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/9b72c894/zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java
new file mode 100644
index 0000000..e9f2bf8
--- /dev/null
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.integration;
+
+
+import org.apache.zeppelin.AbstractZeppelinIT;
+import org.apache.zeppelin.WebDriverManager;
+import org.hamcrest.CoreMatchers;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ErrorCollector;
+import org.openqa.selenium.By;
+import org.openqa.selenium.TimeoutException;
+import org.openqa.selenium.WebElement;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.zeppelin.AbstractZeppelinIT.HelperKeys.*;
+import static org.openqa.selenium.Keys.*;
+
+public class SparkParagraphIT extends AbstractZeppelinIT {
+ private static final Logger LOG = LoggerFactory.getLogger(SparkParagraphIT.class);
+
+
+ @Rule
+ public ErrorCollector collector = new ErrorCollector();
+
+ @Before
+ public void startUp() {
+ if (!endToEndTestEnabled()) {
+ return;
+ }
+ driver = WebDriverManager.getWebDriver();
+ createNewNote();
+ waitForParagraph(1, "READY");
+ }
+
+ @After
+ public void tearDown() {
+ if (!endToEndTestEnabled()) {
+ return;
+ }
+ deleteTestNotebook(driver);
+ driver.quit();
+ }
+
+ @Test
+ public void testSpark() throws Exception {
+ if (!endToEndTestEnabled()) {
+ return;
+ }
+ try {
+ WebElement paragraph1Editor = driver.findElement(By.xpath(getParagraphXPath(1) + "//textarea"));
+ paragraph1Editor.sendKeys("sc.version");
+ paragraph1Editor.sendKeys(SHIFT_ENTER);
+
+ waitForParagraph(1, "FINISHED");
+ WebElement paragraph1Result = driver.findElement(By.xpath(
+ getParagraphXPath(1) + "//div[@class=\"tableDisplay\"]"));
+ Float sparkVersion = Float.parseFloat(paragraph1Result.getText().split("= ")[1].substring(0, 3));
+
+ WebElement paragraph2Editor = driver.findElement(By.xpath(getParagraphXPath(2) + "//textarea"));
+
+
+ /*
+ equivalent of
+ import org.apache.commons.io.IOUtils
+ import java.net.URL
+ import java.nio.charset.Charset
+ val bankText = sc.parallelize(IOUtils.toString(new URL("https://s3.amazonaws.com/apache-zeppelin/tutorial/bank/bank.csv"),Charset.forName("utf8")).split("\n"))
+ case class Bank(age: Integer, job: String, marital: String, education: String, balance: Integer)
+
+ val bank = bankText.map(s => s.split(";")).filter(s => s(0) != "\"age\"").map(s => Bank(s(0).toInt,s(1).replaceAll("\"", ""),s(2).replaceAll("\"", ""),s(3).replaceAll("\"", ""),s(5).replaceAll("\"", "").toInt)).toDF()
+ bank.registerTempTable("bank")
+ */
+ paragraph2Editor.sendKeys("import org.apache.commons.io.IOUtils" +
+ ENTER +
+
+ "import java.net.URL" +
+ ENTER +
+
+ "import java.nio.charset.Charset" +
+ ENTER +
+
+ "val bankText = sc.parallelize" + OPEN_PARENTHESIS +
+ "IOUtils.toString" + OPEN_PARENTHESIS + "new URL" + OPEN_PARENTHESIS
+ + "\"https://s3.amazonaws.com/apache" + SUBTRACT + "zeppelin/tutorial/bank/bank." +
+ "csv\"),Charset.forName" + OPEN_PARENTHESIS + "\"utf8\"))" +
+ ".split" + OPEN_PARENTHESIS + "\"\\n\"))" +
+ ENTER +
+
+ "case class Bank" + OPEN_PARENTHESIS +
+ "age: Integer, job: String, marital: String, education: String, balance: Integer)" +
+ ENTER +
+ ENTER +
+
+ "val bank = bankText.map" + OPEN_PARENTHESIS + "s => s.split" +
+ OPEN_PARENTHESIS + "\";\")).filter" + OPEN_PARENTHESIS +
+ "s => s" + OPEN_PARENTHESIS + "0) " + EXCLAMATION +
+ "= \"\\\"age\\\"\").map" + OPEN_PARENTHESIS +
+ "s => Bank" + OPEN_PARENTHESIS + "s" + OPEN_PARENTHESIS +
+ "0).toInt,s" + OPEN_PARENTHESIS + "1).replaceAll" +
+ OPEN_PARENTHESIS + "\"\\\"\", \"\")," +
+ "s" + OPEN_PARENTHESIS + "2).replaceAll" +
+ OPEN_PARENTHESIS + "\"\\\"\", \"\")," +
+ "s" + OPEN_PARENTHESIS + "3).replaceAll" +
+ OPEN_PARENTHESIS + "\"\\\"\", \"\")," +
+ "s" + OPEN_PARENTHESIS + "5).replaceAll" +
+ OPEN_PARENTHESIS + "\"\\\"\", \"\").toInt" + ")" +
+ ")" + (sparkVersion < 1.3f ? "" : ".toDF" + OPEN_PARENTHESIS + ")") +
+ ENTER +
+
+ "bank.registerTempTable" + OPEN_PARENTHESIS + "\"bank\")"
+ );
+ paragraph2Editor.sendKeys("" + END + BACK_SPACE + BACK_SPACE +
+ BACK_SPACE + BACK_SPACE + BACK_SPACE + BACK_SPACE +
+ BACK_SPACE + BACK_SPACE + BACK_SPACE + BACK_SPACE + BACK_SPACE);
+
+ paragraph2Editor.sendKeys(SHIFT_ENTER);
+
+ try {
+ waitForParagraph(2, "FINISHED");
+ } catch (TimeoutException e) {
+ waitForParagraph(2, "ERROR");
+ collector.checkThat("2nd Paragraph from SparkParagraphIT of testSpark status:",
+ "ERROR", CoreMatchers.equalTo("FINISHED")
+ );
+ }
+
+ WebElement paragraph2Result = driver.findElement(By.xpath(
+ getParagraphXPath(2) + "//div[@class=\"tableDisplay\"]"));
+
+ collector.checkThat("2nd Paragraph from SparkParagraphIT of testSpark result: ",
+ paragraph2Result.getText().toString(), CoreMatchers.containsString(
+ "import org.apache.commons.io.IOUtils"
+ )
+ );
+
+ } catch (Exception e) {
+ handleException("Exception in SparkParagraphIT while testSpark", e);
+ }
+ }
+
+ @Test
+ public void testPySpark() throws Exception {
+ if (!endToEndTestEnabled()) {
+ return;
+ }
+ try {
+ WebElement paragraph1Editor = driver.findElement(By.xpath(getParagraphXPath(1) + "//textarea"));
+
+ paragraph1Editor.sendKeys(PERCENTAGE + "pyspark" + ENTER +
+ "for x in range" + OPEN_PARENTHESIS + "0, 3):" + ENTER +
+ " print \"test loop " + PERCENTAGE + "d\" " +
+ PERCENTAGE + " " + OPEN_PARENTHESIS + "x)" + ENTER);
+
+ paragraph1Editor.sendKeys(SHIFT_ENTER);
+
+ try {
+ waitForParagraph(1, "FINISHED");
+ } catch (TimeoutException e) {
+ waitForParagraph(1, "ERROR");
+ collector.checkThat("Paragraph from SparkParagraphIT of testPySpark status: ",
+ "ERROR", CoreMatchers.equalTo("FINISHED")
+ );
+ }
+
+ WebElement paragraph1Result = driver.findElement(By.xpath(
+ getParagraphXPath(1) + "//div[@class=\"tableDisplay\"]"));
+ collector.checkThat("Paragraph from SparkParagraphIT of testPySpark result: ",
+ paragraph1Result.getText().toString(), CoreMatchers.equalTo("test loop 0\ntest loop 1\ntest loop 2")
+ );
+
+ } catch (Exception e) {
+ handleException("Exception in SparkParagraphIT while testPySpark", e);
+ }
+ }
+
+ @Test
+ public void testSqlSpark() throws Exception {
+ if (!endToEndTestEnabled()) {
+ return;
+ }
+ try {
+ WebElement paragraph1Editor = driver.findElement(By.xpath(getParagraphXPath(1) + "//textarea"));
+
+ paragraph1Editor.sendKeys(PERCENTAGE + "sql" + ENTER +
+ "select * from bank limit 1");
+
+ paragraph1Editor.sendKeys(SHIFT_ENTER);
+
+ try {
+ waitForParagraph(1, "FINISHED");
+ } catch (TimeoutException e) {
+ waitForParagraph(1, "ERROR");
+ collector.checkThat("Paragraph from SparkParagraphIT of testSqlSpark status: ",
+ "ERROR", CoreMatchers.equalTo("FINISHED")
+ );
+ }
+
+ WebElement paragraph1Result = driver.findElement(By.xpath(
+ getParagraphXPath(1) + "//div[@class=\"tableDisplay\"]"));
+ collector.checkThat("Paragraph from SparkParagraphIT of testSqlSpark result: ",
+ paragraph1Result.getText().toString(), CoreMatchers.equalTo("age job marital education balance\n" +
+ "30 unemployed married primary 1,787")
+ );
+ } catch (Exception e) {
+ handleException("Exception in SparkParagraphIT while testSqlSpark", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/9b72c894/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java
index 4dd20e8..f01a5de 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java
@@ -196,9 +196,7 @@ public class ZeppelinIT extends AbstractZeppelinIT {
System.out.println("testCreateNotebook Test executed");
} catch (Exception e) {
- LOG.error("Exception in ZeppelinIT while testAngularDisplay ", e);
- File scrFile = ((TakesScreenshot)driver).getScreenshotAs(OutputType.FILE);
- throw e;
+ handleException("Exception in ZeppelinIT while testAngularDisplay ", e);
}
}
@@ -254,9 +252,7 @@ public class ZeppelinIT extends AbstractZeppelinIT {
driver.findElement(By.xpath("//button[contains(.,'Save')]")).submit();
driver.switchTo().alert().accept();
} catch (Exception e) {
- LOG.error("Exception in ZeppelinIT while testSparkInterpreterDependencyLoading ", e);
- File scrFile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE);
- throw e;
+ handleException("Exception in ZeppelinIT while testSparkInterpreterDependencyLoading ", e);
}
}
}