You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by fe...@apache.org on 2017/01/05 06:46:38 UTC
zeppelin git commit: ZEPPELIN-1866. No meaningful exception for pig
interpreter under tez engine
Repository: zeppelin
Updated Branches:
refs/heads/master a43815352 -> 1e7c05373
ZEPPELIN-1866. No meaningful exception for pig interpreter under tez engine
### What is this PR for?
This PR is to fix the bug that there's no meaningful exception when using tez engine.
### What type of PR is it?
[Bug Fix | Improvement ]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-1866
### How should this be tested?
Unit test is added and also manually test it.
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <zj...@apache.org>
Closes #1809 from zjffdu/ZEPPELIN-1866 and squashes the following commits:
34901ef [Jeff Zhang] ZEPPELIN-1866. No meaningful exception for pig interpreter under tez engine
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/1e7c0537
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/1e7c0537
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/1e7c0537
Branch: refs/heads/master
Commit: 1e7c0537390f8630788db2006a5bafe359ef5a15
Parents: a438153
Author: Jeff Zhang <zj...@apache.org>
Authored: Wed Dec 28 17:44:47 2016 +0800
Committer: Felix Cheung <fe...@apache.org>
Committed: Wed Jan 4 22:46:23 2017 -0800
----------------------------------------------------------------------
pig/pom.xml | 7 +
.../java/org/apache/zeppelin/pig/PigUtils.java | 9 ++
.../apache/zeppelin/pig/PigInterpreterTest.java | 13 +-
.../zeppelin/pig/PigInterpreterTezTest.java | 152 +++++++++++++++++++
pig/src/test/resources/core-site.xml | 3 +
5 files changed, 177 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/1e7c0537/pig/pom.xml
----------------------------------------------------------------------
diff --git a/pig/pom.xml b/pig/pom.xml
index f8050cd..1dbfa14 100644
--- a/pig/pom.xml
+++ b/pig/pom.xml
@@ -177,6 +177,13 @@
</execution>
</executions>
</plugin>
+
+ <plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <forkMode>always</forkMode>
+ </configuration>
+ </plugin>
</plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/1e7c0537/pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java
----------------------------------------------------------------------
diff --git a/pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java b/pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java
index d444e02..3398281 100644
--- a/pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java
+++ b/pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java
@@ -171,6 +171,15 @@ public class PigUtils {
private static String extractFromTezPigStats(TezPigScriptStats stats) {
try {
+ if (stats.getReturnCode() == PigRunner.ReturnCode.UNKNOWN) {
+ LOGGER.warn("unknown return code, can't display the results");
+ return null;
+ }
+ if (stats.getPigContext() == null) {
+ LOGGER.warn("unknown exec type, don't display the results");
+ return null;
+ }
+
Field userIdField = PigStats.class.getDeclaredField("userId");
userIdField.setAccessible(true);
String userId = (String) (userIdField.get(stats));
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/1e7c0537/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java
----------------------------------------------------------------------
diff --git a/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java
index 2f1aab2..efcbb58 100644
--- a/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java
+++ b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java
@@ -41,10 +41,10 @@ public class PigInterpreterTest {
private PigInterpreter pigInterpreter;
private InterpreterContext context;
- @Before
- public void setUp() {
+ private void setUpLocal(boolean includeJobStats) {
Properties properties = new Properties();
properties.put("zeppelin.pig.execType", "local");
+ properties.put("zeppelin.pig.includeJobStats", includeJobStats + "");
pigInterpreter = new PigInterpreter(properties);
pigInterpreter.open();
context = new InterpreterContext(null, "paragraph_id", null, null, null, null, null, null, null, null,
@@ -58,6 +58,8 @@ public class PigInterpreterTest {
@Test
public void testBasics() throws IOException {
+ setUpLocal(false);
+
String content = "1\tandy\n"
+ "2\tpeter\n";
File tmpFile = File.createTempFile("zeppelin", "test");
@@ -101,11 +103,7 @@ public class PigInterpreterTest {
@Test
public void testIncludeJobStats() throws IOException {
- Properties properties = new Properties();
- properties.put("zeppelin.pig.execType", "local");
- properties.put("zeppelin.pig.includeJobStats", "true");
- pigInterpreter = new PigInterpreter(properties);
- pigInterpreter.open();
+ setUpLocal(true);
String content = "1\tandy\n"
+ "2\tpeter\n";
@@ -152,4 +150,5 @@ public class PigInterpreterTest {
assertTrue(result.message().get(0).getData().contains("Counters:"));
assertTrue(result.message().get(0).getData().contains("Input path does not exist"));
}
+
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/1e7c0537/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTezTest.java
----------------------------------------------------------------------
diff --git a/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTezTest.java b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTezTest.java
new file mode 100644
index 0000000..e742fd8
--- /dev/null
+++ b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTezTest.java
@@ -0,0 +1,152 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.zeppelin.pig;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResult.Code;
+import org.apache.zeppelin.interpreter.InterpreterResult.Type;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class PigInterpreterTezTest {
+
+ private PigInterpreter pigInterpreter;
+ private InterpreterContext context;
+
+ public void setUpTez(boolean includeJobStats) {
+ Properties properties = new Properties();
+ properties.put("zeppelin.pig.execType", "tez_local");
+ properties.put("zeppelin.pig.includeJobStats", includeJobStats + "");
+ pigInterpreter = new PigInterpreter(properties);
+ pigInterpreter.open();
+ context = new InterpreterContext(null, "paragraph_id", null, null, null, null, null, null, null, null,
+ null, null);
+
+ }
+ @After
+ public void tearDown() {
+ pigInterpreter.close();
+ }
+
+ @Test
+ public void testBasics() throws IOException {
+ setUpTez(false);
+
+ String content = "1\tandy\n"
+ + "2\tpeter\n";
+ File tmpFile = File.createTempFile("zeppelin", "test");
+ FileWriter writer = new FileWriter(tmpFile);
+ IOUtils.write(content, writer);
+ writer.close();
+
+ // simple pig script using dump
+ String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';"
+ + "dump a;";
+ InterpreterResult result = pigInterpreter.interpret(pigscript, context);
+ assertEquals(Type.TEXT, result.message().get(0).getType());
+ assertEquals(Code.SUCCESS, result.code());
+ assertTrue(result.message().get(0).getData().contains("(1,andy)\n(2,peter)"));
+
+ // describe
+ pigscript = "a = load '" + tmpFile.getAbsolutePath() + "' as (id: int, name: bytearray);"
+ + "describe a;";
+ result = pigInterpreter.interpret(pigscript, context);
+ assertEquals(Type.TEXT, result.message().get(0).getType());
+ assertEquals(Code.SUCCESS, result.code());
+ assertTrue(result.message().get(0).getData().contains("a: {id: int,name: bytearray}"));
+
+ // syntax error (compilation error)
+ pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';"
+ + "describe a;";
+ result = pigInterpreter.interpret(pigscript, context);
+ assertEquals(Type.TEXT, result.message().get(0).getType());
+ assertEquals(Code.ERROR, result.code());
+ assertTrue(result.message().get(0).getData().contains("Syntax error, unexpected symbol at or near 'a'"));
+
+ // syntax error
+ pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';"
+ + "foreach a generate $0;";
+ result = pigInterpreter.interpret(pigscript, context);
+ assertEquals(Type.TEXT, result.message().get(0).getType());
+ assertEquals(Code.ERROR, result.code());
+ assertTrue(result.message().get(0).getData().contains("expecting one of"));
+ }
+
+ @Test
+ public void testIncludeJobStats() throws IOException {
+ setUpTez(true);
+
+ String content = "1\tandy\n"
+ + "2\tpeter\n";
+ File tmpFile = File.createTempFile("zeppelin", "test");
+ FileWriter writer = new FileWriter(tmpFile);
+ IOUtils.write(content, writer);
+ writer.close();
+
+ // simple pig script using dump
+ String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';"
+ + "dump a;";
+ InterpreterResult result = pigInterpreter.interpret(pigscript, context);
+ assertEquals(Type.TEXT, result.message().get(0).getType());
+ assertEquals(Code.SUCCESS, result.code());
+ assertTrue(result.message().get(0).getData().contains("Vertex Stats"));
+ assertTrue(result.message().get(0).getData().contains("(1,andy)\n(2,peter)"));
+
+ // describe
+ pigscript = "a = load '" + tmpFile.getAbsolutePath() + "' as (id: int, name: bytearray);"
+ + "describe a;";
+ result = pigInterpreter.interpret(pigscript, context);
+ assertEquals(Type.TEXT, result.message().get(0).getType());
+ assertEquals(Code.SUCCESS, result.code());
+ // no job is launched, so no jobStats
+ assertTrue(!result.message().get(0).getData().contains("Vertex Stats"));
+ assertTrue(result.message().get(0).getData().contains("a: {id: int,name: bytearray}"));
+
+ // syntax error (compilation error)
+ pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';"
+ + "describe a;";
+ result = pigInterpreter.interpret(pigscript, context);
+ assertEquals(Type.TEXT, result.message().get(0).getType());
+ assertEquals(Code.ERROR, result.code());
+ // no job is launched, so no jobStats
+ assertTrue(!result.message().get(0).getData().contains("Vertex Stats"));
+ assertTrue(result.message().get(0).getData().contains("Syntax error, unexpected symbol at or near 'a'"));
+
+ // execution error
+ pigscript = "a = load 'invalid_path';"
+ + "dump a;";
+ result = pigInterpreter.interpret(pigscript, context);
+ assertEquals(Type.TEXT, result.message().get(0).getType());
+ assertEquals(Code.ERROR, result.code());
+ assertTrue(!result.message().get(0).getData().contains("Vertex Stats"));
+ assertTrue(result.message().get(0).getData().contains("Input path does not exist"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/1e7c0537/pig/src/test/resources/core-site.xml
----------------------------------------------------------------------
diff --git a/pig/src/test/resources/core-site.xml b/pig/src/test/resources/core-site.xml
new file mode 100644
index 0000000..f1fe563
--- /dev/null
+++ b/pig/src/test/resources/core-site.xml
@@ -0,0 +1,3 @@
+<configuration>
+
+</configuration>
\ No newline at end of file