You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by jo...@apache.org on 2018/08/29 10:07:37 UTC
[41/50] [abbrv] zeppelin git commit: Revert "[ZEPPELIN-3740] Adopt
`google-java-format` and `fmt-maven-plugin`"
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/neo4j/src/test/java/org/apache/zeppelin/graph/neo4j/Neo4jCypherInterpreterTest.java
----------------------------------------------------------------------
diff --git a/neo4j/src/test/java/org/apache/zeppelin/graph/neo4j/Neo4jCypherInterpreterTest.java b/neo4j/src/test/java/org/apache/zeppelin/graph/neo4j/Neo4jCypherInterpreterTest.java
index e398568..24bd513 100644
--- a/neo4j/src/test/java/org/apache/zeppelin/graph/neo4j/Neo4jCypherInterpreterTest.java
+++ b/neo4j/src/test/java/org/apache/zeppelin/graph/neo4j/Neo4jCypherInterpreterTest.java
@@ -16,12 +16,7 @@
*/
package org.apache.zeppelin.graph.neo4j;
-import static org.junit.Assert.assertEquals;
-
import com.google.gson.Gson;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Properties;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.graph.neo4j.Neo4jConnectionManager.Neo4jAuthType;
import org.apache.zeppelin.interpreter.InterpreterContext;
@@ -39,6 +34,12 @@ import org.junit.runners.MethodSorters;
import org.neo4j.harness.ServerControls;
import org.neo4j.harness.TestServerBuilders;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class Neo4jCypherInterpreterTest {
@@ -54,27 +55,25 @@ public class Neo4jCypherInterpreterTest {
private static final String REL_KNOWS = "KNOWS";
private static final String CYPHER_FOREACH =
- "FOREACH (x in range(1,1000) | CREATE (:%s{name: \"name\" + x, age: %s}))";
- private static final String CHPHER_UNWIND =
- "UNWIND range(1,1000) as x "
- + "MATCH (n), (m) WHERE id(n) = x AND id(m) = toInt(rand() * 1000) "
- + "CREATE (n)-[:%s]->(m)";
+ "FOREACH (x in range(1,1000) | CREATE (:%s{name: \"name\" + x, age: %s}))";
+ private static final String CHPHER_UNWIND = "UNWIND range(1,1000) as x "
+ + "MATCH (n), (m) WHERE id(n) = x AND id(m) = toInt(rand() * 1000) "
+ + "CREATE (n)-[:%s]->(m)";
@BeforeClass
public static void setUpNeo4jServer() throws Exception {
- server =
- TestServerBuilders.newInProcessBuilder()
- .withConfig("dbms.security.auth_enabled", "false")
- .withFixture(String.format(CYPHER_FOREACH, LABEL_PERSON, "x % 10"))
- .withFixture(String.format(CHPHER_UNWIND, REL_KNOWS))
- .newServer();
+ server = TestServerBuilders.newInProcessBuilder()
+ .withConfig("dbms.security.auth_enabled", "false")
+ .withFixture(String.format(CYPHER_FOREACH, LABEL_PERSON, "x % 10"))
+ .withFixture(String.format(CHPHER_UNWIND, REL_KNOWS))
+ .newServer();
}
@AfterClass
public static void tearDownNeo4jServer() throws Exception {
server.close();
}
-
+
@Before
public void setUpZeppelin() {
Properties p = new Properties();
@@ -82,8 +81,9 @@ public class Neo4jCypherInterpreterTest {
p.setProperty(Neo4jConnectionManager.NEO4J_AUTH_TYPE, Neo4jAuthType.NONE.toString());
p.setProperty(Neo4jConnectionManager.NEO4J_MAX_CONCURRENCY, "50");
interpreter = new Neo4jCypherInterpreter(p);
- context = InterpreterContext.builder().setInterpreterOut(new InterpreterOutput(null)).build();
- ;
+ context = InterpreterContext.builder()
+ .setInterpreterOut(new InterpreterOutput(null))
+ .build();;
}
@After
@@ -94,18 +94,17 @@ public class Neo4jCypherInterpreterTest {
@Test
public void testTableWithArray() {
interpreter.open();
- InterpreterResult result =
- interpreter.interpret("return 'a' as colA, 'b' as colB, [1, 2, 3] as colC", context);
+ InterpreterResult result = interpreter.interpret(
+ "return 'a' as colA, 'b' as colB, [1, 2, 3] as colC", context);
assertEquals(Code.SUCCESS, result.code());
final String tableResult = "colA\tcolB\tcolC\n\"a\"\t\"b\"\t[1,2,3]\n";
assertEquals(tableResult, result.toString().replace("%table ", StringUtils.EMPTY));
-
- result =
- interpreter.interpret(
+
+ result = interpreter.interpret(
"return 'a' as colA, 'b' as colB, [{key: \"value\"}, {key: 1}] as colC", context);
assertEquals(Code.SUCCESS, result.code());
final String tableResultWithMap =
- "colA\tcolB\tcolC\n\"a\"\t\"b\"\t[{\"key\":\"value\"},{\"key\":1}]\n";
+ "colA\tcolB\tcolC\n\"a\"\t\"b\"\t[{\"key\":\"value\"},{\"key\":1}]\n";
assertEquals(tableResultWithMap, result.toString().replace("%table ", StringUtils.EMPTY));
}
@@ -120,12 +119,9 @@ public class Neo4jCypherInterpreterTest {
@Test
public void testRenderTable() {
interpreter.open();
- InterpreterResult result =
- interpreter.interpret(
- "MATCH (n:Person) "
- + "WHERE n.name IN ['name1', 'name2', 'name3'] "
- + "RETURN n.name AS name, n.age AS age",
- context);
+ InterpreterResult result = interpreter.interpret("MATCH (n:Person) "
+ + "WHERE n.name IN ['name1', 'name2', 'name3'] "
+ + "RETURN n.name AS name, n.age AS age", context);
assertEquals(Code.SUCCESS, result.code());
final String tableResult = "name\tage\n\"name1\"\t1\n\"name2\"\t2\n\"name3\"\t3\n";
assertEquals(tableResult, result.toString().replace("%table ", StringUtils.EMPTY));
@@ -135,15 +131,12 @@ public class Neo4jCypherInterpreterTest {
public void testRenderMap() {
interpreter.open();
final String jsonQuery =
- "RETURN {key: \"value\", listKey: [{inner: \"Map1\"}, {inner: \"Map2\"}]} as object";
+ "RETURN {key: \"value\", listKey: [{inner: \"Map1\"}, {inner: \"Map2\"}]} as object";
final String objectKey = "object.key";
final String objectListKey = "object.listKey";
InterpreterResult result = interpreter.interpret(jsonQuery, context);
assertEquals(Code.SUCCESS, result.code());
- String[] rows =
- result
- .toString()
- .replace("%table ", StringUtils.EMPTY)
+ String[] rows = result.toString().replace("%table ", StringUtils.EMPTY)
.split(Neo4jCypherInterpreter.NEW_LINE);
assertEquals(rows.length, 2);
List<String> header = Arrays.asList(rows[0].split(Neo4jCypherInterpreter.TAB));
@@ -152,19 +145,15 @@ public class Neo4jCypherInterpreterTest {
List<String> row = Arrays.asList(rows[1].split(Neo4jCypherInterpreter.TAB));
assertEquals(row.size(), header.size());
assertEquals(row.get(header.indexOf(objectKey)), "value");
- assertEquals(
- row.get(header.indexOf(objectListKey)), "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]");
+ assertEquals(row.get(header.indexOf(objectListKey)),
+ "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]");
- final String query =
- "WITH [{key: \"value\", listKey: [{inner: \"Map1\"}, {inner: \"Map2\"}]},"
+ final String query = "WITH [{key: \"value\", listKey: [{inner: \"Map1\"}, {inner: \"Map2\"}]},"
+ "{key: \"value2\", listKey: [{inner: \"Map12\"}, {inner: \"Map22\"}]}] "
+ "AS array UNWIND array AS object RETURN object";
result = interpreter.interpret(query, context);
assertEquals(Code.SUCCESS, result.code());
- rows =
- result
- .toString()
- .replace("%table ", StringUtils.EMPTY)
+ rows = result.toString().replace("%table ", StringUtils.EMPTY)
.split(Neo4jCypherInterpreter.NEW_LINE);
assertEquals(rows.length, 3);
header = Arrays.asList(rows[0].split(Neo4jCypherInterpreter.TAB));
@@ -173,24 +162,20 @@ public class Neo4jCypherInterpreterTest {
row = Arrays.asList(rows[1].split(Neo4jCypherInterpreter.TAB));
assertEquals(row.size(), header.size());
assertEquals(row.get(header.indexOf(objectKey)), "value");
- assertEquals(
- row.get(header.indexOf(objectListKey)), "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]");
+ assertEquals(row.get(header.indexOf(objectListKey)),
+ "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]");
row = Arrays.asList(rows[2].split(Neo4jCypherInterpreter.TAB));
assertEquals(row.size(), header.size());
assertEquals(row.get(header.indexOf(objectKey)), "value2");
- assertEquals(
- row.get(header.indexOf(objectListKey)), "[{\"inner\":\"Map12\"},{\"inner\":\"Map22\"}]");
+ assertEquals(row.get(header.indexOf(objectListKey)),
+ "[{\"inner\":\"Map12\"},{\"inner\":\"Map22\"}]");
- final String jsonListWithNullQuery =
- "WITH [{key: \"value\", listKey: null},"
+ final String jsonListWithNullQuery = "WITH [{key: \"value\", listKey: null},"
+ "{key: \"value2\", listKey: [{inner: \"Map1\"}, {inner: \"Map2\"}]}] "
+ "AS array UNWIND array AS object RETURN object";
result = interpreter.interpret(jsonListWithNullQuery, context);
assertEquals(Code.SUCCESS, result.code());
- rows =
- result
- .toString()
- .replace("%table ", StringUtils.EMPTY)
+ rows = result.toString().replace("%table ", StringUtils.EMPTY)
.split(Neo4jCypherInterpreter.NEW_LINE);
assertEquals(rows.length, 3);
header = Arrays.asList(rows[0].split(Neo4jCypherInterpreter.TAB, -1));
@@ -204,19 +189,15 @@ public class Neo4jCypherInterpreterTest {
row = Arrays.asList(rows[2].split(Neo4jCypherInterpreter.TAB, -1));
assertEquals(row.size(), header.size());
assertEquals(row.get(header.indexOf(objectKey)), "value2");
- assertEquals(
- row.get(header.indexOf(objectListKey)), "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]");
-
- final String jsonListWithoutListKeyQuery =
- "WITH [{key: \"value\"},"
+ assertEquals(row.get(header.indexOf(objectListKey)),
+ "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]");
+
+ final String jsonListWithoutListKeyQuery = "WITH [{key: \"value\"},"
+ "{key: \"value2\", listKey: [{inner: \"Map1\"}, {inner: \"Map2\"}]}] "
+ "AS array UNWIND array AS object RETURN object";
result = interpreter.interpret(jsonListWithoutListKeyQuery, context);
assertEquals(Code.SUCCESS, result.code());
- rows =
- result
- .toString()
- .replace("%table ", StringUtils.EMPTY)
+ rows = result.toString().replace("%table ", StringUtils.EMPTY)
.split(Neo4jCypherInterpreter.NEW_LINE);
assertEquals(rows.length, 3);
header = Arrays.asList(rows[0].split(Neo4jCypherInterpreter.TAB, -1));
@@ -229,18 +210,17 @@ public class Neo4jCypherInterpreterTest {
row = Arrays.asList(rows[2].split(Neo4jCypherInterpreter.TAB, -1));
assertEquals(row.size(), header.size());
assertEquals(row.get(header.indexOf(objectKey)), "value2");
- assertEquals(
- row.get(header.indexOf(objectListKey)), "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]");
+ assertEquals(row.get(header.indexOf(objectListKey)),
+ "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]");
}
@Test
public void testRenderNetwork() {
interpreter.open();
- InterpreterResult result =
- interpreter.interpret("MATCH (n)-[r:KNOWS]-(m) RETURN n, r, m LIMIT 1", context);
- GraphResult.Graph graph =
- gson.fromJson(
- result.toString().replace("%network ", StringUtils.EMPTY), GraphResult.Graph.class);
+ InterpreterResult result = interpreter.interpret(
+ "MATCH (n)-[r:KNOWS]-(m) RETURN n, r, m LIMIT 1", context);
+ GraphResult.Graph graph = gson.fromJson(result.toString().replace("%network ",
+ StringUtils.EMPTY), GraphResult.Graph.class);
assertEquals(2, graph.getNodes().size());
assertEquals(true, graph.getNodes().iterator().next().getLabel().equals(LABEL_PERSON));
assertEquals(1, graph.getEdges().size());
@@ -264,9 +244,8 @@ public class Neo4jCypherInterpreterTest {
assertEquals(Code.SUCCESS, result.code());
assertEquals(errorMsgEmpty, result.toString());
- result =
- interpreter.interpret(
- "MATCH (n:Person{name: }) RETURN n.name AS name, n.age AS age", context);
+ result = interpreter.interpret("MATCH (n:Person{name: }) RETURN n.name AS name, n.age AS age",
+ context);
assertEquals(Code.ERROR, result.code());
}
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/pig/pom.xml
----------------------------------------------------------------------
diff --git a/pig/pom.xml b/pig/pom.xml
index 4553b5c..571d198 100644
--- a/pig/pom.xml
+++ b/pig/pom.xml
@@ -190,6 +190,13 @@
<forkMode>always</forkMode>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <configuration>
+ <skip>false</skip>
+ </configuration>
+ </plugin>
</plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/pig/src/main/java/org/apache/zeppelin/pig/BasePigInterpreter.java
----------------------------------------------------------------------
diff --git a/pig/src/main/java/org/apache/zeppelin/pig/BasePigInterpreter.java b/pig/src/main/java/org/apache/zeppelin/pig/BasePigInterpreter.java
index 97d15de..9503aa7 100644
--- a/pig/src/main/java/org/apache/zeppelin/pig/BasePigInterpreter.java
+++ b/pig/src/main/java/org/apache/zeppelin/pig/BasePigInterpreter.java
@@ -17,24 +17,28 @@
package org.apache.zeppelin.pig;
-import java.lang.reflect.Field;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.pig.PigServer;
import org.apache.pig.backend.BackendException;
import org.apache.pig.backend.hadoop.executionengine.HExecutionEngine;
import org.apache.pig.backend.hadoop.executionengine.Launcher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.Field;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-/** */
+/**
+ *
+ */
public abstract class BasePigInterpreter extends Interpreter {
private static final Logger LOGGER = LoggerFactory.getLogger(BasePigInterpreter.class);
@@ -56,7 +60,7 @@ public abstract class BasePigInterpreter extends Interpreter {
for (String jobId : jobIds) {
LOGGER.info("Kill jobId:" + jobId);
HExecutionEngine engine =
- (HExecutionEngine) getPigServer().getPigContext().getExecutionEngine();
+ (HExecutionEngine) getPigServer().getPigContext().getExecutionEngine();
try {
Field launcherField = HExecutionEngine.class.getDeclaredField("launcher");
launcherField.setAccessible(true);
@@ -68,8 +72,8 @@ public abstract class BasePigInterpreter extends Interpreter {
}
}
} else {
- LOGGER.warn(
- "No PigScriptListener found, can not cancel paragraph:" + context.getParagraphId());
+ LOGGER.warn("No PigScriptListener found, can not cancel paragraph:"
+ + context.getParagraphId());
}
}
@@ -89,15 +93,14 @@ public abstract class BasePigInterpreter extends Interpreter {
@Override
public Scheduler getScheduler() {
- return SchedulerFactory.singleton()
- .createOrGetFIFOScheduler(PigInterpreter.class.getName() + this.hashCode());
+ return SchedulerFactory.singleton().createOrGetFIFOScheduler(
+ PigInterpreter.class.getName() + this.hashCode());
}
public abstract PigServer getPigServer();
/**
* Use paragraph title if it exists, else use the last line of pig script.
- *
* @param cmd
* @param context
* @return
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/pig/src/main/java/org/apache/zeppelin/pig/PigInterpreter.java
----------------------------------------------------------------------
diff --git a/pig/src/main/java/org/apache/zeppelin/pig/PigInterpreter.java b/pig/src/main/java/org/apache/zeppelin/pig/PigInterpreter.java
index 5f79e97..4fc0676 100644
--- a/pig/src/main/java/org/apache/zeppelin/pig/PigInterpreter.java
+++ b/pig/src/main/java/org/apache/zeppelin/pig/PigInterpreter.java
@@ -17,11 +17,6 @@
package org.apache.zeppelin.pig;
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.Map;
-import java.util.Properties;
import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.pig.PigServer;
@@ -29,13 +24,22 @@ import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.tools.pigscript.parser.ParseException;
import org.apache.pig.tools.pigstats.PigStats;
import org.apache.pig.tools.pigstats.ScriptState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.Map;
+import java.util.Properties;
+
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-/** Pig interpreter for Zeppelin. */
+/**
+ * Pig interpreter for Zeppelin.
+ */
public class PigInterpreter extends BasePigInterpreter {
private static final Logger LOGGER = LoggerFactory.getLogger(PigInterpreter.class);
@@ -60,10 +64,8 @@ public class PigInterpreter extends BasePigInterpreter {
pigServer = new PigServer(execType);
for (Map.Entry entry : getProperties().entrySet()) {
if (!entry.getKey().toString().startsWith("zeppelin.")) {
- pigServer
- .getPigContext()
- .getProperties()
- .setProperty(entry.getKey().toString(), entry.getValue().toString());
+ pigServer.getPigContext().getProperties().setProperty(entry.getKey().toString(),
+ entry.getValue().toString());
}
}
} catch (IOException e) {
@@ -77,6 +79,7 @@ public class PigInterpreter extends BasePigInterpreter {
pigServer = null;
}
+
@Override
public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) {
// remember the origial stdout, because we will redirect stdout to capture
@@ -143,7 +146,10 @@ public class PigInterpreter extends BasePigInterpreter {
return new InterpreterResult(Code.SUCCESS, outputBuilder.toString());
}
+
public PigServer getPigServer() {
return pigServer;
}
+
}
+
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java
----------------------------------------------------------------------
diff --git a/pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java b/pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java
index 0c6eef6..97c2f6e 100644
--- a/pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java
+++ b/pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java
@@ -17,12 +17,6 @@
package org.apache.zeppelin.pig;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Properties;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.pig.PigServer;
@@ -32,15 +26,25 @@ import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.tools.pigscript.parser.ParseException;
import org.apache.pig.tools.pigstats.PigStats;
import org.apache.pig.tools.pigstats.ScriptState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Properties;
+
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.ResultMessages;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-/** */
+/**
+ *
+ */
public class PigQueryInterpreter extends BasePigInterpreter {
private static final Logger LOGGER = LoggerFactory.getLogger(PigQueryInterpreter.class);
private static final String MAX_RESULTS = "zeppelin.pig.maxResult";
@@ -58,7 +62,9 @@ public class PigQueryInterpreter extends BasePigInterpreter {
}
@Override
- public void close() {}
+ public void close() {
+
+ }
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/pig/src/main/java/org/apache/zeppelin/pig/PigScriptListener.java
----------------------------------------------------------------------
diff --git a/pig/src/main/java/org/apache/zeppelin/pig/PigScriptListener.java b/pig/src/main/java/org/apache/zeppelin/pig/PigScriptListener.java
index 0e6068a..8ff1bf8 100644
--- a/pig/src/main/java/org/apache/zeppelin/pig/PigScriptListener.java
+++ b/pig/src/main/java/org/apache/zeppelin/pig/PigScriptListener.java
@@ -17,8 +17,6 @@
package org.apache.zeppelin.pig;
-import java.util.HashSet;
-import java.util.Set;
import org.apache.pig.impl.plan.OperatorPlan;
import org.apache.pig.tools.pigstats.JobStats;
import org.apache.pig.tools.pigstats.OutputStats;
@@ -26,7 +24,12 @@ import org.apache.pig.tools.pigstats.PigProgressNotificationListener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-/** */
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ *
+ */
public class PigScriptListener implements PigProgressNotificationListener {
private static final Logger LOGGER = LoggerFactory.getLogger(PigScriptListener.class);
@@ -34,13 +37,19 @@ public class PigScriptListener implements PigProgressNotificationListener {
private int progress;
@Override
- public void initialPlanNotification(String scriptId, OperatorPlan<?> plan) {}
+ public void initialPlanNotification(String scriptId, OperatorPlan<?> plan) {
+
+ }
@Override
- public void launchStartedNotification(String scriptId, int numJobsToLaunch) {}
+ public void launchStartedNotification(String scriptId, int numJobsToLaunch) {
+
+ }
@Override
- public void jobsSubmittedNotification(String scriptId, int numJobsSubmitted) {}
+ public void jobsSubmittedNotification(String scriptId, int numJobsSubmitted) {
+
+ }
@Override
public void jobStartedNotification(String scriptId, String assignedJobId) {
@@ -48,13 +57,19 @@ public class PigScriptListener implements PigProgressNotificationListener {
}
@Override
- public void jobFinishedNotification(String scriptId, JobStats jobStats) {}
+ public void jobFinishedNotification(String scriptId, JobStats jobStats) {
+
+ }
@Override
- public void jobFailedNotification(String scriptId, JobStats jobStats) {}
+ public void jobFailedNotification(String scriptId, JobStats jobStats) {
+
+ }
@Override
- public void outputCompletedNotification(String scriptId, OutputStats outputStats) {}
+ public void outputCompletedNotification(String scriptId, OutputStats outputStats) {
+
+ }
@Override
public void progressUpdatedNotification(String scriptId, int progress) {
@@ -63,7 +78,9 @@ public class PigScriptListener implements PigProgressNotificationListener {
}
@Override
- public void launchCompletedNotification(String scriptId, int numJobsSucceeded) {}
+ public void launchCompletedNotification(String scriptId, int numJobsSucceeded) {
+
+ }
public Set<String> getJobIds() {
return jobIds;
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java
----------------------------------------------------------------------
diff --git a/pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java b/pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java
index 28fd763..1c48250 100644
--- a/pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java
+++ b/pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java
@@ -17,16 +17,19 @@
package org.apache.zeppelin.pig;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-/** */
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.List;
+
+/**
+ *
+ */
public class PigUtils {
private static final Logger LOGGER = LoggerFactory.getLogger(PigUtils.class);
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterSparkTest.java
----------------------------------------------------------------------
diff --git a/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterSparkTest.java b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterSparkTest.java
index 1fd5c82..ea1a3f8 100644
--- a/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterSparkTest.java
+++ b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterSparkTest.java
@@ -1,31 +1,37 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * <p>http://www.apache.org/licenses/LICENSE-2.0
- *
- * <p>Unless required by applicable law or agreed to in writing, software distributed under the
- * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing permissions and
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.zeppelin.pig;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import org.apache.commons.io.IOUtils;
+import org.junit.After;
+import org.junit.Test;
+
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Properties;
-import org.apache.commons.io.IOUtils;
+
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.junit.After;
-import org.junit.Test;
public class PigInterpreterSparkTest {
private PigInterpreter pigInterpreter;
@@ -38,8 +44,8 @@ public class PigInterpreterSparkTest {
pigInterpreter = new PigInterpreter(properties);
pigInterpreter.open();
context = InterpreterContext.builder().setParagraphId("paragraphId").build();
- }
+ }
@After
public void tearDown() {
pigInterpreter.close();
@@ -49,44 +55,41 @@ public class PigInterpreterSparkTest {
public void testBasics() throws IOException {
setUpSpark(false);
- String content = "1\tandy\n" + "2\tpeter\n";
+ String content = "1\tandy\n"
+ + "2\tpeter\n";
File tmpFile = File.createTempFile("zeppelin", "test");
FileWriter writer = new FileWriter(tmpFile);
IOUtils.write(content, writer);
writer.close();
// simple pig script using dump
- String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" + "dump a;";
+ String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';"
+ + "dump a;";
InterpreterResult result = pigInterpreter.interpret(pigscript, context);
assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertTrue(result.message().get(0).getData().contains("(1,andy)\n(2,peter)"));
// describe
- pigscript =
- "a = load '"
- + tmpFile.getAbsolutePath()
- + "' as (id: int, name: bytearray);"
- + "describe a;";
+ pigscript = "a = load '" + tmpFile.getAbsolutePath() + "' as (id: int, name: bytearray);"
+ + "describe a;";
result = pigInterpreter.interpret(pigscript, context);
assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertTrue(result.message().get(0).getData().contains("a: {id: int,name: bytearray}"));
// syntax error (compilation error)
- pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';" + "describe a;";
+ pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';"
+ + "describe a;";
result = pigInterpreter.interpret(pigscript, context);
assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
assertEquals(InterpreterResult.Code.ERROR, result.code());
- assertTrue(
- result
- .message()
- .get(0)
- .getData()
- .contains("Syntax error, unexpected symbol at or near 'a'"));
+ assertTrue(result.message().get(0).getData().contains(
+ "Syntax error, unexpected symbol at or near 'a'"));
// syntax error
- pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" + "foreach a generate $0;";
+ pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';"
+ + "foreach a generate $0;";
result = pigInterpreter.interpret(pigscript, context);
assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
assertEquals(InterpreterResult.Code.ERROR, result.code());
@@ -97,14 +100,16 @@ public class PigInterpreterSparkTest {
public void testIncludeJobStats() throws IOException {
setUpSpark(true);
- String content = "1\tandy\n" + "2\tpeter\n";
+ String content = "1\tandy\n"
+ + "2\tpeter\n";
File tmpFile = File.createTempFile("zeppelin", "test");
FileWriter writer = new FileWriter(tmpFile);
IOUtils.write(content, writer);
writer.close();
// simple pig script using dump
- String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" + "dump a;";
+ String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';"
+ + "dump a;";
InterpreterResult result = pigInterpreter.interpret(pigscript, context);
assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
@@ -112,11 +117,8 @@ public class PigInterpreterSparkTest {
assertTrue(result.message().get(0).getData().contains("(1,andy)\n(2,peter)"));
// describe
- pigscript =
- "a = load '"
- + tmpFile.getAbsolutePath()
- + "' as (id: int, name: bytearray);"
- + "describe a;";
+ pigscript = "a = load '" + tmpFile.getAbsolutePath() + "' as (id: int, name: bytearray);"
+ + "describe a;";
result = pigInterpreter.interpret(pigscript, context);
assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
@@ -124,23 +126,24 @@ public class PigInterpreterSparkTest {
assertTrue(result.message().get(0).getData().contains("a: {id: int,name: bytearray}"));
// syntax error (compilation error)
- pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';" + "describe a;";
+ pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';"
+ + "describe a;";
result = pigInterpreter.interpret(pigscript, context);
assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
assertEquals(InterpreterResult.Code.ERROR, result.code());
// no job is launched, so no jobStats
- assertTrue(
- result
- .message()
- .get(0)
- .getData()
- .contains("Syntax error, unexpected symbol at or near 'a'"));
+ assertTrue(result.message().get(0).getData().contains(
+ "Syntax error, unexpected symbol at or near 'a'"));
// execution error
- pigscript = "a = load 'invalid_path';" + "dump a;";
+ pigscript = "a = load 'invalid_path';"
+ + "dump a;";
result = pigInterpreter.interpret(pigscript, context);
assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
assertEquals(InterpreterResult.Code.ERROR, result.code());
assertTrue(result.message().get(0).getData().contains("Failed to read data from"));
}
+
}
+
+
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java
----------------------------------------------------------------------
diff --git a/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java
index f1f6ad8..5a21bb3 100644
--- a/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java
+++ b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java
@@ -1,33 +1,39 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * <p>http://www.apache.org/licenses/LICENSE-2.0
- *
- * <p>Unless required by applicable law or agreed to in writing, software distributed under the
- * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing permissions and
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.zeppelin.pig;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import org.apache.commons.io.IOUtils;
+import org.junit.After;
+import org.junit.Test;
+
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Properties;
-import org.apache.commons.io.IOUtils;
+
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
-import org.junit.After;
-import org.junit.Test;
public class PigInterpreterTest {
@@ -52,24 +58,23 @@ public class PigInterpreterTest {
public void testBasics() throws IOException {
setUpLocal(false);
- String content = "1\tandy\n" + "2\tpeter\n";
+ String content = "1\tandy\n"
+ + "2\tpeter\n";
File tmpFile = File.createTempFile("zeppelin", "test");
FileWriter writer = new FileWriter(tmpFile);
IOUtils.write(content, writer);
writer.close();
// simple pig script using dump
- String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" + "dump a;";
+ String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';"
+ + "dump a;";
InterpreterResult result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
assertEquals(Code.SUCCESS, result.code());
assertTrue(result.message().get(0).getData().contains("(1,andy)\n(2,peter)"));
// describe
- pigscript =
- "a = load '"
- + tmpFile.getAbsolutePath()
- + "' as (id: int, name: bytearray);"
+ pigscript = "a = load '" + tmpFile.getAbsolutePath() + "' as (id: int, name: bytearray);"
+ "describe a;";
result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
@@ -77,37 +82,38 @@ public class PigInterpreterTest {
assertTrue(result.message().get(0).getData().contains("a: {id: int,name: bytearray}"));
// syntax error (compilation error)
- pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';" + "describe a;";
+ pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';"
+ + "describe a;";
result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
assertEquals(Code.ERROR, result.code());
- assertTrue(
- result
- .message()
- .get(0)
- .getData()
- .contains("Syntax error, unexpected symbol at or near 'a'"));
+ assertTrue(result.message().get(0).getData().contains(
+ "Syntax error, unexpected symbol at or near 'a'"));
// execution error
- pigscript = "a = load 'invalid_path';" + "dump a;";
+ pigscript = "a = load 'invalid_path';"
+ + "dump a;";
result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
assertEquals(Code.ERROR, result.code());
assertTrue(result.message().get(0).getData().contains("Input path does not exist"));
}
+
@Test
public void testIncludeJobStats() throws IOException {
setUpLocal(true);
- String content = "1\tandy\n" + "2\tpeter\n";
+ String content = "1\tandy\n"
+ + "2\tpeter\n";
File tmpFile = File.createTempFile("zeppelin", "test");
FileWriter writer = new FileWriter(tmpFile);
IOUtils.write(content, writer);
writer.close();
// simple pig script using dump
- String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" + "dump a;";
+ String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';"
+ + "dump a;";
InterpreterResult result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
assertEquals(Code.SUCCESS, result.code());
@@ -115,10 +121,7 @@ public class PigInterpreterTest {
assertTrue(result.message().get(0).getData().contains("(1,andy)\n(2,peter)"));
// describe
- pigscript =
- "a = load '"
- + tmpFile.getAbsolutePath()
- + "' as (id: int, name: bytearray);"
+ pigscript = "a = load '" + tmpFile.getAbsolutePath() + "' as (id: int, name: bytearray);"
+ "describe a;";
result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
@@ -128,25 +131,24 @@ public class PigInterpreterTest {
assertTrue(result.message().get(0).getData().contains("a: {id: int,name: bytearray}"));
// syntax error (compilation error)
- pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';" + "describe a;";
+ pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';"
+ + "describe a;";
result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
assertEquals(Code.ERROR, result.code());
// no job is launched, so no jobStats
assertTrue(!result.message().get(0).getData().contains("Counters:"));
- assertTrue(
- result
- .message()
- .get(0)
- .getData()
- .contains("Syntax error, unexpected symbol at or near 'a'"));
+ assertTrue(result.message().get(0).getData().contains(
+ "Syntax error, unexpected symbol at or near 'a'"));
// execution error
- pigscript = "a = load 'invalid_path';" + "dump a;";
+ pigscript = "a = load 'invalid_path';"
+ + "dump a;";
result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
assertEquals(Code.ERROR, result.code());
assertTrue(result.message().get(0).getData().contains("Counters:"));
assertTrue(result.message().get(0).getData().contains("Input path does not exist"));
}
+
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTezTest.java
----------------------------------------------------------------------
diff --git a/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTezTest.java b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTezTest.java
index 14d9686..ec09a88 100644
--- a/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTezTest.java
+++ b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTezTest.java
@@ -1,33 +1,39 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * <p>http://www.apache.org/licenses/LICENSE-2.0
- *
- * <p>Unless required by applicable law or agreed to in writing, software distributed under the
- * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing permissions and
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.zeppelin.pig;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import org.apache.commons.io.IOUtils;
+import org.junit.After;
+import org.junit.Test;
+
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Properties;
-import org.apache.commons.io.IOUtils;
+
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
-import org.junit.After;
-import org.junit.Test;
public class PigInterpreterTezTest {
@@ -42,8 +48,8 @@ public class PigInterpreterTezTest {
pigInterpreter = new PigInterpreter(properties);
pigInterpreter.open();
context = InterpreterContext.builder().setParagraphId("paragraphId").build();
- }
+ }
@After
public void tearDown() {
pigInterpreter.close();
@@ -53,52 +59,45 @@ public class PigInterpreterTezTest {
public void testBasics() throws IOException {
setUpTez(false);
- assertEquals(
- "test",
- pigInterpreter
- .getPigServer()
- .getPigContext()
- .getProperties()
+ assertEquals("test",
+ pigInterpreter.getPigServer().getPigContext().getProperties()
.getProperty("tez.queue.name"));
-
- String content = "1\tandy\n" + "2\tpeter\n";
+
+ String content = "1\tandy\n"
+ + "2\tpeter\n";
File tmpFile = File.createTempFile("zeppelin", "test");
FileWriter writer = new FileWriter(tmpFile);
IOUtils.write(content, writer);
writer.close();
// simple pig script using dump
- String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" + "dump a;";
+ String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';"
+ + "dump a;";
InterpreterResult result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
assertEquals(Code.SUCCESS, result.code());
assertTrue(result.message().get(0).getData().contains("(1,andy)\n(2,peter)"));
// describe
- pigscript =
- "a = load '"
- + tmpFile.getAbsolutePath()
- + "' as (id: int, name: bytearray);"
- + "describe a;";
+ pigscript = "a = load '" + tmpFile.getAbsolutePath() + "' as (id: int, name: bytearray);"
+ + "describe a;";
result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
assertEquals(Code.SUCCESS, result.code());
assertTrue(result.message().get(0).getData().contains("a: {id: int,name: bytearray}"));
// syntax error (compilation error)
- pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';" + "describe a;";
+ pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';"
+ + "describe a;";
result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
assertEquals(Code.ERROR, result.code());
- assertTrue(
- result
- .message()
- .get(0)
- .getData()
- .contains("Syntax error, unexpected symbol at or near 'a'"));
+ assertTrue(result.message().get(0).getData().contains(
+ "Syntax error, unexpected symbol at or near 'a'"));
// syntax error
- pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" + "foreach a generate $0;";
+ pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';"
+ + "foreach a generate $0;";
result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
assertEquals(Code.ERROR, result.code());
@@ -109,14 +108,16 @@ public class PigInterpreterTezTest {
public void testIncludeJobStats() throws IOException {
setUpTez(true);
- String content = "1\tandy\n" + "2\tpeter\n";
+ String content = "1\tandy\n"
+ + "2\tpeter\n";
File tmpFile = File.createTempFile("zeppelin", "test");
FileWriter writer = new FileWriter(tmpFile);
IOUtils.write(content, writer);
writer.close();
// simple pig script using dump
- String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" + "dump a;";
+ String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';"
+ + "dump a;";
InterpreterResult result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
assertEquals(Code.SUCCESS, result.code());
@@ -124,11 +125,8 @@ public class PigInterpreterTezTest {
assertTrue(result.message().get(0).getData().contains("(1,andy)\n(2,peter)"));
// describe
- pigscript =
- "a = load '"
- + tmpFile.getAbsolutePath()
- + "' as (id: int, name: bytearray);"
- + "describe a;";
+ pigscript = "a = load '" + tmpFile.getAbsolutePath() + "' as (id: int, name: bytearray);"
+ + "describe a;";
result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
assertEquals(Code.SUCCESS, result.code());
@@ -137,21 +135,19 @@ public class PigInterpreterTezTest {
assertTrue(result.message().get(0).getData().contains("a: {id: int,name: bytearray}"));
// syntax error (compilation error)
- pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';" + "describe a;";
+ pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';"
+ + "describe a;";
result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
assertEquals(Code.ERROR, result.code());
// no job is launched, so no jobStats
assertTrue(!result.message().get(0).getData().contains("Vertex Stats"));
- assertTrue(
- result
- .message()
- .get(0)
- .getData()
- .contains("Syntax error, unexpected symbol at or near 'a'"));
+ assertTrue(result.message().get(0).getData().contains(
+ "Syntax error, unexpected symbol at or near 'a'"));
// execution error
- pigscript = "a = load 'invalid_path';" + "dump a;";
+ pigscript = "a = load 'invalid_path';"
+ + "dump a;";
result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
assertEquals(Code.ERROR, result.code());
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java
----------------------------------------------------------------------
diff --git a/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java b/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java
index a850174..01f0a20 100644
--- a/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java
+++ b/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java
@@ -1,40 +1,48 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * <p>http://www.apache.org/licenses/LICENSE-2.0
- *
- * <p>Unless required by applicable law or agreed to in writing, software distributed under the
- * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing permissions and
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.zeppelin.pig;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import org.apache.commons.io.IOUtils;
+import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
-import org.apache.commons.io.IOUtils;
+
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-/** */
+/**
+ *
+ */
public class PigQueryInterpreterTest {
private Interpreter pigInterpreter;
@@ -70,28 +78,23 @@ public class PigQueryInterpreterTest {
@Test
public void testBasics() throws IOException, InterpreterException {
- String content = "andy\tmale\t10\n" + "peter\tmale\t20\n" + "amy\tfemale\t14\n";
+ String content = "andy\tmale\t10\n"
+ + "peter\tmale\t20\n"
+ + "amy\tfemale\t14\n";
File tmpFile = File.createTempFile("zeppelin", "test");
FileWriter writer = new FileWriter(tmpFile);
IOUtils.write(content, writer);
writer.close();
// run script in PigInterpreter
- String pigscript =
- "a = load '"
- + tmpFile.getAbsolutePath()
- + "' as (name, gender, age);\n"
+ String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "' as (name, gender, age);\n"
+ "a2 = load 'invalid_path' as (name, gender, age);\n"
+ "dump a;";
InterpreterResult result = pigInterpreter.interpret(pigscript, context);
assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
- assertTrue(
- result
- .message()
- .get(0)
- .getData()
- .contains("(andy,male,10)\n(peter,male,20)\n(amy,female,14)"));
+ assertTrue(result.message().get(0).getData().contains(
+ "(andy,male,10)\n(peter,male,20)\n(amy,female,14)"));
// run single line query in PigQueryInterpreter
String query = "foreach a generate name, age;";
@@ -115,18 +118,13 @@ public class PigQueryInterpreterTest {
assertEquals("group\tcol_1\nmale\t2\nfemale\t1\n", result.message().get(0).getData());
// syntax error in PigQueryInterpereter
- query =
- "b = group a by invalid_column;\nforeach b generate group as gender, "
- + "COUNT($1) as count;";
+ query = "b = group a by invalid_column;\nforeach b generate group as gender, " +
+ "COUNT($1) as count;";
result = pigQueryInterpreter.interpret(query, context);
assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
assertEquals(InterpreterResult.Code.ERROR, result.code());
- assertTrue(
- result
- .message()
- .get(0)
- .getData()
- .contains("Projected field [invalid_column] does not exist in schema"));
+ assertTrue(result.message().get(0).getData().contains(
+ "Projected field [invalid_column] does not exist in schema"));
// execution error in PigQueryInterpreter
query = "foreach a2 generate name, age;";
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a9d3cc0..60af8bc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -147,7 +147,6 @@
<plugin.download.version>1.3.0</plugin.download.version>
<plugin.deploy.version>2.8.2</plugin.deploy.version>
<plugin.shade.version>3.1.1</plugin.shade.version>
- <plugin.fmt.version>2.5.1</plugin.fmt.version>
<PermGen>64m</PermGen>
<MaxPermGen>512m</MaxPermGen>
@@ -398,6 +397,46 @@
</plugin>
<plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <version>2.17</version>
+ <configuration>
+ <skip>true</skip>
+ <failOnViolation>false</failOnViolation>
+ <includeTestSourceDirectory>true</includeTestSourceDirectory>
+ <sourceDirectories>${basedir}/src/main/java,${basedir}/src/main/scala</sourceDirectories>
+ <testSourceDirectory>${basedir}/src/test/java</testSourceDirectory>
+ <configLocation>_tools/checkstyle.xml</configLocation>
+ <outputFile>${basedir}/target/checkstyle-output.xml</outputFile>
+ <inputEncoding>${project.build.sourceEncoding}</inputEncoding>
+ <outputEncoding>${project.reporting.outputEncoding}</outputEncoding>
+ </configuration>
+ <executions>
+ <execution>
+ <id>checkstyle-fail-build</id>
+ <phase>validate</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ <configuration>
+ <failOnViolation>true</failOnViolation>
+ <excludes>org/apache/zeppelin/interpreter/thrift/*,org/apache/zeppelin/scio/avro/*,org/apache/zeppelin/python/proto/*</excludes>
+ </configuration>
+ </execution>
+ <execution>
+ <id>checkstyle-gen-html-report</id>
+ <phase>install</phase>
+ <goals>
+ <goal>checkstyle-aggregate</goal>
+ </goals>
+ <configuration>
+ <excludes>org/apache/zeppelin/interpreter/thrift/*,org/apache/zeppelin/scio/avro/*,org/apache/zeppelin/python/proto/*</excludes>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
<artifactId>maven-resources-plugin</artifactId>
<version>${plugin.resource.version}</version>
<executions>
@@ -530,29 +569,14 @@
</configuration>
</plugin>
-->
- <plugin>
- <groupId>com.coveo</groupId>
- <artifactId>fmt-maven-plugin</artifactId>
- </plugin>
</plugins>
<pluginManagement>
<plugins>
<plugin>
- <groupId>com.coveo</groupId>
- <artifactId>fmt-maven-plugin</artifactId>
- <version>${plugin.fmt.version}</version>
- <configuration>
- <displayLimit>10000</displayLimit>
- </configuration>
- <executions>
- <execution>
- <phase>validate</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <version>${plugin.checkstyle.version}</version>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/python/pom.xml
----------------------------------------------------------------------
diff --git a/python/pom.xml b/python/pom.xml
index b60e423..d11c165 100644
--- a/python/pom.xml
+++ b/python/pom.xml
@@ -175,6 +175,14 @@
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <configuration>
+ <skip>false</skip>
+ </configuration>
+ </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/python/src/main/java/org/apache/zeppelin/python/IPythonClient.java
----------------------------------------------------------------------
diff --git a/python/src/main/java/org/apache/zeppelin/python/IPythonClient.java b/python/src/main/java/org/apache/zeppelin/python/IPythonClient.java
index 200fcf6..b9c897b 100644
--- a/python/src/main/java/org/apache/zeppelin/python/IPythonClient.java
+++ b/python/src/main/java/org/apache/zeppelin/python/IPythonClient.java
@@ -1,30 +1,25 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
package org.apache.zeppelin.python;
import io.grpc.ManagedChannel;
import io.grpc.ManagedChannelBuilder;
import io.grpc.stub.StreamObserver;
-import java.io.IOException;
-import java.security.SecureRandom;
-import java.util.Iterator;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.zeppelin.interpreter.util.InterpreterOutputStream;
import org.apache.zeppelin.python.proto.CancelRequest;
@@ -42,7 +37,15 @@ import org.apache.zeppelin.python.proto.StopRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-/** Grpc client for IPython kernel */
+import java.io.IOException;
+import java.security.SecureRandom;
+import java.util.Iterator;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Grpc client for IPython kernel
+ */
public class IPythonClient {
private static final Logger LOGGER = LoggerFactory.getLogger(IPythonClient.class.getName());
@@ -53,12 +56,16 @@ public class IPythonClient {
private SecureRandom random = new SecureRandom();
- /** Construct client for accessing RouteGuide server at {@code host:port}. */
+ /**
+ * Construct client for accessing RouteGuide server at {@code host:port}.
+ */
public IPythonClient(String host, int port) {
this(ManagedChannelBuilder.forAddress(host, port).usePlaintext(true));
}
- /** Construct client for accessing RouteGuide server using the existing channel. */
+ /**
+ * Construct client for accessing RouteGuide server using the existing channel.
+ */
public IPythonClient(ManagedChannelBuilder<?> channelBuilder) {
channel = channelBuilder.build();
blockingStub = IPythonGrpc.newBlockingStub(channel);
@@ -71,91 +78,86 @@ public class IPythonClient {
// execute the code and make the output as streaming by writing it to InterpreterOutputStream
// one by one.
- public ExecuteResponse stream_execute(
- ExecuteRequest request, final InterpreterOutputStream interpreterOutput) {
- final ExecuteResponse.Builder finalResponseBuilder =
- ExecuteResponse.newBuilder().setStatus(ExecuteStatus.SUCCESS);
+ public ExecuteResponse stream_execute(ExecuteRequest request,
+ final InterpreterOutputStream interpreterOutput) {
+ final ExecuteResponse.Builder finalResponseBuilder = ExecuteResponse.newBuilder()
+ .setStatus(ExecuteStatus.SUCCESS);
final AtomicBoolean completedFlag = new AtomicBoolean(false);
LOGGER.debug("stream_execute code:\n" + request.getCode());
- asyncStub.execute(
- request,
- new StreamObserver<ExecuteResponse>() {
- int index = 0;
- boolean isPreviousOutputImage = false;
-
- @Override
- public void onNext(ExecuteResponse executeResponse) {
- if (executeResponse.getType() == OutputType.TEXT) {
- try {
- LOGGER.debug("Interpreter Streaming Output: " + executeResponse.getOutput());
- if (isPreviousOutputImage) {
- // add '\n' when switch from image to text
- interpreterOutput.write("\n%text ".getBytes());
- }
- isPreviousOutputImage = false;
- interpreterOutput.write(executeResponse.getOutput().getBytes());
- interpreterOutput.getInterpreterOutput().flush();
- } catch (IOException e) {
- LOGGER.error("Unexpected IOException", e);
- }
- }
- if (executeResponse.getType() == OutputType.IMAGE) {
- try {
- LOGGER.debug("Interpreter Streaming Output: IMAGE_DATA");
- if (index != 0) {
- // add '\n' if this is the not the first element. otherwise it would mix the image
- // with the text
- interpreterOutput.write("\n".getBytes());
- }
- interpreterOutput.write(("%img " + executeResponse.getOutput()).getBytes());
- interpreterOutput.getInterpreterOutput().flush();
- isPreviousOutputImage = true;
- } catch (IOException e) {
- LOGGER.error("Unexpected IOException", e);
- }
+ asyncStub.execute(request, new StreamObserver<ExecuteResponse>() {
+ int index = 0;
+ boolean isPreviousOutputImage = false;
+
+ @Override
+ public void onNext(ExecuteResponse executeResponse) {
+ if (executeResponse.getType() == OutputType.TEXT) {
+ try {
+ LOGGER.debug("Interpreter Streaming Output: " + executeResponse.getOutput());
+ if (isPreviousOutputImage) {
+ // add '\n' when switch from image to text
+ interpreterOutput.write("\n%text ".getBytes());
}
- if (executeResponse.getStatus() == ExecuteStatus.ERROR) {
- // set the finalResponse to ERROR if any ERROR happens, otherwise the finalResponse
- // would
- // be SUCCESS.
- finalResponseBuilder.setStatus(ExecuteStatus.ERROR);
+ isPreviousOutputImage = false;
+ interpreterOutput.write(executeResponse.getOutput().getBytes());
+ interpreterOutput.getInterpreterOutput().flush();
+ } catch (IOException e) {
+ LOGGER.error("Unexpected IOException", e);
+ }
+ }
+ if (executeResponse.getType() == OutputType.IMAGE) {
+ try {
+ LOGGER.debug("Interpreter Streaming Output: IMAGE_DATA");
+ if (index != 0) {
+ // add '\n' if this is the not the first element. otherwise it would mix the image
+ // with the text
+ interpreterOutput.write("\n".getBytes());
}
- index++;
+ interpreterOutput.write(("%img " + executeResponse.getOutput()).getBytes());
+ interpreterOutput.getInterpreterOutput().flush();
+ isPreviousOutputImage = true;
+ } catch (IOException e) {
+ LOGGER.error("Unexpected IOException", e);
}
+ }
+ if (executeResponse.getStatus() == ExecuteStatus.ERROR) {
+ // set the finalResponse to ERROR if any ERROR happens, otherwise the finalResponse would
+ // be SUCCESS.
+ finalResponseBuilder.setStatus(ExecuteStatus.ERROR);
+ }
+ index++;
+ }
- @Override
- public void onError(Throwable throwable) {
- try {
- interpreterOutput
- .getInterpreterOutput()
- .write(ExceptionUtils.getStackTrace(throwable));
- interpreterOutput.getInterpreterOutput().flush();
- } catch (IOException e) {
- LOGGER.error("Unexpected IOException", e);
- }
- LOGGER.error("Fail to call IPython grpc", throwable);
- finalResponseBuilder.setStatus(ExecuteStatus.ERROR);
+ @Override
+ public void onError(Throwable throwable) {
+ try {
+ interpreterOutput.getInterpreterOutput().write(ExceptionUtils.getStackTrace(throwable));
+ interpreterOutput.getInterpreterOutput().flush();
+ } catch (IOException e) {
+ LOGGER.error("Unexpected IOException", e);
+ }
+ LOGGER.error("Fail to call IPython grpc", throwable);
+ finalResponseBuilder.setStatus(ExecuteStatus.ERROR);
- completedFlag.set(true);
- synchronized (completedFlag) {
- completedFlag.notify();
- }
- }
+ completedFlag.set(true);
+ synchronized (completedFlag) {
+ completedFlag.notify();
+ }
+ }
- @Override
- public void onCompleted() {
- synchronized (completedFlag) {
- try {
- LOGGER.debug("stream_execute is completed");
- interpreterOutput.getInterpreterOutput().flush();
- } catch (IOException e) {
- LOGGER.error("Unexpected IOException", e);
- }
- completedFlag.set(true);
- completedFlag.notify();
- }
+ @Override
+ public void onCompleted() {
+ synchronized (completedFlag) {
+ try {
+ LOGGER.debug("stream_execute is completed");
+ interpreterOutput.getInterpreterOutput().flush();
+ } catch (IOException e) {
+ LOGGER.error("Unexpected IOException", e);
}
- });
+ completedFlag.set(true);
+ completedFlag.notify();
+ }
+ }
+ });
synchronized (completedFlag) {
if (!completedFlag.get()) {
@@ -202,12 +204,14 @@ public class IPythonClient {
asyncStub.stop(request, null);
}
+
public static void main(String[] args) {
IPythonClient client = new IPythonClient("localhost", 50053);
client.status(StatusRequest.newBuilder().build());
- ExecuteResponse response =
- client.block_execute(ExecuteRequest.newBuilder().setCode("abcd=2").build());
+ ExecuteResponse response = client.block_execute(ExecuteRequest.newBuilder().
+ setCode("abcd=2").build());
System.out.println(response.getOutput());
+
}
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java
----------------------------------------------------------------------
diff --git a/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java
index aebed5c..3c646ae 100644
--- a/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java
+++ b/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java
@@ -18,16 +18,6 @@
package org.apache.zeppelin.python;
import io.grpc.ManagedChannelBuilder;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.nio.file.Files;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
import org.apache.commons.exec.CommandLine;
import org.apache.commons.exec.DefaultExecutor;
import org.apache.commons.exec.ExecuteException;
@@ -62,7 +52,20 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import py4j.GatewayServer;
-/** IPython Interpreter for Zeppelin */
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * IPython Interpreter for Zeppelin
+ */
public class IPythonInterpreter extends Interpreter implements ExecuteResultHandler {
private static final Logger LOGGER = LoggerFactory.getLogger(IPythonInterpreter.class);
@@ -87,8 +90,8 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand
}
/**
- * Sub class can customize the interpreter by adding more python packages under PYTHONPATH. e.g.
- * PySparkInterpreter
+ * Sub class can customize the interpreter by adding more python packages under PYTHONPATH.
+ * e.g. PySparkInterpreter
*
* @param additionalPythonPath
*/
@@ -98,8 +101,8 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand
}
/**
- * Sub class can customize the interpreter by running additional python init code. e.g.
- * PySparkInterpreter
+ * Sub class can customize the interpreter by running additional python init code.
+ * e.g. PySparkInterpreter
*
* @param additionalPythonInitFile
*/
@@ -128,22 +131,18 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand
LOGGER.info("Python Exec: " + pythonExecutable);
String checkPrerequisiteResult = checkIPythonPrerequisite(pythonExecutable);
if (!StringUtils.isEmpty(checkPrerequisiteResult)) {
- throw new InterpreterException(
- "IPython prerequisite is not meet: " + checkPrerequisiteResult);
+ throw new InterpreterException("IPython prerequisite is not meet: " +
+ checkPrerequisiteResult);
}
- ipythonLaunchTimeout =
- Long.parseLong(getProperty("zeppelin.ipython.launch.timeout", "30000"));
+ ipythonLaunchTimeout = Long.parseLong(
+ getProperty("zeppelin.ipython.launch.timeout", "30000"));
this.zeppelinContext = buildZeppelinContext();
int ipythonPort = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces();
int jvmGatewayPort = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces();
- int message_size =
- Integer.parseInt(
- getProperty("zeppelin.ipython.grpc.message_size", 32 * 1024 * 1024 + ""));
- ipythonClient =
- new IPythonClient(
- ManagedChannelBuilder.forAddress("127.0.0.1", ipythonPort)
- .usePlaintext(true)
- .maxInboundMessageSize(message_size));
+ int message_size = Integer.parseInt(getProperty("zeppelin.ipython.grpc.message_size",
+ 32 * 1024 * 1024 + ""));
+ ipythonClient = new IPythonClient(ManagedChannelBuilder.forAddress("127.0.0.1", ipythonPort)
+ .usePlaintext(true).maxInboundMessageSize(message_size));
this.usePy4JAuth = Boolean.parseBoolean(getProperty("zeppelin.py4j.useAuth", "true"));
this.secret = PythonUtils.createSecret(256);
launchIPythonKernel(ipythonPort);
@@ -154,8 +153,8 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand
}
/**
- * non-empty return value mean the errors when checking ipython prerequisite. empty value mean
- * IPython prerequisite is meet.
+ * non-empty return value mean the errors when checking ipython prerequisite.
+ * empty value mean IPython prerequisite is meet.
*
* @param pythonExec
* @return
@@ -171,7 +170,8 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand
Process proc = processBuilder.start();
int ret = proc.waitFor();
if (ret != 0) {
- return "Fail to run pip freeze.\n" + IOUtils.toString(new FileInputStream(stderrFile));
+ return "Fail to run pip freeze.\n" +
+ IOUtils.toString(new FileInputStream(stderrFile));
}
String freezeOutput = IOUtils.toString(new FileInputStream(stdoutFile));
if (!freezeOutput.contains("jupyter-client=")) {
@@ -206,34 +206,26 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand
InputStream input =
getClass().getClassLoader().getResourceAsStream("grpc/python/zeppelin_python.py");
List<String> lines = IOUtils.readLines(input);
- ExecuteResponse response =
- ipythonClient.block_execute(
- ExecuteRequest.newBuilder()
- .setCode(
- StringUtils.join(lines, System.lineSeparator())
- .replace("${JVM_GATEWAY_PORT}", jvmGatewayPort + "")
- .replace("${JVM_GATEWAY_ADDRESS}", serverAddress))
- .build());
+ ExecuteResponse response = ipythonClient.block_execute(ExecuteRequest.newBuilder()
+ .setCode(StringUtils.join(lines, System.lineSeparator())
+ .replace("${JVM_GATEWAY_PORT}", jvmGatewayPort + "")
+ .replace("${JVM_GATEWAY_ADDRESS}", serverAddress)).build());
if (response.getStatus() == ExecuteStatus.ERROR) {
throw new IOException("Fail to setup JVMGateway\n" + response.getOutput());
}
- input = getClass().getClassLoader().getResourceAsStream("python/zeppelin_context.py");
+ input =
+ getClass().getClassLoader().getResourceAsStream("python/zeppelin_context.py");
lines = IOUtils.readLines(input);
- response =
- ipythonClient.block_execute(
- ExecuteRequest.newBuilder()
- .setCode(StringUtils.join(lines, System.lineSeparator()))
- .build());
+ response = ipythonClient.block_execute(ExecuteRequest.newBuilder()
+ .setCode(StringUtils.join(lines, System.lineSeparator())).build());
if (response.getStatus() == ExecuteStatus.ERROR) {
throw new IOException("Fail to import ZeppelinContext\n" + response.getOutput());
}
- response =
- ipythonClient.block_execute(
- ExecuteRequest.newBuilder()
- .setCode("z = __zeppelin__ = PyZeppelinContext(intp.getZeppelinContext(), gateway)")
- .build());
+ response = ipythonClient.block_execute(ExecuteRequest.newBuilder()
+ .setCode("z = __zeppelin__ = PyZeppelinContext(intp.getZeppelinContext(), gateway)")
+ .build());
if (response.getStatus() == ExecuteStatus.ERROR) {
throw new IOException("Fail to setup ZeppelinContext\n" + response.getOutput());
}
@@ -241,31 +233,27 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand
if (additionalPythonInitFile != null) {
input = getClass().getClassLoader().getResourceAsStream(additionalPythonInitFile);
lines = IOUtils.readLines(input);
- response =
- ipythonClient.block_execute(
- ExecuteRequest.newBuilder()
- .setCode(
- StringUtils.join(lines, System.lineSeparator())
- .replace("${JVM_GATEWAY_PORT}", jvmGatewayPort + "")
- .replace("${JVM_GATEWAY_ADDRESS}", serverAddress))
- .build());
+ response = ipythonClient.block_execute(ExecuteRequest.newBuilder()
+ .setCode(StringUtils.join(lines, System.lineSeparator())
+ .replace("${JVM_GATEWAY_PORT}", jvmGatewayPort + "")
+ .replace("${JVM_GATEWAY_ADDRESS}", serverAddress)).build());
if (response.getStatus() == ExecuteStatus.ERROR) {
- throw new IOException(
- "Fail to run additional Python init file: "
- + additionalPythonInitFile
- + "\n"
- + response.getOutput());
+ throw new IOException("Fail to run additional Python init file: "
+ + additionalPythonInitFile + "\n" + response.getOutput());
}
}
}
- private void launchIPythonKernel(int ipythonPort) throws IOException {
+
+ private void launchIPythonKernel(int ipythonPort)
+ throws IOException {
LOGGER.info("Launching IPython Kernel at port: " + ipythonPort);
// copy the python scripts to a temp directory, then launch ipython kernel in that folder
File pythonWorkDir = Files.createTempDirectory("zeppelin_ipython").toFile();
String[] ipythonScripts = {"ipython_server.py", "ipython_pb2.py", "ipython_pb2_grpc.py"};
for (String ipythonScript : ipythonScripts) {
- URL url = getClass().getClassLoader().getResource("grpc/python" + "/" + ipythonScript);
+ URL url = getClass().getClassLoader().getResource("grpc/python"
+ + "/" + ipythonScript);
FileUtils.copyURLToFile(url, new File(pythonWorkDir, ipythonScript));
}
@@ -279,10 +267,10 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand
executor.setWatchdog(watchDog);
if (useBuiltinPy4j) {
- // TODO(zjffdu) don't do hard code on py4j here
+ //TODO(zjffdu) don't do hard code on py4j here
File py4jDestFile = new File(pythonWorkDir, "py4j-src-0.10.7.zip");
- FileUtils.copyURLToFile(
- getClass().getClassLoader().getResource("python/py4j-src-0.10.7.zip"), py4jDestFile);
+ FileUtils.copyURLToFile(getClass().getClassLoader().getResource(
+ "python/py4j-src-0.10.7.zip"), py4jDestFile);
if (additionalPythonPath != null) {
// put the py4j at the end, because additionalPythonPath may already contain py4j.
// e.g. PySparkInterpreter
@@ -318,8 +306,8 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand
}
if ((System.currentTimeMillis() - startTime) > ipythonLaunchTimeout) {
- throw new IOException(
- "Fail to launch IPython Kernel in " + ipythonLaunchTimeout / 1000 + " seconds");
+ throw new IOException("Fail to launch IPython Kernel in " + ipythonLaunchTimeout / 1000
+ + " seconds");
}
}
}
@@ -357,15 +345,15 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand
zeppelinContext.setInterpreterContext(context);
interpreterOutput.setInterpreterOutput(context.out);
ExecuteResponse response =
- ipythonClient.stream_execute(
- ExecuteRequest.newBuilder().setCode(st).build(), interpreterOutput);
+ ipythonClient.stream_execute(ExecuteRequest.newBuilder().setCode(st).build(),
+ interpreterOutput);
try {
interpreterOutput.getInterpreterOutput().flush();
} catch (IOException e) {
throw new RuntimeException("Fail to write output", e);
}
- InterpreterResult result =
- new InterpreterResult(InterpreterResult.Code.valueOf(response.getStatus().name()));
+ InterpreterResult result = new InterpreterResult(
+ InterpreterResult.Code.valueOf(response.getStatus().name()));
return result;
}
@@ -385,17 +373,14 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand
}
@Override
- public List<InterpreterCompletion> completion(
- String buf, int cursor, InterpreterContext interpreterContext) {
+ public List<InterpreterCompletion> completion(String buf, int cursor,
+ InterpreterContext interpreterContext) {
LOGGER.debug("Call completion for: " + buf);
List<InterpreterCompletion> completions = new ArrayList<>();
CompletionResponse response =
ipythonClient.complete(
- CompletionRequest.getDefaultInstance()
- .newBuilder()
- .setCode(buf)
- .setCursor(cursor)
- .build());
+ CompletionRequest.getDefaultInstance().newBuilder().setCode(buf)
+ .setCursor(cursor).build());
for (int i = 0; i < response.getMatchesCount(); i++) {
String match = response.getMatches(i);
int lastIndexOfDot = match.lastIndexOf(".");