You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ma...@apache.org on 2014/09/12 15:26:07 UTC
svn commit: r1624546 [1/2] - in /pig/branches/spark: ./
src/META-INF/services/
src/org/apache/pig/backend/hadoop/executionengine/spark/
src/org/apache/pig/backend/hadoop/executionengine/spark/converter/
src/org/apache/pig/tools/pigstats/
Author: mayur
Date: Fri Sep 12 13:26:06 2014
New Revision: 1624546
URL: http://svn.apache.org/r1624546
Log:
PIG-4167: Initial implementation of Pig on Spark
Added:
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/BroadCastClient.java
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/BroadCastServer.java
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkExecType.java
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkExecutionEngine.java
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkUtil.java
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/DistinctConverter.java
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/FilterConverter.java
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/ForEachConverter.java
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/GlobalRearrangeConverter.java
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/IteratorTransform.java
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LimitConverter.java
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LoadConverter.java
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LocalRearrangeConverter.java
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/POConverter.java
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/POOutputConsumerIterator.java
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/PackageConverter.java
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SortConverter.java
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SplitConverter.java
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/StoreConverter.java
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/UnionConverter.java
pig/branches/spark/src/org/apache/pig/tools/pigstats/SparkStats.java
Modified:
pig/branches/spark/ivy.xml
pig/branches/spark/src/META-INF/services/org.apache.pig.ExecType
Modified: pig/branches/spark/ivy.xml
URL: http://svn.apache.org/viewvc/pig/branches/spark/ivy.xml?rev=1624546&r1=1624545&r2=1624546&view=diff
==============================================================================
--- pig/branches/spark/ivy.xml (original)
+++ pig/branches/spark/ivy.xml Fri Sep 12 13:26:06 2014
@@ -248,12 +248,15 @@
<dependency org="org.antlr" name="antlr" rev="${antlr.version}" conf="compile->master"/>
<dependency org="org.antlr" name="antlr-runtime" rev="${antlr.version}" conf="compile->default"/>
<dependency org="org.antlr" name="ST4" rev="${stringtemplate.version}" conf="compile->default"/>
- <dependency org="org.apache.zookeeper" name="zookeeper" rev="${zookeeper.version}" conf="compile->master"/>
- <dependency org="org.jboss.netty" name="netty" rev="3.2.2.Final" conf="test->master"/>
+ <dependency org="org.apache.zookeeper" name="zookeeper" rev="${zookeeper.version}" conf="compile->master">
+ <exclude org="org.jboss.netty" module="netty"/>
+ </dependency>
+
+ <!-- dependency org="org.jboss.netty" name="netty" rev="3.2.2.Final" conf="test->master"/ -->
<dependency org="dk.brics.automaton" name="automaton" rev="1.11-8" conf="compile->default"/>
<dependency org="org.jruby" name="jruby-complete" rev="${jruby.version}" conf="compile->master"/>
- <dependency org="asm" name="asm" rev="${asm.version}" conf="compile->default"/>
+ <!-- dependency org="asm" name="asm" rev="${asm.version}" conf="compile->default"/ -->
<!-- HBase dependency in format for releases up to 0.94 (including) -->
<dependency org="org.apache.hbase" name="hbase" rev="${hbase94.version}" conf="hbase94->master">
@@ -431,6 +434,11 @@
<dependency org="com.twitter" name="parquet-pig-bundle" rev="${parquet-pig-bundle.version}" conf="compile->master"/>
+ <!-- for Spark integration -->
+ <dependency org="org.apache.spark" name="spark-core_2.10" rev="0.9.0-incubating" conf="compile->default">
+ <exclude org="org.eclipse.jetty.orbit" module="javax.servlet"/>
+ </dependency>
+
<!-- for Tez integration -->
<dependency org="org.apache.tez" name="tez" rev="${tez.version}"
conf="hadoop23->master"/>
Modified: pig/branches/spark/src/META-INF/services/org.apache.pig.ExecType
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/META-INF/services/org.apache.pig.ExecType?rev=1624546&r1=1624545&r2=1624546&view=diff
==============================================================================
--- pig/branches/spark/src/META-INF/services/org.apache.pig.ExecType (original)
+++ pig/branches/spark/src/META-INF/services/org.apache.pig.ExecType Fri Sep 12 13:26:06 2014
@@ -13,6 +13,7 @@
#
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.LocalExecType
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRExecType
-org.apache.pig.backend.hadoop.executionengine.tez.TezLocalExecType
-org.apache.pig.backend.hadoop.executionengine.tez.TezExecType
+#org.apache.pig.backend.hadoop.executionengine.tez.TezLocalExecType
+#org.apache.pig.backend.hadoop.executionengine.tez.TezExecType
+org.apache.pig.backend.hadoop.executionengine.spark.SparkExecType
Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/BroadCastClient.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/BroadCastClient.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/BroadCastClient.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/BroadCastClient.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,51 @@
+package org.apache.pig.backend.hadoop.executionengine.spark;
+
+import java.net.Socket;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.InputStream;
+import java.io.ObjectInputStream;
+import java.io.OutputStream;
+
+/* TCPClient to receive Objects from TCPServer */
+
+public class BroadCastClient {
+
+ private String host;
+ private int port;
+
+ public BroadCastClient(String host, int port) {
+
+ this.host = host;
+ this.port = port;
+
+ }
+
+ public Object getBroadCastMessage(String request) {
+
+ Object response = null;
+
+ try {
+
+ System.out.println("Connecting to " + host + " on port " + port);
+ Socket client = new Socket(host, port);
+
+ OutputStream outToServer = client.getOutputStream();
+ DataOutputStream out = new DataOutputStream(outToServer);
+
+ out.writeUTF(request);
+ ObjectInputStream inFromServer = new ObjectInputStream(
+ client.getInputStream());
+
+ response = inFromServer.readObject();
+ System.out.println("Server says " + response);
+ client.close();
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ return response;
+ }
+}
Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/BroadCastServer.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/BroadCastServer.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/BroadCastServer.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/BroadCastServer.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,86 @@
+package org.apache.pig.backend.hadoop.executionengine.spark;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.ObjectOutputStream;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.util.HashMap;
+import java.util.Map;
+
+/* TCPServer thread to serve missed Objects */
+
+public class BroadCastServer extends Thread {
+
+ private ServerSocket serverSocket;
+ private static Map<Object, Object> storage = null;
+
+ int port;
+
+ public BroadCastServer() {
+ }
+
+ public BroadCastServer(int port) throws IOException {
+
+ this.port = port;
+ serverSocket = new ServerSocket(port);
+
+ }
+
+ public void run() {
+ while (true) {
+ try {
+
+ System.out.println("Waiting for client on port "
+ + serverSocket.getLocalPort() + "...");
+
+ Socket server = serverSocket.accept();
+ System.out.println("Just connected to "
+ + server.getRemoteSocketAddress());
+ DataInputStream in = new DataInputStream(
+ server.getInputStream());
+ String request = in.readUTF();
+ System.out.println("Executor asking for :" + request);
+ ObjectOutputStream out = new ObjectOutputStream(
+ server.getOutputStream());
+
+ if (storage.get(request) != null) {
+
+ out.writeObject(storage.get(request));
+
+ } else {
+
+ out.writeUTF("Requested resource not available!!");
+
+ }
+
+ server.close();
+
+ } catch (Exception s) {
+ s.printStackTrace();
+ break;
+ }
+ }
+ }
+
+ public void startBroadcastServer(int port) {
+
+ try {
+
+ storage = new HashMap<Object, Object>();
+ Thread t = new BroadCastServer(port);
+ t.start();
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ public void addResource(String reference, Object resource) {
+
+ storage.put(reference, resource);
+
+ }
+
+}
Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkExecType.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkExecType.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkExecType.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkExecType.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,46 @@
+package org.apache.pig.backend.hadoop.executionengine.spark;
+
+import java.util.Properties;
+
+import org.apache.pig.ExecType;
+import org.apache.pig.backend.executionengine.ExecutionEngine;
+import org.apache.pig.impl.PigContext;
+
+public class SparkExecType implements ExecType {
+
+ private static final long serialVersionUID = 1L;
+ private static final String mode = "SPARK";
+
+ @Override
+ public boolean accepts(Properties properties) {
+ String execTypeSpecified = properties.getProperty("exectype", "")
+ .toUpperCase();
+ if (execTypeSpecified.equals(mode))
+ return true;
+ return false;
+ }
+
+ @Override
+ public ExecutionEngine getExecutionEngine(PigContext pigContext) {
+ return new SparkExecutionEngine(pigContext);
+ }
+
+ @Override
+ public Class<? extends ExecutionEngine> getExecutionEngineClass() {
+ return SparkExecutionEngine.class;
+ }
+
+ @Override
+ public boolean isLocal() {
+ return false;
+ }
+
+ @Override
+ public String name() {
+ return "SPARK";
+ }
+
+ public String toString() {
+ return name();
+ }
+}
Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkExecutionEngine.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkExecutionEngine.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkExecutionEngine.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkExecutionEngine.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,30 @@
+package org.apache.pig.backend.hadoop.executionengine.spark;
+
+import java.util.UUID;
+
+import org.apache.pig.backend.hadoop.executionengine.HExecutionEngine;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.tools.pigstats.PigStats;
+import org.apache.pig.tools.pigstats.ScriptState;
+import org.apache.pig.tools.pigstats.mapreduce.MRScriptState;
+import org.apache.pig.tools.pigstats.mapreduce.SimplePigStats;
+
+public class SparkExecutionEngine extends HExecutionEngine {
+
+ public SparkExecutionEngine(PigContext pigContext) {
+ super(pigContext);
+ this.launcher = new SparkLauncher();
+ }
+
+ @Override
+ public ScriptState instantiateScriptState() {
+ MRScriptState ss = new MRScriptState(UUID.randomUUID().toString());
+ ss.setPigContext(pigContext);
+ return ss;
+ }
+
+ @Override
+ public PigStats instantiatePigStats() {
+ return new SimplePigStats();
+ }
+}
Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,278 @@
+package org.apache.pig.backend.hadoop.executionengine.spark;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.collect.Lists;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.pig.PigConstants;
+import org.apache.pig.PigException;
+import org.apache.pig.backend.BackendException;
+import org.apache.pig.backend.hadoop.executionengine.Launcher;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.POPackageAnnotator;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFilter;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POGlobalRearrange;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLimit;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POUnion;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.util.PlanHelper;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.DistinctConverter;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.FilterConverter;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.ForEachConverter;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.GlobalRearrangeConverter;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.LimitConverter;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.LoadConverter;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.LocalRearrangeConverter;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.POConverter;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.PackageConverter;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.SortConverter;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.SplitConverter;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.StoreConverter;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.UnionConverter;
+import org.apache.pig.data.SchemaTupleBackend;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.plan.OperatorKey;
+import org.apache.pig.tools.pigstats.PigStats;
+import org.apache.pig.tools.pigstats.SparkStats;
+
+import org.apache.spark.rdd.RDD;
+import org.apache.spark.scheduler.JobLogger;
+import org.apache.spark.scheduler.StatsReportListener;
+import org.apache.spark.SparkContext;
+import org.apache.spark.api.java.JavaSparkContext;
+
+/**
+ * Main class that launches pig for Spark
+ */
+public class SparkLauncher extends Launcher {
+
+ private static final Log LOG = LogFactory.getLog(SparkLauncher.class);
+
+ // Our connection to Spark. It needs to be static so that it can be reused
+ // across jobs, because a
+ // new SparkLauncher gets created for each job.
+ private static SparkContext sparkContext = null;
+
+ public static BroadCastServer bcaster;
+
+ // An object that handle cache calls in the operator graph. This is again
+ // static because we want
+ // it to be shared across SparkLaunchers. It gets cleared whenever we close
+ // the SparkContext.
+ // private static CacheConverter cacheConverter = null;
+
+ @Override
+ public PigStats launchPig(PhysicalPlan physicalPlan, String grpName,
+ PigContext pigContext) throws Exception {
+ LOG.info("!!!!!!!!!! Launching Spark (woot) !!!!!!!!!!!!");
+ LOG.debug(physicalPlan);
+ Configuration c = SparkUtil.newJobConf(pigContext);
+ c.set(PigConstants.LOCAL_CODE_DIR, System.getProperty("java.io.tmpdir"));
+
+ SchemaTupleBackend.initialize(c, pigContext);
+ // ///////
+ // stolen from MapReduceLauncher
+ MRCompiler mrCompiler = new MRCompiler(physicalPlan, pigContext);
+ mrCompiler.compile();
+ MROperPlan plan = mrCompiler.getMRPlan();
+ POPackageAnnotator pkgAnnotator = new POPackageAnnotator(plan);
+ pkgAnnotator.visit();
+ // // this one: not sure
+ // KeyTypeDiscoveryVisitor kdv = new KeyTypeDiscoveryVisitor(plan);
+ // kdv.visit();
+
+ // ///////
+
+ if (System.getenv("BROADCAST_PORT") == null
+ && System.getenv("BROADCAST_MASTER_IP") == null) {
+ LOG.error("Missing BROADCAST_POST/BROADCAST_HOST in the environment.");
+ } else {
+ bcaster = new BroadCastServer();
+ bcaster.startBroadcastServer(Integer.parseInt(System
+ .getenv("BROADCAST_PORT")));
+ }
+
+ startSparkIfNeeded();
+
+ // initialize the supported converters
+ Map<Class<? extends PhysicalOperator>, POConverter> convertMap = new HashMap<Class<? extends PhysicalOperator>, POConverter>();
+
+ convertMap.put(POLoad.class, new LoadConverter(pigContext,
+ physicalPlan, sparkContext));
+ convertMap.put(POStore.class, new StoreConverter(pigContext));
+ convertMap.put(POForEach.class, new ForEachConverter());
+ convertMap.put(POFilter.class, new FilterConverter());
+ convertMap.put(POPackage.class, new PackageConverter());
+ // convertMap.put(POCache.class, cacheConverter);
+ convertMap.put(POLocalRearrange.class, new LocalRearrangeConverter());
+ convertMap.put(POGlobalRearrange.class, new GlobalRearrangeConverter());
+ convertMap.put(POLimit.class, new LimitConverter());
+ convertMap.put(PODistinct.class, new DistinctConverter());
+ convertMap.put(POUnion.class, new UnionConverter(sparkContext));
+ convertMap.put(POSort.class, new SortConverter());
+ convertMap.put(POSplit.class, new SplitConverter());
+
+ Map<OperatorKey, RDD<Tuple>> rdds = new HashMap<OperatorKey, RDD<Tuple>>();
+
+ SparkStats stats = new SparkStats();
+ LinkedList<POStore> stores = PlanHelper.getPhysicalOperators(
+ physicalPlan, POStore.class);
+ for (POStore poStore : stores) {
+ physicalToRDD(physicalPlan, poStore, rdds, convertMap);
+ stats.addOutputInfo(poStore, 1, 1, true, c); // TODO: use real
+ // values
+ }
+
+ return stats;
+ }
+
+ private static void startSparkIfNeeded() throws PigException {
+ if (sparkContext == null) {
+ String master = System.getenv("SPARK_MASTER");
+ if (master == null) {
+ LOG.info("SPARK_MASTER not specified, using \"local\"");
+ master = "local";
+ }
+
+ String sparkHome = System.getenv("SPARK_HOME"); // It's okay if this
+ // is null for local
+ // mode
+ String sparkJarsSetting = System.getenv("SPARK_JARS");
+ String pigJar = System.getenv("SPARK_PIG_JAR");
+ String[] sparkJars = sparkJarsSetting == null ? new String[] {}
+ : sparkJarsSetting.split(",");
+
+ // TODO: Don't hardcode this JAR
+ List<String> jars = Lists.asList(pigJar, sparkJars);
+
+ if (!master.startsWith("local") && !master.equals("yarn-client")) {
+ // Check that we have the Mesos native library and Spark home
+ // are set
+ if (sparkHome == null) {
+ System.err
+ .println("You need to set SPARK_HOME to run on a Mesos cluster!");
+ throw new PigException("SPARK_HOME is not set");
+ }
+ /*
+ * if (System.getenv("MESOS_NATIVE_LIBRARY") == null) {
+ *
+ * System.err.println(
+ * "You need to set MESOS_NATIVE_LIBRARY to run on a Mesos cluster!"
+ * ); throw new PigException("MESOS_NATIVE_LIBRARY is not set");
+ * }
+ *
+ * // Tell Spark to use Mesos in coarse-grained mode (only
+ * affects Spark 0.6+; no impact on others)
+ * System.setProperty("spark.mesos.coarse", "true");
+ */
+ }
+
+ // For coarse-grained Mesos mode, tell it an upper bound on how many
+ // cores to grab in total;
+ // we conservatively set this to 32 unless the user set the
+ // SPARK_MAX_CPUS environment variable.
+ if (System.getenv("SPARK_MAX_CPUS") != null) {
+ int maxCores = 32;
+ maxCores = Integer.parseInt(System.getenv("SPARK_MAX_CPUS"));
+ System.setProperty("spark.cores.max", "" + maxCores);
+ }
+ System.setProperty("spark.cores.max", "1");
+ System.setProperty("spark.executor.memory", "" + "512m");
+ System.setProperty("spark.shuffle.memoryFraction", "0.0");
+ System.setProperty("spark.storage.memoryFraction", "0.0");
+ JavaSparkContext javaContext = new JavaSparkContext(master,
+ "Spork", sparkHome, jars.toArray(new String[jars.size()]));
+ sparkContext = javaContext.sc();
+ sparkContext.addSparkListener(new StatsReportListener());
+ sparkContext.addSparkListener(new JobLogger());
+ // cacheConverter = new CacheConverter();
+ }
+ }
+
+ // You can use this in unit tests to stop the SparkContext between tests.
+ static void stopSpark() {
+ if (sparkContext != null) {
+ sparkContext.stop();
+ sparkContext = null;
+ // cacheConverter = null;
+ }
+ }
+
+ private void physicalToRDD(PhysicalPlan plan,
+ PhysicalOperator physicalOperator,
+ Map<OperatorKey, RDD<Tuple>> rdds,
+ Map<Class<? extends PhysicalOperator>, POConverter> convertMap)
+ throws IOException {
+
+ RDD<Tuple> nextRDD = null;
+ List<PhysicalOperator> predecessors = plan
+ .getPredecessors(physicalOperator);
+ List<RDD<Tuple>> predecessorRdds = Lists.newArrayList();
+ if (predecessors != null) {
+ for (PhysicalOperator predecessor : predecessors) {
+ physicalToRDD(plan, predecessor, rdds, convertMap);
+ predecessorRdds.add(rdds.get(predecessor.getOperatorKey()));
+ }
+ }
+
+ POConverter converter = convertMap.get(physicalOperator.getClass());
+ if (converter == null) {
+ throw new IllegalArgumentException(
+ "Spork unsupported PhysicalOperator: " + physicalOperator);
+ }
+
+ LOG.info("Converting operator "
+ + physicalOperator.getClass().getSimpleName() + " "
+ + physicalOperator);
+ nextRDD = converter.convert(predecessorRdds, physicalOperator);
+
+ if (POStore.class.equals(physicalOperator.getClass())) {
+ return;
+ }
+
+ if (nextRDD == null) {
+ throw new IllegalArgumentException(
+ "RDD should not be null after PhysicalOperator: "
+ + physicalOperator);
+ }
+
+ rdds.put(physicalOperator.getOperatorKey(), nextRDD);
+ }
+
+ @Override
+ public void explain(PhysicalPlan pp, PigContext pc, PrintStream ps,
+ String format, boolean verbose) throws IOException {
+ }
+
+ @Override
+ public void kill() throws BackendException {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public void killJob(String jobID, Configuration conf)
+ throws BackendException {
+ // TODO Auto-generated method stub
+
+ }
+}
Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkUtil.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkUtil.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkUtil.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkUtil.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,85 @@
+package org.apache.pig.backend.hadoop.executionengine.spark;
+
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POGlobalRearrange;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.util.ObjectSerializer;
+import org.apache.pig.impl.util.UDFContext;
+
+import scala.Tuple2;
+import scala.Product2;
+import scala.collection.JavaConversions;
+import scala.collection.Seq;
+//import scala.reflect.ClassManifest;
+//import scala.reflect.ClassManifest$;
+import scala.reflect.ClassTag;
+import scala.reflect.ClassTag$;
+import org.apache.spark.rdd.RDD;
+
+import java.io.IOException;
+import java.util.List;
+
+public class SparkUtil {
+
+ public static <T> ClassTag<T> getManifest(Class<T> clazz) {
+ return ClassTag$.MODULE$.apply(clazz);
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <K, V> ClassTag<Tuple2<K, V>> getTuple2Manifest() {
+ return (ClassTag<Tuple2<K, V>>) (Object) getManifest(Tuple2.class);
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <K, V> ClassTag<Product2<K, V>> getProduct2Manifest() {
+ return (ClassTag<Product2<K, V>>) (Object) getManifest(Product2.class);
+ }
+
+ public static JobConf newJobConf(PigContext pigContext) throws IOException {
+ JobConf jobConf = new JobConf(
+ ConfigurationUtil.toConfiguration(pigContext.getProperties()));
+ jobConf.set("pig.pigContext", ObjectSerializer.serialize(pigContext));
+ UDFContext.getUDFContext().serialize(jobConf);
+ jobConf.set("udf.import.list",
+ ObjectSerializer.serialize(PigContext.getPackageImportList()));
+ return jobConf;
+ }
+
+ public static <T> Seq<T> toScalaSeq(List<T> list) {
+ return JavaConversions.asScalaBuffer(list);
+ }
+
+ public static void assertPredecessorSize(List<RDD<Tuple>> predecessors,
+ PhysicalOperator physicalOperator, int size) {
+ if (predecessors.size() != size) {
+ throw new RuntimeException("Should have " + size
+ + " predecessors for " + physicalOperator.getClass()
+ + ". Got : " + predecessors.size());
+ }
+ }
+
+ public static void assertPredecessorSizeGreaterThan(
+ List<RDD<Tuple>> predecessors, PhysicalOperator physicalOperator,
+ int size) {
+ if (predecessors.size() <= size) {
+ throw new RuntimeException("Should have greater than" + size
+ + " predecessors for " + physicalOperator.getClass()
+ + ". Got : " + predecessors.size());
+ }
+ }
+
+ public static int getParallelism(List<RDD<Tuple>> predecessors,
+ PhysicalOperator physicalOperator) {
+ int parallelism = physicalOperator.getRequestedParallelism();
+ if (parallelism <= 0) {
+ // Parallelism wasn't set in Pig, so set it to whatever Spark thinks
+ // is reasonable.
+ parallelism = predecessors.get(0).context().defaultParallelism();
+ }
+ return parallelism;
+ }
+
+}
Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/DistinctConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/DistinctConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/DistinctConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/DistinctConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,83 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct;
+import org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil;
+import org.apache.pig.data.Tuple;
+
+import scala.Function1;
+import scala.Function2;
+import scala.Tuple2;
+//import scala.reflect.ClassManifest;
+import scala.reflect.ClassTag;
+import scala.runtime.AbstractFunction1;
+import scala.runtime.AbstractFunction2;
+import org.apache.spark.rdd.PairRDDFunctions;
+import org.apache.spark.rdd.RDD;
+
+@SuppressWarnings({ "serial" })
+public class DistinctConverter implements POConverter<Tuple, Tuple, PODistinct> {
+ private static final Log LOG = LogFactory.getLog(DistinctConverter.class);
+
+ private static final Function1<Tuple, Tuple2<Tuple, Object>> TO_KEY_VALUE_FUNCTION = new ToKeyValueFunction();
+ private static final Function2<Object, Object, Object> MERGE_VALUES_FUNCTION = new MergeValuesFunction();
+ private static final Function1<Tuple2<Tuple, Object>, Tuple> TO_VALUE_FUNCTION = new ToValueFunction();
+
+ @Override
+ public RDD<Tuple> convert(List<RDD<Tuple>> predecessors,
+ PODistinct poDistinct) throws IOException {
+ SparkUtil.assertPredecessorSize(predecessors, poDistinct, 1);
+ RDD<Tuple> rdd = predecessors.get(0);
+
+ ClassTag<Tuple2<Tuple, Object>> tuple2ClassManifest = SparkUtil
+ .<Tuple, Object> getTuple2Manifest();
+
+ RDD<Tuple2<Tuple, Object>> rddPairs = rdd.map(TO_KEY_VALUE_FUNCTION,
+ tuple2ClassManifest);
+ PairRDDFunctions<Tuple, Object> pairRDDFunctions = new PairRDDFunctions<Tuple, Object>(
+ rddPairs, SparkUtil.getManifest(Tuple.class),
+ SparkUtil.getManifest(Object.class));
+ int parallelism = SparkUtil.getParallelism(predecessors, poDistinct);
+ return pairRDDFunctions.reduceByKey(MERGE_VALUES_FUNCTION, parallelism)
+ .map(TO_VALUE_FUNCTION, SparkUtil.getManifest(Tuple.class));
+ }
+
+ private static final class ToKeyValueFunction extends
+ AbstractFunction1<Tuple, Tuple2<Tuple, Object>> implements
+ Serializable {
+ @Override
+ public Tuple2<Tuple, Object> apply(Tuple t) {
+ if (LOG.isDebugEnabled())
+ LOG.debug("DistinctConverter.ToKeyValueFunction in " + t);
+ Tuple key = t;
+ Object value = null; // value
+ // (key, value)
+ Tuple2<Tuple, Object> out = new Tuple2<Tuple, Object>(key, value);
+ if (LOG.isDebugEnabled())
+ LOG.debug("DistinctConverter.ToKeyValueFunction out " + out);
+ return out;
+ }
+ }
+
+ private static final class MergeValuesFunction extends
+ AbstractFunction2<Object, Object, Object> implements Serializable {
+ @Override
+ public Object apply(Object arg0, Object arg1) {
+ return null;
+ }
+ }
+
+ private static final class ToValueFunction extends
+ AbstractFunction1<Tuple2<Tuple, Object>, Tuple> implements
+ Serializable {
+ @Override
+ public Tuple apply(Tuple2<Tuple, Object> input) {
+ return input._1;
+ }
+ }
+}
Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/FilterConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/FilterConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/FilterConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/FilterConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,67 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.Serializable;
+import java.util.List;
+
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFilter;
+import org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil;
+import org.apache.pig.data.Tuple;
+
+import scala.runtime.AbstractFunction1;
+import org.apache.spark.rdd.RDD;
+
+/**
+ * Converter that converts an RDD to a filtered RRD using POFilter
+ */
+@SuppressWarnings({ "serial" })
+public class FilterConverter implements POConverter<Tuple, Tuple, POFilter> {
+
+ @Override
+ public RDD<Tuple> convert(List<RDD<Tuple>> predecessors,
+ POFilter physicalOperator) {
+ SparkUtil.assertPredecessorSize(predecessors, physicalOperator, 1);
+ RDD<Tuple> rdd = predecessors.get(0);
+ FilterFunction filterFunction = new FilterFunction(physicalOperator);
+ return rdd.filter(filterFunction);
+ }
+
+ private static class FilterFunction extends
+ AbstractFunction1<Tuple, Object> implements Serializable {
+
+ private POFilter poFilter;
+
+ private FilterFunction(POFilter poFilter) {
+ this.poFilter = poFilter;
+ }
+
+ @Override
+ public Boolean apply(Tuple v1) {
+ Result result;
+ try {
+ poFilter.setInputs(null);
+ poFilter.attachInput(v1);
+ result = poFilter.getNextTuple();
+ } catch (ExecException e) {
+ throw new RuntimeException("Couldn't filter tuple", e);
+ }
+
+ if (result == null) {
+ return false;
+ }
+
+ switch (result.returnStatus) {
+ case POStatus.STATUS_OK:
+ return true;
+ case POStatus.STATUS_EOP: // TODO: probably also ok for EOS,
+ // END_OF_BATCH
+ return false;
+ default:
+ throw new RuntimeException(
+ "Unexpected response code from filter: " + result);
+ }
+ }
+ }
+}
Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/ForEachConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/ForEachConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/ForEachConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/ForEachConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,59 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.Serializable;
+import java.util.List;
+
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach;
+import org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil;
+import org.apache.pig.data.Tuple;
+
+import scala.collection.Iterator;
+import scala.collection.JavaConversions;
+import org.apache.spark.api.java.function.Function;
+import org.apache.spark.rdd.RDD;
+
+/**
+ * Convert that is able to convert an RRD to another RRD using a POForEach
+ */
+@SuppressWarnings({ "serial" })
+public class ForEachConverter implements POConverter<Tuple, Tuple, POForEach> {
+
+ @Override
+ public RDD<Tuple> convert(List<RDD<Tuple>> predecessors,
+ POForEach physicalOperator) {
+ SparkUtil.assertPredecessorSize(predecessors, physicalOperator, 1);
+ RDD<Tuple> rdd = predecessors.get(0);
+ ForEachFunction forEachFunction = new ForEachFunction(physicalOperator);
+ return rdd.mapPartitions(forEachFunction, true,
+ SparkUtil.getManifest(Tuple.class));
+ }
+
+ private static class ForEachFunction extends
+ Function<Iterator<Tuple>, Iterator<Tuple>> implements Serializable {
+
+ private POForEach poForEach;
+
+ private ForEachFunction(POForEach poForEach) {
+ this.poForEach = poForEach;
+ }
+
+ public Iterator<Tuple> call(Iterator<Tuple> i) {
+ final java.util.Iterator<Tuple> input = JavaConversions
+ .asJavaIterator(i);
+ Iterator<Tuple> output = JavaConversions
+ .asScalaIterator(new POOutputConsumerIterator(input) {
+ protected void attach(Tuple tuple) {
+ poForEach.setInputs(null);
+ poForEach.attachInput(tuple);
+ }
+
+ protected Result getNextResult() throws ExecException {
+ return poForEach.getNextTuple();
+ }
+ });
+ return output;
+ }
+ }
+}
Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/GlobalRearrangeConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/GlobalRearrangeConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/GlobalRearrangeConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/GlobalRearrangeConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,230 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POGlobalRearrange;
+import org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.spark.HashPartitioner;
+import org.apache.spark.api.java.function.Function;
+import org.apache.spark.rdd.CoGroupedRDD;
+import org.apache.spark.rdd.RDD;
+
+import scala.Product2;
+import scala.Tuple2;
+import scala.collection.JavaConversions;
+import scala.collection.Seq;
+//import scala.reflect.ClassManifest;
+import scala.reflect.ClassTag;
+
+@SuppressWarnings({ "serial" })
+public class GlobalRearrangeConverter implements
+ POConverter<Tuple, Tuple, POGlobalRearrange> {
+ private static final Log LOG = LogFactory
+ .getLog(GlobalRearrangeConverter.class);
+
+ private static final TupleFactory tf = TupleFactory.getInstance();
+
+ // GROUP FUNCTIONS
+ private static final ToKeyValueFunction TO_KEY_VALUE_FUNCTION = new ToKeyValueFunction();
+ private static final GetKeyFunction GET_KEY_FUNCTION = new GetKeyFunction();
+ // COGROUP FUNCTIONS
+ private static final GroupTupleFunction GROUP_TUPLE_FUNCTION = new GroupTupleFunction();
+ private static final ToGroupKeyValueFunction TO_GROUP_KEY_VALUE_FUNCTION = new ToGroupKeyValueFunction();
+
+ @Override
+ public RDD<Tuple> convert(List<RDD<Tuple>> predecessors,
+ POGlobalRearrange physicalOperator) throws IOException {
+ SparkUtil.assertPredecessorSizeGreaterThan(predecessors,
+ physicalOperator, 0);
+ int parallelism = SparkUtil.getParallelism(predecessors,
+ physicalOperator);
+
+ String reducers = System.getenv("SPARK_REDUCERS");
+ if (reducers != null) {
+ parallelism = Integer.parseInt(reducers);
+ }
+ LOG.info("Parallelism for Spark groupBy: " + parallelism);
+
+ if (predecessors.size() == 1) {
+ // GROUP
+ return predecessors
+ .get(0)
+ // group by key
+ .groupBy(GET_KEY_FUNCTION, parallelism,
+ SparkUtil.getManifest(Object.class))
+ // convert result to a tuple (key, { values })
+ .map(GROUP_TUPLE_FUNCTION,
+ SparkUtil.getManifest(Tuple.class));
+ } else {
+ // COGROUP
+ // each pred returns (index, key, value)
+ ClassTag<Tuple2<Object, Tuple>> tuple2ClassManifest = SparkUtil
+ .<Object, Tuple> getTuple2Manifest();
+
+ List<RDD<Tuple2<Object, Tuple>>> rddPairs = new ArrayList();
+ for (RDD<Tuple> rdd : predecessors) {
+ RDD<Tuple2<Object, Tuple>> rddPair = rdd.map(
+ TO_KEY_VALUE_FUNCTION, tuple2ClassManifest);
+ rddPairs.add(rddPair);
+ }
+
+ // Something's wrong with the type parameters of CoGroupedRDD
+ // key and value are the same type ???
+ CoGroupedRDD<Object> coGroupedRDD = new CoGroupedRDD<Object>(
+ (Seq<RDD<? extends Product2<Object, ?>>>) (Object) (JavaConversions
+ .asScalaBuffer(rddPairs).toSeq()),
+ new HashPartitioner(parallelism));
+
+ RDD<Tuple2<Object, Seq<Seq<Tuple>>>> rdd = (RDD<Tuple2<Object, Seq<Seq<Tuple>>>>) (Object) coGroupedRDD;
+ return rdd.map(TO_GROUP_KEY_VALUE_FUNCTION,
+ SparkUtil.getManifest(Tuple.class));
+ }
+ }
+
+ private static class GetKeyFunction extends Function<Tuple, Object>
+ implements Serializable {
+
+ @Override
+ public Object call(Tuple t) {
+ try {
+ LOG.debug("GetKeyFunction in " + t);
+ // see PigGenericMapReduce For the key
+ Object key = t.get(1);
+ LOG.debug("GetKeyFunction out " + key);
+ return key;
+ } catch (ExecException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private static class GroupTupleFunction extends
+ Function<Tuple2<Object, Seq<Tuple>>, Tuple> implements Serializable {
+
+ @Override
+ public Tuple call(Tuple2<Object, Seq<Tuple>> v1) {
+ try {
+ LOG.debug("GroupTupleFunction in " + v1);
+ Tuple tuple = tf.newTuple(2);
+ tuple.set(0, v1._1()); // the (index, key) tuple
+ tuple.set(1, JavaConversions.asJavaCollection(v1._2())
+ .iterator()); // the Seq<Tuple> aka bag of values
+ LOG.debug("GroupTupleFunction out " + tuple);
+ return tuple;
+ } catch (ExecException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private static class ToKeyValueFunction extends
+ Function<Tuple, Tuple2<Object, Tuple>> implements Serializable {
+
+ @Override
+ public Tuple2<Object, Tuple> call(Tuple t) {
+ try {
+ // (index, key, value)
+ LOG.debug("ToKeyValueFunction in " + t);
+ Object key = t.get(1);
+ Tuple value = (Tuple) t.get(2); // value
+ // (key, value)
+ Tuple2<Object, Tuple> out = new Tuple2<Object, Tuple>(key,
+ value);
+ LOG.debug("ToKeyValueFunction out " + out);
+ return out;
+ } catch (ExecException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private static class ToGroupKeyValueFunction extends
+ Function<Tuple2<Object, Seq<Seq<Tuple>>>, Tuple> implements
+ Serializable {
+
+ @Override
+ public Tuple call(Tuple2<Object, Seq<Seq<Tuple>>> input) {
+ try {
+ LOG.debug("ToGroupKeyValueFunction2 in " + input);
+ final Object key = input._1();
+ Seq<Seq<Tuple>> bags = input._2();
+ Iterable<Seq<Tuple>> bagsList = JavaConversions
+ .asJavaIterable(bags);
+ int i = 0;
+ List<Iterator<Tuple>> tupleIterators = new ArrayList();
+ for (Seq<Tuple> bag : bagsList) {
+ Iterator<Tuple> iterator = JavaConversions
+ .asJavaCollection(bag).iterator();
+ final int index = i;
+ tupleIterators.add(new IteratorTransform<Tuple, Tuple>(
+ iterator) {
+ @Override
+ protected Tuple transform(Tuple next) {
+ try {
+ Tuple tuple = tf.newTuple(3);
+ tuple.set(0, index);
+ tuple.set(1, key);
+ tuple.set(2, next);
+ return tuple;
+ } catch (ExecException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ });
+ ++i;
+ }
+ Tuple out = tf.newTuple(2);
+ out.set(0, key);
+ out.set(1, new IteratorUnion<Tuple>(tupleIterators.iterator()));
+ LOG.debug("ToGroupKeyValueFunction2 out " + out);
+ return out;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private static class IteratorUnion<T> implements Iterator<T> {
+
+ private final Iterator<Iterator<T>> iterators;
+
+ private Iterator<T> current;
+
+ public IteratorUnion(Iterator<Iterator<T>> iterators) {
+ super();
+ this.iterators = iterators;
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (current != null && current.hasNext()) {
+ return true;
+ } else if (iterators.hasNext()) {
+ current = iterators.next();
+ return hasNext();
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public T next() {
+ return current.next();
+ }
+
+ @Override
+ public void remove() {
+ current.remove();
+ }
+
+ }
+}
Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/IteratorTransform.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/IteratorTransform.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/IteratorTransform.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/IteratorTransform.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,30 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.util.Iterator;
+
+abstract class IteratorTransform<IN, OUT> implements Iterator<OUT> {
+ private Iterator<IN> delegate;
+
+ public IteratorTransform(Iterator<IN> delegate) {
+ super();
+ this.delegate = delegate;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return delegate.hasNext();
+ }
+
+ @Override
+ public OUT next() {
+ return transform(delegate.next());
+ }
+
+ abstract protected OUT transform(IN next);
+
+ @Override
+ public void remove() {
+ delegate.remove();
+ }
+
+}
\ No newline at end of file
Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LimitConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LimitConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LimitConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LimitConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,64 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.List;
+
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLimit;
+import org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil;
+import org.apache.pig.data.Tuple;
+
+import scala.collection.Iterator;
+import scala.collection.JavaConversions;
+
+import org.apache.spark.api.java.function.Function;
+import org.apache.spark.rdd.RDD;
+import org.apache.spark.api.java.JavaSparkContext;
+
+@SuppressWarnings({ "serial" })
+public class LimitConverter implements POConverter<Tuple, Tuple, POLimit> {
+
+ @Override
+ public RDD<Tuple> convert(List<RDD<Tuple>> predecessors, POLimit poLimit)
+ throws IOException {
+ SparkUtil.assertPredecessorSize(predecessors, poLimit, 1);
+ RDD<Tuple> rdd = predecessors.get(0);
+ LimitFunction limitFunction = new LimitFunction(poLimit);
+ RDD<Tuple> rdd2 = rdd.coalesce(1, false);
+ return rdd2.mapPartitions(limitFunction, false,
+ SparkUtil.getManifest(Tuple.class));
+ }
+
+ private static class LimitFunction extends
+ Function<Iterator<Tuple>, Iterator<Tuple>> implements Serializable {
+
+ private final POLimit poLimit;
+
+ public LimitFunction(POLimit poLimit) {
+ this.poLimit = poLimit;
+ }
+
+ @Override
+ public Iterator<Tuple> call(Iterator<Tuple> i) {
+ final java.util.Iterator<Tuple> tuples = JavaConversions
+ .asJavaIterator(i);
+
+ return JavaConversions
+ .asScalaIterator(new POOutputConsumerIterator(tuples) {
+
+ protected void attach(Tuple tuple) {
+ poLimit.setInputs(null);
+ poLimit.attachInput(tuple);
+ }
+
+ protected Result getNextResult() throws ExecException {
+ return poLimit.getNextTuple();
+ }
+ });
+ }
+
+ }
+
+}
Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LoadConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LoadConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LoadConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LoadConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,131 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.pig.LoadFunc;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigInputFormat;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad;
+import org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.io.FileSpec;
+import org.apache.pig.impl.plan.OperatorKey;
+import org.apache.pig.impl.util.ObjectSerializer;
+
+import scala.Function1;
+import scala.Tuple2;
+import scala.runtime.AbstractFunction1;
+import org.apache.spark.rdd.RDD;
+import org.apache.spark.SparkContext;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Converter that loads data via POLoad and converts it to RRD<Tuple>. Abuses
+ * the interface a bit in that there is no inoput RRD to convert in this case.
+ * Instead input is the source path of the POLoad.
+ *
+ */
+@SuppressWarnings({ "serial" })
+public class LoadConverter implements POConverter<Tuple, Tuple, POLoad> {
+
+ private static final ToTupleFunction TO_TUPLE_FUNCTION = new ToTupleFunction();
+
+ private PigContext pigContext;
+ private PhysicalPlan physicalPlan;
+ private SparkContext sparkContext;
+
+ public LoadConverter(PigContext pigContext, PhysicalPlan physicalPlan,
+ SparkContext sparkContext) {
+ this.pigContext = pigContext;
+ this.physicalPlan = physicalPlan;
+ this.sparkContext = sparkContext;
+ }
+
+ @Override
+ public RDD<Tuple> convert(List<RDD<Tuple>> predecessorRdds, POLoad poLoad)
+ throws IOException {
+ // if (predecessors.size()!=0) {
+ // throw new
+ // RuntimeException("Should not have predecessors for Load. Got : "+predecessors);
+ // }
+
+ JobConf loadJobConf = SparkUtil.newJobConf(pigContext);
+ configureLoader(physicalPlan, poLoad, loadJobConf);
+
+ // don't know why but just doing this cast for now
+ RDD<Tuple2<Text, Tuple>> hadoopRDD = sparkContext.newAPIHadoopFile(
+ poLoad.getLFile().getFileName(), PigInputFormat.class,
+ Text.class, Tuple.class, loadJobConf);
+
+ // map to get just RDD<Tuple>
+ return hadoopRDD.map(TO_TUPLE_FUNCTION,
+ SparkUtil.getManifest(Tuple.class));
+ }
+
+ private static class ToTupleFunction extends
+ AbstractFunction1<Tuple2<Text, Tuple>, Tuple> implements
+ Function1<Tuple2<Text, Tuple>, Tuple>, Serializable {
+
+ @Override
+ public Tuple apply(Tuple2<Text, Tuple> v1) {
+ return v1._2();
+ }
+ }
+
+ /**
+ * stolen from JobControlCompiler TODO: refactor it to share this
+ *
+ * @param physicalPlan
+ * @param poLoad
+ * @param jobConf
+ * @return
+ * @throws java.io.IOException
+ */
+ private static JobConf configureLoader(PhysicalPlan physicalPlan,
+ POLoad poLoad, JobConf jobConf) throws IOException {
+
+ Job job = new Job(jobConf);
+ LoadFunc loadFunc = poLoad.getLoadFunc();
+
+ loadFunc.setLocation(poLoad.getLFile().getFileName(), job);
+
+ // stolen from JobControlCompiler
+ ArrayList<FileSpec> pigInputs = new ArrayList<FileSpec>();
+ // Store the inp filespecs
+ pigInputs.add(poLoad.getLFile());
+
+ ArrayList<List<OperatorKey>> inpTargets = Lists.newArrayList();
+ ArrayList<String> inpSignatures = Lists.newArrayList();
+ ArrayList<Long> inpLimits = Lists.newArrayList();
+ // Store the target operators for tuples read
+ // from this input
+ List<PhysicalOperator> loadSuccessors = physicalPlan
+ .getSuccessors(poLoad);
+ List<OperatorKey> loadSuccessorsKeys = Lists.newArrayList();
+ if (loadSuccessors != null) {
+ for (PhysicalOperator loadSuccessor : loadSuccessors) {
+ loadSuccessorsKeys.add(loadSuccessor.getOperatorKey());
+ }
+ }
+ inpTargets.add(loadSuccessorsKeys);
+ inpSignatures.add(poLoad.getSignature());
+ inpLimits.add(poLoad.getLimit());
+
+ jobConf.set("pig.inputs", ObjectSerializer.serialize(pigInputs));
+ jobConf.set("pig.inpTargets", ObjectSerializer.serialize(inpTargets));
+ jobConf.set("pig.inpSignatures",
+ ObjectSerializer.serialize(inpSignatures));
+ jobConf.set("pig.inpLimits", ObjectSerializer.serialize(inpLimits));
+
+ return jobConf;
+ }
+}
Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LocalRearrangeConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LocalRearrangeConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LocalRearrangeConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LocalRearrangeConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,79 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange;
+import org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil;
+import org.apache.pig.data.Tuple;
+
+import scala.runtime.AbstractFunction1;
+import org.apache.spark.rdd.RDD;
+
+@SuppressWarnings({ "serial" })
+public class LocalRearrangeConverter implements
+ POConverter<Tuple, Tuple, POLocalRearrange> {
+ private static final Log LOG = LogFactory
+ .getLog(GlobalRearrangeConverter.class);
+
+ @Override
+ public RDD<Tuple> convert(List<RDD<Tuple>> predecessors,
+ POLocalRearrange physicalOperator) throws IOException {
+ SparkUtil.assertPredecessorSize(predecessors, physicalOperator, 1);
+ RDD<Tuple> rdd = predecessors.get(0);
+ // call local rearrange to get key and value
+ return rdd.map(new LocalRearrangeFunction(physicalOperator),
+ SparkUtil.getManifest(Tuple.class));
+
+ }
+
+ private static class LocalRearrangeFunction extends
+ AbstractFunction1<Tuple, Tuple> implements Serializable {
+
+ private final POLocalRearrange physicalOperator;
+
+ public LocalRearrangeFunction(POLocalRearrange physicalOperator) {
+ this.physicalOperator = physicalOperator;
+ }
+
+ @Override
+ public Tuple apply(Tuple t) {
+ Result result;
+ try {
+ physicalOperator.setInputs(null);
+ physicalOperator.attachInput(t);
+ result = physicalOperator.getNextTuple();
+
+ if (result == null) {
+ throw new RuntimeException(
+ "Null response found for LocalRearange on tuple: "
+ + t);
+ }
+
+ switch (result.returnStatus) {
+ case POStatus.STATUS_OK:
+ // (index, key, value without keys)
+ Tuple resultTuple = (Tuple) result.result;
+ if (LOG.isDebugEnabled())
+ LOG.debug("LocalRearrangeFunction out " + resultTuple);
+ return resultTuple;
+ default:
+ throw new RuntimeException(
+ "Unexpected response code from operator "
+ + physicalOperator + " : " + result);
+ }
+ } catch (ExecException e) {
+ throw new RuntimeException(
+ "Couldn't do LocalRearange on tuple: " + t, e);
+ }
+ }
+
+ }
+
+}
Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/POConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/POConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/POConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/POConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,16 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
+
+import org.apache.spark.rdd.RDD;
+
+/**
+ * Given an RDD and a PhysicalOperater, and implementation of this class can
+ * convert the RDD to another RDD.
+ */
+public interface POConverter<IN, OUT, T extends PhysicalOperator> {
+ RDD<OUT> convert(List<RDD<IN>> rdd, T physicalOperator) throws IOException;
+}
Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/POOutputConsumerIterator.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/POOutputConsumerIterator.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/POOutputConsumerIterator.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/POOutputConsumerIterator.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,85 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
+import org.apache.pig.data.Tuple;
+
+abstract class POOutputConsumerIterator implements java.util.Iterator<Tuple> {
+ private final java.util.Iterator<Tuple> input;
+ private Result result = null;
+ private boolean returned = true;
+ private boolean finished = false;
+
+ POOutputConsumerIterator(java.util.Iterator<Tuple> input) {
+ this.input = input;
+ }
+
+ abstract protected void attach(Tuple tuple);
+
+ abstract protected Result getNextResult() throws ExecException;
+
+ private void readNext() {
+ try {
+ if (result != null && !returned) {
+ return;
+ }
+ // see PigGenericMapBase
+ if (result == null) {
+ if (!input.hasNext()) {
+ finished = true;
+ return;
+ }
+ Tuple v1 = input.next();
+ attach(v1);
+ }
+ result = getNextResult();
+ returned = false;
+ switch (result.returnStatus) {
+ case POStatus.STATUS_OK:
+ returned = false;
+ break;
+ case POStatus.STATUS_NULL:
+ returned = true; // skip: see PigGenericMapBase
+ readNext();
+ break;
+ case POStatus.STATUS_EOP:
+ finished = !input.hasNext();
+ if (!finished) {
+ result = null;
+ readNext();
+ }
+ break;
+ case POStatus.STATUS_ERR:
+ throw new RuntimeException("Error while processing " + result);
+ }
+ } catch (ExecException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ readNext();
+ return !finished;
+ }
+
+ @Override
+ public Tuple next() {
+ readNext();
+ if (finished) {
+ throw new RuntimeException("Passed the end. call hasNext() first");
+ }
+ if (result == null || result.returnStatus != POStatus.STATUS_OK) {
+ throw new RuntimeException("Unexpected response code in ForEach: "
+ + result);
+ }
+ returned = true;
+ return (Tuple) result.result;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+}
\ No newline at end of file
Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/PackageConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/PackageConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/PackageConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/PackageConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,124 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage;
+import org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.io.NullableTuple;
+import org.apache.pig.impl.io.PigNullableWritable;
+
+import scala.runtime.AbstractFunction1;
+import org.apache.spark.rdd.RDD;
+
+@SuppressWarnings({ "serial" })
+public class PackageConverter implements POConverter<Tuple, Tuple, POPackage> {
+ private static final Log LOG = LogFactory.getLog(PackageConverter.class);
+
+ @Override
+ public RDD<Tuple> convert(List<RDD<Tuple>> predecessors,
+ POPackage physicalOperator) throws IOException {
+ SparkUtil.assertPredecessorSize(predecessors, physicalOperator, 1);
+ RDD<Tuple> rdd = predecessors.get(0);
+ // package will generate the group from the result of the local
+ // rearrange
+ return rdd.map(new PackageFunction(physicalOperator),
+ SparkUtil.getManifest(Tuple.class));
+ }
+
+ private static class PackageFunction extends
+ AbstractFunction1<Tuple, Tuple> implements Serializable {
+
+ private final POPackage physicalOperator;
+
+ public PackageFunction(POPackage physicalOperator) {
+ this.physicalOperator = physicalOperator;
+ }
+
+ @Override
+ public Tuple apply(final Tuple t) {
+ // (key, Seq<Tuple>:{(index, key, value without key)})
+ if (LOG.isDebugEnabled())
+ LOG.debug("PackageFunction in " + t);
+ Result result;
+ try {
+ PigNullableWritable key = new PigNullableWritable() {
+
+ public Object getValueAsPigType() {
+ try {
+ Object keyTuple = t.get(0);
+ return keyTuple;
+ } catch (ExecException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ final Iterator<Tuple> bagIterator = (Iterator<Tuple>) t.get(1);
+ Iterator<NullableTuple> iterator = new Iterator<NullableTuple>() {
+ public boolean hasNext() {
+ return bagIterator.hasNext();
+ }
+
+ public NullableTuple next() {
+ try {
+ // we want the value and index only
+ Tuple next = bagIterator.next();
+ NullableTuple nullableTuple = new NullableTuple(
+ (Tuple) next.get(2));
+ nullableTuple.setIndex(((Number) next.get(0))
+ .byteValue());
+ return nullableTuple;
+ } catch (ExecException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ physicalOperator.setInputs(null);
+ physicalOperator.attachInput(key, iterator);
+ result = physicalOperator.getNextTuple();
+ } catch (ExecException e) {
+ throw new RuntimeException(
+ "Couldn't do Package on tuple: " + t, e);
+ }
+
+ if (result == null) {
+ throw new RuntimeException(
+ "Null response found for Package on tuple: " + t);
+ }
+ Tuple out;
+ switch (result.returnStatus) {
+ case POStatus.STATUS_OK:
+ // (key, {(value)...})
+ if (LOG.isDebugEnabled())
+ LOG.debug("PackageFunction out " + result.result);
+ out = (Tuple) result.result;
+ break;
+ case POStatus.STATUS_NULL:
+ out = null;
+ break;
+ default:
+ throw new RuntimeException(
+ "Unexpected response code from operator "
+ + physicalOperator + " : " + result + " "
+ + result.returnStatus);
+ }
+ if (LOG.isDebugEnabled())
+ LOG.debug("PackageFunction out " + out);
+ return out;
+ }
+
+ }
+
+}
Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SortConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SortConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SortConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SortConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,96 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort.SortComparator;
+import org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil;
+import org.apache.pig.data.Tuple;
+
+import scala.Tuple2;
+import scala.runtime.AbstractFunction1;
+
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.function.FlatMapFunction;
+import org.apache.spark.rdd.RDD;
+
+@SuppressWarnings("serial")
+public class SortConverter implements POConverter<Tuple, Tuple, POSort> {
+ private static final Log LOG = LogFactory.getLog(SortConverter.class);
+
+ private static final FlatMapFunction<Iterator<Tuple2<Tuple, Object>>, Tuple> TO_VALUE_FUNCTION = new ToValueFunction();
+
+ @Override
+ public RDD<Tuple> convert(List<RDD<Tuple>> predecessors, POSort sortOperator)
+ throws IOException {
+ SparkUtil.assertPredecessorSize(predecessors, sortOperator, 1);
+ RDD<Tuple> rdd = predecessors.get(0);
+ RDD<Tuple2<Tuple, Object>> rddPair = rdd.map(new ToKeyValueFunction(),
+ SparkUtil.<Tuple, Object> getTuple2Manifest());
+
+ JavaPairRDD<Tuple, Object> r = new JavaPairRDD<Tuple, Object>(rddPair,
+ SparkUtil.getManifest(Tuple.class),
+ SparkUtil.getManifest(Object.class));
+
+ JavaPairRDD<Tuple, Object> sorted = r.sortByKey(
+ sortOperator.new SortComparator(), true);
+ JavaRDD<Tuple> mapped = sorted.mapPartitions(TO_VALUE_FUNCTION);
+
+ return mapped.rdd();
+ }
+
+ private static class ToValueFunction extends
+ FlatMapFunction<Iterator<Tuple2<Tuple, Object>>, Tuple> implements
+ Serializable {
+
+ private class Tuple2TransformIterable implements Iterable<Tuple> {
+
+ Iterator<Tuple2<Tuple, Object>> in;
+
+ Tuple2TransformIterable(Iterator<Tuple2<Tuple, Object>> input) {
+ in = input;
+ }
+
+ public Iterator<Tuple> iterator() {
+ return new IteratorTransform<Tuple2<Tuple, Object>, Tuple>(in) {
+ @Override
+ protected Tuple transform(Tuple2<Tuple, Object> next) {
+ return next._1();
+ }
+ };
+ }
+ }
+
+ @Override
+ public Iterable<Tuple> call(Iterator<Tuple2<Tuple, Object>> input) {
+ return new Tuple2TransformIterable(input);
+ }
+ }
+
+ private static class ToKeyValueFunction extends
+ AbstractFunction1<Tuple, Tuple2<Tuple, Object>> implements
+ Serializable {
+
+ @Override
+ public Tuple2<Tuple, Object> apply(Tuple t) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Sort ToKeyValueFunction in " + t);
+ }
+ Tuple key = t;
+ Object value = null;
+ // (key, value)
+ Tuple2<Tuple, Object> out = new Tuple2<Tuple, Object>(key, value);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Sort ToKeyValueFunction out " + out);
+ }
+ return out;
+ }
+ }
+
+}
Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SplitConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SplitConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SplitConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SplitConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,22 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit;
+import org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil;
+import org.apache.pig.data.Tuple;
+
+import org.apache.spark.rdd.RDD;
+
+public class SplitConverter implements POConverter<Tuple, Tuple, POSplit> {
+
+ @Override
+ public RDD<Tuple> convert(List<RDD<Tuple>> predecessors, POSplit poSplit)
+ throws IOException {
+ SparkUtil.assertPredecessorSize(predecessors, poSplit, 1);
+ return predecessors.get(0);
+ }
+
+}
Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/StoreConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/StoreConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/StoreConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/StoreConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,94 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.pig.StoreFuncInterface;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputFormat;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
+import org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.util.ObjectSerializer;
+
+import scala.Tuple2;
+import scala.collection.JavaConversions;
+import scala.runtime.AbstractFunction1;
+import org.apache.spark.rdd.PairRDDFunctions;
+import org.apache.spark.rdd.RDD;
+import org.apache.spark.api.java.function.Function;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Converter that takes a POStore and stores it's content.
+ */
+@SuppressWarnings({ "serial" })
+public class StoreConverter implements
+ POConverter<Tuple, Tuple2<Text, Tuple>, POStore> {
+
+ private static final FromTupleFunction FROM_TUPLE_FUNCTION = new FromTupleFunction();
+
+ private PigContext pigContext;
+
+ public StoreConverter(PigContext pigContext) {
+ this.pigContext = pigContext;
+ }
+
+ @Override
+ public RDD<Tuple2<Text, Tuple>> convert(List<RDD<Tuple>> predecessors,
+ POStore physicalOperator) throws IOException {
+ SparkUtil.assertPredecessorSize(predecessors, physicalOperator, 1);
+ RDD<Tuple> rdd = predecessors.get(0);
+ // convert back to KV pairs
+ RDD<Tuple2<Text, Tuple>> rddPairs = rdd.map(FROM_TUPLE_FUNCTION,
+ SparkUtil.<Text, Tuple> getTuple2Manifest());
+ PairRDDFunctions<Text, Tuple> pairRDDFunctions = new PairRDDFunctions<Text, Tuple>(
+ rddPairs, SparkUtil.getManifest(Text.class),
+ SparkUtil.getManifest(Tuple.class));
+
+ JobConf storeJobConf = SparkUtil.newJobConf(pigContext);
+ POStore poStore = configureStorer(storeJobConf, physicalOperator);
+
+ pairRDDFunctions.saveAsNewAPIHadoopFile(poStore.getSFile()
+ .getFileName(), Text.class, Tuple.class, PigOutputFormat.class,
+ storeJobConf);
+
+ return rddPairs;
+ }
+
+ private static POStore configureStorer(JobConf jobConf,
+ PhysicalOperator physicalOperator) throws IOException {
+ ArrayList<POStore> storeLocations = Lists.newArrayList();
+ POStore poStore = (POStore) physicalOperator;
+ storeLocations.add(poStore);
+ StoreFuncInterface sFunc = poStore.getStoreFunc();
+ sFunc.setStoreLocation(poStore.getSFile().getFileName(),
+ new org.apache.hadoop.mapreduce.Job(jobConf));
+ poStore.setInputs(null);
+ poStore.setParentPlan(null);
+
+ jobConf.set(JobControlCompiler.PIG_MAP_STORES,
+ ObjectSerializer.serialize(Lists.newArrayList()));
+ jobConf.set(JobControlCompiler.PIG_REDUCE_STORES,
+ ObjectSerializer.serialize(storeLocations));
+ return poStore;
+ }
+
+ private static class FromTupleFunction extends
+ Function<Tuple, Tuple2<Text, Tuple>> implements Serializable {
+
+ private static Text EMPTY_TEXT = new Text();
+
+ public Tuple2<Text, Tuple> call(Tuple v1) {
+ return new Tuple2<Text, Tuple>(EMPTY_TEXT, v1);
+ }
+ }
+}
Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/UnionConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/UnionConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/UnionConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/UnionConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,34 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POUnion;
+import org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil;
+import org.apache.pig.data.Tuple;
+
+import scala.collection.JavaConversions;
+import org.apache.spark.rdd.UnionRDD;
+import org.apache.spark.rdd.RDD;
+import org.apache.spark.SparkContext;
+
+public class UnionConverter implements POConverter<Tuple, Tuple, POUnion> {
+
+ private final SparkContext sc;
+
+ public UnionConverter(SparkContext sc) {
+ this.sc = sc;
+ }
+
+ @Override
+ public RDD<Tuple> convert(List<RDD<Tuple>> predecessors,
+ POUnion physicalOperator) throws IOException {
+ SparkUtil.assertPredecessorSizeGreaterThan(predecessors,
+ physicalOperator, 0);
+ UnionRDD<Tuple> unionRDD = new UnionRDD<Tuple>(sc,
+ JavaConversions.asScalaBuffer(predecessors),
+ SparkUtil.getManifest(Tuple.class));
+ return unionRDD;
+ }
+
+}