You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ma...@apache.org on 2014/09/12 15:26:07 UTC

svn commit: r1624546 [1/2] - in /pig/branches/spark: ./ src/META-INF/services/ src/org/apache/pig/backend/hadoop/executionengine/spark/ src/org/apache/pig/backend/hadoop/executionengine/spark/converter/ src/org/apache/pig/tools/pigstats/

Author: mayur
Date: Fri Sep 12 13:26:06 2014
New Revision: 1624546

URL: http://svn.apache.org/r1624546
Log:
PIG-4167: Initial implementation of Pig on Spark

Added:
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/BroadCastClient.java
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/BroadCastServer.java
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkExecType.java
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkExecutionEngine.java
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkUtil.java
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/DistinctConverter.java
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/FilterConverter.java
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/ForEachConverter.java
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/GlobalRearrangeConverter.java
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/IteratorTransform.java
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LimitConverter.java
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LoadConverter.java
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LocalRearrangeConverter.java
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/POConverter.java
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/POOutputConsumerIterator.java
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/PackageConverter.java
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SortConverter.java
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SplitConverter.java
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/StoreConverter.java
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/UnionConverter.java
    pig/branches/spark/src/org/apache/pig/tools/pigstats/SparkStats.java
Modified:
    pig/branches/spark/ivy.xml
    pig/branches/spark/src/META-INF/services/org.apache.pig.ExecType

Modified: pig/branches/spark/ivy.xml
URL: http://svn.apache.org/viewvc/pig/branches/spark/ivy.xml?rev=1624546&r1=1624545&r2=1624546&view=diff
==============================================================================
--- pig/branches/spark/ivy.xml (original)
+++ pig/branches/spark/ivy.xml Fri Sep 12 13:26:06 2014
@@ -248,12 +248,15 @@
     <dependency org="org.antlr" name="antlr" rev="${antlr.version}" conf="compile->master"/>
     <dependency org="org.antlr" name="antlr-runtime" rev="${antlr.version}" conf="compile->default"/>
     <dependency org="org.antlr" name="ST4" rev="${stringtemplate.version}" conf="compile->default"/>
-    <dependency org="org.apache.zookeeper" name="zookeeper" rev="${zookeeper.version}" conf="compile->master"/>
-    <dependency org="org.jboss.netty" name="netty" rev="3.2.2.Final" conf="test->master"/>
+    <dependency org="org.apache.zookeeper" name="zookeeper" rev="${zookeeper.version}" conf="compile->master">
+    	<exclude org="org.jboss.netty" module="netty"/>
+    </dependency>
+
+    <!-- dependency org="org.jboss.netty" name="netty" rev="3.2.2.Final" conf="test->master"/ -->
     <dependency org="dk.brics.automaton" name="automaton" rev="1.11-8" conf="compile->default"/>
 
     <dependency org="org.jruby" name="jruby-complete" rev="${jruby.version}" conf="compile->master"/>
-    <dependency org="asm" name="asm" rev="${asm.version}" conf="compile->default"/>
+    <!-- dependency org="asm" name="asm" rev="${asm.version}" conf="compile->default"/ -->
 
     <!-- HBase dependency in format for releases up to 0.94 (including) -->
     <dependency org="org.apache.hbase" name="hbase" rev="${hbase94.version}" conf="hbase94->master">
@@ -431,6 +434,11 @@
 
     <dependency org="com.twitter" name="parquet-pig-bundle" rev="${parquet-pig-bundle.version}" conf="compile->master"/>
 
+    <!-- for Spark integration -->
+    <dependency org="org.apache.spark" name="spark-core_2.10" rev="0.9.0-incubating" conf="compile->default">
+        <exclude org="org.eclipse.jetty.orbit" module="javax.servlet"/>
+    </dependency>
+
     <!-- for Tez integration -->
     <dependency org="org.apache.tez" name="tez" rev="${tez.version}"
        conf="hadoop23->master"/>

Modified: pig/branches/spark/src/META-INF/services/org.apache.pig.ExecType
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/META-INF/services/org.apache.pig.ExecType?rev=1624546&r1=1624545&r2=1624546&view=diff
==============================================================================
--- pig/branches/spark/src/META-INF/services/org.apache.pig.ExecType (original)
+++ pig/branches/spark/src/META-INF/services/org.apache.pig.ExecType Fri Sep 12 13:26:06 2014
@@ -13,6 +13,7 @@
 #
 org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.LocalExecType
 org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRExecType
-org.apache.pig.backend.hadoop.executionengine.tez.TezLocalExecType
-org.apache.pig.backend.hadoop.executionengine.tez.TezExecType
+#org.apache.pig.backend.hadoop.executionengine.tez.TezLocalExecType
+#org.apache.pig.backend.hadoop.executionengine.tez.TezExecType
+org.apache.pig.backend.hadoop.executionengine.spark.SparkExecType
 

Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/BroadCastClient.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/BroadCastClient.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/BroadCastClient.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/BroadCastClient.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,51 @@
+package org.apache.pig.backend.hadoop.executionengine.spark;
+
+import java.net.Socket;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.InputStream;
+import java.io.ObjectInputStream;
+import java.io.OutputStream;
+
+/* TCPClient to receive Objects from TCPServer */
+
+public class BroadCastClient {
+
+    private String host;
+    private int port;
+
+    public BroadCastClient(String host, int port) {
+
+        this.host = host;
+        this.port = port;
+
+    }
+
+    public Object getBroadCastMessage(String request) {
+
+        Object response = null;
+
+        try {
+
+            System.out.println("Connecting to " + host + " on port " + port);
+            Socket client = new Socket(host, port);
+
+            OutputStream outToServer = client.getOutputStream();
+            DataOutputStream out = new DataOutputStream(outToServer);
+
+            out.writeUTF(request);
+            ObjectInputStream inFromServer = new ObjectInputStream(
+                    client.getInputStream());
+
+            response = inFromServer.readObject();
+            System.out.println("Server says " + response);
+            client.close();
+
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+
+        return response;
+    }
+}

Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/BroadCastServer.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/BroadCastServer.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/BroadCastServer.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/BroadCastServer.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,86 @@
+package org.apache.pig.backend.hadoop.executionengine.spark;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.ObjectOutputStream;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.util.HashMap;
+import java.util.Map;
+
+/* TCPServer thread to serve missed Objects */
+
+public class BroadCastServer extends Thread {
+
+    private ServerSocket serverSocket;
+    private static Map<Object, Object> storage = null;
+
+    int port;
+
+    public BroadCastServer() {
+    }
+
+    public BroadCastServer(int port) throws IOException {
+
+        this.port = port;
+        serverSocket = new ServerSocket(port);
+
+    }
+
+    public void run() {
+        while (true) {
+            try {
+
+                System.out.println("Waiting for client on port "
+                        + serverSocket.getLocalPort() + "...");
+
+                Socket server = serverSocket.accept();
+                System.out.println("Just connected to "
+                        + server.getRemoteSocketAddress());
+                DataInputStream in = new DataInputStream(
+                        server.getInputStream());
+                String request = in.readUTF();
+                System.out.println("Executor asking for :" + request);
+                ObjectOutputStream out = new ObjectOutputStream(
+                        server.getOutputStream());
+
+                if (storage.get(request) != null) {
+
+                    out.writeObject(storage.get(request));
+
+                } else {
+
+                    out.writeUTF("Requested resource not available!!");
+
+                }
+
+                server.close();
+
+            } catch (Exception s) {
+                s.printStackTrace();
+                break;
+            }
+        }
+    }
+
+    public void startBroadcastServer(int port) {
+
+        try {
+
+            storage = new HashMap<Object, Object>();
+            Thread t = new BroadCastServer(port);
+            t.start();
+
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    public void addResource(String reference, Object resource) {
+
+        storage.put(reference, resource);
+
+    }
+
+}

Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkExecType.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkExecType.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkExecType.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkExecType.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,46 @@
+package org.apache.pig.backend.hadoop.executionengine.spark;
+
+import java.util.Properties;
+
+import org.apache.pig.ExecType;
+import org.apache.pig.backend.executionengine.ExecutionEngine;
+import org.apache.pig.impl.PigContext;
+
+public class SparkExecType implements ExecType {
+
+    private static final long serialVersionUID = 1L;
+    private static final String mode = "SPARK";
+
+    @Override
+    public boolean accepts(Properties properties) {
+        String execTypeSpecified = properties.getProperty("exectype", "")
+                .toUpperCase();
+        if (execTypeSpecified.equals(mode))
+            return true;
+        return false;
+    }
+
+    @Override
+    public ExecutionEngine getExecutionEngine(PigContext pigContext) {
+        return new SparkExecutionEngine(pigContext);
+    }
+
+    @Override
+    public Class<? extends ExecutionEngine> getExecutionEngineClass() {
+        return SparkExecutionEngine.class;
+    }
+
+    @Override
+    public boolean isLocal() {
+        return false;
+    }
+
+    @Override
+    public String name() {
+        return "SPARK";
+    }
+
+    public String toString() {
+        return name();
+    }
+}

Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkExecutionEngine.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkExecutionEngine.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkExecutionEngine.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkExecutionEngine.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,30 @@
+package org.apache.pig.backend.hadoop.executionengine.spark;
+
+import java.util.UUID;
+
+import org.apache.pig.backend.hadoop.executionengine.HExecutionEngine;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.tools.pigstats.PigStats;
+import org.apache.pig.tools.pigstats.ScriptState;
+import org.apache.pig.tools.pigstats.mapreduce.MRScriptState;
+import org.apache.pig.tools.pigstats.mapreduce.SimplePigStats;
+
+public class SparkExecutionEngine extends HExecutionEngine {
+
+    public SparkExecutionEngine(PigContext pigContext) {
+        super(pigContext);
+        this.launcher = new SparkLauncher();
+    }
+
+    @Override
+    public ScriptState instantiateScriptState() {
+        MRScriptState ss = new MRScriptState(UUID.randomUUID().toString());
+        ss.setPigContext(pigContext);
+        return ss;
+    }
+
+    @Override
+    public PigStats instantiatePigStats() {
+        return new SimplePigStats();
+    }
+}

Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,278 @@
+package org.apache.pig.backend.hadoop.executionengine.spark;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.collect.Lists;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.pig.PigConstants;
+import org.apache.pig.PigException;
+import org.apache.pig.backend.BackendException;
+import org.apache.pig.backend.hadoop.executionengine.Launcher;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.POPackageAnnotator;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFilter;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POGlobalRearrange;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLimit;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POUnion;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.util.PlanHelper;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.DistinctConverter;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.FilterConverter;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.ForEachConverter;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.GlobalRearrangeConverter;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.LimitConverter;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.LoadConverter;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.LocalRearrangeConverter;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.POConverter;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.PackageConverter;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.SortConverter;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.SplitConverter;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.StoreConverter;
+import org.apache.pig.backend.hadoop.executionengine.spark.converter.UnionConverter;
+import org.apache.pig.data.SchemaTupleBackend;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.plan.OperatorKey;
+import org.apache.pig.tools.pigstats.PigStats;
+import org.apache.pig.tools.pigstats.SparkStats;
+
+import org.apache.spark.rdd.RDD;
+import org.apache.spark.scheduler.JobLogger;
+import org.apache.spark.scheduler.StatsReportListener;
+import org.apache.spark.SparkContext;
+import org.apache.spark.api.java.JavaSparkContext;
+
+/**
+ * Main class that launches pig for Spark
+ */
+public class SparkLauncher extends Launcher {
+
+    private static final Log LOG = LogFactory.getLog(SparkLauncher.class);
+
+    // Our connection to Spark. It needs to be static so that it can be reused
+    // across jobs, because a
+    // new SparkLauncher gets created for each job.
+    private static SparkContext sparkContext = null;
+
+    public static BroadCastServer bcaster;
+
+    // An object that handle cache calls in the operator graph. This is again
+    // static because we want
+    // it to be shared across SparkLaunchers. It gets cleared whenever we close
+    // the SparkContext.
+    // private static CacheConverter cacheConverter = null;
+
+    @Override
+    public PigStats launchPig(PhysicalPlan physicalPlan, String grpName,
+            PigContext pigContext) throws Exception {
+        LOG.info("!!!!!!!!!!  Launching Spark (woot) !!!!!!!!!!!!");
+        LOG.debug(physicalPlan);
+        Configuration c = SparkUtil.newJobConf(pigContext);
+        c.set(PigConstants.LOCAL_CODE_DIR, System.getProperty("java.io.tmpdir"));
+
+        SchemaTupleBackend.initialize(c, pigContext);
+        // ///////
+        // stolen from MapReduceLauncher
+        MRCompiler mrCompiler = new MRCompiler(physicalPlan, pigContext);
+        mrCompiler.compile();
+        MROperPlan plan = mrCompiler.getMRPlan();
+        POPackageAnnotator pkgAnnotator = new POPackageAnnotator(plan);
+        pkgAnnotator.visit();
+        // // this one: not sure
+        // KeyTypeDiscoveryVisitor kdv = new KeyTypeDiscoveryVisitor(plan);
+        // kdv.visit();
+
+        // ///////
+
+        if (System.getenv("BROADCAST_PORT") == null
+                && System.getenv("BROADCAST_MASTER_IP") == null) {
+            LOG.error("Missing BROADCAST_POST/BROADCAST_HOST in the environment.");            
+        } else {
+            bcaster = new BroadCastServer();
+            bcaster.startBroadcastServer(Integer.parseInt(System
+                    .getenv("BROADCAST_PORT")));
+        }
+
+        startSparkIfNeeded();
+
+        // initialize the supported converters
+        Map<Class<? extends PhysicalOperator>, POConverter> convertMap = new HashMap<Class<? extends PhysicalOperator>, POConverter>();
+
+        convertMap.put(POLoad.class, new LoadConverter(pigContext,
+                physicalPlan, sparkContext));
+        convertMap.put(POStore.class, new StoreConverter(pigContext));
+        convertMap.put(POForEach.class, new ForEachConverter());
+        convertMap.put(POFilter.class, new FilterConverter());
+        convertMap.put(POPackage.class, new PackageConverter());
+        // convertMap.put(POCache.class, cacheConverter);
+        convertMap.put(POLocalRearrange.class, new LocalRearrangeConverter());
+        convertMap.put(POGlobalRearrange.class, new GlobalRearrangeConverter());
+        convertMap.put(POLimit.class, new LimitConverter());
+        convertMap.put(PODistinct.class, new DistinctConverter());
+        convertMap.put(POUnion.class, new UnionConverter(sparkContext));
+        convertMap.put(POSort.class, new SortConverter());
+        convertMap.put(POSplit.class, new SplitConverter());
+
+        Map<OperatorKey, RDD<Tuple>> rdds = new HashMap<OperatorKey, RDD<Tuple>>();
+
+        SparkStats stats = new SparkStats();
+        LinkedList<POStore> stores = PlanHelper.getPhysicalOperators(
+                physicalPlan, POStore.class);
+        for (POStore poStore : stores) {
+            physicalToRDD(physicalPlan, poStore, rdds, convertMap);
+            stats.addOutputInfo(poStore, 1, 1, true, c); // TODO: use real
+            // values
+        }
+
+        return stats;
+    }
+
+    private static void startSparkIfNeeded() throws PigException {
+        if (sparkContext == null) {
+            String master = System.getenv("SPARK_MASTER");
+            if (master == null) {
+                LOG.info("SPARK_MASTER not specified, using \"local\"");
+                master = "local";
+            }
+
+            String sparkHome = System.getenv("SPARK_HOME"); // It's okay if this
+            // is null for local
+            // mode
+            String sparkJarsSetting = System.getenv("SPARK_JARS");
+            String pigJar = System.getenv("SPARK_PIG_JAR");
+            String[] sparkJars = sparkJarsSetting == null ? new String[] {}
+                    : sparkJarsSetting.split(",");
+
+            // TODO: Don't hardcode this JAR
+            List<String> jars = Lists.asList(pigJar, sparkJars);
+
+            if (!master.startsWith("local") && !master.equals("yarn-client")) {
+                // Check that we have the Mesos native library and Spark home
+                // are set
+                if (sparkHome == null) {
+                    System.err
+                            .println("You need to set SPARK_HOME to run on a Mesos cluster!");
+                    throw new PigException("SPARK_HOME is not set");
+                }
+                /*
+                 * if (System.getenv("MESOS_NATIVE_LIBRARY") == null) {
+                 *
+                 * System.err.println(
+                 * "You need to set MESOS_NATIVE_LIBRARY to run on a Mesos cluster!"
+                 * ); throw new PigException("MESOS_NATIVE_LIBRARY is not set");
+                 * }
+                 *
+                 * // Tell Spark to use Mesos in coarse-grained mode (only
+                 * affects Spark 0.6+; no impact on others)
+                 * System.setProperty("spark.mesos.coarse", "true");
+                 */
+            }
+
+            // For coarse-grained Mesos mode, tell it an upper bound on how many
+            // cores to grab in total;
+            // we conservatively set this to 32 unless the user set the
+            // SPARK_MAX_CPUS environment variable.
+            if (System.getenv("SPARK_MAX_CPUS") != null) {
+                int maxCores = 32;
+                maxCores = Integer.parseInt(System.getenv("SPARK_MAX_CPUS"));
+                System.setProperty("spark.cores.max", "" + maxCores);
+            }
+            System.setProperty("spark.cores.max", "1");
+            System.setProperty("spark.executor.memory", "" + "512m");
+            System.setProperty("spark.shuffle.memoryFraction", "0.0");
+            System.setProperty("spark.storage.memoryFraction", "0.0");
+            JavaSparkContext javaContext = new JavaSparkContext(master,
+                    "Spork", sparkHome, jars.toArray(new String[jars.size()]));
+            sparkContext = javaContext.sc();
+            sparkContext.addSparkListener(new StatsReportListener());
+            sparkContext.addSparkListener(new JobLogger());
+            // cacheConverter = new CacheConverter();
+        }
+    }
+
+    // You can use this in unit tests to stop the SparkContext between tests.
+    static void stopSpark() {
+        if (sparkContext != null) {
+            sparkContext.stop();
+            sparkContext = null;
+            // cacheConverter = null;
+        }
+    }
+
+    private void physicalToRDD(PhysicalPlan plan,
+            PhysicalOperator physicalOperator,
+            Map<OperatorKey, RDD<Tuple>> rdds,
+            Map<Class<? extends PhysicalOperator>, POConverter> convertMap)
+            throws IOException {
+
+        RDD<Tuple> nextRDD = null;
+        List<PhysicalOperator> predecessors = plan
+                .getPredecessors(physicalOperator);
+        List<RDD<Tuple>> predecessorRdds = Lists.newArrayList();
+        if (predecessors != null) {
+            for (PhysicalOperator predecessor : predecessors) {
+                physicalToRDD(plan, predecessor, rdds, convertMap);
+                predecessorRdds.add(rdds.get(predecessor.getOperatorKey()));
+            }
+        }
+
+        POConverter converter = convertMap.get(physicalOperator.getClass());
+        if (converter == null) {
+            throw new IllegalArgumentException(
+                    "Spork unsupported PhysicalOperator: " + physicalOperator);
+        }
+
+        LOG.info("Converting operator "
+                + physicalOperator.getClass().getSimpleName() + " "
+                + physicalOperator);
+        nextRDD = converter.convert(predecessorRdds, physicalOperator);
+
+        if (POStore.class.equals(physicalOperator.getClass())) {
+            return;
+        }
+
+        if (nextRDD == null) {
+            throw new IllegalArgumentException(
+                    "RDD should not be null after PhysicalOperator: "
+                            + physicalOperator);
+        }
+
+        rdds.put(physicalOperator.getOperatorKey(), nextRDD);
+    }
+
+    @Override
+    public void explain(PhysicalPlan pp, PigContext pc, PrintStream ps,
+            String format, boolean verbose) throws IOException {
+    }
+
+    @Override
+    public void kill() throws BackendException {
+        // TODO Auto-generated method stub
+
+    }
+
+    @Override
+    public void killJob(String jobID, Configuration conf)
+            throws BackendException {
+        // TODO Auto-generated method stub
+
+    }
+}

Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkUtil.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkUtil.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkUtil.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkUtil.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,85 @@
+package org.apache.pig.backend.hadoop.executionengine.spark;
+
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POGlobalRearrange;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.util.ObjectSerializer;
+import org.apache.pig.impl.util.UDFContext;
+
+import scala.Tuple2;
+import scala.Product2;
+import scala.collection.JavaConversions;
+import scala.collection.Seq;
+//import scala.reflect.ClassManifest;
+//import scala.reflect.ClassManifest$;
+import scala.reflect.ClassTag;
+import scala.reflect.ClassTag$;
+import org.apache.spark.rdd.RDD;
+
+import java.io.IOException;
+import java.util.List;
+
+public class SparkUtil {
+
+    public static <T> ClassTag<T> getManifest(Class<T> clazz) {
+        return ClassTag$.MODULE$.apply(clazz);
+    }
+
+    @SuppressWarnings("unchecked")
+    public static <K, V> ClassTag<Tuple2<K, V>> getTuple2Manifest() {
+        return (ClassTag<Tuple2<K, V>>) (Object) getManifest(Tuple2.class);
+    }
+
+    @SuppressWarnings("unchecked")
+    public static <K, V> ClassTag<Product2<K, V>> getProduct2Manifest() {
+        return (ClassTag<Product2<K, V>>) (Object) getManifest(Product2.class);
+    }
+
+    public static JobConf newJobConf(PigContext pigContext) throws IOException {
+        JobConf jobConf = new JobConf(
+                ConfigurationUtil.toConfiguration(pigContext.getProperties()));
+        jobConf.set("pig.pigContext", ObjectSerializer.serialize(pigContext));
+        UDFContext.getUDFContext().serialize(jobConf);
+        jobConf.set("udf.import.list",
+                ObjectSerializer.serialize(PigContext.getPackageImportList()));
+        return jobConf;
+    }
+
+    public static <T> Seq<T> toScalaSeq(List<T> list) {
+        return JavaConversions.asScalaBuffer(list);
+    }
+
+    public static void assertPredecessorSize(List<RDD<Tuple>> predecessors,
+            PhysicalOperator physicalOperator, int size) {
+        if (predecessors.size() != size) {
+            throw new RuntimeException("Should have " + size
+                    + " predecessors for " + physicalOperator.getClass()
+                    + ". Got : " + predecessors.size());
+        }
+    }
+
+    public static void assertPredecessorSizeGreaterThan(
+            List<RDD<Tuple>> predecessors, PhysicalOperator physicalOperator,
+            int size) {
+        if (predecessors.size() <= size) {
+            throw new RuntimeException("Should have greater than" + size
+                    + " predecessors for " + physicalOperator.getClass()
+                    + ". Got : " + predecessors.size());
+        }
+    }
+
+    public static int getParallelism(List<RDD<Tuple>> predecessors,
+            PhysicalOperator physicalOperator) {
+        int parallelism = physicalOperator.getRequestedParallelism();
+        if (parallelism <= 0) {
+            // Parallelism wasn't set in Pig, so set it to whatever Spark thinks
+            // is reasonable.
+            parallelism = predecessors.get(0).context().defaultParallelism();
+        }
+        return parallelism;
+    }
+
+}

Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/DistinctConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/DistinctConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/DistinctConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/DistinctConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,83 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct;
+import org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil;
+import org.apache.pig.data.Tuple;
+
+import scala.Function1;
+import scala.Function2;
+import scala.Tuple2;
+//import scala.reflect.ClassManifest;
+import scala.reflect.ClassTag;
+import scala.runtime.AbstractFunction1;
+import scala.runtime.AbstractFunction2;
+import org.apache.spark.rdd.PairRDDFunctions;
+import org.apache.spark.rdd.RDD;
+
+@SuppressWarnings({ "serial" })
+public class DistinctConverter implements POConverter<Tuple, Tuple, PODistinct> {
+    private static final Log LOG = LogFactory.getLog(DistinctConverter.class);
+
+    private static final Function1<Tuple, Tuple2<Tuple, Object>> TO_KEY_VALUE_FUNCTION = new ToKeyValueFunction();
+    private static final Function2<Object, Object, Object> MERGE_VALUES_FUNCTION = new MergeValuesFunction();
+    private static final Function1<Tuple2<Tuple, Object>, Tuple> TO_VALUE_FUNCTION = new ToValueFunction();
+
+    @Override
+    public RDD<Tuple> convert(List<RDD<Tuple>> predecessors,
+            PODistinct poDistinct) throws IOException {
+        SparkUtil.assertPredecessorSize(predecessors, poDistinct, 1);
+        RDD<Tuple> rdd = predecessors.get(0);
+
+        ClassTag<Tuple2<Tuple, Object>> tuple2ClassManifest = SparkUtil
+                .<Tuple, Object> getTuple2Manifest();
+
+        RDD<Tuple2<Tuple, Object>> rddPairs = rdd.map(TO_KEY_VALUE_FUNCTION,
+                tuple2ClassManifest);
+        PairRDDFunctions<Tuple, Object> pairRDDFunctions = new PairRDDFunctions<Tuple, Object>(
+                rddPairs, SparkUtil.getManifest(Tuple.class),
+                SparkUtil.getManifest(Object.class));
+        int parallelism = SparkUtil.getParallelism(predecessors, poDistinct);
+        return pairRDDFunctions.reduceByKey(MERGE_VALUES_FUNCTION, parallelism)
+                .map(TO_VALUE_FUNCTION, SparkUtil.getManifest(Tuple.class));
+    }
+
+    private static final class ToKeyValueFunction extends
+            AbstractFunction1<Tuple, Tuple2<Tuple, Object>> implements
+            Serializable {
+        @Override
+        public Tuple2<Tuple, Object> apply(Tuple t) {
+            if (LOG.isDebugEnabled())
+                LOG.debug("DistinctConverter.ToKeyValueFunction in " + t);
+            Tuple key = t;
+            Object value = null; // value
+            // (key, value)
+            Tuple2<Tuple, Object> out = new Tuple2<Tuple, Object>(key, value);
+            if (LOG.isDebugEnabled())
+                LOG.debug("DistinctConverter.ToKeyValueFunction out " + out);
+            return out;
+        }
+    }
+
+    private static final class MergeValuesFunction extends
+            AbstractFunction2<Object, Object, Object> implements Serializable {
+        @Override
+        public Object apply(Object arg0, Object arg1) {
+            return null;
+        }
+    }
+
+    private static final class ToValueFunction extends
+            AbstractFunction1<Tuple2<Tuple, Object>, Tuple> implements
+            Serializable {
+        @Override
+        public Tuple apply(Tuple2<Tuple, Object> input) {
+            return input._1;
+        }
+    }
+}

Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/FilterConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/FilterConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/FilterConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/FilterConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,67 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.Serializable;
+import java.util.List;
+
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFilter;
+import org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil;
+import org.apache.pig.data.Tuple;
+
+import scala.runtime.AbstractFunction1;
+import org.apache.spark.rdd.RDD;
+
+/**
+ * Converter that converts an RDD to a filtered RRD using POFilter
+ */
+@SuppressWarnings({ "serial" })
+public class FilterConverter implements POConverter<Tuple, Tuple, POFilter> {
+
+    @Override
+    public RDD<Tuple> convert(List<RDD<Tuple>> predecessors,
+            POFilter physicalOperator) {
+        SparkUtil.assertPredecessorSize(predecessors, physicalOperator, 1);
+        RDD<Tuple> rdd = predecessors.get(0);
+        FilterFunction filterFunction = new FilterFunction(physicalOperator);
+        return rdd.filter(filterFunction);
+    }
+
+    private static class FilterFunction extends
+            AbstractFunction1<Tuple, Object> implements Serializable {
+
+        private POFilter poFilter;
+
+        private FilterFunction(POFilter poFilter) {
+            this.poFilter = poFilter;
+        }
+
+        @Override
+        public Boolean apply(Tuple v1) {
+            Result result;
+            try {
+                poFilter.setInputs(null);
+                poFilter.attachInput(v1);
+                result = poFilter.getNextTuple();
+            } catch (ExecException e) {
+                throw new RuntimeException("Couldn't filter tuple", e);
+            }
+
+            if (result == null) {
+                return false;
+            }
+
+            switch (result.returnStatus) {
+            case POStatus.STATUS_OK:
+                return true;
+            case POStatus.STATUS_EOP: // TODO: probably also ok for EOS,
+                                      // END_OF_BATCH
+                return false;
+            default:
+                throw new RuntimeException(
+                        "Unexpected response code from filter: " + result);
+            }
+        }
+    }
+}

Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/ForEachConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/ForEachConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/ForEachConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/ForEachConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,59 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.Serializable;
+import java.util.List;
+
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach;
+import org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil;
+import org.apache.pig.data.Tuple;
+
+import scala.collection.Iterator;
+import scala.collection.JavaConversions;
+import org.apache.spark.api.java.function.Function;
+import org.apache.spark.rdd.RDD;
+
+/**
+ * Convert that is able to convert an RRD to another RRD using a POForEach
+ */
+@SuppressWarnings({ "serial" })
+public class ForEachConverter implements POConverter<Tuple, Tuple, POForEach> {
+
+    @Override
+    public RDD<Tuple> convert(List<RDD<Tuple>> predecessors,
+            POForEach physicalOperator) {
+        SparkUtil.assertPredecessorSize(predecessors, physicalOperator, 1);
+        RDD<Tuple> rdd = predecessors.get(0);
+        ForEachFunction forEachFunction = new ForEachFunction(physicalOperator);
+        return rdd.mapPartitions(forEachFunction, true,
+                SparkUtil.getManifest(Tuple.class));
+    }
+
+    private static class ForEachFunction extends
+            Function<Iterator<Tuple>, Iterator<Tuple>> implements Serializable {
+
+        private POForEach poForEach;
+
+        private ForEachFunction(POForEach poForEach) {
+            this.poForEach = poForEach;
+        }
+
+        public Iterator<Tuple> call(Iterator<Tuple> i) {
+            final java.util.Iterator<Tuple> input = JavaConversions
+                    .asJavaIterator(i);
+            Iterator<Tuple> output = JavaConversions
+                    .asScalaIterator(new POOutputConsumerIterator(input) {
+                        protected void attach(Tuple tuple) {
+                            poForEach.setInputs(null);
+                            poForEach.attachInput(tuple);
+                        }
+
+                        protected Result getNextResult() throws ExecException {
+                            return poForEach.getNextTuple();
+                        }
+                    });
+            return output;
+        }
+    }
+}

Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/GlobalRearrangeConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/GlobalRearrangeConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/GlobalRearrangeConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/GlobalRearrangeConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,230 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POGlobalRearrange;
+import org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.spark.HashPartitioner;
+import org.apache.spark.api.java.function.Function;
+import org.apache.spark.rdd.CoGroupedRDD;
+import org.apache.spark.rdd.RDD;
+
+import scala.Product2;
+import scala.Tuple2;
+import scala.collection.JavaConversions;
+import scala.collection.Seq;
+//import scala.reflect.ClassManifest;
+import scala.reflect.ClassTag;
+
+@SuppressWarnings({ "serial" })
+public class GlobalRearrangeConverter implements
+        POConverter<Tuple, Tuple, POGlobalRearrange> {
+    private static final Log LOG = LogFactory
+            .getLog(GlobalRearrangeConverter.class);
+
+    private static final TupleFactory tf = TupleFactory.getInstance();
+
+    // GROUP FUNCTIONS
+    private static final ToKeyValueFunction TO_KEY_VALUE_FUNCTION = new ToKeyValueFunction();
+    private static final GetKeyFunction GET_KEY_FUNCTION = new GetKeyFunction();
+    // COGROUP FUNCTIONS
+    private static final GroupTupleFunction GROUP_TUPLE_FUNCTION = new GroupTupleFunction();
+    private static final ToGroupKeyValueFunction TO_GROUP_KEY_VALUE_FUNCTION = new ToGroupKeyValueFunction();
+
+    @Override
+    public RDD<Tuple> convert(List<RDD<Tuple>> predecessors,
+            POGlobalRearrange physicalOperator) throws IOException {
+        SparkUtil.assertPredecessorSizeGreaterThan(predecessors,
+                physicalOperator, 0);
+        int parallelism = SparkUtil.getParallelism(predecessors,
+                physicalOperator);
+
+        String reducers = System.getenv("SPARK_REDUCERS");
+        if (reducers != null) {
+            parallelism = Integer.parseInt(reducers);
+        }
+        LOG.info("Parallelism for Spark groupBy: " + parallelism);
+
+        if (predecessors.size() == 1) {
+            // GROUP
+            return predecessors
+                    .get(0)
+                    // group by key
+                    .groupBy(GET_KEY_FUNCTION, parallelism,
+                            SparkUtil.getManifest(Object.class))
+                    // convert result to a tuple (key, { values })
+                    .map(GROUP_TUPLE_FUNCTION,
+                            SparkUtil.getManifest(Tuple.class));
+        } else {
+            // COGROUP
+            // each pred returns (index, key, value)
+            ClassTag<Tuple2<Object, Tuple>> tuple2ClassManifest = SparkUtil
+                    .<Object, Tuple> getTuple2Manifest();
+
+            List<RDD<Tuple2<Object, Tuple>>> rddPairs = new ArrayList();
+            for (RDD<Tuple> rdd : predecessors) {
+                RDD<Tuple2<Object, Tuple>> rddPair = rdd.map(
+                        TO_KEY_VALUE_FUNCTION, tuple2ClassManifest);
+                rddPairs.add(rddPair);
+            }
+
+            // Something's wrong with the type parameters of CoGroupedRDD
+            // key and value are the same type ???
+            CoGroupedRDD<Object> coGroupedRDD = new CoGroupedRDD<Object>(
+                    (Seq<RDD<? extends Product2<Object, ?>>>) (Object) (JavaConversions
+                            .asScalaBuffer(rddPairs).toSeq()),
+                    new HashPartitioner(parallelism));
+
+            RDD<Tuple2<Object, Seq<Seq<Tuple>>>> rdd = (RDD<Tuple2<Object, Seq<Seq<Tuple>>>>) (Object) coGroupedRDD;
+            return rdd.map(TO_GROUP_KEY_VALUE_FUNCTION,
+                    SparkUtil.getManifest(Tuple.class));
+        }
+    }
+
+    private static class GetKeyFunction extends Function<Tuple, Object>
+            implements Serializable {
+
+        @Override
+        public Object call(Tuple t) {
+            try {
+                LOG.debug("GetKeyFunction in " + t);
+                // see PigGenericMapReduce For the key
+                Object key = t.get(1);
+                LOG.debug("GetKeyFunction out " + key);
+                return key;
+            } catch (ExecException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+    private static class GroupTupleFunction extends
+            Function<Tuple2<Object, Seq<Tuple>>, Tuple> implements Serializable {
+
+        @Override
+        public Tuple call(Tuple2<Object, Seq<Tuple>> v1) {
+            try {
+                LOG.debug("GroupTupleFunction in " + v1);
+                Tuple tuple = tf.newTuple(2);
+                tuple.set(0, v1._1()); // the (index, key) tuple
+                tuple.set(1, JavaConversions.asJavaCollection(v1._2())
+                        .iterator()); // the Seq<Tuple> aka bag of values
+                LOG.debug("GroupTupleFunction out " + tuple);
+                return tuple;
+            } catch (ExecException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+    private static class ToKeyValueFunction extends
+            Function<Tuple, Tuple2<Object, Tuple>> implements Serializable {
+
+        @Override
+        public Tuple2<Object, Tuple> call(Tuple t) {
+            try {
+                // (index, key, value)
+                LOG.debug("ToKeyValueFunction in " + t);
+                Object key = t.get(1);
+                Tuple value = (Tuple) t.get(2); // value
+                // (key, value)
+                Tuple2<Object, Tuple> out = new Tuple2<Object, Tuple>(key,
+                        value);
+                LOG.debug("ToKeyValueFunction out " + out);
+                return out;
+            } catch (ExecException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+    private static class ToGroupKeyValueFunction extends
+            Function<Tuple2<Object, Seq<Seq<Tuple>>>, Tuple> implements
+            Serializable {
+
+        @Override
+        public Tuple call(Tuple2<Object, Seq<Seq<Tuple>>> input) {
+            try {
+                LOG.debug("ToGroupKeyValueFunction2 in " + input);
+                final Object key = input._1();
+                Seq<Seq<Tuple>> bags = input._2();
+                Iterable<Seq<Tuple>> bagsList = JavaConversions
+                        .asJavaIterable(bags);
+                int i = 0;
+                List<Iterator<Tuple>> tupleIterators = new ArrayList();
+                for (Seq<Tuple> bag : bagsList) {
+                    Iterator<Tuple> iterator = JavaConversions
+                            .asJavaCollection(bag).iterator();
+                    final int index = i;
+                    tupleIterators.add(new IteratorTransform<Tuple, Tuple>(
+                            iterator) {
+                        @Override
+                        protected Tuple transform(Tuple next) {
+                            try {
+                                Tuple tuple = tf.newTuple(3);
+                                tuple.set(0, index);
+                                tuple.set(1, key);
+                                tuple.set(2, next);
+                                return tuple;
+                            } catch (ExecException e) {
+                                throw new RuntimeException(e);
+                            }
+                        }
+                    });
+                    ++i;
+                }
+                Tuple out = tf.newTuple(2);
+                out.set(0, key);
+                out.set(1, new IteratorUnion<Tuple>(tupleIterators.iterator()));
+                LOG.debug("ToGroupKeyValueFunction2 out " + out);
+                return out;
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+    private static class IteratorUnion<T> implements Iterator<T> {
+
+        private final Iterator<Iterator<T>> iterators;
+
+        private Iterator<T> current;
+
+        public IteratorUnion(Iterator<Iterator<T>> iterators) {
+            super();
+            this.iterators = iterators;
+        }
+
+        @Override
+        public boolean hasNext() {
+            if (current != null && current.hasNext()) {
+                return true;
+            } else if (iterators.hasNext()) {
+                current = iterators.next();
+                return hasNext();
+            } else {
+                return false;
+            }
+        }
+
+        @Override
+        public T next() {
+            return current.next();
+        }
+
+        @Override
+        public void remove() {
+            current.remove();
+        }
+
+    }
+}

Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/IteratorTransform.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/IteratorTransform.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/IteratorTransform.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/IteratorTransform.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,30 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.util.Iterator;
+
+abstract class IteratorTransform<IN, OUT> implements Iterator<OUT> {
+    private Iterator<IN> delegate;
+
+    public IteratorTransform(Iterator<IN> delegate) {
+        super();
+        this.delegate = delegate;
+    }
+
+    @Override
+    public boolean hasNext() {
+        return delegate.hasNext();
+    }
+
+    @Override
+    public OUT next() {
+        return transform(delegate.next());
+    }
+
+    abstract protected OUT transform(IN next);
+
+    @Override
+    public void remove() {
+        delegate.remove();
+    }
+
+}
\ No newline at end of file

Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LimitConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LimitConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LimitConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LimitConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,64 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.List;
+
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLimit;
+import org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil;
+import org.apache.pig.data.Tuple;
+
+import scala.collection.Iterator;
+import scala.collection.JavaConversions;
+
+import org.apache.spark.api.java.function.Function;
+import org.apache.spark.rdd.RDD;
+import org.apache.spark.api.java.JavaSparkContext;
+
+@SuppressWarnings({ "serial" })
+public class LimitConverter implements POConverter<Tuple, Tuple, POLimit> {
+
+    @Override
+    public RDD<Tuple> convert(List<RDD<Tuple>> predecessors, POLimit poLimit)
+            throws IOException {
+        SparkUtil.assertPredecessorSize(predecessors, poLimit, 1);
+        RDD<Tuple> rdd = predecessors.get(0);
+        LimitFunction limitFunction = new LimitFunction(poLimit);
+        RDD<Tuple> rdd2 = rdd.coalesce(1, false);
+        return rdd2.mapPartitions(limitFunction, false,
+                SparkUtil.getManifest(Tuple.class));
+    }
+
+    private static class LimitFunction extends
+            Function<Iterator<Tuple>, Iterator<Tuple>> implements Serializable {
+
+        private final POLimit poLimit;
+
+        public LimitFunction(POLimit poLimit) {
+            this.poLimit = poLimit;
+        }
+
+        @Override
+        public Iterator<Tuple> call(Iterator<Tuple> i) {
+            final java.util.Iterator<Tuple> tuples = JavaConversions
+                    .asJavaIterator(i);
+
+            return JavaConversions
+                    .asScalaIterator(new POOutputConsumerIterator(tuples) {
+
+                        protected void attach(Tuple tuple) {
+                            poLimit.setInputs(null);
+                            poLimit.attachInput(tuple);
+                        }
+
+                        protected Result getNextResult() throws ExecException {
+                            return poLimit.getNextTuple();
+                        }
+                    });
+        }
+
+    }
+
+}

Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LoadConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LoadConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LoadConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LoadConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,131 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.pig.LoadFunc;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigInputFormat;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad;
+import org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.io.FileSpec;
+import org.apache.pig.impl.plan.OperatorKey;
+import org.apache.pig.impl.util.ObjectSerializer;
+
+import scala.Function1;
+import scala.Tuple2;
+import scala.runtime.AbstractFunction1;
+import org.apache.spark.rdd.RDD;
+import org.apache.spark.SparkContext;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Converter that loads data via POLoad and converts it to RRD&lt;Tuple>. Abuses
+ * the interface a bit in that there is no inoput RRD to convert in this case.
+ * Instead input is the source path of the POLoad.
+ *
+ */
+@SuppressWarnings({ "serial" })
+public class LoadConverter implements POConverter<Tuple, Tuple, POLoad> {
+
+    private static final ToTupleFunction TO_TUPLE_FUNCTION = new ToTupleFunction();
+
+    private PigContext pigContext;
+    private PhysicalPlan physicalPlan;
+    private SparkContext sparkContext;
+
+    public LoadConverter(PigContext pigContext, PhysicalPlan physicalPlan,
+            SparkContext sparkContext) {
+        this.pigContext = pigContext;
+        this.physicalPlan = physicalPlan;
+        this.sparkContext = sparkContext;
+    }
+
+    @Override
+    public RDD<Tuple> convert(List<RDD<Tuple>> predecessorRdds, POLoad poLoad)
+            throws IOException {
+        // if (predecessors.size()!=0) {
+        // throw new
+        // RuntimeException("Should not have predecessors for Load. Got : "+predecessors);
+        // }
+
+        JobConf loadJobConf = SparkUtil.newJobConf(pigContext);
+        configureLoader(physicalPlan, poLoad, loadJobConf);
+
+        // don't know why but just doing this cast for now
+        RDD<Tuple2<Text, Tuple>> hadoopRDD = sparkContext.newAPIHadoopFile(
+                poLoad.getLFile().getFileName(), PigInputFormat.class,
+                Text.class, Tuple.class, loadJobConf);
+
+        // map to get just RDD<Tuple>
+        return hadoopRDD.map(TO_TUPLE_FUNCTION,
+                SparkUtil.getManifest(Tuple.class));
+    }
+
+    private static class ToTupleFunction extends
+            AbstractFunction1<Tuple2<Text, Tuple>, Tuple> implements
+            Function1<Tuple2<Text, Tuple>, Tuple>, Serializable {
+
+        @Override
+        public Tuple apply(Tuple2<Text, Tuple> v1) {
+            return v1._2();
+        }
+    }
+
+    /**
+     * stolen from JobControlCompiler TODO: refactor it to share this
+     *
+     * @param physicalPlan
+     * @param poLoad
+     * @param jobConf
+     * @return
+     * @throws java.io.IOException
+     */
+    private static JobConf configureLoader(PhysicalPlan physicalPlan,
+            POLoad poLoad, JobConf jobConf) throws IOException {
+
+        Job job = new Job(jobConf);
+        LoadFunc loadFunc = poLoad.getLoadFunc();
+
+        loadFunc.setLocation(poLoad.getLFile().getFileName(), job);
+
+        // stolen from JobControlCompiler
+        ArrayList<FileSpec> pigInputs = new ArrayList<FileSpec>();
+        // Store the inp filespecs
+        pigInputs.add(poLoad.getLFile());
+
+        ArrayList<List<OperatorKey>> inpTargets = Lists.newArrayList();
+        ArrayList<String> inpSignatures = Lists.newArrayList();
+        ArrayList<Long> inpLimits = Lists.newArrayList();
+        // Store the target operators for tuples read
+        // from this input
+        List<PhysicalOperator> loadSuccessors = physicalPlan
+                .getSuccessors(poLoad);
+        List<OperatorKey> loadSuccessorsKeys = Lists.newArrayList();
+        if (loadSuccessors != null) {
+            for (PhysicalOperator loadSuccessor : loadSuccessors) {
+                loadSuccessorsKeys.add(loadSuccessor.getOperatorKey());
+            }
+        }
+        inpTargets.add(loadSuccessorsKeys);
+        inpSignatures.add(poLoad.getSignature());
+        inpLimits.add(poLoad.getLimit());
+
+        jobConf.set("pig.inputs", ObjectSerializer.serialize(pigInputs));
+        jobConf.set("pig.inpTargets", ObjectSerializer.serialize(inpTargets));
+        jobConf.set("pig.inpSignatures",
+                ObjectSerializer.serialize(inpSignatures));
+        jobConf.set("pig.inpLimits", ObjectSerializer.serialize(inpLimits));
+
+        return jobConf;
+    }
+}

Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LocalRearrangeConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LocalRearrangeConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LocalRearrangeConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/LocalRearrangeConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,79 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange;
+import org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil;
+import org.apache.pig.data.Tuple;
+
+import scala.runtime.AbstractFunction1;
+import org.apache.spark.rdd.RDD;
+
+@SuppressWarnings({ "serial" })
+public class LocalRearrangeConverter implements
+        POConverter<Tuple, Tuple, POLocalRearrange> {
+    private static final Log LOG = LogFactory
+            .getLog(GlobalRearrangeConverter.class);
+
+    @Override
+    public RDD<Tuple> convert(List<RDD<Tuple>> predecessors,
+            POLocalRearrange physicalOperator) throws IOException {
+        SparkUtil.assertPredecessorSize(predecessors, physicalOperator, 1);
+        RDD<Tuple> rdd = predecessors.get(0);
+        // call local rearrange to get key and value
+        return rdd.map(new LocalRearrangeFunction(physicalOperator),
+                SparkUtil.getManifest(Tuple.class));
+
+    }
+
+    private static class LocalRearrangeFunction extends
+            AbstractFunction1<Tuple, Tuple> implements Serializable {
+
+        private final POLocalRearrange physicalOperator;
+
+        public LocalRearrangeFunction(POLocalRearrange physicalOperator) {
+            this.physicalOperator = physicalOperator;
+        }
+
+        @Override
+        public Tuple apply(Tuple t) {
+            Result result;
+            try {
+                physicalOperator.setInputs(null);
+                physicalOperator.attachInput(t);
+                result = physicalOperator.getNextTuple();
+
+                if (result == null) {
+                    throw new RuntimeException(
+                            "Null response found for LocalRearange on tuple: "
+                                    + t);
+                }
+
+                switch (result.returnStatus) {
+                case POStatus.STATUS_OK:
+                    // (index, key, value without keys)
+                    Tuple resultTuple = (Tuple) result.result;
+                    if (LOG.isDebugEnabled())
+                        LOG.debug("LocalRearrangeFunction out " + resultTuple);
+                    return resultTuple;
+                default:
+                    throw new RuntimeException(
+                            "Unexpected response code from operator "
+                                    + physicalOperator + " : " + result);
+                }
+            } catch (ExecException e) {
+                throw new RuntimeException(
+                        "Couldn't do LocalRearange on tuple: " + t, e);
+            }
+        }
+
+    }
+
+}

Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/POConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/POConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/POConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/POConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,16 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
+
+import org.apache.spark.rdd.RDD;
+
+/**
+ * Given an RDD and a PhysicalOperater, and implementation of this class can
+ * convert the RDD to another RDD.
+ */
+public interface POConverter<IN, OUT, T extends PhysicalOperator> {
+    RDD<OUT> convert(List<RDD<IN>> rdd, T physicalOperator) throws IOException;
+}

Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/POOutputConsumerIterator.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/POOutputConsumerIterator.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/POOutputConsumerIterator.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/POOutputConsumerIterator.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,85 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
+import org.apache.pig.data.Tuple;
+
+abstract class POOutputConsumerIterator implements java.util.Iterator<Tuple> {
+    private final java.util.Iterator<Tuple> input;
+    private Result result = null;
+    private boolean returned = true;
+    private boolean finished = false;
+
+    POOutputConsumerIterator(java.util.Iterator<Tuple> input) {
+        this.input = input;
+    }
+
+    abstract protected void attach(Tuple tuple);
+
+    abstract protected Result getNextResult() throws ExecException;
+
+    private void readNext() {
+        try {
+            if (result != null && !returned) {
+                return;
+            }
+            // see PigGenericMapBase
+            if (result == null) {
+                if (!input.hasNext()) {
+                    finished = true;
+                    return;
+                }
+                Tuple v1 = input.next();
+                attach(v1);
+            }
+            result = getNextResult();
+            returned = false;
+            switch (result.returnStatus) {
+            case POStatus.STATUS_OK:
+                returned = false;
+                break;
+            case POStatus.STATUS_NULL:
+                returned = true; // skip: see PigGenericMapBase
+                readNext();
+                break;
+            case POStatus.STATUS_EOP:
+                finished = !input.hasNext();
+                if (!finished) {
+                    result = null;
+                    readNext();
+                }
+                break;
+            case POStatus.STATUS_ERR:
+                throw new RuntimeException("Error while processing " + result);
+            }
+        } catch (ExecException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public boolean hasNext() {
+        readNext();
+        return !finished;
+    }
+
+    @Override
+    public Tuple next() {
+        readNext();
+        if (finished) {
+            throw new RuntimeException("Passed the end. call hasNext() first");
+        }
+        if (result == null || result.returnStatus != POStatus.STATUS_OK) {
+            throw new RuntimeException("Unexpected response code in ForEach: "
+                    + result);
+        }
+        returned = true;
+        return (Tuple) result.result;
+    }
+
+    @Override
+    public void remove() {
+        throw new UnsupportedOperationException();
+    }
+}
\ No newline at end of file

Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/PackageConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/PackageConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/PackageConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/PackageConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,124 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage;
+import org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.io.NullableTuple;
+import org.apache.pig.impl.io.PigNullableWritable;
+
+import scala.runtime.AbstractFunction1;
+import org.apache.spark.rdd.RDD;
+
+@SuppressWarnings({ "serial" })
+public class PackageConverter implements POConverter<Tuple, Tuple, POPackage> {
+    private static final Log LOG = LogFactory.getLog(PackageConverter.class);
+
+    @Override
+    public RDD<Tuple> convert(List<RDD<Tuple>> predecessors,
+            POPackage physicalOperator) throws IOException {
+        SparkUtil.assertPredecessorSize(predecessors, physicalOperator, 1);
+        RDD<Tuple> rdd = predecessors.get(0);
+        // package will generate the group from the result of the local
+        // rearrange
+        return rdd.map(new PackageFunction(physicalOperator),
+                SparkUtil.getManifest(Tuple.class));
+    }
+
+    private static class PackageFunction extends
+            AbstractFunction1<Tuple, Tuple> implements Serializable {
+
+        private final POPackage physicalOperator;
+
+        public PackageFunction(POPackage physicalOperator) {
+            this.physicalOperator = physicalOperator;
+        }
+
+        @Override
+        public Tuple apply(final Tuple t) {
+            // (key, Seq<Tuple>:{(index, key, value without key)})
+            if (LOG.isDebugEnabled())
+                LOG.debug("PackageFunction in " + t);
+            Result result;
+            try {
+                PigNullableWritable key = new PigNullableWritable() {
+
+                    public Object getValueAsPigType() {
+                        try {
+                            Object keyTuple = t.get(0);
+                            return keyTuple;
+                        } catch (ExecException e) {
+                            throw new RuntimeException(e);
+                        }
+                    }
+                };
+                final Iterator<Tuple> bagIterator = (Iterator<Tuple>) t.get(1);
+                Iterator<NullableTuple> iterator = new Iterator<NullableTuple>() {
+                    public boolean hasNext() {
+                        return bagIterator.hasNext();
+                    }
+
+                    public NullableTuple next() {
+                        try {
+                            // we want the value and index only
+                            Tuple next = bagIterator.next();
+                            NullableTuple nullableTuple = new NullableTuple(
+                                    (Tuple) next.get(2));
+                            nullableTuple.setIndex(((Number) next.get(0))
+                                    .byteValue());
+                            return nullableTuple;
+                        } catch (ExecException e) {
+                            throw new RuntimeException(e);
+                        }
+                    }
+
+                    public void remove() {
+                        throw new UnsupportedOperationException();
+                    }
+                };
+                physicalOperator.setInputs(null);
+                physicalOperator.attachInput(key, iterator);
+                result = physicalOperator.getNextTuple();
+            } catch (ExecException e) {
+                throw new RuntimeException(
+                        "Couldn't do Package on tuple: " + t, e);
+            }
+
+            if (result == null) {
+                throw new RuntimeException(
+                        "Null response found for Package on tuple: " + t);
+            }
+            Tuple out;
+            switch (result.returnStatus) {
+            case POStatus.STATUS_OK:
+                // (key, {(value)...})
+                if (LOG.isDebugEnabled())
+                    LOG.debug("PackageFunction out " + result.result);
+                out = (Tuple) result.result;
+                break;
+            case POStatus.STATUS_NULL:
+                out = null;
+                break;
+            default:
+                throw new RuntimeException(
+                        "Unexpected response code from operator "
+                                + physicalOperator + " : " + result + " "
+                                + result.returnStatus);
+            }
+            if (LOG.isDebugEnabled())
+                LOG.debug("PackageFunction out " + out);
+            return out;
+        }
+
+    }
+
+}

Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SortConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SortConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SortConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SortConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,96 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort.SortComparator;
+import org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil;
+import org.apache.pig.data.Tuple;
+
+import scala.Tuple2;
+import scala.runtime.AbstractFunction1;
+
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.function.FlatMapFunction;
+import org.apache.spark.rdd.RDD;
+
+@SuppressWarnings("serial")
+public class SortConverter implements POConverter<Tuple, Tuple, POSort> {
+    private static final Log LOG = LogFactory.getLog(SortConverter.class);
+
+    private static final FlatMapFunction<Iterator<Tuple2<Tuple, Object>>, Tuple> TO_VALUE_FUNCTION = new ToValueFunction();
+
+    @Override
+    public RDD<Tuple> convert(List<RDD<Tuple>> predecessors, POSort sortOperator)
+            throws IOException {
+        SparkUtil.assertPredecessorSize(predecessors, sortOperator, 1);
+        RDD<Tuple> rdd = predecessors.get(0);
+        RDD<Tuple2<Tuple, Object>> rddPair = rdd.map(new ToKeyValueFunction(),
+                SparkUtil.<Tuple, Object> getTuple2Manifest());
+
+        JavaPairRDD<Tuple, Object> r = new JavaPairRDD<Tuple, Object>(rddPair,
+                SparkUtil.getManifest(Tuple.class),
+                SparkUtil.getManifest(Object.class));
+
+        JavaPairRDD<Tuple, Object> sorted = r.sortByKey(
+                sortOperator.new SortComparator(), true);
+        JavaRDD<Tuple> mapped = sorted.mapPartitions(TO_VALUE_FUNCTION);
+
+        return mapped.rdd();
+    }
+
+    private static class ToValueFunction extends
+            FlatMapFunction<Iterator<Tuple2<Tuple, Object>>, Tuple> implements
+            Serializable {
+
+        private class Tuple2TransformIterable implements Iterable<Tuple> {
+
+            Iterator<Tuple2<Tuple, Object>> in;
+
+            Tuple2TransformIterable(Iterator<Tuple2<Tuple, Object>> input) {
+                in = input;
+            }
+
+            public Iterator<Tuple> iterator() {
+                return new IteratorTransform<Tuple2<Tuple, Object>, Tuple>(in) {
+                    @Override
+                    protected Tuple transform(Tuple2<Tuple, Object> next) {
+                        return next._1();
+                    }
+                };
+            }
+        }
+
+        @Override
+        public Iterable<Tuple> call(Iterator<Tuple2<Tuple, Object>> input) {
+            return new Tuple2TransformIterable(input);
+        }
+    }
+
+    private static class ToKeyValueFunction extends
+            AbstractFunction1<Tuple, Tuple2<Tuple, Object>> implements
+            Serializable {
+
+        @Override
+        public Tuple2<Tuple, Object> apply(Tuple t) {
+            if (LOG.isDebugEnabled()) {
+                LOG.debug("Sort ToKeyValueFunction in " + t);
+            }
+            Tuple key = t;
+            Object value = null;
+            // (key, value)
+            Tuple2<Tuple, Object> out = new Tuple2<Tuple, Object>(key, value);
+            if (LOG.isDebugEnabled()) {
+                LOG.debug("Sort ToKeyValueFunction out " + out);
+            }
+            return out;
+        }
+    }
+
+}

Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SplitConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SplitConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SplitConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SplitConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,22 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit;
+import org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil;
+import org.apache.pig.data.Tuple;
+
+import org.apache.spark.rdd.RDD;
+
+public class SplitConverter implements POConverter<Tuple, Tuple, POSplit> {
+
+    @Override
+    public RDD<Tuple> convert(List<RDD<Tuple>> predecessors, POSplit poSplit)
+            throws IOException {
+        SparkUtil.assertPredecessorSize(predecessors, poSplit, 1);
+        return predecessors.get(0);
+    }
+
+}

Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/StoreConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/StoreConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/StoreConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/StoreConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,94 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.pig.StoreFuncInterface;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputFormat;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
+import org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.util.ObjectSerializer;
+
+import scala.Tuple2;
+import scala.collection.JavaConversions;
+import scala.runtime.AbstractFunction1;
+import org.apache.spark.rdd.PairRDDFunctions;
+import org.apache.spark.rdd.RDD;
+import org.apache.spark.api.java.function.Function;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Converter that takes a POStore and stores it's content.
+ */
+@SuppressWarnings({ "serial" })
+public class StoreConverter implements
+        POConverter<Tuple, Tuple2<Text, Tuple>, POStore> {
+
+    private static final FromTupleFunction FROM_TUPLE_FUNCTION = new FromTupleFunction();
+
+    private PigContext pigContext;
+
+    public StoreConverter(PigContext pigContext) {
+        this.pigContext = pigContext;
+    }
+
+    @Override
+    public RDD<Tuple2<Text, Tuple>> convert(List<RDD<Tuple>> predecessors,
+            POStore physicalOperator) throws IOException {
+        SparkUtil.assertPredecessorSize(predecessors, physicalOperator, 1);
+        RDD<Tuple> rdd = predecessors.get(0);
+        // convert back to KV pairs
+        RDD<Tuple2<Text, Tuple>> rddPairs = rdd.map(FROM_TUPLE_FUNCTION,
+                SparkUtil.<Text, Tuple> getTuple2Manifest());
+        PairRDDFunctions<Text, Tuple> pairRDDFunctions = new PairRDDFunctions<Text, Tuple>(
+                rddPairs, SparkUtil.getManifest(Text.class),
+                SparkUtil.getManifest(Tuple.class));
+
+        JobConf storeJobConf = SparkUtil.newJobConf(pigContext);
+        POStore poStore = configureStorer(storeJobConf, physicalOperator);
+
+        pairRDDFunctions.saveAsNewAPIHadoopFile(poStore.getSFile()
+                .getFileName(), Text.class, Tuple.class, PigOutputFormat.class,
+                storeJobConf);
+
+        return rddPairs;
+    }
+
+    private static POStore configureStorer(JobConf jobConf,
+            PhysicalOperator physicalOperator) throws IOException {
+        ArrayList<POStore> storeLocations = Lists.newArrayList();
+        POStore poStore = (POStore) physicalOperator;
+        storeLocations.add(poStore);
+        StoreFuncInterface sFunc = poStore.getStoreFunc();
+        sFunc.setStoreLocation(poStore.getSFile().getFileName(),
+                new org.apache.hadoop.mapreduce.Job(jobConf));
+        poStore.setInputs(null);
+        poStore.setParentPlan(null);
+
+        jobConf.set(JobControlCompiler.PIG_MAP_STORES,
+                ObjectSerializer.serialize(Lists.newArrayList()));
+        jobConf.set(JobControlCompiler.PIG_REDUCE_STORES,
+                ObjectSerializer.serialize(storeLocations));
+        return poStore;
+    }
+
+    private static class FromTupleFunction extends
+            Function<Tuple, Tuple2<Text, Tuple>> implements Serializable {
+
+        private static Text EMPTY_TEXT = new Text();
+
+        public Tuple2<Text, Tuple> call(Tuple v1) {
+            return new Tuple2<Text, Tuple>(EMPTY_TEXT, v1);
+        }
+    }
+}

Added: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/UnionConverter.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/UnionConverter.java?rev=1624546&view=auto
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/UnionConverter.java (added)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/UnionConverter.java Fri Sep 12 13:26:06 2014
@@ -0,0 +1,34 @@
+package org.apache.pig.backend.hadoop.executionengine.spark.converter;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POUnion;
+import org.apache.pig.backend.hadoop.executionengine.spark.SparkUtil;
+import org.apache.pig.data.Tuple;
+
+import scala.collection.JavaConversions;
+import org.apache.spark.rdd.UnionRDD;
+import org.apache.spark.rdd.RDD;
+import org.apache.spark.SparkContext;
+
+public class UnionConverter implements POConverter<Tuple, Tuple, POUnion> {
+
+    private final SparkContext sc;
+
+    public UnionConverter(SparkContext sc) {
+        this.sc = sc;
+    }
+
+    @Override
+    public RDD<Tuple> convert(List<RDD<Tuple>> predecessors,
+            POUnion physicalOperator) throws IOException {
+        SparkUtil.assertPredecessorSizeGreaterThan(predecessors,
+                physicalOperator, 0);
+        UnionRDD<Tuple> unionRDD = new UnionRDD<Tuple>(sc,
+                JavaConversions.asScalaBuffer(predecessors),
+                SparkUtil.getManifest(Tuple.class));
+        return unionRDD;
+    }
+
+}