You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by ni...@apache.org on 2013/07/31 20:55:48 UTC

[3/5] GIRAPH-717: HiveJythonRunner with support for pure Jython value types (nitay)

http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-core/src/test/java/org/apache/giraph/jython/TestJython.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/test/java/org/apache/giraph/jython/TestJython.java b/giraph-core/src/test/java/org/apache/giraph/jython/TestJython.java
deleted file mode 100644
index b5110bb..0000000
--- a/giraph-core/src/test/java/org/apache/giraph/jython/TestJython.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.giraph.jython;
-
-import org.apache.giraph.conf.GiraphConfiguration;
-import org.apache.giraph.conf.GiraphTypes;
-import org.apache.giraph.edge.ByteArrayEdges;
-import org.apache.giraph.graph.Language;
-import org.apache.giraph.io.formats.IdWithValueTextOutputFormat;
-import org.apache.giraph.io.formats.IntNullTextEdgeInputFormat;
-import org.apache.giraph.scripting.DeployType;
-import org.apache.giraph.scripting.ScriptLoader;
-import org.apache.giraph.utils.InternalVertexRunner;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.NullWritable;
-import org.junit.Test;
-import org.python.core.PyClass;
-import org.python.core.PyDictionary;
-import org.python.core.PyInteger;
-import org.python.core.PyList;
-import org.python.core.PyObject;
-import org.python.util.PythonInterpreter;
-
-import com.google.common.collect.Maps;
-
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class TestJython {
-  private static final double DELTA = 0.0000001;
-
-  @Test
-  public void testBasic() {
-    String jython =
-        "class Foo:\n" +
-        "    def __init__(self):\n" +
-        "        self.map = {\"32\": 32, \"4.3\": 4.3}\n" +
-        "        self.list = [ 2, 9, 11 ]\n" +
-        "        self.ival = 17\n" +
-        "\n" +
-        "def get_map(foo):\n" +
-        "    return foo.map\n" +
-        "\n" +
-        "def get_list(foo):\n" +
-        "    return foo.list\n" +
-        "\n" +
-        "def get_ival(foo):\n" +
-        "    return foo.ival\n" +
-        "";
-
-    PythonInterpreter interpreter = new PythonInterpreter();
-    interpreter.exec(jython);
-
-    PyObject fooClass = interpreter.get("Foo");
-    assertTrue(fooClass instanceof PyClass);
-
-    PyObject getMapFunc = interpreter.get("get_map");
-    PyObject getListFunc = interpreter.get("get_list");
-    PyObject getIValFunc = interpreter.get("get_ival");
-
-    PyObject foo = fooClass.__call__();
-
-    PyObject mapResult = getMapFunc.__call__(foo);
-    assertTrue(mapResult instanceof PyDictionary);
-    PyDictionary pyMapResult = ((PyDictionary) mapResult);
-    assertEquals(2, pyMapResult.size());
-    Object thirtyTwo = pyMapResult.get("32");
-    assertTrue(thirtyTwo instanceof Integer);
-    assertEquals(32, ((Integer) thirtyTwo).intValue());
-    Object fourPointThree = pyMapResult.get("4.3");
-    assertTrue(fourPointThree instanceof Double);
-    assertEquals(4.3, (Double) fourPointThree, DELTA);
-
-    PyObject listResult = getListFunc.__call__(foo);
-    assertTrue(listResult instanceof PyList);
-    PyList pyListResult = (PyList) listResult;
-    assertEquals(3, pyListResult.size());
-    assertEquals(2, pyListResult.get(0));
-    assertEquals(9, pyListResult.get(1));
-    assertEquals(11, pyListResult.get(2));
-
-    PyObject ivalResult = getIValFunc.__call__(foo);
-    assertTrue(ivalResult instanceof PyInteger);
-    assertEquals(17, ((PyInteger) ivalResult).getValue());
-  }
-
-  @Test
-  public void testCountEdges() throws Exception {
-    String[] edges = new String[] {
-        "1 2",
-        "2 3",
-        "2 4",
-        "4 1"
-    };
-
-    GiraphConfiguration conf = new GiraphConfiguration();
-    GiraphTypes types = new GiraphTypes(IntWritable.class, IntWritable.class,
-        NullWritable.class, NullWritable.class, NullWritable.class);
-    types.writeIfUnset(conf);
-    ScriptLoader.setScriptsToLoad(conf,
-        "org/apache/giraph/jython/count-edges.py",
-        DeployType.RESOURCE, Language.JYTHON);
-    JythonUtils.init(conf, "CountEdges");
-    conf.setOutEdgesClass(ByteArrayEdges.class);
-    conf.setEdgeInputFormatClass(IntNullTextEdgeInputFormat.class);
-    conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
-    Iterable<String> results = InternalVertexRunner.run(conf, null, edges);
-
-    Map<Integer, Integer> values = parseResults(results);
-
-    // Check that all vertices with outgoing edges have been created
-    assertEquals(3, values.size());
-    // Check the number of edges for each vertex
-    assertEquals(1, (int) values.get(1));
-    assertEquals(2, (int) values.get(2));
-    assertEquals(1, (int) values.get(4));
-  }
-
-  private static Map<Integer, Integer> parseResults(Iterable<String> results) {
-    Map<Integer, Integer> values = Maps.newHashMap();
-    for (String line : results) {
-      String[] tokens = line.split("\\s+");
-      int id = Integer.valueOf(tokens[0]);
-      int value = Integer.valueOf(tokens[1]);
-      values.put(id, value);
-    }
-    return values;
-  }
-}

http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-core/src/test/java/org/apache/giraph/jython/TestJythonBasic.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/test/java/org/apache/giraph/jython/TestJythonBasic.java b/giraph-core/src/test/java/org/apache/giraph/jython/TestJythonBasic.java
new file mode 100644
index 0000000..5f8db66
--- /dev/null
+++ b/giraph-core/src/test/java/org/apache/giraph/jython/TestJythonBasic.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.jython;
+
+import org.junit.Test;
+import org.python.core.PyClass;
+import org.python.core.PyDictionary;
+import org.python.core.PyInteger;
+import org.python.core.PyList;
+import org.python.core.PyObject;
+import org.python.util.PythonInterpreter;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TestJythonBasic {
+  private static final double DELTA = 0.0000001;
+
+  @Test
+  public void testBasic() {
+    String jython =
+        "class Foo:\n" +
+        "    def __init__(self):\n" +
+        "        self.map = {\"32\": 32, \"4.3\": 4.3}\n" +
+        "        self.list = [ 2, 9, 11 ]\n" +
+        "        self.ival = 17\n" +
+        "\n" +
+        "def get_map(foo):\n" +
+        "    return foo.map\n" +
+        "\n" +
+        "def get_list(foo):\n" +
+        "    return foo.list\n" +
+        "\n" +
+        "def get_ival(foo):\n" +
+        "    return foo.ival\n" +
+        "";
+
+    PythonInterpreter interpreter = new PythonInterpreter();
+    interpreter.exec(jython);
+
+    PyObject fooClass = interpreter.get("Foo");
+    assertTrue(fooClass instanceof PyClass);
+
+    PyObject getMapFunc = interpreter.get("get_map");
+    PyObject getListFunc = interpreter.get("get_list");
+    PyObject getIValFunc = interpreter.get("get_ival");
+
+    PyObject foo = fooClass.__call__();
+
+    PyObject mapResult = getMapFunc.__call__(foo);
+    assertTrue(mapResult instanceof PyDictionary);
+    PyDictionary pyMapResult = ((PyDictionary) mapResult);
+    assertEquals(2, pyMapResult.size());
+    Object thirtyTwo = pyMapResult.get("32");
+    assertTrue(thirtyTwo instanceof Integer);
+    assertEquals(32, ((Integer) thirtyTwo).intValue());
+    Object fourPointThree = pyMapResult.get("4.3");
+    assertTrue(fourPointThree instanceof Double);
+    assertEquals(4.3, (Double) fourPointThree, DELTA);
+
+    PyObject listResult = getListFunc.__call__(foo);
+    assertTrue(listResult instanceof PyList);
+    PyList pyListResult = (PyList) listResult;
+    assertEquals(3, pyListResult.size());
+    assertEquals(2, pyListResult.get(0));
+    assertEquals(9, pyListResult.get(1));
+    assertEquals(11, pyListResult.get(2));
+
+    PyObject ivalResult = getIValFunc.__call__(foo);
+    assertTrue(ivalResult instanceof PyInteger);
+    assertEquals(17, ((PyInteger) ivalResult).getValue());
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-core/src/test/java/org/apache/giraph/jython/TestJythonComputation.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/test/java/org/apache/giraph/jython/TestJythonComputation.java b/giraph-core/src/test/java/org/apache/giraph/jython/TestJythonComputation.java
new file mode 100644
index 0000000..5feabaf
--- /dev/null
+++ b/giraph-core/src/test/java/org/apache/giraph/jython/TestJythonComputation.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.jython;
+
+import org.apache.giraph.conf.GiraphConfiguration;
+import org.apache.giraph.conf.GiraphTypes;
+import org.apache.giraph.edge.ByteArrayEdges;
+import org.apache.giraph.graph.Language;
+import org.apache.giraph.io.formats.IdWithValueTextOutputFormat;
+import org.apache.giraph.io.formats.IntNullTextEdgeInputFormat;
+import org.apache.giraph.scripting.DeployType;
+import org.apache.giraph.scripting.ScriptLoader;
+import org.apache.giraph.utils.InternalVertexRunner;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.junit.Test;
+
+import com.google.common.collect.Maps;
+
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestJythonComputation {
+  @Test
+  public void testCountEdges() throws Exception {
+    String[] edges = new String[] {
+        "1 2",
+        "2 3",
+        "2 4",
+        "4 1"
+    };
+
+    GiraphConfiguration conf = new GiraphConfiguration();
+    GiraphTypes types = new GiraphTypes(IntWritable.class, IntWritable.class,
+        NullWritable.class, NullWritable.class, NullWritable.class);
+    types.writeIfUnset(conf);
+    ScriptLoader.setScriptsToLoad(conf,
+        "org/apache/giraph/jython/count-edges.py",
+        DeployType.RESOURCE, Language.JYTHON);
+    JythonUtils.init(conf, "CountEdges");
+    conf.setOutEdgesClass(ByteArrayEdges.class);
+    conf.setEdgeInputFormatClass(IntNullTextEdgeInputFormat.class);
+    conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
+    Iterable<String> results = InternalVertexRunner.run(conf, null, edges);
+
+    Map<Integer, Integer> values = parseResults(results);
+
+    // Check that all vertices with outgoing edges have been created
+    assertEquals(3, values.size());
+    // Check the number of edges for each vertex
+    assertEquals(1, (int) values.get(1));
+    assertEquals(2, (int) values.get(2));
+    assertEquals(1, (int) values.get(4));
+  }
+
+  private static Map<Integer, Integer> parseResults(Iterable<String> results) {
+    Map<Integer, Integer> values = Maps.newHashMap();
+    for (String line : results) {
+      String[] tokens = line.split("\\s+");
+      int id = Integer.valueOf(tokens[0]);
+      int value = Integer.valueOf(tokens[1]);
+      values.put(id, value);
+    }
+    return values;
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-core/src/test/java/org/apache/giraph/jython/TestJythonWritableWrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/test/java/org/apache/giraph/jython/TestJythonWritableWrapper.java b/giraph-core/src/test/java/org/apache/giraph/jython/TestJythonWritableWrapper.java
new file mode 100644
index 0000000..3d34571
--- /dev/null
+++ b/giraph-core/src/test/java/org/apache/giraph/jython/TestJythonWritableWrapper.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.jython;
+
+import org.apache.giraph.jython.wrappers.JythonWritableWrapper;
+import org.junit.Test;
+import org.python.core.PyClass;
+import org.python.core.PyFunction;
+import org.python.core.PyInteger;
+import org.python.core.PyMethod;
+import org.python.core.PyObject;
+import org.python.util.PythonInterpreter;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TestJythonWritableWrapper {
+  @Test
+  public void testWrap() throws IOException {
+    String jython =
+        "class Foo:\n" +
+        "    def __init__(self):\n" +
+        "        self.val = 17\n" +
+        "\n" +
+        "    def do_add(self, x):\n" +
+        "        self.val += x\n" +
+        "\n" +
+        "    def do_add_squared(self, x):\n" +
+        "        self.do_add(x * x)\n" +
+        "\n" +
+        "    def new_other(self):\n" +
+        "        self.other_val = 3\n" +
+        "\n" +
+        "def outside_add_squared(foo, x):\n" +
+        "    foo.do_add_squared(x)\n" +
+        "\n";
+
+    PythonInterpreter interpreter = new PythonInterpreter();
+    interpreter.exec(jython);
+
+    PyObject fooClass = interpreter.get("Foo");
+    assertTrue(fooClass instanceof PyClass);
+    PyObject foo = fooClass.__call__();
+
+    PyObject fooVal = foo.__getattr__("val");
+    assertTrue(fooVal instanceof PyInteger);
+    PyInteger val = (PyInteger) fooVal;
+    assertEquals(17, val.getValue());
+
+    PyObject function = interpreter.get("outside_add_squared");
+    assertTrue("method class: " + function.getClass(), function instanceof PyFunction);
+    function.__call__(foo, new PyInteger(3));
+
+    fooVal = foo.__getattr__("val");
+    assertTrue(fooVal instanceof PyInteger);
+    val = (PyInteger) fooVal;
+    assertEquals(26, val.getValue());
+
+    JythonWritableWrapper wrappedFoo = new JythonWritableWrapper(foo);
+    PyObject newOtherMethod = wrappedFoo.__getattr__("new_other");
+
+    assertTrue(newOtherMethod instanceof PyMethod);
+    newOtherMethod.__call__();
+
+    function.__call__(wrappedFoo, new PyInteger(2));
+
+    fooVal = foo.__getattr__("val");
+    assertTrue(fooVal instanceof PyInteger);
+    val = (PyInteger) fooVal;
+    assertEquals(30, val.getValue());
+
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    DataOutputStream dos = new DataOutputStream(baos);
+    wrappedFoo.write(dos);
+
+    byte[] data = baos.toByteArray();
+    ByteArrayInputStream bais = new ByteArrayInputStream(data);
+    DataInputStream dis = new DataInputStream(bais);
+
+    PyObject foo2 = fooClass.__call__();
+
+    PyObject foo2Val = foo2.__getattr__("val");
+    assertTrue(foo2Val instanceof PyInteger);
+    PyInteger val2 = (PyInteger) foo2Val;
+    assertEquals(17, val2.getValue());
+
+    JythonWritableWrapper wrappedFoo2 = new JythonWritableWrapper(foo2);
+
+    foo2Val = wrappedFoo2.getPyObject().__getattr__("val");
+    assertTrue(foo2Val instanceof PyInteger);
+    val2 = (PyInteger) foo2Val;
+    assertEquals(17, val2.getValue());
+
+    wrappedFoo2.readFields(dis);
+
+    foo2Val = wrappedFoo2.getPyObject().__getattr__("val");
+    assertTrue(foo2Val instanceof PyInteger);
+    val2 = (PyInteger) foo2Val;
+    assertEquals(30, val2.getValue());
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-core/src/test/java/org/apache/giraph/master/TestComputationCombinerTypes.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/test/java/org/apache/giraph/master/TestComputationCombinerTypes.java b/giraph-core/src/test/java/org/apache/giraph/master/TestComputationCombinerTypes.java
index f9bd4e4..b62775f 100644
--- a/giraph-core/src/test/java/org/apache/giraph/master/TestComputationCombinerTypes.java
+++ b/giraph-core/src/test/java/org/apache/giraph/master/TestComputationCombinerTypes.java
@@ -21,6 +21,7 @@ package org.apache.giraph.master;
 import org.apache.giraph.combiner.Combiner;
 import org.apache.giraph.conf.GiraphConfiguration;
 import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.giraph.graph.AbstractComputation;
 import org.apache.giraph.graph.Computation;
 import org.apache.giraph.graph.Vertex;
 import org.apache.giraph.utils.IntNoOpComputation;
@@ -110,7 +111,7 @@ public class TestComputationCombinerTypes {
 
   public static class NoOpComputation<I extends WritableComparable,
       V extends Writable, E extends Writable, M1 extends Writable,
-      M2 extends Writable> extends Computation<I, V, E, M1, M2> {
+      M2 extends Writable> extends AbstractComputation<I, V, E, M1, M2> {
     @Override
     public void compute(Vertex<I, V, E> vertex,
         Iterable<M1> messages) throws IOException {

http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-core/src/test/java/org/apache/giraph/master/TestSwitchClasses.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/test/java/org/apache/giraph/master/TestSwitchClasses.java b/giraph-core/src/test/java/org/apache/giraph/master/TestSwitchClasses.java
index 4069972..6b0ed35 100644
--- a/giraph-core/src/test/java/org/apache/giraph/master/TestSwitchClasses.java
+++ b/giraph-core/src/test/java/org/apache/giraph/master/TestSwitchClasses.java
@@ -20,7 +20,7 @@ package org.apache.giraph.master;
 
 import org.apache.giraph.combiner.Combiner;
 import org.apache.giraph.conf.GiraphConfiguration;
-import org.apache.giraph.graph.Computation;
+import org.apache.giraph.graph.AbstractComputation;
 import org.apache.giraph.graph.Vertex;
 import org.apache.giraph.utils.InternalVertexRunner;
 import org.apache.giraph.utils.TestGraph;
@@ -136,8 +136,8 @@ public class TestSwitchClasses {
     }
   }
 
-  public static class Computation1 extends Computation<IntWritable,
-      StatusValue, IntWritable, IntWritable, IntWritable> {
+  public static class Computation1 extends AbstractComputation<IntWritable,
+        StatusValue, IntWritable, IntWritable, IntWritable> {
     @Override
     public void compute(Vertex<IntWritable, StatusValue, IntWritable> vertex,
         Iterable<IntWritable> messages) throws IOException {
@@ -150,8 +150,8 @@ public class TestSwitchClasses {
     }
   }
 
-  public static class Computation2 extends Computation<IntWritable,
-      StatusValue, IntWritable, IntWritable, DoubleWritable> {
+  public static class Computation2 extends AbstractComputation<IntWritable,
+        StatusValue, IntWritable, IntWritable, DoubleWritable> {
     @Override
     public void compute(Vertex<IntWritable, StatusValue, IntWritable> vertex,
         Iterable<IntWritable> messages) throws IOException {
@@ -164,8 +164,8 @@ public class TestSwitchClasses {
     }
   }
 
-  public static class Computation3 extends Computation<IntWritable,
-      StatusValue, IntWritable, DoubleWritable, IntWritable> {
+  public static class Computation3 extends AbstractComputation<IntWritable,
+        StatusValue, IntWritable, DoubleWritable, IntWritable> {
     @Override
     public void compute(
         Vertex<IntWritable, StatusValue, IntWritable> vertex,

http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-core/src/test/java/org/apache/giraph/utils/TestReflectionUtils.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/test/java/org/apache/giraph/utils/TestReflectionUtils.java b/giraph-core/src/test/java/org/apache/giraph/utils/TestReflectionUtils.java
index 1c3eed3..261e7c2 100644
--- a/giraph-core/src/test/java/org/apache/giraph/utils/TestReflectionUtils.java
+++ b/giraph-core/src/test/java/org/apache/giraph/utils/TestReflectionUtils.java
@@ -29,6 +29,7 @@ import org.apache.giraph.factories.EdgeValueFactory;
 import org.apache.giraph.factories.MessageValueFactory;
 import org.apache.giraph.factories.VertexIdFactory;
 import org.apache.giraph.factories.VertexValueFactory;
+import org.apache.giraph.graph.AbstractComputation;
 import org.apache.giraph.graph.BasicComputation;
 import org.apache.giraph.graph.Computation;
 import org.apache.giraph.graph.DefaultVertexResolver;
@@ -47,19 +48,19 @@ import static org.junit.Assert.assertEquals;
 public class TestReflectionUtils {
   @Test
   public void testPackagePath() {
-    assertEquals("/org/apache/giraph/utils",
+    assertEquals("org/apache/giraph/utils",
         ReflectionUtils.getPackagePath(TestReflectionUtils.class));
-    assertEquals("/org/apache/giraph/utils",
+    assertEquals("org/apache/giraph/utils",
         ReflectionUtils.getPackagePath(getClass()));
-    assertEquals("/org/apache/giraph/utils",
+    assertEquals("org/apache/giraph/utils",
         ReflectionUtils.getPackagePath(this));
   }
 
   private static class IntTypes implements TypesHolder<IntWritable,
       IntWritable, IntWritable, IntWritable, IntWritable> { }
 
-  private static class IntComputation extends Computation<IntWritable,
-        IntWritable, IntWritable, IntWritable, IntWritable> {
+  private static class IntComputation extends AbstractComputation<IntWritable,
+          IntWritable, IntWritable, IntWritable, IntWritable> {
     @Override
     public void compute(Vertex<IntWritable, IntWritable, IntWritable> vertex,
         Iterable<IntWritable> messages) throws IOException {

http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-core/src/test/resources/org/apache/giraph/jython/count-edges.py
----------------------------------------------------------------------
diff --git a/giraph-core/src/test/resources/org/apache/giraph/jython/count-edges.py b/giraph-core/src/test/resources/org/apache/giraph/jython/count-edges.py
index 4cad945..443480a 100644
--- a/giraph-core/src/test/resources/org/apache/giraph/jython/count-edges.py
+++ b/giraph-core/src/test/resources/org/apache/giraph/jython/count-edges.py
@@ -1,7 +1,7 @@
-from org.apache.giraph.graph import BasicComputation
+from org.apache.giraph.jython import JythonComputation
 from org.apache.hadoop.io import IntWritable
 
-class CountEdges(BasicComputation):
+class CountEdges(JythonComputation):
   def compute(self, vertex, messages):
     num_edges = vertex.getNumEdges()
     vertex.setValue(IntWritable(num_edges))

http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-examples/src/test/java/org/apache/giraph/TestBspBasic.java
----------------------------------------------------------------------
diff --git a/giraph-examples/src/test/java/org/apache/giraph/TestBspBasic.java b/giraph-examples/src/test/java/org/apache/giraph/TestBspBasic.java
index f9e77af..115da7e 100644
--- a/giraph-examples/src/test/java/org/apache/giraph/TestBspBasic.java
+++ b/giraph-examples/src/test/java/org/apache/giraph/TestBspBasic.java
@@ -153,7 +153,7 @@ public class
         immutableClassesGiraphConfiguration.createEdgeValue();
     Writable messageValue =
         immutableClassesGiraphConfiguration.getOutgoingMessageValueFactory()
-            .createMessageValue();
+            .newInstance();
     assertSame(vertexValue.getClass(), NullWritable.class);
     assertSame(vertexValue, edgeValue);
     assertSame(edgeValue, messageValue);

http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-examples/src/test/java/org/apache/giraph/vertex/TestComputationTypes.java
----------------------------------------------------------------------
diff --git a/giraph-examples/src/test/java/org/apache/giraph/vertex/TestComputationTypes.java b/giraph-examples/src/test/java/org/apache/giraph/vertex/TestComputationTypes.java
index fe354fa..4f74fcb 100644
--- a/giraph-examples/src/test/java/org/apache/giraph/vertex/TestComputationTypes.java
+++ b/giraph-examples/src/test/java/org/apache/giraph/vertex/TestComputationTypes.java
@@ -104,7 +104,12 @@ public class TestComputationTypes {
       public void initialize(ImmutableClassesGiraphConfiguration conf) { }
 
       @Override
-      public DoubleWritable createVertexValue() {
+      public Class<DoubleWritable> getValueClass() {
+        return DoubleWritable.class;
+      }
+
+      @Override
+      public DoubleWritable newInstance() {
         return new DoubleWritable();
       }
     }

http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-hcatalog/src/main/java/org/apache/giraph/io/hcatalog/HCatGiraphRunner.java
----------------------------------------------------------------------
diff --git a/giraph-hcatalog/src/main/java/org/apache/giraph/io/hcatalog/HCatGiraphRunner.java b/giraph-hcatalog/src/main/java/org/apache/giraph/io/hcatalog/HCatGiraphRunner.java
index 6135c7d..25d100b 100644
--- a/giraph-hcatalog/src/main/java/org/apache/giraph/io/hcatalog/HCatGiraphRunner.java
+++ b/giraph-hcatalog/src/main/java/org/apache/giraph/io/hcatalog/HCatGiraphRunner.java
@@ -26,9 +26,9 @@ import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
 import org.apache.giraph.graph.Computation;
 import org.apache.giraph.io.EdgeInputFormat;
-import org.apache.giraph.job.GiraphJob;
 import org.apache.giraph.io.VertexInputFormat;
 import org.apache.giraph.io.VertexOutputFormat;
+import org.apache.giraph.job.GiraphJob;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.util.Tool;

http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-hive/src/main/java/org/apache/giraph/hive/column/HiveReadableColumn.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/column/HiveReadableColumn.java b/giraph-hive/src/main/java/org/apache/giraph/hive/column/HiveReadableColumn.java
new file mode 100644
index 0000000..93bd2c2
--- /dev/null
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/column/HiveReadableColumn.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.hive.column;
+
+import com.facebook.hiveio.common.HiveType;
+import com.facebook.hiveio.record.HiveReadableRecord;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A single column from a Hive record.
+ */
+public class HiveReadableColumn {
+  /** The Hive record */
+  private HiveReadableRecord record;
+  /** The column index to use */
+  private int index;
+
+  public int getIndex() {
+    return index;
+  }
+
+  public void setIndex(int index) {
+    this.index = index;
+  }
+
+  public HiveReadableRecord getRecord() {
+    return record;
+  }
+
+  public void setRecord(HiveReadableRecord record) {
+    this.record = record;
+  }
+
+  public boolean isNull() {
+    return record.isNull(index);
+  }
+
+  /**
+   * Get type for this column
+   *
+   * @return {@link HiveType}
+   */
+  public HiveType hiveType() {
+    return record.columnType(index);
+  }
+
+  /**
+   * Get column value
+   *
+   * Regular data columns from the tables should always be placed first, and
+   * then partition value columns.
+   *
+   * If you know the type of the column and it is a primitive you should use
+   * one of the calls below as it will likely be more efficient.
+   *
+   * @return Object for column
+   * @deprecated use {@link #get(com.facebook.hiveio.common.HiveType)}
+   *             or one of the getX() methods
+   */
+  @Deprecated
+  public Object get() {
+    return record.get(index);
+  }
+
+  /**
+   * Get column value
+   *
+   * Regular data columns from the tables should always be placed first, and
+   * then partition value columns.
+   *
+   * You should probably be using one of getX() methods below instead.
+   *
+   * @param hiveType HiveType
+   * @return Object for column
+   */
+  public Object get(HiveType hiveType) {
+    return record.get(index, hiveType);
+  }
+
+  /**
+   * Get boolean value
+   *
+   * @return boolean at index
+   */
+  public boolean getBoolean() {
+    return record.getBoolean(index);
+  }
+
+  /**
+   * Get byte value
+   *
+   * @return byte at index
+   */
+  public byte getByte() {
+    return record.getByte(index);
+  }
+
+  /**
+   * Get short value
+   *
+   * @return short at index
+   */
+  public short getShort() {
+    return record.getShort(index);
+  }
+
+  /**
+   * Get int value
+   *
+   * @return int at index
+   */
+  public int getInt() {
+    return record.getInt(index);
+  }
+
+  /**
+   * Get long value
+   *
+   * @return long at index
+   */
+  public long getLong() {
+    return record.getLong(index);
+  }
+
+  /**
+   * Get float value
+   *
+   * @return float at index
+   */
+  public float getFloat() {
+    return record.getFloat(index);
+  }
+
+  /**
+   * Get double value
+   *
+   * @return double at index
+   */
+  public double getDouble() {
+    return record.getDouble(index);
+  }
+
+  /**
+   * Get String column value
+   * Note that partition values are all strings.
+   *
+   * @return String at index
+   */
+  public String getString() {
+    return record.getString(index);
+  }
+
+  /**
+   * Get List column value
+   * Note that partition values are all strings.
+   *
+   * @param <T> item type
+   * @return List at index
+   */
+  public <T> List<T> getList() {
+    return record.getList(index);
+  }
+
+  /**
+   * Get Map column value
+   * Note that partition values are all strings.
+   *
+   * @param <K> key type
+   * @param <V> value type
+   * @return Map at index
+   */
+  public <K, V> Map<K, V> getMap() {
+    return record.getMap(index);
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-hive/src/main/java/org/apache/giraph/hive/column/HiveWritableColumn.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/column/HiveWritableColumn.java b/giraph-hive/src/main/java/org/apache/giraph/hive/column/HiveWritableColumn.java
new file mode 100644
index 0000000..5763126
--- /dev/null
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/column/HiveWritableColumn.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.hive.column;
+
+import com.facebook.hiveio.common.HiveType;
+import com.facebook.hiveio.record.HiveWritableRecord;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A single column to write to a Hive record.
+ */
+public class HiveWritableColumn {
+  /** The Hive record */
+  private HiveWritableRecord record;
+  /** The column index to use */
+  private int index;
+
+  public int getIndex() {
+    return index;
+  }
+
+  public void setIndex(int index) {
+    this.index = index;
+  }
+
+  public HiveWritableRecord getRecord() {
+    return record;
+  }
+
+  public void setRecord(HiveWritableRecord record) {
+    this.record = record;
+  }
+
+  /**
+   * Set value for column.
+   *
+   * @param value Data for column
+   * @deprecated
+   *  use {@link #set(Object, com.facebook.hiveio.common.HiveType)}
+   *  or one of the setX() methods
+   */
+  @Deprecated
+  public void set(Object value) {
+    record.set(index, value);
+  }
+
+  /**
+   * Set value with type for column.
+   *
+   * @param value data for column
+   * @param hiveType expected hive type
+   */
+  public void set(Object value, HiveType hiveType) {
+    record.set(index, value, hiveType);
+  }
+
+  /**
+   * Set boolean value for column.
+   *
+   * @param value Data for column
+   */
+  public void setBoolean(boolean value) {
+    record.setBoolean(index, value);
+  }
+
+  /**
+   * Set byte value for column.
+   *
+   * @param value Data for column
+   */
+  public void setByte(byte value) {
+    record.setByte(index, value);
+  }
+
+  /**
+   * Set short value for column.
+   *
+   * @param value Data for column
+   */
+  public void setShort(short value) {
+    record.setShort(index, value);
+  }
+
+  /**
+   * Set int value for column.
+   *
+   * @param value Data for column
+   */
+  public void setInt(int value) {
+    record.setInt(index, value);
+  }
+
+  /**
+   * Set long value for column.
+   *
+   * @param value Data for column
+   */
+  public void setLong(long value) {
+    record.setLong(index, value);
+  }
+
+  /**
+   * Set float value for column.
+   *
+   * @param value Data for column
+   */
+  public void setFloat(float value) {
+    record.setFloat(index, value);
+  }
+
+  /**
+   * Set double value for column.
+   *
+   * @param value Data for column
+   */
+  public void setDouble(double value) {
+    record.setDouble(index, value);
+  }
+
+  /**
+   * Set double value for column.
+   *
+   * @param value Data for column
+   */
+  public void setString(String value) {
+    record.setString(index, value);
+  }
+
+  /**
+   * Set List value for column.
+   *
+   * @param data Data for column
+   */
+  public void setList(List data) {
+    record.setList(index, data);
+  }
+
+  /**
+   * Set Map value for column.
+   *
+   * @param data Data for column
+   */
+  public void setMap(Map data) {
+    record.setMap(index, data);
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-hive/src/main/java/org/apache/giraph/hive/column/package-info.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/column/package-info.java b/giraph-hive/src/main/java/org/apache/giraph/hive/column/package-info.java
new file mode 100644
index 0000000..2077c4f
--- /dev/null
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/column/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Package of Hive per-column IO related things.
+ */
+package org.apache.giraph.hive.column;

http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-hive/src/main/java/org/apache/giraph/hive/common/GiraphHiveConstants.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/common/GiraphHiveConstants.java b/giraph-hive/src/main/java/org/apache/giraph/hive/common/GiraphHiveConstants.java
index e492694..c7ad63b 100644
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/common/GiraphHiveConstants.java
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/common/GiraphHiveConstants.java
@@ -57,6 +57,27 @@ public class GiraphHiveConstants {
       new StrConfOption("giraph.hive.output.vertex.partition", "",
           "Vertex output partition");
 
+  /** Vertex ID hive reader */
+  public static final StrConfOption VERTEX_ID_READER_JYTHON_NAME =
+      new StrConfOption("giraph.hive.jython.vertex.id.reader", null,
+          "Vertex ID hive reader");
+  /** Vertex ID hive writer */
+  public static final StrConfOption VERTEX_ID_WRITER_JYTHON_NAME =
+      new StrConfOption("giraph.hive.jython.vertex.id.writer", null,
+          "Vertex ID hive writer");
+  /** Vertex value hive reader */
+  public static final StrConfOption VERTEX_VALUE_READER_JYTHON_NAME =
+      new StrConfOption("giraph.hive.jython.vertex.value.reader", null,
+          "Vertex value hive reader");
+  /** Vertex value hive writer */
+  public static final StrConfOption VERTEX_VALUE_WRITER_JYTHON_NAME =
+      new StrConfOption("giraph.hive.jython.vertex.value.writer", null,
+          "Vertex value hive writer");
+  /** Edge value hive reader */
+  public static final StrConfOption EDGE_VALUE_READER_JYTHON_NAME =
+      new StrConfOption("giraph.hive.jython.edge.value.reader", null,
+          "Edge value hive reader");
+
   /** Don't construct */
   protected GiraphHiveConstants() { }
 }

http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-hive/src/main/java/org/apache/giraph/hive/common/HiveUtils.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/common/HiveUtils.java b/giraph-hive/src/main/java/org/apache/giraph/hive/common/HiveUtils.java
index ae2a402..6e1d62b 100644
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/common/HiveUtils.java
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/common/HiveUtils.java
@@ -42,18 +42,17 @@ import com.google.common.collect.Maps;
 
 import java.io.File;
 import java.io.IOException;
+import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 
-import static com.google.common.base.Preconditions.checkNotNull;
 import static java.lang.System.getenv;
 import static org.apache.giraph.hive.common.GiraphHiveConstants.HIVE_EDGE_INPUT;
 import static org.apache.giraph.hive.common.GiraphHiveConstants.HIVE_VERTEX_INPUT;
 import static org.apache.giraph.hive.common.GiraphHiveConstants.VERTEX_TO_HIVE_CLASS;
-import static org.apache.giraph.utils.ReflectionUtils.newInstance;
 
 /**
  * Utility methods for Hive IO
@@ -184,6 +183,34 @@ public class HiveUtils {
    */
   public static void addHiveSiteCustomXmlToTmpFiles(Configuration conf) {
     addToHiveFromClassLoader(conf, "hive-site-custom.xml");
+    addToHiveFromEnv(conf, "HIVE_HOME", "conf/hive-site.xml");
+  }
+
+  /**
+   * Add a file to Configuration tmpfiles from environment variable
+   *
+   * @param conf Configuration
+   * @param envKey environment variable key
+   * @param path search path
+   * @return true if file found and added, false otherwise
+   */
+  private static boolean addToHiveFromEnv(Configuration conf,
+      String envKey, String path) {
+    String envValue = getenv(envKey);
+    if (envValue == null) {
+      return false;
+    }
+    File file = new File(envValue, path);
+    if (file.exists()) {
+      LOG.info("addToHiveFromEnv: Adding " + file.getPath() +
+          " to Configuration tmpfiles");
+    }
+    try {
+      addToStringCollection(conf, "tmpfiles", file.toURI().toURL().toString());
+    } catch (MalformedURLException e) {
+      LOG.error("Failed to get URL for file " + file);
+    }
+    return true;
   }
 
   /**
@@ -191,16 +218,20 @@ public class HiveUtils {
    *
    * @param conf Configuration
    * @param name file name
+   * @return true if file found in class loader, false otherwise
    */
-  private static void addToHiveFromClassLoader(Configuration conf,
+  private static boolean addToHiveFromClassLoader(Configuration conf,
       String name) {
     URL url = conf.getClassLoader().getResource(name);
-    checkNotNull(url);
+    if (url == null) {
+      return false;
+    }
     if (LOG.isInfoEnabled()) {
       LOG.info("addToHiveFromClassLoader: Adding " + name + " at " +
           url + " to Configuration tmpfiles");
     }
     addToStringCollection(conf, "tmpfiles", url.toString());
+    return true;
   }
 
   /**
@@ -212,12 +243,12 @@ public class HiveUtils {
   public static void addHadoopClasspathToTmpJars(Configuration conf) {
     // Or, more effectively, we can provide all the jars client needed to
     // the workers as well
-    String[] hadoopJars = getenv("HADOOP_CLASSPATH").split(File.pathSeparator);
+    String hadoopClasspath = getenv("HADOOP_CLASSPATH");
+    if (hadoopClasspath == null) {
+      return;
+    }
+    String[] hadoopJars = hadoopClasspath.split(File.pathSeparator);
     if (hadoopJars.length > 0) {
-      if (LOG.isInfoEnabled()) {
-        LOG.info("addHadoopClasspathToTmpJars: Adding HADOOP_CLASSPATH jars " +
-            "at " + Arrays.toString(hadoopJars) + " to Configuration tmpjars");
-      }
       List<String> hadoopJarURLs = Lists.newArrayList();
       for (String jarPath : hadoopJars) {
         File file = new File(jarPath);
@@ -308,9 +339,7 @@ public class HiveUtils {
       throw new IOException(VERTEX_TO_HIVE_CLASS.getKey() +
           " not set in conf");
     }
-    VertexToHive<I, V, E> vertexToHive = newInstance(klass, conf);
-    HiveTableSchemas.configure(vertexToHive, schema);
-    return vertexToHive;
+    return newInstance(klass, conf, schema);
   }
 
   /**
@@ -332,9 +361,7 @@ public class HiveUtils {
       throw new IllegalArgumentException(
           HIVE_EDGE_INPUT.getClassOpt().getKey() + " not set in conf");
     }
-    HiveToEdge hiveToEdge = ReflectionUtils.newInstance(klass, conf);
-    HiveTableSchemas.configure(hiveToEdge, schema);
-    return hiveToEdge;
+    return newInstance(klass, conf, schema);
   }
 
   /**
@@ -356,8 +383,29 @@ public class HiveUtils {
       throw new IllegalArgumentException(
           HIVE_VERTEX_INPUT.getClassOpt().getKey() + " not set in conf");
     }
-    HiveToVertex hiveToVertex = ReflectionUtils.newInstance(klass, conf);
-    HiveTableSchemas.configure(hiveToVertex, schema);
-    return hiveToVertex;
+    return newInstance(klass, conf, schema);
+  }
+
+  /**
+   * Create a new instance of a class, configuring it and setting the Hive table
+   * schema if it supports those types.
+   *
+   * @param klass Class to create
+   * @param conf {@link ImmutableClassesGiraphConfiguration} to configure with
+   * @param schema {@link HiveTableSchema} from Hive to set
+   * @param <I> Vertex ID
+   * @param <V> Vertex Value
+   * @param <E> Edge Value
+   * @param <T> type being created
+   * @return new object of type <T>
+   */
+  public static
+  <I extends WritableComparable, V extends Writable, E extends Writable, T>
+  T newInstance(Class<T> klass,
+      ImmutableClassesGiraphConfiguration<I, V, E> conf,
+      HiveTableSchema schema) {
+    T object = ReflectionUtils.<T>newInstance(klass, conf);
+    HiveTableSchemas.configure(object, schema);
+    return object;
   }
 }

http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-hive/src/main/java/org/apache/giraph/hive/common/LanguageAndType.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/common/LanguageAndType.java b/giraph-hive/src/main/java/org/apache/giraph/hive/common/LanguageAndType.java
new file mode 100644
index 0000000..7a2789a
--- /dev/null
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/common/LanguageAndType.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.hive.common;
+
+import org.apache.giraph.graph.Language;
+
+/**
+ * A class/type with the language.
+ */
+public class LanguageAndType {
+  /** Language programmed in */
+  private final Language language;
+  /** Java class name */
+  private final Class javaClass;
+  /** Jython class name */
+  private final String jythonClassName;
+
+  /**
+   * Constructor
+   *
+   * @param language Language
+   * @param javaClass java class
+   * @param jythonClassName jython class name
+   */
+  private LanguageAndType(Language language, Class javaClass,
+      String jythonClassName) {
+    this.javaClass = javaClass;
+    this.language = language;
+    this.jythonClassName = jythonClassName;
+  }
+
+  /**
+   * create for jython class
+   *
+   * @param name jython class name
+   * @return new instance
+   */
+  public static LanguageAndType jython(String name) {
+    return new LanguageAndType(Language.JYTHON, null, name);
+  }
+
+  /**
+   * create for java class
+   *
+   * @param klass class
+   * @return new instance
+   */
+  public static LanguageAndType java(Class klass) {
+    return new LanguageAndType(Language.JAVA, klass, null);
+  }
+
+  public Class getJavaClass() {
+    return javaClass;
+  }
+
+  public String getJythonClassName() {
+    return jythonClassName;
+  }
+
+  public Language getLanguage() {
+    return language;
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/AbstractHiveToEdge.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/AbstractHiveToEdge.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/AbstractHiveToEdge.java
index a3c323e..9fe5a39 100644
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/AbstractHiveToEdge.java
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/AbstractHiveToEdge.java
@@ -17,7 +17,6 @@
  */
 package org.apache.giraph.hive.input.edge;
 
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
 import org.apache.giraph.hive.common.DefaultConfigurableAndTableSchemaAware;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
@@ -33,12 +32,6 @@ public abstract class AbstractHiveToEdge<I extends WritableComparable,
     extends DefaultConfigurableAndTableSchemaAware<I, Writable, E>
     implements HiveToEdge<I, E> {
   @Override
-  public ImmutableClassesGiraphConfiguration<I, Writable, E>
-  getConf() {
-    return super.getConf();
-  }
-
-  @Override
   public final void remove() {
     throw new UnsupportedOperationException();
   }

http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/TypedHiveToEdge.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/TypedHiveToEdge.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/TypedHiveToEdge.java
deleted file mode 100644
index ac62827..0000000
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/TypedHiveToEdge.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.giraph.hive.input.edge;
-
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
-import org.apache.giraph.conf.StrConfOption;
-import org.apache.giraph.hive.types.HiveValueReader;
-import org.apache.giraph.hive.types.HiveVertexIdReader;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-
-import com.facebook.hiveio.input.HiveInputDescription;
-import com.facebook.hiveio.record.HiveReadableRecord;
-import com.facebook.hiveio.schema.HiveTableSchema;
-
-import java.util.Iterator;
-
-import static org.apache.giraph.hive.types.TypedValueReader.createValueReader;
-import static org.apache.giraph.hive.types.TypedVertexIdReader.createIdReader;
-
-/**
- * A {@link HiveToEdge} using {@link org.apache.giraph.types.WritableWrapper}s
- *
- * @param <I> Vertex ID
- * @param <E> Edge Value
- */
-public class TypedHiveToEdge<I extends WritableComparable, E extends Writable>
-    extends SimpleHiveToEdge<I, E> {
-  /** Source ID column name in Hive */
-  public static final StrConfOption EDGE_SOURCE_ID_COLUMN =
-      new StrConfOption("hive.input.edge.source.id.column", null,
-          "Source Vertex ID column");
-  /** Target ID column name in Hive */
-  public static final StrConfOption EDGE_TARGET_ID_COLUMN =
-      new StrConfOption("hive.input.edge.target.id.column", null,
-          "Target Vertex ID column");
-  /** Edge Value column name in Hive */
-  public static final StrConfOption EDGE_VALUE_COLUMN =
-      new StrConfOption("hive.input.edge.value.column", null,
-          "Edge Value column");
-
-  /** Source ID reader */
-  private HiveVertexIdReader<I> sourceIdReader;
-  /** Target ID reader */
-  private HiveVertexIdReader<I> targetIdReader;
-  /** Edge Value reader */
-  private HiveValueReader<E> vertexValueReader;
-
-  @Override
-  public void checkInput(HiveInputDescription inputDesc,
-      HiveTableSchema schema) { }
-
-  @Override
-  public void initializeRecords(Iterator<HiveReadableRecord> records) {
-    super.initializeRecords(records);
-
-    HiveTableSchema schema = getTableSchema();
-    ImmutableClassesGiraphConfiguration conf = getConf();
-
-    sourceIdReader = createIdReader(conf, EDGE_SOURCE_ID_COLUMN, schema);
-    targetIdReader = createIdReader(conf, EDGE_TARGET_ID_COLUMN, schema);
-    vertexValueReader = createValueReader(conf, EDGE_VALUE_COLUMN, schema);
-  }
-
-  @Override
-  public E getEdgeValue(HiveReadableRecord record) {
-    return vertexValueReader.readValue(record);
-  }
-
-  @Override
-  public I getSourceVertexId(
-      HiveReadableRecord record) {
-    return sourceIdReader.readId(record);
-  }
-
-  @Override
-  public I getTargetVertexId(
-      HiveReadableRecord record) {
-    return targetIdReader.readId(record);
-  }
-}
-

http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertex.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertex.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertex.java
index 45cc209..87f3cfe 100644
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertex.java
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertex.java
@@ -18,7 +18,6 @@
 
 package org.apache.giraph.hive.input.vertex;
 
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
 import org.apache.giraph.hive.common.DefaultConfigurableAndTableSchemaAware;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
@@ -35,11 +34,6 @@ public abstract class AbstractHiveToVertex<I extends WritableComparable,
     extends DefaultConfigurableAndTableSchemaAware<I, V, E>
     implements HiveToVertex<I, V, E> {
   @Override
-  public ImmutableClassesGiraphConfiguration<I, V, E> getConf() {
-    return super.getConf();
-  }
-
-  @Override
   public final void remove() {
     throw new UnsupportedOperationException();
   }

http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/TypedHiveToVertex.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/TypedHiveToVertex.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/TypedHiveToVertex.java
deleted file mode 100644
index 5ca7d34..0000000
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/TypedHiveToVertex.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.giraph.hive.input.vertex;
-
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
-import org.apache.giraph.conf.StrConfOption;
-import org.apache.giraph.edge.Edge;
-import org.apache.giraph.hive.types.HiveValueReader;
-import org.apache.giraph.hive.types.HiveVertexIdReader;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-
-import com.facebook.hiveio.input.HiveInputDescription;
-import com.facebook.hiveio.record.HiveReadableRecord;
-import com.facebook.hiveio.schema.HiveTableSchema;
-import com.google.common.collect.ImmutableList;
-
-import java.util.Iterator;
-
-import static org.apache.giraph.hive.types.TypedValueReader.createValueReader;
-import static org.apache.giraph.hive.types.TypedVertexIdReader.createIdReader;
-
-/**
- * A {@link HiveToVertex} using {@link org.apache.giraph.types.WritableWrapper}
- *
- * @param <I> Vertex ID
- * @param <V> Vertex Value
- * @param <E> Edge Value
- */
-public class TypedHiveToVertex<I extends WritableComparable, V extends Writable,
-    E extends Writable> extends SimpleHiveToVertex<I, V, E> {
-  /** Source ID column name in Hive */
-  public static final StrConfOption VERTEX_ID_COLUMN =
-      new StrConfOption("hive.input.vertex.id.column", null,
-          "Vertex ID column");
-  /** Target ID column name in Hive */
-  public static final StrConfOption VERTEX_VALUE_COLUMN =
-      new StrConfOption("hive.input.vertex.value.column", null,
-          "Vertex Value column");
-  /** Edge Value column name in Hive */
-  public static final StrConfOption EDGES_COLUMN =
-      new StrConfOption("hive.input.vertex.edges.column", null,
-          "Edges column");
-
-  /** Vertex ID reader */
-  private HiveVertexIdReader<I> vertexIdReader;
-  /** Vertex Value reader */
-  private HiveValueReader<V> vertexValueReader;
-
-  @Override
-  public void checkInput(HiveInputDescription inputDesc,
-      HiveTableSchema schema) { }
-
-  @Override
-  public void initializeRecords(Iterator<HiveReadableRecord> records) {
-    super.initializeRecords(records);
-
-    HiveTableSchema schema = getTableSchema();
-    ImmutableClassesGiraphConfiguration conf = getConf();
-
-    vertexIdReader = createIdReader(conf, VERTEX_ID_COLUMN, schema);
-    vertexValueReader = createValueReader(conf, VERTEX_VALUE_COLUMN, schema);
-  }
-
-  @Override
-  public Iterable<Edge<I, E>> getEdges(HiveReadableRecord record) {
-    return ImmutableList.of();
-  }
-
-  @Override
-  public I getVertexId(HiveReadableRecord record) {
-    return vertexIdReader.readId(record);
-  }
-
-  @Override
-  public V getVertexValue(HiveReadableRecord record) {
-    return vertexValueReader.readValue(record);
-  }
-}

http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-hive/src/main/java/org/apache/giraph/hive/jython/HiveJythonRunner.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/jython/HiveJythonRunner.java b/giraph-hive/src/main/java/org/apache/giraph/hive/jython/HiveJythonRunner.java
new file mode 100644
index 0000000..3d7afec
--- /dev/null
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/jython/HiveJythonRunner.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.hive.jython;
+
+import org.apache.giraph.graph.Language;
+import org.apache.giraph.job.GiraphJob;
+import org.apache.giraph.jython.JythonJob;
+import org.apache.giraph.scripting.DeployType;
+import org.apache.giraph.scripting.ScriptLoader;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Logger;
+import org.python.util.PythonInterpreter;
+
+import com.facebook.hiveio.HiveIO;
+
+import java.util.Arrays;
+
+import static org.apache.giraph.hive.jython.HiveJythonUtils.parseJythonFiles;
+import static org.apache.giraph.utils.DistributedCacheUtils.copyAndAdd;
+
+/**
+ * Runner for jobs written in Jython
+ */
+public class HiveJythonRunner implements Tool {
+  /** Logger */
+  private static final Logger LOG = Logger.getLogger(HiveJythonRunner.class);
+  /** Configuration */
+  private static HiveConf CONF = new HiveConf();
+
+  @Override public int run(String[] args) throws Exception {
+    args = HiveJythonUtils.processArgs(args, CONF);
+    LOG.info("Processed hive options now have args: " + Arrays.toString(args));
+
+    HiveIO.init(CONF, false);
+
+    PythonInterpreter interpreter = new PythonInterpreter();
+
+    JythonJob jythonJob = parseJythonFiles(interpreter, args);
+
+    logOptions();
+
+    for (String arg : args) {
+      Path remoteScriptPath = copyAndAdd(new Path(arg), CONF);
+      ScriptLoader.addScriptToLoad(CONF, remoteScriptPath.toString(),
+          DeployType.DISTRIBUTED_CACHE, Language.JYTHON);
+    }
+
+    String name = HiveJythonUtils.writeJythonJobToConf(jythonJob, CONF,
+       interpreter);
+
+    GiraphJob job = new GiraphJob(CONF, name);
+    return job.run(true) ? 0 : -1;
+  }
+
+  /**
+   * Log options used
+   */
+  private static void logOptions() {
+    StringBuilder sb = new StringBuilder(100);
+    appendEnvVars(sb, "JAVA_HOME", "MAPRED_POOL_NAME");
+    appendEnvVars(sb, "HADOOP_HOME", "HIVE_HOME");
+    LOG.info("Environment:\n" + sb);
+  }
+
+  /**
+   * Append environment variables to StringBuilder
+   *
+   * @param sb StringBuilder
+   * @param names vararg of env keys
+   */
+  private static void appendEnvVars(StringBuilder sb, String ... names) {
+    for (String name : names) {
+      sb.append(name).append("=").append(System.getenv(name)).append("\n");
+    }
+  }
+
+  /**
+   * Set the static configuration stored
+   *
+   * @param conf Configuration
+   */
+  public static void setStaticConf(Configuration conf) {
+    if (conf instanceof HiveConf) {
+      HiveJythonRunner.CONF = (HiveConf) conf;
+    } else {
+      HiveJythonRunner.CONF = new HiveConf(conf, HiveJythonRunner.class);
+    }
+  }
+
+  @Override public void setConf(Configuration conf) {
+    setStaticConf(conf);
+  }
+
+  @Override public Configuration getConf() {
+    return CONF;
+  }
+
+  /**
+   * Entry point
+   *
+   * @param args command line args
+   * @throws Exception
+   */
+  public static void main(String[] args) throws Exception {
+    System.exit(ToolRunner.run(new HiveJythonRunner(), args));
+  }
+}