You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by ni...@apache.org on 2013/07/31 20:55:48 UTC
[3/5] GIRAPH-717: HiveJythonRunner with support for pure Jython value
types (nitay)
http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-core/src/test/java/org/apache/giraph/jython/TestJython.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/test/java/org/apache/giraph/jython/TestJython.java b/giraph-core/src/test/java/org/apache/giraph/jython/TestJython.java
deleted file mode 100644
index b5110bb..0000000
--- a/giraph-core/src/test/java/org/apache/giraph/jython/TestJython.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.giraph.jython;
-
-import org.apache.giraph.conf.GiraphConfiguration;
-import org.apache.giraph.conf.GiraphTypes;
-import org.apache.giraph.edge.ByteArrayEdges;
-import org.apache.giraph.graph.Language;
-import org.apache.giraph.io.formats.IdWithValueTextOutputFormat;
-import org.apache.giraph.io.formats.IntNullTextEdgeInputFormat;
-import org.apache.giraph.scripting.DeployType;
-import org.apache.giraph.scripting.ScriptLoader;
-import org.apache.giraph.utils.InternalVertexRunner;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.NullWritable;
-import org.junit.Test;
-import org.python.core.PyClass;
-import org.python.core.PyDictionary;
-import org.python.core.PyInteger;
-import org.python.core.PyList;
-import org.python.core.PyObject;
-import org.python.util.PythonInterpreter;
-
-import com.google.common.collect.Maps;
-
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class TestJython {
- private static final double DELTA = 0.0000001;
-
- @Test
- public void testBasic() {
- String jython =
- "class Foo:\n" +
- " def __init__(self):\n" +
- " self.map = {\"32\": 32, \"4.3\": 4.3}\n" +
- " self.list = [ 2, 9, 11 ]\n" +
- " self.ival = 17\n" +
- "\n" +
- "def get_map(foo):\n" +
- " return foo.map\n" +
- "\n" +
- "def get_list(foo):\n" +
- " return foo.list\n" +
- "\n" +
- "def get_ival(foo):\n" +
- " return foo.ival\n" +
- "";
-
- PythonInterpreter interpreter = new PythonInterpreter();
- interpreter.exec(jython);
-
- PyObject fooClass = interpreter.get("Foo");
- assertTrue(fooClass instanceof PyClass);
-
- PyObject getMapFunc = interpreter.get("get_map");
- PyObject getListFunc = interpreter.get("get_list");
- PyObject getIValFunc = interpreter.get("get_ival");
-
- PyObject foo = fooClass.__call__();
-
- PyObject mapResult = getMapFunc.__call__(foo);
- assertTrue(mapResult instanceof PyDictionary);
- PyDictionary pyMapResult = ((PyDictionary) mapResult);
- assertEquals(2, pyMapResult.size());
- Object thirtyTwo = pyMapResult.get("32");
- assertTrue(thirtyTwo instanceof Integer);
- assertEquals(32, ((Integer) thirtyTwo).intValue());
- Object fourPointThree = pyMapResult.get("4.3");
- assertTrue(fourPointThree instanceof Double);
- assertEquals(4.3, (Double) fourPointThree, DELTA);
-
- PyObject listResult = getListFunc.__call__(foo);
- assertTrue(listResult instanceof PyList);
- PyList pyListResult = (PyList) listResult;
- assertEquals(3, pyListResult.size());
- assertEquals(2, pyListResult.get(0));
- assertEquals(9, pyListResult.get(1));
- assertEquals(11, pyListResult.get(2));
-
- PyObject ivalResult = getIValFunc.__call__(foo);
- assertTrue(ivalResult instanceof PyInteger);
- assertEquals(17, ((PyInteger) ivalResult).getValue());
- }
-
- @Test
- public void testCountEdges() throws Exception {
- String[] edges = new String[] {
- "1 2",
- "2 3",
- "2 4",
- "4 1"
- };
-
- GiraphConfiguration conf = new GiraphConfiguration();
- GiraphTypes types = new GiraphTypes(IntWritable.class, IntWritable.class,
- NullWritable.class, NullWritable.class, NullWritable.class);
- types.writeIfUnset(conf);
- ScriptLoader.setScriptsToLoad(conf,
- "org/apache/giraph/jython/count-edges.py",
- DeployType.RESOURCE, Language.JYTHON);
- JythonUtils.init(conf, "CountEdges");
- conf.setOutEdgesClass(ByteArrayEdges.class);
- conf.setEdgeInputFormatClass(IntNullTextEdgeInputFormat.class);
- conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
- Iterable<String> results = InternalVertexRunner.run(conf, null, edges);
-
- Map<Integer, Integer> values = parseResults(results);
-
- // Check that all vertices with outgoing edges have been created
- assertEquals(3, values.size());
- // Check the number of edges for each vertex
- assertEquals(1, (int) values.get(1));
- assertEquals(2, (int) values.get(2));
- assertEquals(1, (int) values.get(4));
- }
-
- private static Map<Integer, Integer> parseResults(Iterable<String> results) {
- Map<Integer, Integer> values = Maps.newHashMap();
- for (String line : results) {
- String[] tokens = line.split("\\s+");
- int id = Integer.valueOf(tokens[0]);
- int value = Integer.valueOf(tokens[1]);
- values.put(id, value);
- }
- return values;
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-core/src/test/java/org/apache/giraph/jython/TestJythonBasic.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/test/java/org/apache/giraph/jython/TestJythonBasic.java b/giraph-core/src/test/java/org/apache/giraph/jython/TestJythonBasic.java
new file mode 100644
index 0000000..5f8db66
--- /dev/null
+++ b/giraph-core/src/test/java/org/apache/giraph/jython/TestJythonBasic.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.jython;
+
+import org.junit.Test;
+import org.python.core.PyClass;
+import org.python.core.PyDictionary;
+import org.python.core.PyInteger;
+import org.python.core.PyList;
+import org.python.core.PyObject;
+import org.python.util.PythonInterpreter;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TestJythonBasic {
+ private static final double DELTA = 0.0000001;
+
+ @Test
+ public void testBasic() {
+ String jython =
+ "class Foo:\n" +
+ " def __init__(self):\n" +
+ " self.map = {\"32\": 32, \"4.3\": 4.3}\n" +
+ " self.list = [ 2, 9, 11 ]\n" +
+ " self.ival = 17\n" +
+ "\n" +
+ "def get_map(foo):\n" +
+ " return foo.map\n" +
+ "\n" +
+ "def get_list(foo):\n" +
+ " return foo.list\n" +
+ "\n" +
+ "def get_ival(foo):\n" +
+ " return foo.ival\n" +
+ "";
+
+ PythonInterpreter interpreter = new PythonInterpreter();
+ interpreter.exec(jython);
+
+ PyObject fooClass = interpreter.get("Foo");
+ assertTrue(fooClass instanceof PyClass);
+
+ PyObject getMapFunc = interpreter.get("get_map");
+ PyObject getListFunc = interpreter.get("get_list");
+ PyObject getIValFunc = interpreter.get("get_ival");
+
+ PyObject foo = fooClass.__call__();
+
+ PyObject mapResult = getMapFunc.__call__(foo);
+ assertTrue(mapResult instanceof PyDictionary);
+ PyDictionary pyMapResult = ((PyDictionary) mapResult);
+ assertEquals(2, pyMapResult.size());
+ Object thirtyTwo = pyMapResult.get("32");
+ assertTrue(thirtyTwo instanceof Integer);
+ assertEquals(32, ((Integer) thirtyTwo).intValue());
+ Object fourPointThree = pyMapResult.get("4.3");
+ assertTrue(fourPointThree instanceof Double);
+ assertEquals(4.3, (Double) fourPointThree, DELTA);
+
+ PyObject listResult = getListFunc.__call__(foo);
+ assertTrue(listResult instanceof PyList);
+ PyList pyListResult = (PyList) listResult;
+ assertEquals(3, pyListResult.size());
+ assertEquals(2, pyListResult.get(0));
+ assertEquals(9, pyListResult.get(1));
+ assertEquals(11, pyListResult.get(2));
+
+ PyObject ivalResult = getIValFunc.__call__(foo);
+ assertTrue(ivalResult instanceof PyInteger);
+ assertEquals(17, ((PyInteger) ivalResult).getValue());
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-core/src/test/java/org/apache/giraph/jython/TestJythonComputation.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/test/java/org/apache/giraph/jython/TestJythonComputation.java b/giraph-core/src/test/java/org/apache/giraph/jython/TestJythonComputation.java
new file mode 100644
index 0000000..5feabaf
--- /dev/null
+++ b/giraph-core/src/test/java/org/apache/giraph/jython/TestJythonComputation.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.jython;
+
+import org.apache.giraph.conf.GiraphConfiguration;
+import org.apache.giraph.conf.GiraphTypes;
+import org.apache.giraph.edge.ByteArrayEdges;
+import org.apache.giraph.graph.Language;
+import org.apache.giraph.io.formats.IdWithValueTextOutputFormat;
+import org.apache.giraph.io.formats.IntNullTextEdgeInputFormat;
+import org.apache.giraph.scripting.DeployType;
+import org.apache.giraph.scripting.ScriptLoader;
+import org.apache.giraph.utils.InternalVertexRunner;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.junit.Test;
+
+import com.google.common.collect.Maps;
+
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestJythonComputation {
+ @Test
+ public void testCountEdges() throws Exception {
+ String[] edges = new String[] {
+ "1 2",
+ "2 3",
+ "2 4",
+ "4 1"
+ };
+
+ GiraphConfiguration conf = new GiraphConfiguration();
+ GiraphTypes types = new GiraphTypes(IntWritable.class, IntWritable.class,
+ NullWritable.class, NullWritable.class, NullWritable.class);
+ types.writeIfUnset(conf);
+ ScriptLoader.setScriptsToLoad(conf,
+ "org/apache/giraph/jython/count-edges.py",
+ DeployType.RESOURCE, Language.JYTHON);
+ JythonUtils.init(conf, "CountEdges");
+ conf.setOutEdgesClass(ByteArrayEdges.class);
+ conf.setEdgeInputFormatClass(IntNullTextEdgeInputFormat.class);
+ conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
+ Iterable<String> results = InternalVertexRunner.run(conf, null, edges);
+
+ Map<Integer, Integer> values = parseResults(results);
+
+ // Check that all vertices with outgoing edges have been created
+ assertEquals(3, values.size());
+ // Check the number of edges for each vertex
+ assertEquals(1, (int) values.get(1));
+ assertEquals(2, (int) values.get(2));
+ assertEquals(1, (int) values.get(4));
+ }
+
+ private static Map<Integer, Integer> parseResults(Iterable<String> results) {
+ Map<Integer, Integer> values = Maps.newHashMap();
+ for (String line : results) {
+ String[] tokens = line.split("\\s+");
+ int id = Integer.valueOf(tokens[0]);
+ int value = Integer.valueOf(tokens[1]);
+ values.put(id, value);
+ }
+ return values;
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-core/src/test/java/org/apache/giraph/jython/TestJythonWritableWrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/test/java/org/apache/giraph/jython/TestJythonWritableWrapper.java b/giraph-core/src/test/java/org/apache/giraph/jython/TestJythonWritableWrapper.java
new file mode 100644
index 0000000..3d34571
--- /dev/null
+++ b/giraph-core/src/test/java/org/apache/giraph/jython/TestJythonWritableWrapper.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.jython;
+
+import org.apache.giraph.jython.wrappers.JythonWritableWrapper;
+import org.junit.Test;
+import org.python.core.PyClass;
+import org.python.core.PyFunction;
+import org.python.core.PyInteger;
+import org.python.core.PyMethod;
+import org.python.core.PyObject;
+import org.python.util.PythonInterpreter;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TestJythonWritableWrapper {
+ @Test
+ public void testWrap() throws IOException {
+ String jython =
+ "class Foo:\n" +
+ " def __init__(self):\n" +
+ " self.val = 17\n" +
+ "\n" +
+ " def do_add(self, x):\n" +
+ " self.val += x\n" +
+ "\n" +
+ " def do_add_squared(self, x):\n" +
+ " self.do_add(x * x)\n" +
+ "\n" +
+ " def new_other(self):\n" +
+ " self.other_val = 3\n" +
+ "\n" +
+ "def outside_add_squared(foo, x):\n" +
+ " foo.do_add_squared(x)\n" +
+ "\n";
+
+ PythonInterpreter interpreter = new PythonInterpreter();
+ interpreter.exec(jython);
+
+ PyObject fooClass = interpreter.get("Foo");
+ assertTrue(fooClass instanceof PyClass);
+ PyObject foo = fooClass.__call__();
+
+ PyObject fooVal = foo.__getattr__("val");
+ assertTrue(fooVal instanceof PyInteger);
+ PyInteger val = (PyInteger) fooVal;
+ assertEquals(17, val.getValue());
+
+ PyObject function = interpreter.get("outside_add_squared");
+ assertTrue("method class: " + function.getClass(), function instanceof PyFunction);
+ function.__call__(foo, new PyInteger(3));
+
+ fooVal = foo.__getattr__("val");
+ assertTrue(fooVal instanceof PyInteger);
+ val = (PyInteger) fooVal;
+ assertEquals(26, val.getValue());
+
+ JythonWritableWrapper wrappedFoo = new JythonWritableWrapper(foo);
+ PyObject newOtherMethod = wrappedFoo.__getattr__("new_other");
+
+ assertTrue(newOtherMethod instanceof PyMethod);
+ newOtherMethod.__call__();
+
+ function.__call__(wrappedFoo, new PyInteger(2));
+
+ fooVal = foo.__getattr__("val");
+ assertTrue(fooVal instanceof PyInteger);
+ val = (PyInteger) fooVal;
+ assertEquals(30, val.getValue());
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ wrappedFoo.write(dos);
+
+ byte[] data = baos.toByteArray();
+ ByteArrayInputStream bais = new ByteArrayInputStream(data);
+ DataInputStream dis = new DataInputStream(bais);
+
+ PyObject foo2 = fooClass.__call__();
+
+ PyObject foo2Val = foo2.__getattr__("val");
+ assertTrue(foo2Val instanceof PyInteger);
+ PyInteger val2 = (PyInteger) foo2Val;
+ assertEquals(17, val2.getValue());
+
+ JythonWritableWrapper wrappedFoo2 = new JythonWritableWrapper(foo2);
+
+ foo2Val = wrappedFoo2.getPyObject().__getattr__("val");
+ assertTrue(foo2Val instanceof PyInteger);
+ val2 = (PyInteger) foo2Val;
+ assertEquals(17, val2.getValue());
+
+ wrappedFoo2.readFields(dis);
+
+ foo2Val = wrappedFoo2.getPyObject().__getattr__("val");
+ assertTrue(foo2Val instanceof PyInteger);
+ val2 = (PyInteger) foo2Val;
+ assertEquals(30, val2.getValue());
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-core/src/test/java/org/apache/giraph/master/TestComputationCombinerTypes.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/test/java/org/apache/giraph/master/TestComputationCombinerTypes.java b/giraph-core/src/test/java/org/apache/giraph/master/TestComputationCombinerTypes.java
index f9bd4e4..b62775f 100644
--- a/giraph-core/src/test/java/org/apache/giraph/master/TestComputationCombinerTypes.java
+++ b/giraph-core/src/test/java/org/apache/giraph/master/TestComputationCombinerTypes.java
@@ -21,6 +21,7 @@ package org.apache.giraph.master;
import org.apache.giraph.combiner.Combiner;
import org.apache.giraph.conf.GiraphConfiguration;
import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.giraph.graph.AbstractComputation;
import org.apache.giraph.graph.Computation;
import org.apache.giraph.graph.Vertex;
import org.apache.giraph.utils.IntNoOpComputation;
@@ -110,7 +111,7 @@ public class TestComputationCombinerTypes {
public static class NoOpComputation<I extends WritableComparable,
V extends Writable, E extends Writable, M1 extends Writable,
- M2 extends Writable> extends Computation<I, V, E, M1, M2> {
+ M2 extends Writable> extends AbstractComputation<I, V, E, M1, M2> {
@Override
public void compute(Vertex<I, V, E> vertex,
Iterable<M1> messages) throws IOException {
http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-core/src/test/java/org/apache/giraph/master/TestSwitchClasses.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/test/java/org/apache/giraph/master/TestSwitchClasses.java b/giraph-core/src/test/java/org/apache/giraph/master/TestSwitchClasses.java
index 4069972..6b0ed35 100644
--- a/giraph-core/src/test/java/org/apache/giraph/master/TestSwitchClasses.java
+++ b/giraph-core/src/test/java/org/apache/giraph/master/TestSwitchClasses.java
@@ -20,7 +20,7 @@ package org.apache.giraph.master;
import org.apache.giraph.combiner.Combiner;
import org.apache.giraph.conf.GiraphConfiguration;
-import org.apache.giraph.graph.Computation;
+import org.apache.giraph.graph.AbstractComputation;
import org.apache.giraph.graph.Vertex;
import org.apache.giraph.utils.InternalVertexRunner;
import org.apache.giraph.utils.TestGraph;
@@ -136,8 +136,8 @@ public class TestSwitchClasses {
}
}
- public static class Computation1 extends Computation<IntWritable,
- StatusValue, IntWritable, IntWritable, IntWritable> {
+ public static class Computation1 extends AbstractComputation<IntWritable,
+ StatusValue, IntWritable, IntWritable, IntWritable> {
@Override
public void compute(Vertex<IntWritable, StatusValue, IntWritable> vertex,
Iterable<IntWritable> messages) throws IOException {
@@ -150,8 +150,8 @@ public class TestSwitchClasses {
}
}
- public static class Computation2 extends Computation<IntWritable,
- StatusValue, IntWritable, IntWritable, DoubleWritable> {
+ public static class Computation2 extends AbstractComputation<IntWritable,
+ StatusValue, IntWritable, IntWritable, DoubleWritable> {
@Override
public void compute(Vertex<IntWritable, StatusValue, IntWritable> vertex,
Iterable<IntWritable> messages) throws IOException {
@@ -164,8 +164,8 @@ public class TestSwitchClasses {
}
}
- public static class Computation3 extends Computation<IntWritable,
- StatusValue, IntWritable, DoubleWritable, IntWritable> {
+ public static class Computation3 extends AbstractComputation<IntWritable,
+ StatusValue, IntWritable, DoubleWritable, IntWritable> {
@Override
public void compute(
Vertex<IntWritable, StatusValue, IntWritable> vertex,
http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-core/src/test/java/org/apache/giraph/utils/TestReflectionUtils.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/test/java/org/apache/giraph/utils/TestReflectionUtils.java b/giraph-core/src/test/java/org/apache/giraph/utils/TestReflectionUtils.java
index 1c3eed3..261e7c2 100644
--- a/giraph-core/src/test/java/org/apache/giraph/utils/TestReflectionUtils.java
+++ b/giraph-core/src/test/java/org/apache/giraph/utils/TestReflectionUtils.java
@@ -29,6 +29,7 @@ import org.apache.giraph.factories.EdgeValueFactory;
import org.apache.giraph.factories.MessageValueFactory;
import org.apache.giraph.factories.VertexIdFactory;
import org.apache.giraph.factories.VertexValueFactory;
+import org.apache.giraph.graph.AbstractComputation;
import org.apache.giraph.graph.BasicComputation;
import org.apache.giraph.graph.Computation;
import org.apache.giraph.graph.DefaultVertexResolver;
@@ -47,19 +48,19 @@ import static org.junit.Assert.assertEquals;
public class TestReflectionUtils {
@Test
public void testPackagePath() {
- assertEquals("/org/apache/giraph/utils",
+ assertEquals("org/apache/giraph/utils",
ReflectionUtils.getPackagePath(TestReflectionUtils.class));
- assertEquals("/org/apache/giraph/utils",
+ assertEquals("org/apache/giraph/utils",
ReflectionUtils.getPackagePath(getClass()));
- assertEquals("/org/apache/giraph/utils",
+ assertEquals("org/apache/giraph/utils",
ReflectionUtils.getPackagePath(this));
}
private static class IntTypes implements TypesHolder<IntWritable,
IntWritable, IntWritable, IntWritable, IntWritable> { }
- private static class IntComputation extends Computation<IntWritable,
- IntWritable, IntWritable, IntWritable, IntWritable> {
+ private static class IntComputation extends AbstractComputation<IntWritable,
+ IntWritable, IntWritable, IntWritable, IntWritable> {
@Override
public void compute(Vertex<IntWritable, IntWritable, IntWritable> vertex,
Iterable<IntWritable> messages) throws IOException {
http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-core/src/test/resources/org/apache/giraph/jython/count-edges.py
----------------------------------------------------------------------
diff --git a/giraph-core/src/test/resources/org/apache/giraph/jython/count-edges.py b/giraph-core/src/test/resources/org/apache/giraph/jython/count-edges.py
index 4cad945..443480a 100644
--- a/giraph-core/src/test/resources/org/apache/giraph/jython/count-edges.py
+++ b/giraph-core/src/test/resources/org/apache/giraph/jython/count-edges.py
@@ -1,7 +1,7 @@
-from org.apache.giraph.graph import BasicComputation
+from org.apache.giraph.jython import JythonComputation
from org.apache.hadoop.io import IntWritable
-class CountEdges(BasicComputation):
+class CountEdges(JythonComputation):
def compute(self, vertex, messages):
num_edges = vertex.getNumEdges()
vertex.setValue(IntWritable(num_edges))
http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-examples/src/test/java/org/apache/giraph/TestBspBasic.java
----------------------------------------------------------------------
diff --git a/giraph-examples/src/test/java/org/apache/giraph/TestBspBasic.java b/giraph-examples/src/test/java/org/apache/giraph/TestBspBasic.java
index f9e77af..115da7e 100644
--- a/giraph-examples/src/test/java/org/apache/giraph/TestBspBasic.java
+++ b/giraph-examples/src/test/java/org/apache/giraph/TestBspBasic.java
@@ -153,7 +153,7 @@ public class
immutableClassesGiraphConfiguration.createEdgeValue();
Writable messageValue =
immutableClassesGiraphConfiguration.getOutgoingMessageValueFactory()
- .createMessageValue();
+ .newInstance();
assertSame(vertexValue.getClass(), NullWritable.class);
assertSame(vertexValue, edgeValue);
assertSame(edgeValue, messageValue);
http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-examples/src/test/java/org/apache/giraph/vertex/TestComputationTypes.java
----------------------------------------------------------------------
diff --git a/giraph-examples/src/test/java/org/apache/giraph/vertex/TestComputationTypes.java b/giraph-examples/src/test/java/org/apache/giraph/vertex/TestComputationTypes.java
index fe354fa..4f74fcb 100644
--- a/giraph-examples/src/test/java/org/apache/giraph/vertex/TestComputationTypes.java
+++ b/giraph-examples/src/test/java/org/apache/giraph/vertex/TestComputationTypes.java
@@ -104,7 +104,12 @@ public class TestComputationTypes {
public void initialize(ImmutableClassesGiraphConfiguration conf) { }
@Override
- public DoubleWritable createVertexValue() {
+ public Class<DoubleWritable> getValueClass() {
+ return DoubleWritable.class;
+ }
+
+ @Override
+ public DoubleWritable newInstance() {
return new DoubleWritable();
}
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-hcatalog/src/main/java/org/apache/giraph/io/hcatalog/HCatGiraphRunner.java
----------------------------------------------------------------------
diff --git a/giraph-hcatalog/src/main/java/org/apache/giraph/io/hcatalog/HCatGiraphRunner.java b/giraph-hcatalog/src/main/java/org/apache/giraph/io/hcatalog/HCatGiraphRunner.java
index 6135c7d..25d100b 100644
--- a/giraph-hcatalog/src/main/java/org/apache/giraph/io/hcatalog/HCatGiraphRunner.java
+++ b/giraph-hcatalog/src/main/java/org/apache/giraph/io/hcatalog/HCatGiraphRunner.java
@@ -26,9 +26,9 @@ import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.giraph.graph.Computation;
import org.apache.giraph.io.EdgeInputFormat;
-import org.apache.giraph.job.GiraphJob;
import org.apache.giraph.io.VertexInputFormat;
import org.apache.giraph.io.VertexOutputFormat;
+import org.apache.giraph.job.GiraphJob;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.util.Tool;
http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-hive/src/main/java/org/apache/giraph/hive/column/HiveReadableColumn.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/column/HiveReadableColumn.java b/giraph-hive/src/main/java/org/apache/giraph/hive/column/HiveReadableColumn.java
new file mode 100644
index 0000000..93bd2c2
--- /dev/null
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/column/HiveReadableColumn.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.hive.column;
+
+import com.facebook.hiveio.common.HiveType;
+import com.facebook.hiveio.record.HiveReadableRecord;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A single column from a Hive record.
+ */
+public class HiveReadableColumn {
+ /** The Hive record */
+ private HiveReadableRecord record;
+ /** The column index to use */
+ private int index;
+
+ public int getIndex() {
+ return index;
+ }
+
+ public void setIndex(int index) {
+ this.index = index;
+ }
+
+ public HiveReadableRecord getRecord() {
+ return record;
+ }
+
+ public void setRecord(HiveReadableRecord record) {
+ this.record = record;
+ }
+
+ public boolean isNull() {
+ return record.isNull(index);
+ }
+
+ /**
+ * Get type for this column
+ *
+ * @return {@link HiveType}
+ */
+ public HiveType hiveType() {
+ return record.columnType(index);
+ }
+
+ /**
+ * Get column value
+ *
+ * Regular data columns from the tables should always be placed first, and
+ * then partition value columns.
+ *
+ * If you know the type of the column and it is a primitive you should use
+ * one of the calls below as it will likely be more efficient.
+ *
+ * @return Object for column
+ * @deprecated use {@link #get(com.facebook.hiveio.common.HiveType)}
+ * or one of the getX() methods
+ */
+ @Deprecated
+ public Object get() {
+ return record.get(index);
+ }
+
+ /**
+ * Get column value
+ *
+ * Regular data columns from the tables should always be placed first, and
+ * then partition value columns.
+ *
+ * You should probably be using one of getX() methods below instead.
+ *
+ * @param hiveType HiveType
+ * @return Object for column
+ */
+ public Object get(HiveType hiveType) {
+ return record.get(index, hiveType);
+ }
+
+ /**
+ * Get boolean value
+ *
+ * @return boolean at index
+ */
+ public boolean getBoolean() {
+ return record.getBoolean(index);
+ }
+
+ /**
+ * Get byte value
+ *
+ * @return byte at index
+ */
+ public byte getByte() {
+ return record.getByte(index);
+ }
+
+ /**
+ * Get short value
+ *
+ * @return short at index
+ */
+ public short getShort() {
+ return record.getShort(index);
+ }
+
+ /**
+ * Get int value
+ *
+ * @return int at index
+ */
+ public int getInt() {
+ return record.getInt(index);
+ }
+
+ /**
+ * Get long value
+ *
+ * @return long at index
+ */
+ public long getLong() {
+ return record.getLong(index);
+ }
+
+ /**
+ * Get float value
+ *
+ * @return float at index
+ */
+ public float getFloat() {
+ return record.getFloat(index);
+ }
+
+ /**
+ * Get double value
+ *
+ * @return double at index
+ */
+ public double getDouble() {
+ return record.getDouble(index);
+ }
+
+ /**
+ * Get String column value
+ * Note that partition values are all strings.
+ *
+ * @return String at index
+ */
+ public String getString() {
+ return record.getString(index);
+ }
+
+ /**
+ * Get List column value
+ * Note that partition values are all strings.
+ *
+ * @param <T> item type
+ * @return List at index
+ */
+ public <T> List<T> getList() {
+ return record.getList(index);
+ }
+
+ /**
+ * Get Map column value
+ * Note that partition values are all strings.
+ *
+ * @param <K> key type
+ * @param <V> value type
+ * @return Map at index
+ */
+ public <K, V> Map<K, V> getMap() {
+ return record.getMap(index);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-hive/src/main/java/org/apache/giraph/hive/column/HiveWritableColumn.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/column/HiveWritableColumn.java b/giraph-hive/src/main/java/org/apache/giraph/hive/column/HiveWritableColumn.java
new file mode 100644
index 0000000..5763126
--- /dev/null
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/column/HiveWritableColumn.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.hive.column;
+
+import com.facebook.hiveio.common.HiveType;
+import com.facebook.hiveio.record.HiveWritableRecord;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A single column to write to a Hive record.
+ */
+public class HiveWritableColumn {
+ /** The Hive record */
+ private HiveWritableRecord record;
+ /** The column index to use */
+ private int index;
+
+ public int getIndex() {
+ return index;
+ }
+
+ public void setIndex(int index) {
+ this.index = index;
+ }
+
+ public HiveWritableRecord getRecord() {
+ return record;
+ }
+
+ public void setRecord(HiveWritableRecord record) {
+ this.record = record;
+ }
+
+ /**
+ * Set value for column.
+ *
+ * @param value Data for column
+ * @deprecated
+ * use {@link #set(Object, com.facebook.hiveio.common.HiveType)}
+ * or one of the setX() methods
+ */
+ @Deprecated
+ public void set(Object value) {
+ record.set(index, value);
+ }
+
+ /**
+ * Set value with type for column.
+ *
+ * @param value data for column
+ * @param hiveType expected hive type
+ */
+ public void set(Object value, HiveType hiveType) {
+ record.set(index, value, hiveType);
+ }
+
+ /**
+ * Set boolean value for column.
+ *
+ * @param value Data for column
+ */
+ public void setBoolean(boolean value) {
+ record.setBoolean(index, value);
+ }
+
+ /**
+ * Set byte value for column.
+ *
+ * @param value Data for column
+ */
+ public void setByte(byte value) {
+ record.setByte(index, value);
+ }
+
+ /**
+ * Set short value for column.
+ *
+ * @param value Data for column
+ */
+ public void setShort(short value) {
+ record.setShort(index, value);
+ }
+
+ /**
+ * Set int value for column.
+ *
+ * @param value Data for column
+ */
+ public void setInt(int value) {
+ record.setInt(index, value);
+ }
+
+ /**
+ * Set long value for column.
+ *
+ * @param value Data for column
+ */
+ public void setLong(long value) {
+ record.setLong(index, value);
+ }
+
+ /**
+ * Set float value for column.
+ *
+ * @param value Data for column
+ */
+ public void setFloat(float value) {
+ record.setFloat(index, value);
+ }
+
+ /**
+ * Set double value for column.
+ *
+ * @param value Data for column
+ */
+ public void setDouble(double value) {
+ record.setDouble(index, value);
+ }
+
+ /**
+ * Set double value for column.
+ *
+ * @param value Data for column
+ */
+ public void setString(String value) {
+ record.setString(index, value);
+ }
+
+ /**
+ * Set List value for column.
+ *
+ * @param data Data for column
+ */
+ public void setList(List data) {
+ record.setList(index, data);
+ }
+
+ /**
+ * Set Map value for column.
+ *
+ * @param data Data for column
+ */
+ public void setMap(Map data) {
+ record.setMap(index, data);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-hive/src/main/java/org/apache/giraph/hive/column/package-info.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/column/package-info.java b/giraph-hive/src/main/java/org/apache/giraph/hive/column/package-info.java
new file mode 100644
index 0000000..2077c4f
--- /dev/null
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/column/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Package of Hive per-column IO related things.
+ */
+package org.apache.giraph.hive.column;
http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-hive/src/main/java/org/apache/giraph/hive/common/GiraphHiveConstants.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/common/GiraphHiveConstants.java b/giraph-hive/src/main/java/org/apache/giraph/hive/common/GiraphHiveConstants.java
index e492694..c7ad63b 100644
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/common/GiraphHiveConstants.java
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/common/GiraphHiveConstants.java
@@ -57,6 +57,27 @@ public class GiraphHiveConstants {
new StrConfOption("giraph.hive.output.vertex.partition", "",
"Vertex output partition");
+ /** Vertex ID hive reader */
+ public static final StrConfOption VERTEX_ID_READER_JYTHON_NAME =
+ new StrConfOption("giraph.hive.jython.vertex.id.reader", null,
+ "Vertex ID hive reader");
+ /** Vertex ID hive writer */
+ public static final StrConfOption VERTEX_ID_WRITER_JYTHON_NAME =
+ new StrConfOption("giraph.hive.jython.vertex.id.writer", null,
+ "Vertex ID hive writer");
+ /** Vertex value hive reader */
+ public static final StrConfOption VERTEX_VALUE_READER_JYTHON_NAME =
+ new StrConfOption("giraph.hive.jython.vertex.value.reader", null,
+ "Vertex value hive reader");
+ /** Vertex value hive writer */
+ public static final StrConfOption VERTEX_VALUE_WRITER_JYTHON_NAME =
+ new StrConfOption("giraph.hive.jython.vertex.value.writer", null,
+ "Vertex value hive writer");
+ /** Edge value hive reader */
+ public static final StrConfOption EDGE_VALUE_READER_JYTHON_NAME =
+ new StrConfOption("giraph.hive.jython.edge.value.reader", null,
+ "Edge value hive reader");
+
/** Don't construct */
protected GiraphHiveConstants() { }
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-hive/src/main/java/org/apache/giraph/hive/common/HiveUtils.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/common/HiveUtils.java b/giraph-hive/src/main/java/org/apache/giraph/hive/common/HiveUtils.java
index ae2a402..6e1d62b 100644
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/common/HiveUtils.java
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/common/HiveUtils.java
@@ -42,18 +42,17 @@ import com.google.common.collect.Maps;
import java.io.File;
import java.io.IOException;
+import java.net.MalformedURLException;
import java.net.URL;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
-import static com.google.common.base.Preconditions.checkNotNull;
import static java.lang.System.getenv;
import static org.apache.giraph.hive.common.GiraphHiveConstants.HIVE_EDGE_INPUT;
import static org.apache.giraph.hive.common.GiraphHiveConstants.HIVE_VERTEX_INPUT;
import static org.apache.giraph.hive.common.GiraphHiveConstants.VERTEX_TO_HIVE_CLASS;
-import static org.apache.giraph.utils.ReflectionUtils.newInstance;
/**
* Utility methods for Hive IO
@@ -184,6 +183,34 @@ public class HiveUtils {
*/
public static void addHiveSiteCustomXmlToTmpFiles(Configuration conf) {
addToHiveFromClassLoader(conf, "hive-site-custom.xml");
+ addToHiveFromEnv(conf, "HIVE_HOME", "conf/hive-site.xml");
+ }
+
+ /**
+ * Add a file to Configuration tmpfiles from environment variable
+ *
+ * @param conf Configuration
+ * @param envKey environment variable key
+ * @param path search path
+ * @return true if file found and added, false otherwise
+ */
+ private static boolean addToHiveFromEnv(Configuration conf,
+ String envKey, String path) {
+ String envValue = getenv(envKey);
+ if (envValue == null) {
+ return false;
+ }
+ File file = new File(envValue, path);
+ if (file.exists()) {
+ LOG.info("addToHiveFromEnv: Adding " + file.getPath() +
+ " to Configuration tmpfiles");
+ }
+ try {
+ addToStringCollection(conf, "tmpfiles", file.toURI().toURL().toString());
+ } catch (MalformedURLException e) {
+ LOG.error("Failed to get URL for file " + file);
+ }
+ return true;
}
/**
@@ -191,16 +218,20 @@ public class HiveUtils {
*
* @param conf Configuration
* @param name file name
+ * @return true if file found in class loader, false otherwise
*/
- private static void addToHiveFromClassLoader(Configuration conf,
+ private static boolean addToHiveFromClassLoader(Configuration conf,
String name) {
URL url = conf.getClassLoader().getResource(name);
- checkNotNull(url);
+ if (url == null) {
+ return false;
+ }
if (LOG.isInfoEnabled()) {
LOG.info("addToHiveFromClassLoader: Adding " + name + " at " +
url + " to Configuration tmpfiles");
}
addToStringCollection(conf, "tmpfiles", url.toString());
+ return true;
}
/**
@@ -212,12 +243,12 @@ public class HiveUtils {
public static void addHadoopClasspathToTmpJars(Configuration conf) {
// Or, more effectively, we can provide all the jars client needed to
// the workers as well
- String[] hadoopJars = getenv("HADOOP_CLASSPATH").split(File.pathSeparator);
+ String hadoopClasspath = getenv("HADOOP_CLASSPATH");
+ if (hadoopClasspath == null) {
+ return;
+ }
+ String[] hadoopJars = hadoopClasspath.split(File.pathSeparator);
if (hadoopJars.length > 0) {
- if (LOG.isInfoEnabled()) {
- LOG.info("addHadoopClasspathToTmpJars: Adding HADOOP_CLASSPATH jars " +
- "at " + Arrays.toString(hadoopJars) + " to Configuration tmpjars");
- }
List<String> hadoopJarURLs = Lists.newArrayList();
for (String jarPath : hadoopJars) {
File file = new File(jarPath);
@@ -308,9 +339,7 @@ public class HiveUtils {
throw new IOException(VERTEX_TO_HIVE_CLASS.getKey() +
" not set in conf");
}
- VertexToHive<I, V, E> vertexToHive = newInstance(klass, conf);
- HiveTableSchemas.configure(vertexToHive, schema);
- return vertexToHive;
+ return newInstance(klass, conf, schema);
}
/**
@@ -332,9 +361,7 @@ public class HiveUtils {
throw new IllegalArgumentException(
HIVE_EDGE_INPUT.getClassOpt().getKey() + " not set in conf");
}
- HiveToEdge hiveToEdge = ReflectionUtils.newInstance(klass, conf);
- HiveTableSchemas.configure(hiveToEdge, schema);
- return hiveToEdge;
+ return newInstance(klass, conf, schema);
}
/**
@@ -356,8 +383,29 @@ public class HiveUtils {
throw new IllegalArgumentException(
HIVE_VERTEX_INPUT.getClassOpt().getKey() + " not set in conf");
}
- HiveToVertex hiveToVertex = ReflectionUtils.newInstance(klass, conf);
- HiveTableSchemas.configure(hiveToVertex, schema);
- return hiveToVertex;
+ return newInstance(klass, conf, schema);
+ }
+
+ /**
+ * Create a new instance of a class, configuring it and setting the Hive table
+ * schema if it supports those types.
+ *
+ * @param klass Class to create
+ * @param conf {@link ImmutableClassesGiraphConfiguration} to configure with
+ * @param schema {@link HiveTableSchema} from Hive to set
+ * @param <I> Vertex ID
+ * @param <V> Vertex Value
+ * @param <E> Edge Value
+ * @param <T> type being created
+ * @return new object of type <T>
+ */
+ public static
+ <I extends WritableComparable, V extends Writable, E extends Writable, T>
+ T newInstance(Class<T> klass,
+ ImmutableClassesGiraphConfiguration<I, V, E> conf,
+ HiveTableSchema schema) {
+ T object = ReflectionUtils.<T>newInstance(klass, conf);
+ HiveTableSchemas.configure(object, schema);
+ return object;
}
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-hive/src/main/java/org/apache/giraph/hive/common/LanguageAndType.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/common/LanguageAndType.java b/giraph-hive/src/main/java/org/apache/giraph/hive/common/LanguageAndType.java
new file mode 100644
index 0000000..7a2789a
--- /dev/null
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/common/LanguageAndType.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.hive.common;
+
+import org.apache.giraph.graph.Language;
+
+/**
+ * A class/type with the language.
+ */
+public class LanguageAndType {
+ /** Language programmed in */
+ private final Language language;
+ /** Java class name */
+ private final Class javaClass;
+ /** Jython class name */
+ private final String jythonClassName;
+
+ /**
+ * Constructor
+ *
+ * @param language Language
+ * @param javaClass java class
+ * @param jythonClassName jython class name
+ */
+ private LanguageAndType(Language language, Class javaClass,
+ String jythonClassName) {
+ this.javaClass = javaClass;
+ this.language = language;
+ this.jythonClassName = jythonClassName;
+ }
+
+ /**
+ * create for jython class
+ *
+ * @param name jython class name
+ * @return new instance
+ */
+ public static LanguageAndType jython(String name) {
+ return new LanguageAndType(Language.JYTHON, null, name);
+ }
+
+ /**
+ * create for java class
+ *
+ * @param klass class
+ * @return new instance
+ */
+ public static LanguageAndType java(Class klass) {
+ return new LanguageAndType(Language.JAVA, klass, null);
+ }
+
+ public Class getJavaClass() {
+ return javaClass;
+ }
+
+ public String getJythonClassName() {
+ return jythonClassName;
+ }
+
+ public Language getLanguage() {
+ return language;
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/AbstractHiveToEdge.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/AbstractHiveToEdge.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/AbstractHiveToEdge.java
index a3c323e..9fe5a39 100644
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/AbstractHiveToEdge.java
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/AbstractHiveToEdge.java
@@ -17,7 +17,6 @@
*/
package org.apache.giraph.hive.input.edge;
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
import org.apache.giraph.hive.common.DefaultConfigurableAndTableSchemaAware;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
@@ -33,12 +32,6 @@ public abstract class AbstractHiveToEdge<I extends WritableComparable,
extends DefaultConfigurableAndTableSchemaAware<I, Writable, E>
implements HiveToEdge<I, E> {
@Override
- public ImmutableClassesGiraphConfiguration<I, Writable, E>
- getConf() {
- return super.getConf();
- }
-
- @Override
public final void remove() {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/TypedHiveToEdge.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/TypedHiveToEdge.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/TypedHiveToEdge.java
deleted file mode 100644
index ac62827..0000000
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/TypedHiveToEdge.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.giraph.hive.input.edge;
-
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
-import org.apache.giraph.conf.StrConfOption;
-import org.apache.giraph.hive.types.HiveValueReader;
-import org.apache.giraph.hive.types.HiveVertexIdReader;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-
-import com.facebook.hiveio.input.HiveInputDescription;
-import com.facebook.hiveio.record.HiveReadableRecord;
-import com.facebook.hiveio.schema.HiveTableSchema;
-
-import java.util.Iterator;
-
-import static org.apache.giraph.hive.types.TypedValueReader.createValueReader;
-import static org.apache.giraph.hive.types.TypedVertexIdReader.createIdReader;
-
-/**
- * A {@link HiveToEdge} using {@link org.apache.giraph.types.WritableWrapper}s
- *
- * @param <I> Vertex ID
- * @param <E> Edge Value
- */
-public class TypedHiveToEdge<I extends WritableComparable, E extends Writable>
- extends SimpleHiveToEdge<I, E> {
- /** Source ID column name in Hive */
- public static final StrConfOption EDGE_SOURCE_ID_COLUMN =
- new StrConfOption("hive.input.edge.source.id.column", null,
- "Source Vertex ID column");
- /** Target ID column name in Hive */
- public static final StrConfOption EDGE_TARGET_ID_COLUMN =
- new StrConfOption("hive.input.edge.target.id.column", null,
- "Target Vertex ID column");
- /** Edge Value column name in Hive */
- public static final StrConfOption EDGE_VALUE_COLUMN =
- new StrConfOption("hive.input.edge.value.column", null,
- "Edge Value column");
-
- /** Source ID reader */
- private HiveVertexIdReader<I> sourceIdReader;
- /** Target ID reader */
- private HiveVertexIdReader<I> targetIdReader;
- /** Edge Value reader */
- private HiveValueReader<E> vertexValueReader;
-
- @Override
- public void checkInput(HiveInputDescription inputDesc,
- HiveTableSchema schema) { }
-
- @Override
- public void initializeRecords(Iterator<HiveReadableRecord> records) {
- super.initializeRecords(records);
-
- HiveTableSchema schema = getTableSchema();
- ImmutableClassesGiraphConfiguration conf = getConf();
-
- sourceIdReader = createIdReader(conf, EDGE_SOURCE_ID_COLUMN, schema);
- targetIdReader = createIdReader(conf, EDGE_TARGET_ID_COLUMN, schema);
- vertexValueReader = createValueReader(conf, EDGE_VALUE_COLUMN, schema);
- }
-
- @Override
- public E getEdgeValue(HiveReadableRecord record) {
- return vertexValueReader.readValue(record);
- }
-
- @Override
- public I getSourceVertexId(
- HiveReadableRecord record) {
- return sourceIdReader.readId(record);
- }
-
- @Override
- public I getTargetVertexId(
- HiveReadableRecord record) {
- return targetIdReader.readId(record);
- }
-}
-
http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertex.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertex.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertex.java
index 45cc209..87f3cfe 100644
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertex.java
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/AbstractHiveToVertex.java
@@ -18,7 +18,6 @@
package org.apache.giraph.hive.input.vertex;
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
import org.apache.giraph.hive.common.DefaultConfigurableAndTableSchemaAware;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
@@ -35,11 +34,6 @@ public abstract class AbstractHiveToVertex<I extends WritableComparable,
extends DefaultConfigurableAndTableSchemaAware<I, V, E>
implements HiveToVertex<I, V, E> {
@Override
- public ImmutableClassesGiraphConfiguration<I, V, E> getConf() {
- return super.getConf();
- }
-
- @Override
public final void remove() {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/TypedHiveToVertex.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/TypedHiveToVertex.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/TypedHiveToVertex.java
deleted file mode 100644
index 5ca7d34..0000000
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/input/vertex/TypedHiveToVertex.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.giraph.hive.input.vertex;
-
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
-import org.apache.giraph.conf.StrConfOption;
-import org.apache.giraph.edge.Edge;
-import org.apache.giraph.hive.types.HiveValueReader;
-import org.apache.giraph.hive.types.HiveVertexIdReader;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-
-import com.facebook.hiveio.input.HiveInputDescription;
-import com.facebook.hiveio.record.HiveReadableRecord;
-import com.facebook.hiveio.schema.HiveTableSchema;
-import com.google.common.collect.ImmutableList;
-
-import java.util.Iterator;
-
-import static org.apache.giraph.hive.types.TypedValueReader.createValueReader;
-import static org.apache.giraph.hive.types.TypedVertexIdReader.createIdReader;
-
-/**
- * A {@link HiveToVertex} using {@link org.apache.giraph.types.WritableWrapper}
- *
- * @param <I> Vertex ID
- * @param <V> Vertex Value
- * @param <E> Edge Value
- */
-public class TypedHiveToVertex<I extends WritableComparable, V extends Writable,
- E extends Writable> extends SimpleHiveToVertex<I, V, E> {
- /** Source ID column name in Hive */
- public static final StrConfOption VERTEX_ID_COLUMN =
- new StrConfOption("hive.input.vertex.id.column", null,
- "Vertex ID column");
- /** Target ID column name in Hive */
- public static final StrConfOption VERTEX_VALUE_COLUMN =
- new StrConfOption("hive.input.vertex.value.column", null,
- "Vertex Value column");
- /** Edge Value column name in Hive */
- public static final StrConfOption EDGES_COLUMN =
- new StrConfOption("hive.input.vertex.edges.column", null,
- "Edges column");
-
- /** Vertex ID reader */
- private HiveVertexIdReader<I> vertexIdReader;
- /** Vertex Value reader */
- private HiveValueReader<V> vertexValueReader;
-
- @Override
- public void checkInput(HiveInputDescription inputDesc,
- HiveTableSchema schema) { }
-
- @Override
- public void initializeRecords(Iterator<HiveReadableRecord> records) {
- super.initializeRecords(records);
-
- HiveTableSchema schema = getTableSchema();
- ImmutableClassesGiraphConfiguration conf = getConf();
-
- vertexIdReader = createIdReader(conf, VERTEX_ID_COLUMN, schema);
- vertexValueReader = createValueReader(conf, VERTEX_VALUE_COLUMN, schema);
- }
-
- @Override
- public Iterable<Edge<I, E>> getEdges(HiveReadableRecord record) {
- return ImmutableList.of();
- }
-
- @Override
- public I getVertexId(HiveReadableRecord record) {
- return vertexIdReader.readId(record);
- }
-
- @Override
- public V getVertexValue(HiveReadableRecord record) {
- return vertexValueReader.readValue(record);
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/d419f8f4/giraph-hive/src/main/java/org/apache/giraph/hive/jython/HiveJythonRunner.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/jython/HiveJythonRunner.java b/giraph-hive/src/main/java/org/apache/giraph/hive/jython/HiveJythonRunner.java
new file mode 100644
index 0000000..3d7afec
--- /dev/null
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/jython/HiveJythonRunner.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.hive.jython;
+
+import org.apache.giraph.graph.Language;
+import org.apache.giraph.job.GiraphJob;
+import org.apache.giraph.jython.JythonJob;
+import org.apache.giraph.scripting.DeployType;
+import org.apache.giraph.scripting.ScriptLoader;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Logger;
+import org.python.util.PythonInterpreter;
+
+import com.facebook.hiveio.HiveIO;
+
+import java.util.Arrays;
+
+import static org.apache.giraph.hive.jython.HiveJythonUtils.parseJythonFiles;
+import static org.apache.giraph.utils.DistributedCacheUtils.copyAndAdd;
+
+/**
+ * Runner for jobs written in Jython
+ */
+public class HiveJythonRunner implements Tool {
+ /** Logger */
+ private static final Logger LOG = Logger.getLogger(HiveJythonRunner.class);
+ /** Configuration */
+ private static HiveConf CONF = new HiveConf();
+
+ @Override public int run(String[] args) throws Exception {
+ args = HiveJythonUtils.processArgs(args, CONF);
+ LOG.info("Processed hive options now have args: " + Arrays.toString(args));
+
+ HiveIO.init(CONF, false);
+
+ PythonInterpreter interpreter = new PythonInterpreter();
+
+ JythonJob jythonJob = parseJythonFiles(interpreter, args);
+
+ logOptions();
+
+ for (String arg : args) {
+ Path remoteScriptPath = copyAndAdd(new Path(arg), CONF);
+ ScriptLoader.addScriptToLoad(CONF, remoteScriptPath.toString(),
+ DeployType.DISTRIBUTED_CACHE, Language.JYTHON);
+ }
+
+ String name = HiveJythonUtils.writeJythonJobToConf(jythonJob, CONF,
+ interpreter);
+
+ GiraphJob job = new GiraphJob(CONF, name);
+ return job.run(true) ? 0 : -1;
+ }
+
+ /**
+ * Log options used
+ */
+ private static void logOptions() {
+ StringBuilder sb = new StringBuilder(100);
+ appendEnvVars(sb, "JAVA_HOME", "MAPRED_POOL_NAME");
+ appendEnvVars(sb, "HADOOP_HOME", "HIVE_HOME");
+ LOG.info("Environment:\n" + sb);
+ }
+
+ /**
+ * Append environment variables to StringBuilder
+ *
+ * @param sb StringBuilder
+ * @param names vararg of env keys
+ */
+ private static void appendEnvVars(StringBuilder sb, String ... names) {
+ for (String name : names) {
+ sb.append(name).append("=").append(System.getenv(name)).append("\n");
+ }
+ }
+
+ /**
+ * Set the static configuration stored
+ *
+ * @param conf Configuration
+ */
+ public static void setStaticConf(Configuration conf) {
+ if (conf instanceof HiveConf) {
+ HiveJythonRunner.CONF = (HiveConf) conf;
+ } else {
+ HiveJythonRunner.CONF = new HiveConf(conf, HiveJythonRunner.class);
+ }
+ }
+
+ @Override public void setConf(Configuration conf) {
+ setStaticConf(conf);
+ }
+
+ @Override public Configuration getConf() {
+ return CONF;
+ }
+
+ /**
+ * Entry point
+ *
+ * @param args command line args
+ * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ System.exit(ToolRunner.run(new HiveJythonRunner(), args));
+ }
+}