You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by ni...@apache.org on 2013/07/08 23:04:08 UTC
[2/2] git commit: updated refs/heads/trunk to c2af204
GIRAPH-705: Type converters and giraph-hive cleanup (nitay)
Project: http://git-wip-us.apache.org/repos/asf/giraph/repo
Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/c2af2048
Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/c2af2048
Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/c2af2048
Branch: refs/heads/trunk
Commit: c2af20485a2acf3bff5175ba79981382236942f4
Parents: 49dbdf7
Author: Nitay Joffe <ni...@apache.org>
Authored: Mon Jul 8 17:01:19 2013 -0400
Committer: Nitay Joffe <ni...@apache.org>
Committed: Mon Jul 8 17:01:19 2013 -0400
----------------------------------------------------------------------
CHANGELOG | 2 +
.../giraph/benchmark/PageRankBenchmark.java | 7 +-
.../org/apache/giraph/jython/JythonUtils.java | 9 +-
.../types/BooleanToBooleanWritableWrapper.java | 31 ++++
.../BooleanWritableToBooleanUnwrapper.java | 31 ++++
.../giraph/types/ByteToByteWritableWrapper.java | 32 ++++
.../giraph/types/ByteToIntWritableWrapper.java | 31 ++++
.../giraph/types/ByteToLongWritableWrapper.java | 31 ++++
.../types/ByteWritableToByteUnwrapper.java | 31 ++++
.../types/ByteWritableToIntUnwrapper.java | 31 ++++
.../types/ByteWritableToLongUnwrapper.java | 31 ++++
.../types/ByteWritableToShortUnwrapper.java | 32 ++++
.../types/DoubleToDoubleWritableWrapper.java | 31 ++++
.../types/DoubleWritableToDoubleUnwrapper.java | 31 ++++
.../types/FloatToDoubleWritableWrapper.java | 31 ++++
.../types/FloatToFloatWritableWrapper.java | 31 ++++
.../types/FloatWritableToDoubleUnwrapper.java | 31 ++++
.../types/FloatWritableToFloatUnwrapper.java | 31 ++++
.../giraph/types/IntToIntWritableWrapper.java | 32 ++++
.../giraph/types/IntToLongWritableWrapper.java | 31 ++++
.../giraph/types/IntWritableToIntUnwrapper.java | 31 ++++
.../types/IntWritableToLongUnwrapper.java | 32 ++++
.../apache/giraph/types/JavaWritablePair.java | 152 +++++++++++++++++
.../giraph/types/LongToLongWritableWrapper.java | 31 ++++
.../types/LongWritableToLongUnwrapper.java | 31 ++++
.../giraph/types/ShortToIntWritableWrapper.java | 32 ++++
.../types/ShortToLongWritableWrapper.java | 31 ++++
.../apache/giraph/types/WritableUnwrapper.java | 36 ++++
.../apache/giraph/types/WritableUnwrappers.java | 89 ++++++++++
.../apache/giraph/types/WritableWrapper.java | 36 ++++
.../apache/giraph/types/WritableWrappers.java | 91 ++++++++++
.../org/apache/giraph/types/package-info.java | 21 +++
.../apache/giraph/utils/ConfigurationUtils.java | 9 +-
.../giraph/utils/DistributedCacheUtils.java | 13 ++
.../org/apache/giraph/jython/TestJython.java | 64 ++++++-
.../apache/giraph/hive/HiveGiraphRunner.java | 78 +--------
.../apache/giraph/hive/common/HiveUtils.java | 169 +++++++++++++++++++
.../hive/input/edge/SimpleHiveToEdge.java | 1 +
.../giraph/hive/input/edge/TypedHiveToEdge.java | 97 +++++++++++
.../hive/input/vertex/TypedHiveToVertex.java | 94 +++++++++++
.../hive/output/AbstractVertexToHive.java | 5 +-
.../giraph/hive/output/HiveVertexWriter.java | 1 +
.../giraph/hive/output/TypedVertexToHive.java | 77 +++++++++
.../apache/giraph/hive/output/VertexToHive.java | 6 +
.../giraph/hive/types/HiveValueReader.java | 62 +++++++
.../giraph/hive/types/HiveValueWriter.java | 59 +++++++
.../giraph/hive/types/HiveVertexIdReader.java | 64 +++++++
.../giraph/hive/types/HiveVertexIdWriter.java | 60 +++++++
.../giraph/hive/types/TypedValueReader.java | 92 ++++++++++
.../giraph/hive/types/TypedValueWriter.java | 84 +++++++++
.../giraph/hive/types/TypedVertexIdReader.java | 93 ++++++++++
.../giraph/hive/types/TypedVertexIdWriter.java | 84 +++++++++
.../apache/giraph/hive/types/package-info.java | 21 +++
53 files changed, 2302 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index ae6e96d..8bfb4dd 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,8 @@
Giraph Change Log
Release 1.1.0 - unreleased
+ GIRAPH-705: Type converters and giraph-hive cleanup (nitay)
+
GIRAPH-704: Specialized message stores (majakabiljo)
GIRAPH-703: create an appropriate way to generate the options.html page using maven.
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankBenchmark.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankBenchmark.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankBenchmark.java
index d88ff0d..413107d 100644
--- a/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankBenchmark.java
+++ b/giraph-core/src/main/java/org/apache/giraph/benchmark/PageRankBenchmark.java
@@ -29,7 +29,6 @@ import org.apache.giraph.jython.DeployType;
import org.apache.giraph.jython.JythonUtils;
import org.apache.giraph.utils.DistributedCacheUtils;
import org.apache.giraph.utils.ReflectionUtils;
-import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.ToolRunner;
@@ -62,14 +61,14 @@ public class PageRankBenchmark extends GiraphBenchmark {
JYTHON_DEPLOY_TYPE.set(conf, DeployType.DISTRIBUTED_CACHE);
String path = BenchmarkOption.SCRIPT_PATH.getOptionValue(cmd);
Path hadoopPath = new Path(path);
- Path remotePath = DistributedCacheUtils.copyToHdfs(hadoopPath, conf);
- DistributedCache.addCacheFile(remotePath.toUri(), conf);
+ Path remotePath = DistributedCacheUtils.copyAndAdd(hadoopPath, conf);
script = remotePath.toString();
} else {
JYTHON_DEPLOY_TYPE.set(conf, DeployType.RESOURCE);
script = ReflectionUtils.getPackagePath(this) + "/page-rank.py";
}
- JythonUtils.init(conf, script, "PageRank", types);
+ types.writeIfUnset(conf);
+ JythonUtils.init(conf, script, "PageRank");
} else {
conf.setComputationClass(PageRankComputation.class);
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/jython/JythonUtils.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/jython/JythonUtils.java b/giraph-core/src/main/java/org/apache/giraph/jython/JythonUtils.java
index f456aa8..77040e3 100644
--- a/giraph-core/src/main/java/org/apache/giraph/jython/JythonUtils.java
+++ b/giraph-core/src/main/java/org/apache/giraph/jython/JythonUtils.java
@@ -17,9 +17,8 @@
*/
package org.apache.giraph.jython;
-import org.apache.giraph.conf.GiraphConfiguration;
-import org.apache.giraph.conf.GiraphTypes;
import org.apache.giraph.graph.Language;
+import org.apache.hadoop.conf.Configuration;
import org.python.core.PyObject;
import static org.apache.giraph.conf.GiraphConstants.COMPUTATION_FACTORY_CLASS;
@@ -61,11 +60,9 @@ public class JythonUtils {
* @param conf Configuration to se
* @param scriptPath Path to Jython script (resource or distributed cache)
* @param klassName Class name of Jython Computation
- * @param types GiraphTypes
*/
- public static void init(GiraphConfiguration conf, String scriptPath,
- String klassName, GiraphTypes types) {
- types.writeIfUnset(conf);
+ public static void init(Configuration conf, String scriptPath,
+ String klassName) {
COMPUTATION_LANGUAGE.set(conf, Language.JYTHON);
COMPUTATION_FACTORY_CLASS.set(conf, JythonComputationFactory.class);
JYTHON_SCRIPT_PATH.set(conf, scriptPath);
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/BooleanToBooleanWritableWrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/BooleanToBooleanWritableWrapper.java b/giraph-core/src/main/java/org/apache/giraph/types/BooleanToBooleanWritableWrapper.java
new file mode 100644
index 0000000..d063750
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/BooleanToBooleanWritableWrapper.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.BooleanWritable;
+
+/**
+ * Converts Booleans to BooleanWritables
+ */
+public class BooleanToBooleanWritableWrapper
+ implements WritableWrapper<BooleanWritable, Boolean> {
+ @Override
+ public BooleanWritable wrap(Boolean javaValue) {
+ return new BooleanWritable(javaValue);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/BooleanWritableToBooleanUnwrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/BooleanWritableToBooleanUnwrapper.java b/giraph-core/src/main/java/org/apache/giraph/types/BooleanWritableToBooleanUnwrapper.java
new file mode 100644
index 0000000..69da496
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/BooleanWritableToBooleanUnwrapper.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.BooleanWritable;
+
+/**
+ * Converts BooleanWritables to Booleans.
+ */
+public class BooleanWritableToBooleanUnwrapper
+ implements WritableUnwrapper<BooleanWritable, Boolean> {
+ @Override
+ public Boolean unwrap(BooleanWritable writableValue) {
+ return writableValue.get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/ByteToByteWritableWrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/ByteToByteWritableWrapper.java b/giraph-core/src/main/java/org/apache/giraph/types/ByteToByteWritableWrapper.java
new file mode 100644
index 0000000..1b36178
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/ByteToByteWritableWrapper.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.ByteWritable;
+
+/**
+ * Converts Bytes to ByteWritables
+ */
+public class ByteToByteWritableWrapper
+ implements WritableWrapper<ByteWritable, Byte> {
+ @Override
+ public ByteWritable wrap(Byte javaValue) {
+ return new ByteWritable(javaValue);
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/ByteToIntWritableWrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/ByteToIntWritableWrapper.java b/giraph-core/src/main/java/org/apache/giraph/types/ByteToIntWritableWrapper.java
new file mode 100644
index 0000000..a2671f3
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/ByteToIntWritableWrapper.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.IntWritable;
+
+/**
+ * Converts Bytes to IntWritables
+ */
+public class ByteToIntWritableWrapper
+ implements WritableWrapper<IntWritable, Byte> {
+ @Override
+ public IntWritable wrap(Byte javaValue) {
+ return new IntWritable(javaValue);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/ByteToLongWritableWrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/ByteToLongWritableWrapper.java b/giraph-core/src/main/java/org/apache/giraph/types/ByteToLongWritableWrapper.java
new file mode 100644
index 0000000..64b0b23
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/ByteToLongWritableWrapper.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.LongWritable;
+
+/**
+ * Converts Bytes to LongWritables
+ */
+public class ByteToLongWritableWrapper
+ implements WritableWrapper<LongWritable, Byte> {
+ @Override
+ public LongWritable wrap(Byte javaValue) {
+ return new LongWritable(javaValue);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/ByteWritableToByteUnwrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/ByteWritableToByteUnwrapper.java b/giraph-core/src/main/java/org/apache/giraph/types/ByteWritableToByteUnwrapper.java
new file mode 100644
index 0000000..9e55673
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/ByteWritableToByteUnwrapper.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.ByteWritable;
+
+/**
+ * Converts ByteWritables to Bytes
+ */
+public class ByteWritableToByteUnwrapper
+ implements WritableUnwrapper<ByteWritable, Byte> {
+ @Override
+ public Byte unwrap(ByteWritable writableValue) {
+ return writableValue.get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/ByteWritableToIntUnwrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/ByteWritableToIntUnwrapper.java b/giraph-core/src/main/java/org/apache/giraph/types/ByteWritableToIntUnwrapper.java
new file mode 100644
index 0000000..6b12067
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/ByteWritableToIntUnwrapper.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.ByteWritable;
+
+/**
+ * Converts Integers to ByteWritables
+ */
+public class ByteWritableToIntUnwrapper
+ implements WritableUnwrapper<ByteWritable, Integer> {
+ @Override
+ public Integer unwrap(ByteWritable writableValue) {
+ return (int) writableValue.get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/ByteWritableToLongUnwrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/ByteWritableToLongUnwrapper.java b/giraph-core/src/main/java/org/apache/giraph/types/ByteWritableToLongUnwrapper.java
new file mode 100644
index 0000000..79f2000
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/ByteWritableToLongUnwrapper.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.ByteWritable;
+
+/**
+ * Converts ByteWritables to Longs
+ */
+public class ByteWritableToLongUnwrapper
+ implements WritableUnwrapper<ByteWritable, Long> {
+ @Override
+ public Long unwrap(ByteWritable writableValue) {
+ return (long) writableValue.get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/ByteWritableToShortUnwrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/ByteWritableToShortUnwrapper.java b/giraph-core/src/main/java/org/apache/giraph/types/ByteWritableToShortUnwrapper.java
new file mode 100644
index 0000000..395a968
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/ByteWritableToShortUnwrapper.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.ByteWritable;
+
+/**
+ * Converts ByteWritables to Shorts
+ */
+public class ByteWritableToShortUnwrapper
+ implements WritableUnwrapper<ByteWritable, Short> {
+ @Override
+ public Short unwrap(ByteWritable writableValue) {
+ return (short) writableValue.get();
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/DoubleToDoubleWritableWrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/DoubleToDoubleWritableWrapper.java b/giraph-core/src/main/java/org/apache/giraph/types/DoubleToDoubleWritableWrapper.java
new file mode 100644
index 0000000..a1fae98
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/DoubleToDoubleWritableWrapper.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.DoubleWritable;
+
+/**
+ * Converts Doubles to DoubleWritables
+ */
+public class DoubleToDoubleWritableWrapper
+ implements WritableWrapper<DoubleWritable, Double> {
+ @Override
+ public DoubleWritable wrap(Double javaValue) {
+ return new DoubleWritable(javaValue);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/DoubleWritableToDoubleUnwrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/DoubleWritableToDoubleUnwrapper.java b/giraph-core/src/main/java/org/apache/giraph/types/DoubleWritableToDoubleUnwrapper.java
new file mode 100644
index 0000000..b9ff640
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/DoubleWritableToDoubleUnwrapper.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.DoubleWritable;
+
+/**
+ * Converts DoubleWritables to Doubles
+ */
+public class DoubleWritableToDoubleUnwrapper
+ implements WritableUnwrapper<DoubleWritable, Double> {
+ @Override
+ public Double unwrap(DoubleWritable writableValue) {
+ return writableValue.get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/FloatToDoubleWritableWrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/FloatToDoubleWritableWrapper.java b/giraph-core/src/main/java/org/apache/giraph/types/FloatToDoubleWritableWrapper.java
new file mode 100644
index 0000000..daa53fa
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/FloatToDoubleWritableWrapper.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.DoubleWritable;
+
+/**
+ * Converts Floats to DoubleWritables
+ */
+public class FloatToDoubleWritableWrapper
+ implements WritableWrapper<DoubleWritable, Float> {
+ @Override
+ public DoubleWritable wrap(Float javaValue) {
+ return new DoubleWritable(javaValue);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/FloatToFloatWritableWrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/FloatToFloatWritableWrapper.java b/giraph-core/src/main/java/org/apache/giraph/types/FloatToFloatWritableWrapper.java
new file mode 100644
index 0000000..4f4dd97
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/FloatToFloatWritableWrapper.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.FloatWritable;
+
+/**
+ * Converts Floats to FloatWritables
+ */
+public class FloatToFloatWritableWrapper
+ implements WritableWrapper<FloatWritable, Float> {
+ @Override
+ public FloatWritable wrap(Float javaValue) {
+ return new FloatWritable(javaValue);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/FloatWritableToDoubleUnwrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/FloatWritableToDoubleUnwrapper.java b/giraph-core/src/main/java/org/apache/giraph/types/FloatWritableToDoubleUnwrapper.java
new file mode 100644
index 0000000..9924aea
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/FloatWritableToDoubleUnwrapper.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.FloatWritable;
+
+/**
+ * Converts FloatWritables to Doubles
+ */
+public class FloatWritableToDoubleUnwrapper
+ implements WritableUnwrapper<FloatWritable, Double> {
+ @Override
+ public Double unwrap(FloatWritable writableValue) {
+ return (double) writableValue.get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/FloatWritableToFloatUnwrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/FloatWritableToFloatUnwrapper.java b/giraph-core/src/main/java/org/apache/giraph/types/FloatWritableToFloatUnwrapper.java
new file mode 100644
index 0000000..d380e09
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/FloatWritableToFloatUnwrapper.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.FloatWritable;
+
+/**
+ * Converts FloatWritables to Floats
+ */
+public class FloatWritableToFloatUnwrapper
+ implements WritableUnwrapper<FloatWritable, Float> {
+ @Override
+ public Float unwrap(FloatWritable writableValue) {
+ return writableValue.get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/IntToIntWritableWrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/IntToIntWritableWrapper.java b/giraph-core/src/main/java/org/apache/giraph/types/IntToIntWritableWrapper.java
new file mode 100644
index 0000000..4d868ac
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/IntToIntWritableWrapper.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.IntWritable;
+
+/**
+ * Converts Integers to IntWritables
+ */
+public class IntToIntWritableWrapper
+ implements WritableWrapper<IntWritable, Integer> {
+ @Override
+ public IntWritable wrap(Integer javaValue) {
+ return new IntWritable(javaValue);
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/IntToLongWritableWrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/IntToLongWritableWrapper.java b/giraph-core/src/main/java/org/apache/giraph/types/IntToLongWritableWrapper.java
new file mode 100644
index 0000000..517107f
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/IntToLongWritableWrapper.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.LongWritable;
+
+/**
+ * Converts Integers to LongWritables
+ */
+public class IntToLongWritableWrapper
+ implements WritableWrapper<LongWritable, Integer> {
+ @Override
+ public LongWritable wrap(Integer javaValue) {
+ return new LongWritable(javaValue);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/IntWritableToIntUnwrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/IntWritableToIntUnwrapper.java b/giraph-core/src/main/java/org/apache/giraph/types/IntWritableToIntUnwrapper.java
new file mode 100644
index 0000000..15b04ca
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/IntWritableToIntUnwrapper.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.IntWritable;
+
+/**
+ * Converts IntWritables to Integers
+ */
+public class IntWritableToIntUnwrapper
+ implements WritableUnwrapper<IntWritable, Integer> {
+ @Override
+ public Integer unwrap(IntWritable writableValue) {
+ return writableValue.get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/IntWritableToLongUnwrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/IntWritableToLongUnwrapper.java b/giraph-core/src/main/java/org/apache/giraph/types/IntWritableToLongUnwrapper.java
new file mode 100644
index 0000000..9d0bd76
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/IntWritableToLongUnwrapper.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.IntWritable;
+
+/**
+ * Converts IntWritables to Longs
+ */
+public class IntWritableToLongUnwrapper
+ implements WritableUnwrapper<IntWritable, Long> {
+ @Override
+ public Long unwrap(IntWritable writableValue) {
+ return (long) writableValue.get();
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/JavaWritablePair.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/JavaWritablePair.java b/giraph-core/src/main/java/org/apache/giraph/types/JavaWritablePair.java
new file mode 100644
index 0000000..752f5c1
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/JavaWritablePair.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.BooleanWritable;
+import org.apache.hadoop.io.ByteWritable;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Writable;
+
+import com.google.common.base.Objects;
+
+/**
+ * Holder for java and writable class pair.
+ *
+ * @param <W> writable class type
+ * @param <J> java class type
+ */
+public class JavaWritablePair<W extends Writable, J> {
+ /** Boolean,BooleanWritable */
+ public static final JavaWritablePair<BooleanWritable, Boolean>
+ BOOLEAN_BOOLEAN_WRITABLE = create(BooleanWritable.class, Boolean.class);
+ /** Byte,ByteWritable */
+ public static final JavaWritablePair<ByteWritable, Byte>
+ BYTE_BYTE_WRITABLE = create(ByteWritable.class, Byte.class);
+ /** Byte,IntWritable */
+ public static final JavaWritablePair<IntWritable, Byte>
+ BYTE_INT_WRITABLE = create(IntWritable.class, Byte.class);
+ /** Byte,LongWritable */
+ public static final JavaWritablePair<LongWritable, Byte>
+ BYTE_LONG_WRITABLE = create(LongWritable.class, Byte.class);
+ /** Double,FloatWritable */
+ public static final JavaWritablePair<FloatWritable, Double>
+ DOUBLE_FLOAT_WRITABLE = create(FloatWritable.class, Double.class);
+ /** Double,DoubleWritable */
+ public static final JavaWritablePair<DoubleWritable, Double>
+ DOUBLE_DOUBLE_WRITABLE = create(DoubleWritable.class, Double.class);
+ /** Float,FloatWritable */
+ public static final JavaWritablePair<FloatWritable, Float>
+ FLOAT_FLOAT_WRITABLE = create(FloatWritable.class, Float.class);
+ /** Float,DoubleWritable */
+ public static final JavaWritablePair<DoubleWritable, Float>
+ FLOAT_DOUBLE_WRITABLE = create(DoubleWritable.class, Float.class);
+ /** Integer,ByteWritable */
+ public static final JavaWritablePair<ByteWritable, Integer>
+ INT_BYTE_WRITABLE = create(ByteWritable.class, Integer.class);
+ /** Integer,IntWritable */
+ public static final JavaWritablePair<IntWritable, Integer>
+ INT_INT_WRITABLE = create(IntWritable.class, Integer.class);
+ /** Integer,LongWritable */
+ public static final JavaWritablePair<LongWritable, Integer>
+ INT_LONG_WRITABLE = create(LongWritable.class, Integer.class);
+ /** Long,ByteWritable */
+ public static final JavaWritablePair<ByteWritable, Long>
+ LONG_BYTE_WRITABLE = create(ByteWritable.class, Long.class);
+ /** Long,IntWritable */
+ public static final JavaWritablePair<IntWritable, Long>
+ LONG_INT_WRITABLE = create(IntWritable.class, Long.class);
+ /** Long,LongWritable */
+ public static final JavaWritablePair<LongWritable, Long>
+ LONG_LONG_WRITABLE = create(LongWritable.class, Long.class);
+ /** Short,ByteWritable */
+ public static final JavaWritablePair<ByteWritable, Short>
+ SHORT_BYTE_WRITABLE = create(ByteWritable.class, Short.class);
+ /** Short,IntWritable */
+ public static final JavaWritablePair<IntWritable, Short>
+ SHORT_INT_WRITABLE = create(IntWritable.class, Short.class);
+ /** Short,LongWritable */
+ public static final JavaWritablePair<LongWritable, Short>
+ SHORT_LONG_WRITABLE = create(LongWritable.class, Short.class);
+
+ /** java class */
+ private final Class<J> javaClass;
+ /** writable class */
+ private final Class<W> writableClass;
+
+ /**
+ * Constructor
+ *
+ * @param javaClass java class
+ * @param writableClass writable class
+ */
+ private JavaWritablePair(Class<W> writableClass, Class<J> javaClass) {
+ this.javaClass = javaClass;
+ this.writableClass = writableClass;
+ }
+
+ /**
+ * Create holder for classes
+ *
+ * @param writableClass writable class
+ * @param javaClass java class
+ * @param <W> writable class type
+ * @param <J> java class type
+ * @return JavaAndWritableClasses
+ */
+ public static <W extends Writable, J> JavaWritablePair<W, J> create(
+ Class<W> writableClass, Class<J> javaClass) {
+ return new JavaWritablePair<W, J>(writableClass, javaClass);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj instanceof JavaWritablePair) {
+ JavaWritablePair other = (JavaWritablePair) obj;
+ return Objects.equal(javaClass, other.javaClass) &&
+ Objects.equal(writableClass, other.writableClass);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(javaClass, writableClass);
+ }
+
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(this)
+ .add("javaClass", javaClass.getSimpleName())
+ .add("writableClass", writableClass.getSimpleName())
+ .toString();
+ }
+
+ public Class<J> getJavaClass() {
+ return javaClass;
+ }
+
+ public Class<W> getWritableClass() {
+ return writableClass;
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/LongToLongWritableWrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/LongToLongWritableWrapper.java b/giraph-core/src/main/java/org/apache/giraph/types/LongToLongWritableWrapper.java
new file mode 100644
index 0000000..2211ccb
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/LongToLongWritableWrapper.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.LongWritable;
+
+/**
+ * Converts Longs to LongWritables
+ */
+public class LongToLongWritableWrapper
+ implements WritableWrapper<LongWritable, Long> {
+ @Override
+ public LongWritable wrap(Long javaValue) {
+ return new LongWritable(javaValue);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/LongWritableToLongUnwrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/LongWritableToLongUnwrapper.java b/giraph-core/src/main/java/org/apache/giraph/types/LongWritableToLongUnwrapper.java
new file mode 100644
index 0000000..5611254
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/LongWritableToLongUnwrapper.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.LongWritable;
+
+/**
+ * Converts LongWritables to Longs
+ */
+public class LongWritableToLongUnwrapper
+ implements WritableUnwrapper<LongWritable, Long> {
+ @Override
+ public Long unwrap(LongWritable writableValue) {
+ return writableValue.get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/ShortToIntWritableWrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/ShortToIntWritableWrapper.java b/giraph-core/src/main/java/org/apache/giraph/types/ShortToIntWritableWrapper.java
new file mode 100644
index 0000000..6df538a
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/ShortToIntWritableWrapper.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.IntWritable;
+
+/**
+ * Converts IntWritables to Shorts
+ */
+public class ShortToIntWritableWrapper
+ implements WritableWrapper<IntWritable, Short> {
+ @Override
+ public IntWritable wrap(Short javaValue) {
+ return new IntWritable(javaValue.intValue());
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/ShortToLongWritableWrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/ShortToLongWritableWrapper.java b/giraph-core/src/main/java/org/apache/giraph/types/ShortToLongWritableWrapper.java
new file mode 100644
index 0000000..4027035
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/ShortToLongWritableWrapper.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.LongWritable;
+
+/**
+ * Converts Shorts to LongWritables
+ */
+public class ShortToLongWritableWrapper
+ implements WritableWrapper<LongWritable, Short> {
+ @Override
+ public LongWritable wrap(Short javaValue) {
+ return new LongWritable(javaValue.intValue());
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/WritableUnwrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/WritableUnwrapper.java b/giraph-core/src/main/java/org/apache/giraph/types/WritableUnwrapper.java
new file mode 100644
index 0000000..49256de
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/WritableUnwrapper.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.Writable;
+
+/**
+ * Converts from Writable type to Java type
+ *
+ * @param <W> Writable type
+ * @param <J> Java type
+ */
+public interface WritableUnwrapper<W extends Writable, J> {
+ /**
+ * Convert from writable type to java type
+ *
+ * @param writableValue writable type
+ * @return java type
+ */
+ J unwrap(W writableValue);
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/WritableUnwrappers.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/WritableUnwrappers.java b/giraph-core/src/main/java/org/apache/giraph/types/WritableUnwrappers.java
new file mode 100644
index 0000000..5d53db5
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/WritableUnwrappers.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.Writable;
+
+import com.google.common.collect.Maps;
+
+import java.util.Map;
+
+import static org.apache.giraph.types.JavaWritablePair.BOOLEAN_BOOLEAN_WRITABLE;
+import static org.apache.giraph.types.JavaWritablePair.BYTE_BYTE_WRITABLE;
+import static org.apache.giraph.types.JavaWritablePair.DOUBLE_DOUBLE_WRITABLE;
+import static org.apache.giraph.types.JavaWritablePair.DOUBLE_FLOAT_WRITABLE;
+import static org.apache.giraph.types.JavaWritablePair.FLOAT_FLOAT_WRITABLE;
+import static org.apache.giraph.types.JavaWritablePair.INT_BYTE_WRITABLE;
+import static org.apache.giraph.types.JavaWritablePair.INT_INT_WRITABLE;
+import static org.apache.giraph.types.JavaWritablePair.LONG_BYTE_WRITABLE;
+import static org.apache.giraph.types.JavaWritablePair.LONG_INT_WRITABLE;
+import static org.apache.giraph.types.JavaWritablePair.LONG_LONG_WRITABLE;
+import static org.apache.giraph.types.JavaWritablePair.SHORT_BYTE_WRITABLE;
+
+/**
+ * Mapping of all the known Writable wrappers.
+ */
+public class WritableUnwrappers {
+ /** Map of (Writable,Java)-type pair to wrapper for those types */
+ private static final Map<JavaWritablePair, WritableUnwrapper> MAP;
+
+ static {
+ MAP = Maps.newHashMap();
+ MAP.put(BOOLEAN_BOOLEAN_WRITABLE, new BooleanWritableToBooleanUnwrapper());
+ MAP.put(BYTE_BYTE_WRITABLE, new ByteWritableToByteUnwrapper());
+ MAP.put(DOUBLE_DOUBLE_WRITABLE, new DoubleWritableToDoubleUnwrapper());
+ MAP.put(DOUBLE_FLOAT_WRITABLE, new FloatWritableToDoubleUnwrapper());
+ MAP.put(FLOAT_FLOAT_WRITABLE, new FloatWritableToFloatUnwrapper());
+ MAP.put(INT_BYTE_WRITABLE, new ByteWritableToIntUnwrapper());
+ MAP.put(INT_INT_WRITABLE, new IntWritableToIntUnwrapper());
+ MAP.put(SHORT_BYTE_WRITABLE, new ByteWritableToShortUnwrapper());
+ MAP.put(LONG_BYTE_WRITABLE, new ByteWritableToLongUnwrapper());
+ MAP.put(LONG_INT_WRITABLE, new IntWritableToLongUnwrapper());
+ MAP.put(LONG_LONG_WRITABLE, new LongWritableToLongUnwrapper());
+ }
+
+ /** Don't construct */
+ private WritableUnwrappers() { }
+
+ /**
+ * Lookup type converter
+ *
+ * @param writableClass class of Writable
+ * @param javaClass class of Java type
+ * @param <W> Writable type
+ * @param <J> Java type
+ * @return {@link WritableUnwrapper}
+ */
+ public static <W extends Writable, J> WritableUnwrapper<W, J> lookup(
+ Class<W> writableClass, Class<J> javaClass) {
+ return lookup(JavaWritablePair.create(writableClass, javaClass));
+ }
+
+ /**
+ * Lookup type converter
+ *
+ * @param classes JavaAndWritableClasses
+ * @param <W> Writable type
+ * @param <J> Java type
+ * @return {@link WritableUnwrapper}
+ */
+ public static <W extends Writable, J> WritableUnwrapper<W, J> lookup(
+ JavaWritablePair<W, J> classes) {
+ return MAP.get(classes);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/WritableWrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/WritableWrapper.java b/giraph-core/src/main/java/org/apache/giraph/types/WritableWrapper.java
new file mode 100644
index 0000000..edb62a5
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/WritableWrapper.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.Writable;
+
+/**
+ * Converts from Java type to Writable type
+ *
+ * @param <W> Writable type
+ * @param <J> Java type
+ */
+public interface WritableWrapper<W extends Writable, J> {
+ /**
+ * Convert from java type to writable type
+ *
+ * @param javaValue java type
+ * @return writable value
+ */
+ W wrap(J javaValue);
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/WritableWrappers.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/WritableWrappers.java b/giraph-core/src/main/java/org/apache/giraph/types/WritableWrappers.java
new file mode 100644
index 0000000..68b458f
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/WritableWrappers.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.types;
+
+import org.apache.hadoop.io.Writable;
+
+import com.google.common.collect.Maps;
+
+import java.util.Map;
+
+import static org.apache.giraph.types.JavaWritablePair.BOOLEAN_BOOLEAN_WRITABLE;
+import static org.apache.giraph.types.JavaWritablePair.BYTE_BYTE_WRITABLE;
+import static org.apache.giraph.types.JavaWritablePair.BYTE_INT_WRITABLE;
+import static org.apache.giraph.types.JavaWritablePair.BYTE_LONG_WRITABLE;
+import static org.apache.giraph.types.JavaWritablePair.DOUBLE_DOUBLE_WRITABLE;
+import static org.apache.giraph.types.JavaWritablePair.FLOAT_DOUBLE_WRITABLE;
+import static org.apache.giraph.types.JavaWritablePair.FLOAT_FLOAT_WRITABLE;
+import static org.apache.giraph.types.JavaWritablePair.INT_INT_WRITABLE;
+import static org.apache.giraph.types.JavaWritablePair.INT_LONG_WRITABLE;
+import static org.apache.giraph.types.JavaWritablePair.LONG_LONG_WRITABLE;
+import static org.apache.giraph.types.JavaWritablePair.SHORT_INT_WRITABLE;
+import static org.apache.giraph.types.JavaWritablePair.SHORT_LONG_WRITABLE;
+
+/**
+ * Mapping of all the known Writable wrappers.
+ */
+public class WritableWrappers {
+ /** Map of (Writable,Java)-type pair to wrapper for those types */
+ private static final Map<JavaWritablePair, WritableWrapper> MAP;
+
+ static {
+ MAP = Maps.newHashMap();
+ MAP.put(BOOLEAN_BOOLEAN_WRITABLE, new BooleanToBooleanWritableWrapper());
+ MAP.put(BYTE_BYTE_WRITABLE, new ByteToByteWritableWrapper());
+ MAP.put(BYTE_INT_WRITABLE, new ByteToIntWritableWrapper());
+ MAP.put(BYTE_LONG_WRITABLE, new ByteToLongWritableWrapper());
+ MAP.put(DOUBLE_DOUBLE_WRITABLE, new DoubleToDoubleWritableWrapper());
+ MAP.put(FLOAT_DOUBLE_WRITABLE, new FloatToDoubleWritableWrapper());
+ MAP.put(FLOAT_FLOAT_WRITABLE, new FloatToFloatWritableWrapper());
+ MAP.put(INT_INT_WRITABLE, new IntToIntWritableWrapper());
+ MAP.put(INT_LONG_WRITABLE, new IntToLongWritableWrapper());
+ MAP.put(LONG_LONG_WRITABLE, new LongToLongWritableWrapper());
+ MAP.put(SHORT_INT_WRITABLE, new ShortToIntWritableWrapper());
+ MAP.put(SHORT_LONG_WRITABLE, new ShortToLongWritableWrapper());
+ }
+
+ /** Don't construct */
+ private WritableWrappers() { }
+
+ /**
+ * Lookup type converter
+ *
+ * @param writableClass class of Writable
+ * @param javaClass class of Java type
+ * @param <W> Writable type
+ * @param <J> Java type
+ * @return {@link WritableWrapper}
+ */
+ public static <W extends Writable, J> WritableWrapper<W, J> lookup(
+ Class<W> writableClass, Class<J> javaClass) {
+ return lookup(JavaWritablePair.create(writableClass, javaClass));
+ }
+
+ /**
+ * Lookup type converter
+ *
+ * @param classes JavaAndWritableClasses
+ * @param <W> Writable type
+ * @param <J> Java type
+ * @return {@link WritableWrapper}
+ */
+ public static <W extends Writable, J> WritableWrapper<W, J> lookup(
+ JavaWritablePair<W, J> classes) {
+ return MAP.get(classes);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/types/package-info.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/types/package-info.java b/giraph-core/src/main/java/org/apache/giraph/types/package-info.java
new file mode 100644
index 0000000..ea3f853
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/types/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Type coercion, inference, and reflection.
+ */
+package org.apache.giraph.types;
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/utils/ConfigurationUtils.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/utils/ConfigurationUtils.java b/giraph-core/src/main/java/org/apache/giraph/utils/ConfigurationUtils.java
index f0a7e4f..aba5131 100644
--- a/giraph-core/src/main/java/org/apache/giraph/utils/ConfigurationUtils.java
+++ b/giraph-core/src/main/java/org/apache/giraph/utils/ConfigurationUtils.java
@@ -43,7 +43,6 @@ import org.apache.giraph.master.MasterCompute;
import org.apache.giraph.partition.Partition;
import org.apache.giraph.worker.WorkerContext;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.Level;
@@ -432,12 +431,10 @@ public final class ConfigurationUtils {
}
Path path = new Path(scriptPath);
- Path remotePath = DistributedCacheUtils.copyToHdfs(path, conf);
+ Path remotePath = DistributedCacheUtils.copyAndAdd(path, conf);
- DistributedCache.addCacheFile(remotePath.toUri(), conf);
-
- GiraphTypes types = GiraphTypes.readFrom(conf);
- JythonUtils.init(conf, scriptPath, jythonClass, types);
+ GiraphTypes.readFrom(conf).writeIfUnset(conf);
+ JythonUtils.init(conf, remotePath.toString(), jythonClass);
}
/**
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/main/java/org/apache/giraph/utils/DistributedCacheUtils.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/utils/DistributedCacheUtils.java b/giraph-core/src/main/java/org/apache/giraph/utils/DistributedCacheUtils.java
index 6abe89b..58c1e02 100644
--- a/giraph-core/src/main/java/org/apache/giraph/utils/DistributedCacheUtils.java
+++ b/giraph-core/src/main/java/org/apache/giraph/utils/DistributedCacheUtils.java
@@ -98,4 +98,17 @@ public class DistributedCacheUtils {
}
return remotePath;
}
+
+ /**
+ * Copy a file to HDFS if it is local, and adds it to the distributed cache.
+ *
+ * @param path path to file
+ * @param conf Configuration
+ * @return remote path to file
+ */
+ public static Path copyAndAdd(Path path, Configuration conf) {
+ Path remotePath = copyToHdfs(path, conf);
+ DistributedCache.addCacheFile(remotePath.toUri(), conf);
+ return remotePath;
+ }
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-core/src/test/java/org/apache/giraph/jython/TestJython.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/test/java/org/apache/giraph/jython/TestJython.java b/giraph-core/src/test/java/org/apache/giraph/jython/TestJython.java
index 245d342..58f25a6 100644
--- a/giraph-core/src/test/java/org/apache/giraph/jython/TestJython.java
+++ b/giraph-core/src/test/java/org/apache/giraph/jython/TestJython.java
@@ -26,6 +26,11 @@ import org.apache.giraph.utils.InternalVertexRunner;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.junit.Test;
+import org.python.core.PyDictionary;
+import org.python.core.PyInteger;
+import org.python.core.PyList;
+import org.python.core.PyObject;
+import org.python.util.PythonInterpreter;
import com.google.common.collect.Maps;
@@ -33,8 +38,64 @@ import java.util.Map;
import static org.apache.giraph.jython.JythonComputationFactory.JYTHON_DEPLOY_TYPE;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
public class TestJython {
+ private static final double DELTA = 0.0000001;
+
+ @Test
+ public void testBasic() {
+ String jython =
+ "class Foo:\n" +
+ " def __init__(self):\n" +
+ " self.map = {\"32\": 32, \"4.3\": 4.3}\n" +
+ " self.list = [ 2, 9, 11 ]\n" +
+ " self.ival = 17\n" +
+ "\n" +
+ "def get_map(foo):\n" +
+ " return foo.map\n" +
+ "\n" +
+ "def get_list(foo):\n" +
+ " return foo.list\n" +
+ "\n" +
+ "def get_ival(foo):\n" +
+ " return foo.ival\n" +
+ "";
+
+ PythonInterpreter interpreter = new PythonInterpreter();
+ interpreter.exec(jython);
+
+ PyObject fooClass = interpreter.get("Foo");
+ PyObject getMapFunc = interpreter.get("get_map");
+ PyObject getListFunc = interpreter.get("get_list");
+ PyObject getIValFunc = interpreter.get("get_ival");
+
+ PyObject foo = fooClass.__call__();
+
+ PyObject mapResult = getMapFunc.__call__(foo);
+ assertTrue(mapResult instanceof PyDictionary);
+ PyDictionary pyMapResult = ((PyDictionary) mapResult);
+ assertEquals(2, pyMapResult.size());
+ Object thirtyTwo = pyMapResult.get("32");
+ assertTrue(thirtyTwo instanceof Integer);
+ assertEquals(32, ((Integer) thirtyTwo).intValue());
+ Object fourPointThree = pyMapResult.get("4.3");
+ assertTrue(fourPointThree instanceof Double);
+ assertEquals(4.3, (Double) fourPointThree, DELTA);
+
+ PyObject listResult = getListFunc.__call__(foo);
+ assertTrue(listResult instanceof PyList);
+ PyList pyListResult = (PyList) listResult;
+ assertEquals(3, pyListResult.size());
+ assertEquals(2, pyListResult.get(0));
+ assertEquals(9, pyListResult.get(1));
+ assertEquals(11, pyListResult.get(2));
+
+ PyObject ivalResult = getIValFunc.__call__(foo);
+ assertTrue(ivalResult instanceof PyInteger);
+ assertEquals(17, ((PyInteger) ivalResult).getValue());
+ }
+
@Test
public void testCountEdges() throws Exception {
String[] edges = new String[] {
@@ -47,7 +108,8 @@ public class TestJython {
GiraphConfiguration conf = new GiraphConfiguration();
GiraphTypes types = new GiraphTypes(IntWritable.class, IntWritable.class,
NullWritable.class, NullWritable.class, NullWritable.class);
- JythonUtils.init(conf, "count-edges.py", "CountEdges", types);
+ types.writeIfUnset(conf);
+ JythonUtils.init(conf, "count-edges.py", "CountEdges");
JYTHON_DEPLOY_TYPE.set(conf, DeployType.RESOURCE);
conf.setOutEdgesClass(ByteArrayEdges.class);
conf.setEdgeInputFormatClass(IntNullTextEdgeInputFormat.class);
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-hive/src/main/java/org/apache/giraph/hive/HiveGiraphRunner.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/HiveGiraphRunner.java b/giraph-hive/src/main/java/org/apache/giraph/hive/HiveGiraphRunner.java
index 589fed6..6b8a8e9 100644
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/HiveGiraphRunner.java
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/HiveGiraphRunner.java
@@ -27,6 +27,7 @@ import org.apache.commons.cli.ParseException;
import org.apache.giraph.conf.GiraphConfiguration;
import org.apache.giraph.conf.GiraphConstants;
import org.apache.giraph.graph.Computation;
+import org.apache.giraph.hive.common.HiveUtils;
import org.apache.giraph.hive.input.edge.HiveEdgeInputFormat;
import org.apache.giraph.hive.input.edge.HiveToEdge;
import org.apache.giraph.hive.input.vertex.HiveToVertex;
@@ -49,9 +50,7 @@ import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
-import java.io.File;
import java.util.Arrays;
-import java.util.Collection;
import java.util.List;
import static org.apache.giraph.hive.common.GiraphHiveConstants.HIVE_EDGE_INPUT;
@@ -260,7 +259,8 @@ public class HiveGiraphRunner implements Tool {
}
// additional configuration for Hive
- adjustConfigurationForHive();
+ HiveUtils.addHadoopClasspathToTmpJars(conf);
+ HiveUtils.addHiveSiteXmlToTmpFiles(conf);
// setup GiraphJob
GiraphJob job = new GiraphJob(getConf(), getClass().getName());
@@ -318,31 +318,6 @@ public class HiveGiraphRunner implements Tool {
}
/**
- * set hive configuration
- */
- private void adjustConfigurationForHive() {
- // when output partitions are used, workers register them to the
- // metastore at cleanup stage, and on HiveConf's initialization, it
- // looks for hive-site.xml.
- addToStringCollection(conf, "tmpfiles", conf.getClassLoader()
- .getResource("hive-site.xml").toString());
-
- // Or, more effectively, we can provide all the jars client needed to
- // the workers as well
- String[] hadoopJars = System.getenv("HADOOP_CLASSPATH").split(
- File.pathSeparator);
- List<String> hadoopJarURLs = Lists.newArrayList();
- for (String jarPath : hadoopJars) {
- File file = new File(jarPath);
- if (file.exists() && file.isFile()) {
- String jarURL = file.toURI().toString();
- hadoopJarURLs.add(jarURL);
- }
- }
- addToStringCollection(conf, "tmpjars", hadoopJarURLs);
- }
-
- /**
* process arguments
* @param args to process
* @return CommandLine instance
@@ -463,7 +438,7 @@ public class HiveGiraphRunner implements Tool {
isVerbose = cmdln.hasOption("verbose");
// pick up -hiveconf arguments
- processHiveConfOptions(cmdln);
+ HiveUtils.processHiveconfOptions(cmdln.getOptionValues("hiveconf"), conf);
processMoreArguments(cmdln);
@@ -471,26 +446,6 @@ public class HiveGiraphRunner implements Tool {
}
/**
- * Process -hiveconf options from command line
- *
- * @param cmdln Command line options
- */
- private void processHiveConfOptions(CommandLine cmdln) {
- for (String hiveconf : cmdln.getOptionValues("hiveconf")) {
- String[] keyval = hiveconf.split("=", 2);
- if (keyval.length == 2) {
- String name = keyval[0];
- String value = keyval[1];
- if (name.equals("tmpjars") || name.equals("tmpfiles")) {
- addToStringCollection(conf, name, value);
- } else {
- conf.set(name, value);
- }
- }
- }
- }
-
- /**
* Add hive-related options to command line parser options
*
* @param options Options to use
@@ -548,31 +503,6 @@ public class HiveGiraphRunner implements Tool {
}
/**
- * add string to collection
- * @param conf Configuration
- * @param name name to add
- * @param values values for collection
- */
- private static void addToStringCollection(Configuration conf, String name,
- String... values) {
- addToStringCollection(conf, name, Arrays.asList(values));
- }
-
- /**
- * add string to collection
- * @param conf Configuration
- * @param name to add
- * @param values values for collection
- */
- private static void addToStringCollection(
- Configuration conf, String name, Collection
- <? extends String> values) {
- Collection<String> tmpfiles = conf.getStringCollection(name);
- tmpfiles.addAll(values);
- conf.setStrings(name, tmpfiles.toArray(new String[tmpfiles.size()]));
- }
-
- /**
*
* @param className to find
* @param base base class
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-hive/src/main/java/org/apache/giraph/hive/common/HiveUtils.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/common/HiveUtils.java b/giraph-hive/src/main/java/org/apache/giraph/hive/common/HiveUtils.java
index 11b060f..ae2a402 100644
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/common/HiveUtils.java
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/common/HiveUtils.java
@@ -19,6 +19,7 @@
package org.apache.giraph.hive.common;
import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.giraph.conf.StrConfOption;
import org.apache.giraph.hive.input.edge.HiveToEdge;
import org.apache.giraph.hive.input.vertex.HiveToVertex;
import org.apache.giraph.hive.output.VertexToHive;
@@ -26,6 +27,7 @@ import org.apache.giraph.utils.ReflectionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
+import org.apache.log4j.Logger;
import org.apache.thrift.TException;
import com.facebook.hiveio.input.HiveApiInputFormat;
@@ -38,10 +40,16 @@ import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import java.io.File;
import java.io.IOException;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.lang.System.getenv;
import static org.apache.giraph.hive.common.GiraphHiveConstants.HIVE_EDGE_INPUT;
import static org.apache.giraph.hive.common.GiraphHiveConstants.HIVE_VERTEX_INPUT;
import static org.apache.giraph.hive.common.GiraphHiveConstants.VERTEX_TO_HIVE_CLASS;
@@ -51,6 +59,9 @@ import static org.apache.giraph.utils.ReflectionUtils.newInstance;
* Utility methods for Hive IO
*/
public class HiveUtils {
+ /** Logger */
+ private static final Logger LOG = Logger.getLogger(HiveUtils.class);
+
/** Do not instantiate */
private HiveUtils() {
}
@@ -120,6 +131,164 @@ public class HiveUtils {
}
/**
+ * Lookup index of column in {@link HiveTableSchema}, or throw if not found.
+ *
+ * @param schema {@link HiveTableSchema}
+ * @param columnName column name
+ * @return column index
+ */
+ public static int columnIndexOrThrow(HiveTableSchema schema,
+ String columnName) {
+ int index = schema.positionOf(columnName);
+ if (index == -1) {
+ throw new IllegalArgumentException("Column " + columnName +
+ " not found in table " + schema.getTableDesc());
+ }
+ return index;
+ }
+
+ /**
+ * Lookup index of column in {@link HiveTableSchema}, or throw if not found.
+ *
+ * @param schema {@link HiveTableSchema}
+ * @param conf {@link Configuration}
+ * @param confOption {@link StrConfOption}
+ * @return column index
+ */
+ public static int columnIndexOrThrow(HiveTableSchema schema,
+ Configuration conf, StrConfOption confOption) {
+ String columnName = confOption.get(conf);
+ if (columnName == null) {
+ throw new IllegalArgumentException("Column " + confOption.getKey() +
+ " not set in configuration");
+ }
+ return columnIndexOrThrow(schema, columnName);
+ }
+
+ /**
+ * Add hive-site.xml file to tmpfiles in Configuration.
+ *
+ * @param conf Configuration
+ */
+ public static void addHiveSiteXmlToTmpFiles(Configuration conf) {
+ // When output partitions are used, workers register them to the
+ // metastore at cleanup stage, and on HiveConf's initialization, it
+ // looks for hive-site.xml.
+ addToHiveFromClassLoader(conf, "hive-site.xml");
+ }
+
+ /**
+ * Add hive-site-custom.xml to tmpfiles in Configuration.
+ *
+ * @param conf Configuration
+ */
+ public static void addHiveSiteCustomXmlToTmpFiles(Configuration conf) {
+ addToHiveFromClassLoader(conf, "hive-site-custom.xml");
+ }
+
+ /**
+ * Add a file to Configuration tmpfiles from ClassLoader resource
+ *
+ * @param conf Configuration
+ * @param name file name
+ */
+ private static void addToHiveFromClassLoader(Configuration conf,
+ String name) {
+ URL url = conf.getClassLoader().getResource(name);
+ checkNotNull(url);
+ if (LOG.isInfoEnabled()) {
+ LOG.info("addToHiveFromClassLoader: Adding " + name + " at " +
+ url + " to Configuration tmpfiles");
+ }
+ addToStringCollection(conf, "tmpfiles", url.toString());
+ }
+
+ /**
+ * Add jars from HADOOP_CLASSPATH environment variable to tmpjars property
+ * in Configuration.
+ *
+ * @param conf Configuration
+ */
+ public static void addHadoopClasspathToTmpJars(Configuration conf) {
+ // Or, more effectively, we can provide all the jars client needed to
+ // the workers as well
+ String[] hadoopJars = getenv("HADOOP_CLASSPATH").split(File.pathSeparator);
+ if (hadoopJars.length > 0) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("addHadoopClasspathToTmpJars: Adding HADOOP_CLASSPATH jars " +
+ "at " + Arrays.toString(hadoopJars) + " to Configuration tmpjars");
+ }
+ List<String> hadoopJarURLs = Lists.newArrayList();
+ for (String jarPath : hadoopJars) {
+ File file = new File(jarPath);
+ if (file.exists() && file.isFile()) {
+ hadoopJarURLs.add(file.toURI().toString());
+ }
+ }
+ HiveUtils.addToStringCollection(conf, "tmpjars", hadoopJarURLs);
+ }
+ }
+
+ /**
+ * Handle -hiveconf options, adding them to Configuration
+ *
+ * @param hiveconfArgs array of hiveconf args
+ * @param conf Configuration
+ */
+ public static void processHiveconfOptions(String[] hiveconfArgs,
+ Configuration conf) {
+ for (String hiveconf : hiveconfArgs) {
+ processHiveconfOption(conf, hiveconf);
+ }
+ }
+
+ /**
+ * Process -hiveconf option, adding it to Configuration appropriately.
+ *
+ * @param conf Configuration
+ * @param hiveconf option to process
+ */
+ public static void processHiveconfOption(Configuration conf,
+ String hiveconf) {
+ String[] keyval = hiveconf.split("=", 2);
+ if (keyval.length == 2) {
+ String name = keyval[0];
+ String value = keyval[1];
+ if (name.equals("tmpjars") || name.equals("tmpfiles")) {
+ addToStringCollection(conf, name, value);
+ } else {
+ conf.set(name, value);
+ }
+ }
+ }
+
+ /**
+ * Add string to collection
+ *
+ * @param conf Configuration
+ * @param key key to add
+ * @param values values for collection
+ */
+ public static void addToStringCollection(Configuration conf, String key,
+ String... values) {
+ addToStringCollection(conf, key, Arrays.asList(values));
+ }
+
+ /**
+ * Add string to collection
+ *
+ * @param conf Configuration
+ * @param key to add
+ * @param values values for collection
+ */
+ public static void addToStringCollection(
+ Configuration conf, String key, Collection<String> values) {
+ Collection<String> strings = conf.getStringCollection(key);
+ strings.addAll(values);
+ conf.setStrings(key, strings.toArray(new String[strings.size()]));
+ }
+
+ /**
* Create a new VertexToHive
*
* @param <I> Vertex ID
http://git-wip-us.apache.org/repos/asf/giraph/blob/c2af2048/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/SimpleHiveToEdge.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/SimpleHiveToEdge.java b/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/SimpleHiveToEdge.java
index 56b38f9..c365a87 100644
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/SimpleHiveToEdge.java
+++ b/giraph-hive/src/main/java/org/apache/giraph/hive/input/edge/SimpleHiveToEdge.java
@@ -84,3 +84,4 @@ public abstract class SimpleHiveToEdge<I extends WritableComparable,
return reusableEdge;
}
}
+