You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by ma...@apache.org on 2015/06/12 22:02:34 UTC
[2/5] git commit: updated refs/heads/trunk to ad27a29
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/output/AbstractVertexToHive.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/output/AbstractVertexToHive.java b/giraph-hive/src/main/java/org/apache/giraph/hive/output/AbstractVertexToHive.java
deleted file mode 100644
index 4046bb2..0000000
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/output/AbstractVertexToHive.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.giraph.hive.output;
-
-import org.apache.giraph.hive.common.DefaultConfigurableAndTableSchemaAware;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-
-/**
- * Base class for VertexToHive implementations
- *
- * @param <I> Vertex ID
- * @param <V> Vertex Value
- * @param <E> Edge Value
- */
-public abstract class AbstractVertexToHive<I extends WritableComparable,
- V extends Writable, E extends Writable>
- extends DefaultConfigurableAndTableSchemaAware<I, V, E>
- implements VertexToHive<I, V, E> {
- @Override
- public void initialize() { }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveRecordSaver.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveRecordSaver.java b/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveRecordSaver.java
deleted file mode 100644
index 3477211..0000000
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveRecordSaver.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.hive.output;
-
-import com.facebook.hiveio.record.HiveWritableRecord;
-
-import java.io.IOException;
-
-/**
- * Interface which can save {@link HiveWritableRecord}s.
- */
-public interface HiveRecordSaver {
- /**
- * Save the record.
- *
- * @param record Record to save.
- * @throws IOException
- * @throws InterruptedException
- */
- void save(HiveWritableRecord record) throws IOException, InterruptedException;
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexOutputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexOutputFormat.java b/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexOutputFormat.java
deleted file mode 100644
index 97f61bd..0000000
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexOutputFormat.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.hive.output;
-
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
-import org.apache.giraph.io.VertexOutputFormat;
-import org.apache.giraph.io.VertexWriter;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.RecordWriter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-
-import com.facebook.hiveio.common.HiveTableDesc;
-import com.facebook.hiveio.output.HiveApiOutputFormat;
-import com.facebook.hiveio.output.HiveOutputDescription;
-import com.facebook.hiveio.record.HiveRecordFactory;
-import com.facebook.hiveio.record.HiveWritableRecord;
-import com.facebook.hiveio.schema.HiveTableSchema;
-
-import java.io.IOException;
-
-import static org.apache.giraph.hive.common.GiraphHiveConstants.HIVE_VERTEX_OUTPUT_DATABASE;
-import static org.apache.giraph.hive.common.GiraphHiveConstants.HIVE_VERTEX_OUTPUT_PARTITION;
-import static org.apache.giraph.hive.common.GiraphHiveConstants.HIVE_VERTEX_OUTPUT_PROFILE_ID;
-import static org.apache.giraph.hive.common.GiraphHiveConstants.HIVE_VERTEX_OUTPUT_TABLE;
-import static org.apache.giraph.hive.common.HiveUtils.newVertexToHive;
-import static org.apache.giraph.hive.common.HiveUtils.parsePartitionValues;
-
-/**
- * VertexOutputFormat using Hive
- *
- * @param <I> Vertex ID
- * @param <V> Vertex Value
- * @param <E> Edge Value
- */
-public class HiveVertexOutputFormat<I extends WritableComparable,
- V extends Writable, E extends Writable>
- extends VertexOutputFormat<I, V, E> {
- /** Underlying Hive OutputFormat used */
- private final HiveApiOutputFormat hiveOutputFormat;
-
- /**
- * Create vertex output format
- */
- public HiveVertexOutputFormat() {
- hiveOutputFormat = new HiveApiOutputFormat();
- }
-
- /**
- * Create HiveOutputDescription from Configuration
- *
- * @return HiveOutputDescription
- */
- private HiveOutputDescription makeOutputDesc() {
- HiveOutputDescription outputDesc = new HiveOutputDescription();
- HiveTableDesc tableDesc = outputDesc.getTableDesc();
- tableDesc.setDatabaseName(HIVE_VERTEX_OUTPUT_DATABASE.get(getConf()));
- tableDesc.setTableName(HIVE_VERTEX_OUTPUT_TABLE.get(getConf()));
- outputDesc.setPartitionValues(
- parsePartitionValues(HIVE_VERTEX_OUTPUT_PARTITION.get(getConf())));
- return outputDesc;
- }
-
- @Override
- public void setConf(
- ImmutableClassesGiraphConfiguration<I, V, E> conf) {
- super.setConf(conf);
- hiveOutputFormat.initialize(makeOutputDesc(),
- HIVE_VERTEX_OUTPUT_PROFILE_ID.get(conf), conf);
- }
-
- @Override
- public VertexWriter<I, V, E> createVertexWriter(TaskAttemptContext context)
- throws IOException, InterruptedException {
- RecordWriter<WritableComparable, HiveWritableRecord> baseWriter =
- hiveOutputFormat.getRecordWriter(context);
- HiveVertexWriter<I, V, E> writer = new HiveVertexWriter<I, V, E>();
- writer.setBaseWriter(baseWriter);
- writer.setTableSchema(hiveOutputFormat.getTableSchema(getConf()));
- return writer;
- }
-
- @Override
- public void checkOutputSpecs(JobContext context)
- throws IOException, InterruptedException {
- hiveOutputFormat.checkOutputSpecs(context);
- HiveTableSchema schema = hiveOutputFormat.getTableSchema(getConf());
- VertexToHive<I, V, E> vertexToHive = newVertexToHive(getConf(), schema);
- vertexToHive.checkOutput(makeOutputDesc(), schema,
- HiveRecordFactory.newWritableRecord(schema));
- }
-
- @Override
- public OutputCommitter getOutputCommitter(TaskAttemptContext context)
- throws IOException, InterruptedException {
- return hiveOutputFormat.getOutputCommitter(context);
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexWriter.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexWriter.java b/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexWriter.java
deleted file mode 100644
index 796bc8f..0000000
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/output/HiveVertexWriter.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.hive.output;
-
-import org.apache.giraph.graph.Vertex;
-import org.apache.giraph.hive.common.HiveUtils;
-import org.apache.giraph.io.VertexWriter;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.mapreduce.RecordWriter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.log4j.Logger;
-
-import com.facebook.hiveio.record.HiveRecordFactory;
-import com.facebook.hiveio.record.HiveWritableRecord;
-import com.facebook.hiveio.schema.HiveTableSchema;
-
-import java.io.IOException;
-
-/**
- * Vertex writer using Hive.
- *
- * @param <I> Vertex ID
- * @param <V> Vertex Value
- * @param <E> Edge Value
- */
-public class HiveVertexWriter<I extends WritableComparable, V extends Writable,
- E extends Writable>
- extends VertexWriter<I, V, E> implements HiveRecordSaver {
- /** Logger */
- private static final Logger LOG = Logger.getLogger(HiveVertexWriter.class);
- /** Underlying Hive RecordWriter used */
- private RecordWriter<WritableComparable, HiveWritableRecord> hiveRecordWriter;
- /** Schema for table in Hive */
- private HiveTableSchema tableSchema;
- /** Reusable {@link HiveRecord} */
- private HiveWritableRecord reusableRecord;
- /** User class to write vertices from a HiveRecord */
- private VertexToHive<I, V, E> vertexToHive;
-
- /**
- * Get underlying Hive record writer used.
- *
- * @return RecordWriter for Hive.
- */
- public RecordWriter<WritableComparable, HiveWritableRecord> getBaseWriter() {
- return hiveRecordWriter;
- }
-
- /**
- * Set underlying Hive record writer used.
- *
- * @param hiveRecordWriter RecordWriter to write to Hive.
- */
- public void setBaseWriter(
- RecordWriter<WritableComparable, HiveWritableRecord> hiveRecordWriter) {
- this.hiveRecordWriter = hiveRecordWriter;
- }
-
- /**
- * Get Hive table schema for table being read from.
- *
- * @return Hive table schema for table
- */
- public HiveTableSchema getTableSchema() {
- return tableSchema;
- }
-
- /**
- * Set Hive schema for table being read from.
- *
- * @param tableSchema Hive table schema
- */
- public void setTableSchema(HiveTableSchema tableSchema) {
- this.tableSchema = tableSchema;
- reusableRecord = HiveRecordFactory.newWritableRecord(tableSchema);
- }
-
- @Override
- public void initialize(TaskAttemptContext context)
- throws IOException, InterruptedException {
- vertexToHive = HiveUtils.newVertexToHive(getConf(), tableSchema);
- vertexToHive.initialize();
- }
-
- @Override
- public void writeVertex(Vertex<I, V, E> vertex)
- throws IOException, InterruptedException {
- vertexToHive.saveVertex(vertex, reusableRecord, this);
- }
-
- @Override
- public void close(TaskAttemptContext context)
- throws IOException, InterruptedException {
- hiveRecordWriter.close(context);
- }
-
- @Override
- public void save(HiveWritableRecord record) throws IOException,
- InterruptedException {
- hiveRecordWriter.write(NullWritable.get(), record);
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/output/SimpleVertexToHive.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/output/SimpleVertexToHive.java b/giraph-hive/src/main/java/org/apache/giraph/hive/output/SimpleVertexToHive.java
deleted file mode 100644
index f6de715..0000000
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/output/SimpleVertexToHive.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.hive.output;
-
-import org.apache.giraph.graph.Vertex;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-
-import com.facebook.hiveio.record.HiveWritableRecord;
-
-import java.io.IOException;
-
-/**
- * Simple implementation of {@link VertexToHive} when each {@link Vertex} is
- * stored to one row in the output.
- *
- * @param <I> Vertex ID
- * @param <V> Vertex Value
- * @param <E> Edge Value
- */
-public abstract class SimpleVertexToHive<I extends WritableComparable,
- V extends Writable, E extends Writable> extends
- AbstractVertexToHive<I, V, E> {
-
- /**
- * Fill the HiveRecord from the Vertex given.
- *
- * @param vertex Vertex to read from.
- * @param record HiveRecord to write to.
- */
- public abstract void fillRecord(Vertex<I, V, E> vertex,
- HiveWritableRecord record);
-
- @Override
- public final void saveVertex(
- Vertex<I, V, E> vertex,
- HiveWritableRecord reusableRecord,
- HiveRecordSaver recordSaver) throws IOException, InterruptedException {
- fillRecord(vertex, reusableRecord);
- recordSaver.save(reusableRecord);
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/output/VertexToHive.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/output/VertexToHive.java b/giraph-hive/src/main/java/org/apache/giraph/hive/output/VertexToHive.java
deleted file mode 100644
index bf13ec1..0000000
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/output/VertexToHive.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.hive.output;
-
-import org.apache.giraph.graph.Vertex;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-
-import com.facebook.hiveio.output.HiveOutputDescription;
-import com.facebook.hiveio.record.HiveWritableRecord;
-import com.facebook.hiveio.schema.HiveTableSchema;
-
-import java.io.IOException;
-
-/**
- * Interface for writing vertices to Hive.
- *
- * @param <I> Vertex ID
- * @param <V> Vertex Value
- * @param <E> Edge Value
- */
-public interface VertexToHive<I extends WritableComparable, V extends Writable,
- E extends Writable> {
- /**
- * User initialization before any saveVertex() calls but after Configuration
- * and HiveTableSchema are guaranteed to be set.
- */
- void initialize();
-
- /**
- * Check the output is valid. This method provides information to the user as
- * early as possible so that they may validate they are using the correct
- * input and fail the job early rather than getting into it and waiting a long
- * time only to find out something was misconfigured.
- *
- * @param outputDesc HiveOutputDescription
- * @param schema Hive table schema
- * @param emptyRecord Example Hive record that you can write to as if you were
- * writing a Vertex. This record will check column types.
- */
- void checkOutput(HiveOutputDescription outputDesc, HiveTableSchema schema,
- HiveWritableRecord emptyRecord);
-
- /**
- * Save vertex to the output. One vertex can be stored to multiple rows in
- * the output.
- *
- * Record you get here is reusable, and the protocol to follow is:
- * - fill the reusableRecord with your data
- * - call recordSaver.save(reusableRecord)
- * - repeat
- * If you don't call save() at all then there won't be any output for this
- * vertex.
- *
- * @param vertex Vertex which we want to save.
- * @param reusableRecord Record to use for writing data to it.
- * @param recordSaver Saver of records
- */
- void saveVertex(Vertex<I, V, E> vertex, HiveWritableRecord reusableRecord,
- HiveRecordSaver recordSaver) throws IOException, InterruptedException;
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/output/examples/HiveOutputIntIntVertex.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/output/examples/HiveOutputIntIntVertex.java b/giraph-hive/src/main/java/org/apache/giraph/hive/output/examples/HiveOutputIntIntVertex.java
deleted file mode 100644
index 29af71f..0000000
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/output/examples/HiveOutputIntIntVertex.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.giraph.hive.output.examples;
-
-import org.apache.giraph.graph.Vertex;
-import org.apache.giraph.hive.output.SimpleVertexToHive;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.NullWritable;
-
-import com.facebook.hiveio.common.HiveType;
-import com.facebook.hiveio.output.HiveOutputDescription;
-import com.facebook.hiveio.record.HiveWritableRecord;
-import com.facebook.hiveio.schema.HiveTableSchema;
-import com.google.common.base.Preconditions;
-
-/**
- * VertexToHive that writes Vertexes with integer IDs and integer values
- */
-public class HiveOutputIntIntVertex extends SimpleVertexToHive<IntWritable,
- IntWritable, NullWritable> {
- @Override public void checkOutput(HiveOutputDescription outputDesc,
- HiveTableSchema schema, HiveWritableRecord emptyRecord) {
- Preconditions.checkArgument(schema.columnType(0) == HiveType.LONG);
- Preconditions.checkArgument(schema.columnType(1) == HiveType.LONG);
- }
-
- @Override public void fillRecord(
- Vertex<IntWritable, IntWritable, NullWritable> vertex,
- HiveWritableRecord record) {
- record.set(0, (long) vertex.getId().get());
- record.set(1, (long) vertex.getValue().get());
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/output/examples/package-info.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/output/examples/package-info.java b/giraph-hive/src/main/java/org/apache/giraph/hive/output/examples/package-info.java
deleted file mode 100644
index 5195e69..0000000
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/output/examples/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Hive output examples.
- */
-package org.apache.giraph.hive.output.examples;
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/output/package-info.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/output/package-info.java b/giraph-hive/src/main/java/org/apache/giraph/hive/output/package-info.java
deleted file mode 100644
index 65d87e3..0000000
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/output/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Hive output things.
- */
-package org.apache.giraph.hive.output;
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/package-info.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/package-info.java b/giraph-hive/src/main/java/org/apache/giraph/hive/package-info.java
deleted file mode 100644
index d828d2a..0000000
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * Package of Giraph configuration related things.
- */
-package org.apache.giraph.hive;
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/primitives/PrimitiveValueReader.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/primitives/PrimitiveValueReader.java b/giraph-hive/src/main/java/org/apache/giraph/hive/primitives/PrimitiveValueReader.java
deleted file mode 100644
index f67db6e..0000000
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/primitives/PrimitiveValueReader.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.giraph.hive.primitives;
-
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
-import org.apache.giraph.conf.StrConfOption;
-import org.apache.giraph.graph.GraphType;
-import org.apache.giraph.hive.values.HiveValueReader;
-import org.apache.giraph.types.WritableWrapper;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Writable;
-
-import com.facebook.hiveio.record.HiveReadableRecord;
-import com.facebook.hiveio.schema.HiveTableSchema;
-import com.google.common.base.Preconditions;
-
-import static org.apache.giraph.hive.common.HiveUtils.columnIndexOrThrow;
-import static org.apache.giraph.types.WritableWrappers.lookup;
-
-/**
- * Reader for Vertex/Edge Values from Hive with known types
- *
- * @param <W> Vertex/Edge Value
- */
-public class PrimitiveValueReader<W extends Writable>
- implements HiveValueReader<W> {
- /** Hive column index */
- private final int columnIndex;
- /** {@link WritableWrapper} for Hive column to Giraph Writable */
- private final WritableWrapper<W, Object> writableWrapper;
-
- /**
- * Constructor
- *
- * @param columnIndex column index
- * @param writableWrapper {@link WritableWrapper}
- */
- public PrimitiveValueReader(int columnIndex,
- WritableWrapper<W, Object> writableWrapper) {
- Preconditions.checkNotNull(writableWrapper);
- this.columnIndex = columnIndex;
- this.writableWrapper = writableWrapper;
- }
-
- /**
- * Create from Configuration with column name and Schema
- *
- * @param conf Configuration
- * @param graphType GraphType
- * @param columnOption StrConfOption for column name
- * @param schema HiveTableSchema
- * @param <W> Graph type
- * @return TypedVertexValueReader
- */
- public static <W extends Writable> HiveValueReader<W> create(
- ImmutableClassesGiraphConfiguration conf, GraphType graphType,
- StrConfOption columnOption, HiveTableSchema schema) {
- Class<W> valueClass = graphType.get(conf);
- if (NullWritable.class.isAssignableFrom(valueClass)) {
- return HiveValueReader.Null.get();
- }
- int columnIndex = columnIndexOrThrow(schema, conf, columnOption);
- Class hiveClass = schema.columnType(columnIndex).javaClass();
- WritableWrapper wrapper = lookup(valueClass, hiveClass);
- return new PrimitiveValueReader(columnIndex, wrapper);
- }
-
- @Override
- public void readFields(W value, HiveReadableRecord record) {
- Object object = record.get(columnIndex);
- writableWrapper.wrap(object, value);
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/primitives/PrimitiveValueWriter.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/primitives/PrimitiveValueWriter.java b/giraph-hive/src/main/java/org/apache/giraph/hive/primitives/PrimitiveValueWriter.java
deleted file mode 100644
index 8cd3ea4..0000000
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/primitives/PrimitiveValueWriter.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.giraph.hive.primitives;
-
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
-import org.apache.giraph.conf.StrConfOption;
-import org.apache.giraph.graph.GraphType;
-import org.apache.giraph.hive.values.HiveValueWriter;
-import org.apache.giraph.types.WritableUnwrapper;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Writable;
-
-import com.facebook.hiveio.record.HiveWritableRecord;
-import com.facebook.hiveio.schema.HiveTableSchema;
-import com.google.common.base.Preconditions;
-
-import static org.apache.giraph.hive.common.HiveUtils.columnIndexOrThrow;
-import static org.apache.giraph.types.WritableUnwrappers.lookup;
-
-/**
- * Writer for graph values (IVEMM) from Hive with known types
- *
- * @param <W> Graph user type (IVEMM)
- */
-public class PrimitiveValueWriter<W extends Writable>
- implements HiveValueWriter<W> {
- /** Hive column index */
- private final int columnIndex;
- /** {@link WritableUnwrapper} for Hive column to Giraph Writable */
- private final WritableUnwrapper<W, Object> writableUnwrapper;
-
- /**
- * Constructor
- *
- * @param columnIndex column index
- * @param writableUnwrapper JavaWritableConverter
- */
- public PrimitiveValueWriter(int columnIndex,
- WritableUnwrapper<W, Object> writableUnwrapper) {
- Preconditions.checkNotNull(writableUnwrapper);
- this.columnIndex = columnIndex;
- this.writableUnwrapper = writableUnwrapper;
- }
-
- /**
- * Create from Configuration with column name and Schema
- *
- * @param <T> Graph Type (IVEMM)
- * @param conf Configuration
- * @param columnOption StrConfOption for column name
- * @param schema HiveTableSchema
- * @param graphType GraphType
- * @return TypedVertexValueReader
- */
- public static <T extends Writable> HiveValueWriter<T> create(
- ImmutableClassesGiraphConfiguration conf, StrConfOption columnOption,
- HiveTableSchema schema, GraphType graphType) {
- Class<T> valueClass = graphType.get(conf);
- if (NullWritable.class.isAssignableFrom(valueClass)) {
- return HiveValueWriter.Null.get();
- }
- int columnIndex = columnIndexOrThrow(schema, conf, columnOption);
- Class hiveClass = schema.columnType(columnIndex).javaClass();
- WritableUnwrapper unwrapper = lookup(valueClass, hiveClass);
- return new PrimitiveValueWriter(columnIndex, unwrapper);
- }
-
- @Override
- public void write(W value, HiveWritableRecord record) {
- Object object = writableUnwrapper.unwrap(value);
- record.set(columnIndex, object);
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/primitives/package-info.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/primitives/package-info.java b/giraph-hive/src/main/java/org/apache/giraph/hive/primitives/package-info.java
deleted file mode 100644
index 53d50ed..0000000
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/primitives/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * HiveIO primitive class oriented things.
- */
-package org.apache.giraph.hive.primitives;
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/values/HiveValueReader.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/values/HiveValueReader.java b/giraph-hive/src/main/java/org/apache/giraph/hive/values/HiveValueReader.java
deleted file mode 100644
index e989e28..0000000
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/values/HiveValueReader.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.giraph.hive.values;
-
-import org.apache.hadoop.io.Writable;
-
-import com.facebook.hiveio.record.HiveReadableRecord;
-
-/**
- * Interface for reading Vertex / Edge values from Hive records
- *
- * @param <T> Value type
- */
-public interface HiveValueReader<T extends Writable> {
- /**
- * Read value from record
- *
- * @param value graph value to read into
- * @param record Hive record
- */
- void readFields(T value, HiveReadableRecord record);
-
- /**
- * Null implementation that return NullWritable
- *
- * @param <W> Writable type
- */
- public class Null<W extends Writable> implements HiveValueReader<W> {
- /** Singleton */
- private static final Null INSTANCE = new Null();
-
- /**
- * Get singleton
- *
- * @return singleton instance
- */
- public static Null get() {
- return INSTANCE;
- }
-
- @Override
- public void readFields(W value, HiveReadableRecord record) { }
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/values/HiveValueWriter.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/values/HiveValueWriter.java b/giraph-hive/src/main/java/org/apache/giraph/hive/values/HiveValueWriter.java
deleted file mode 100644
index 4b624f6..0000000
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/values/HiveValueWriter.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.giraph.hive.values;
-
-import org.apache.hadoop.io.Writable;
-
-import com.facebook.hiveio.record.HiveWritableRecord;
-
-/**
- * Interface for writing Vertex / Edge values from Hive records
- *
- * @param <T> Value type
- */
-public interface HiveValueWriter<T extends Writable> {
- /**
- * Write value to record
- *
- * @param value the value
- * @param record Hive record
- */
- void write(T value, HiveWritableRecord record);
-
- /**
- * Null implementation that does nothing
- *
- * @param <W> Writable type
- */
- public static class Null<W extends Writable> implements HiveValueWriter<W> {
- /** Singleton */
- private static final Null INSTANCE = new Null();
-
- /**
- * Get singleton
- *
- * @return singleton instance
- */
- public static Null get() {
- return INSTANCE;
- }
-
- @Override
- public void write(W value, HiveWritableRecord record) { }
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/main/java/org/apache/giraph/hive/values/package-info.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/main/java/org/apache/giraph/hive/values/package-info.java b/giraph-hive/src/main/java/org/apache/giraph/hive/values/package-info.java
deleted file mode 100644
index 1e8f497..0000000
--- a/giraph-hive/src/main/java/org/apache/giraph/hive/values/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Hive type (vertex ID, vertex value, edge value, etc) related things.
- */
-package org.apache.giraph.hive.values;
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/test/java/org/apache/giraph/hive/GiraphHiveTestBase.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/GiraphHiveTestBase.java b/giraph-hive/src/test/java/org/apache/giraph/hive/GiraphHiveTestBase.java
deleted file mode 100644
index 27a7fe0..0000000
--- a/giraph-hive/src/test/java/org/apache/giraph/hive/GiraphHiveTestBase.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.hive;
-
-import org.junit.BeforeClass;
-
-import com.facebook.hiveio.log.LogHelpers;
-
-public class GiraphHiveTestBase {
- @BeforeClass
- public static void silenceLoggers() {
- LogHelpers.silenceLoggers();
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/test/java/org/apache/giraph/hive/Helpers.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/Helpers.java b/giraph-hive/src/test/java/org/apache/giraph/hive/Helpers.java
deleted file mode 100644
index d2103a7..0000000
--- a/giraph-hive/src/test/java/org/apache/giraph/hive/Helpers.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.giraph.hive;
-
-import org.apache.giraph.conf.GiraphConfiguration;
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
-import org.apache.giraph.io.internal.WrappedVertexOutputFormat;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.HackJobContext;
-import org.apache.hadoop.mapred.HackTaskAttemptContext;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.TaskAttemptID;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-
-import com.google.common.collect.Maps;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Map;
-
-public class Helpers {
- public static InputStream getResource(String name) {
- return Helpers.class.getClassLoader().getResourceAsStream(name);
- }
-
- public static Map<Integer, Double> parseIntDoubleResults(Iterable<String> results) {
- Map<Integer, Double> values = Maps.newHashMap();
- for (String line : results) {
- String[] tokens = line.split("\\s+");
- int id = Integer.valueOf(tokens[0]);
- double value = Double.valueOf(tokens[1]);
- values.put(id, value);
- }
- return values;
- }
-
- public static Map<Integer, Integer> parseIntIntResults(Iterable<String> results) {
- Map<Integer, Integer> values = Maps.newHashMap();
- for (String line : results) {
- String[] tokens = line.split("\\s+");
- int id = Integer.valueOf(tokens[0]);
- int value = Integer.valueOf(tokens[1]);
- values.put(id, value);
- }
- return values;
- }
-
- public static void commitJob(GiraphConfiguration conf)
- throws IOException, InterruptedException {
- ImmutableClassesGiraphConfiguration iconf = new ImmutableClassesGiraphConfiguration(conf);
- WrappedVertexOutputFormat outputFormat = iconf.createWrappedVertexOutputFormat();
- JobConf jobConf = new JobConf(conf);
- TaskAttemptContext
- taskContext = new HackTaskAttemptContext(jobConf, new TaskAttemptID());
- OutputCommitter outputCommitter = outputFormat.getOutputCommitter(
- taskContext);
- JobContext jobContext = new HackJobContext(jobConf, taskContext.getJobID());
- outputCommitter.commitJob(jobContext);
- }
-
- public static JobContext makeJobContext(Configuration conf) {
- JobConf jobConf = new JobConf(conf);
- TaskAttemptContext
- taskContext = new HackTaskAttemptContext(jobConf, new TaskAttemptID());
- return new HackJobContext(jobConf, taskContext.getJobID());
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/test/java/org/apache/giraph/hive/computations/ComputationCountEdges.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/computations/ComputationCountEdges.java b/giraph-hive/src/test/java/org/apache/giraph/hive/computations/ComputationCountEdges.java
deleted file mode 100644
index 14ab478..0000000
--- a/giraph-hive/src/test/java/org/apache/giraph/hive/computations/ComputationCountEdges.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.giraph.hive.computations;
-
-import org.apache.giraph.graph.BasicComputation;
-import org.apache.giraph.graph.Vertex;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.NullWritable;
-
-import java.io.IOException;
-
-public class ComputationCountEdges extends BasicComputation<IntWritable,
- IntWritable, NullWritable, NullWritable> {
- @Override
- public void compute(Vertex<IntWritable, IntWritable, NullWritable> vertex,
- Iterable<NullWritable> messages) throws IOException {
- vertex.setValue(new IntWritable(vertex.getNumEdges()));
- vertex.voteToHalt();
- }
-}
-
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/test/java/org/apache/giraph/hive/computations/ComputationSumEdges.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/computations/ComputationSumEdges.java b/giraph-hive/src/test/java/org/apache/giraph/hive/computations/ComputationSumEdges.java
deleted file mode 100644
index 613095e..0000000
--- a/giraph-hive/src/test/java/org/apache/giraph/hive/computations/ComputationSumEdges.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.giraph.hive.computations;
-
-import org.apache.giraph.edge.Edge;
-import org.apache.giraph.graph.BasicComputation;
-import org.apache.giraph.graph.Vertex;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.NullWritable;
-
-import java.io.IOException;
-
-public class ComputationSumEdges extends BasicComputation<IntWritable,
- DoubleWritable, DoubleWritable, NullWritable> {
- @Override
- public void compute(
- Vertex<IntWritable, DoubleWritable, DoubleWritable> vertex,
- Iterable<NullWritable> messages) throws IOException {
- double sum = 0;
- for (Edge<IntWritable, DoubleWritable> edge : vertex.getEdges()) {
- sum += edge.getValue().get();
- }
- vertex.setValue(new DoubleWritable(sum));
- vertex.voteToHalt();
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/test/java/org/apache/giraph/hive/computations/package-info.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/computations/package-info.java b/giraph-hive/src/test/java/org/apache/giraph/hive/computations/package-info.java
deleted file mode 100644
index 5569bda..0000000
--- a/giraph-hive/src/test/java/org/apache/giraph/hive/computations/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Computation implementations for Hive tests.
- */
-package org.apache.giraph.hive.computations;
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/test/java/org/apache/giraph/hive/input/CheckInputTest.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/input/CheckInputTest.java b/giraph-hive/src/test/java/org/apache/giraph/hive/input/CheckInputTest.java
deleted file mode 100644
index f51531a..0000000
--- a/giraph-hive/src/test/java/org/apache/giraph/hive/input/CheckInputTest.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.hive.input;
-
-import org.apache.giraph.conf.GiraphConfiguration;
-import org.apache.giraph.hive.GiraphHiveTestBase;
-import org.apache.giraph.hive.computations.ComputationCountEdges;
-import org.apache.giraph.hive.input.edge.HiveEdgeInputFormat;
-import org.apache.giraph.hive.input.edge.HiveToEdge;
-import org.apache.giraph.hive.input.edge.examples.HiveIntNullEdge;
-import org.apache.giraph.hive.input.vertex.HiveToVertex;
-import org.apache.giraph.hive.input.vertex.examples.HiveIntNullNullVertex;
-import org.apache.giraph.io.formats.IdWithValueTextOutputFormat;
-import org.apache.giraph.utils.InternalVertexRunner;
-import org.apache.thrift.TException;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.facebook.hiveio.common.HiveMetastores;
-import com.facebook.hiveio.common.HiveType;
-import com.facebook.hiveio.input.HiveInputDescription;
-import com.facebook.hiveio.schema.HiveTableSchema;
-import com.facebook.hiveio.schema.TestSchema;
-import com.facebook.hiveio.testing.LocalHiveServer;
-
-import java.io.IOException;
-
-import static junit.framework.Assert.assertNull;
-import static org.apache.giraph.hive.common.GiraphHiveConstants.HIVE_EDGE_INPUT;
-
-public class CheckInputTest extends GiraphHiveTestBase {
- private LocalHiveServer hiveServer = new LocalHiveServer("giraph-hive");
-
- @Before
- public void setUp() throws IOException, TException {
- hiveServer.init();
- HiveMetastores.setTestClient(hiveServer.getClient());
- }
-
- @Test
- public void testCheckEdge() throws Exception {
- HiveToEdge hiveToEdge = new HiveIntNullEdge();
- HiveInputDescription inputDesc = new HiveInputDescription();
- HiveTableSchema schema = TestSchema.builder()
- .addColumn("foo", HiveType.INT)
- .addColumn("bar", HiveType.INT)
- .build();
- hiveToEdge.checkInput(inputDesc, schema);
-
- schema = TestSchema.builder()
- .addColumn("foo", HiveType.INT)
- .addColumn("bar", HiveType.LONG)
- .build();
- checkEdgeThrows(hiveToEdge, inputDesc, schema);
- }
-
- private void checkEdgeThrows(HiveToEdge hiveToEdge,
- HiveInputDescription inputDesc, HiveTableSchema schema) {
- try {
- hiveToEdge.checkInput(inputDesc, schema);
- } catch (IllegalArgumentException e) {
- return;
- }
- Assert.fail();
- }
-
- @Test
- public void testCheckVertex() throws Exception {
- HiveToVertex hiveToVertex = new HiveIntNullNullVertex();
- HiveInputDescription inputDesc = new HiveInputDescription();
- HiveTableSchema schema = TestSchema.builder()
- .addColumn("foo", HiveType.INT)
- .addColumn("bar", HiveType.LIST)
- .build();
- hiveToVertex.checkInput(inputDesc, schema);
-
- schema = TestSchema.builder()
- .addColumn("foo", HiveType.INT)
- .addColumn("bar", HiveType.STRING)
- .build();
- checkVertexThrows(hiveToVertex, inputDesc, schema);
- }
-
- private void checkVertexThrows(HiveToVertex hiveToVertex,
- HiveInputDescription inputDesc, HiveTableSchema schema) {
- try {
- hiveToVertex.checkInput(inputDesc, schema);
- } catch (IllegalArgumentException e) {
- return;
- }
- Assert.fail();
- }
-
- @Test
- public void testCheckJobThrows() throws Exception {
- String tableName = "test1";
- hiveServer.createTable("CREATE TABLE " + tableName +
- " (i1 BIGINT, i2 INT) " +
- " ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'");
- String[] rows = {
- "1\t2",
- "2\t3",
- "2\t4",
- "4\t1",
- };
- hiveServer.loadData(tableName, rows);
-
- GiraphConfiguration conf = new GiraphConfiguration();
- HIVE_EDGE_INPUT.setTable(conf, tableName);
- HIVE_EDGE_INPUT.setClass(conf, HiveIntNullEdge.class);
- conf.setComputationClass(ComputationCountEdges.class);
- conf.setEdgeInputFormatClass(HiveEdgeInputFormat.class);
- conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
- assertNull(InternalVertexRunner.run(conf, new String[0], new String[0]));
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/test/java/org/apache/giraph/hive/input/HiveEdgeInputTest.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/input/HiveEdgeInputTest.java b/giraph-hive/src/test/java/org/apache/giraph/hive/input/HiveEdgeInputTest.java
deleted file mode 100644
index bf5af0f..0000000
--- a/giraph-hive/src/test/java/org/apache/giraph/hive/input/HiveEdgeInputTest.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.giraph.hive.input;
-
-import org.apache.giraph.conf.GiraphConfiguration;
-import org.apache.giraph.hive.GiraphHiveTestBase;
-import org.apache.giraph.hive.Helpers;
-import org.apache.giraph.hive.computations.ComputationCountEdges;
-import org.apache.giraph.hive.computations.ComputationSumEdges;
-import org.apache.giraph.hive.input.edge.HiveEdgeInputFormat;
-import org.apache.giraph.hive.input.edge.examples.HiveIntDoubleEdge;
-import org.apache.giraph.hive.input.edge.examples.HiveIntNullEdge;
-import org.apache.giraph.io.formats.IdWithValueTextOutputFormat;
-import org.apache.giraph.utils.InternalVertexRunner;
-import org.apache.thrift.TException;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.facebook.hiveio.common.HiveMetastores;
-import com.facebook.hiveio.testing.LocalHiveServer;
-
-import java.io.IOException;
-import java.util.Map;
-
-import static junit.framework.Assert.assertEquals;
-import static org.apache.giraph.hive.common.GiraphHiveConstants.HIVE_EDGE_INPUT;
-
-public class HiveEdgeInputTest extends GiraphHiveTestBase {
- private LocalHiveServer hiveServer = new LocalHiveServer("giraph-hive");
-
- @Before
- public void setUp() throws IOException, TException {
- hiveServer.init();
- HiveMetastores.setTestClient(hiveServer.getClient());
- }
-
- @Test
- public void testEdgeInput() throws Exception {
- String tableName = "test1";
- hiveServer.createTable("CREATE TABLE " + tableName +
- " (i1 INT, i2 INT) " +
- " ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'");
- String[] rows = {
- "1\t2",
- "2\t3",
- "2\t4",
- "4\t1",
- };
- hiveServer.loadData(tableName, rows);
-
- GiraphConfiguration conf = new GiraphConfiguration();
- HIVE_EDGE_INPUT.setTable(conf, tableName);
- HIVE_EDGE_INPUT.setClass(conf, HiveIntNullEdge.class);
- conf.setComputationClass(ComputationCountEdges.class);
- conf.setEdgeInputFormatClass(HiveEdgeInputFormat.class);
- conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
- Iterable<String> output = InternalVertexRunner.run(conf, new String[0], new String[0]);
-
- Map<Integer, Integer> data = Helpers.parseIntIntResults(output);
- assertEquals(3, data.size());
- assertEquals(1, (int) data.get(1));
- assertEquals(2, (int) data.get(2));
- assertEquals(1, (int) data.get(4));
- }
-
- @Test
- public void testEdgeInputWithPartitions() throws Exception {
- String tableName = "test1";
- String partition = "ds='foobar'";
- hiveServer.createTable("CREATE TABLE " + tableName +
- " (i1 INT, i2 INT) " +
- " PARTITIONED BY (ds STRING) " +
- " ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' ");
- String[] rows = {
- "1\t2",
- "2\t3",
- "2\t4",
- "4\t1",
- };
- hiveServer.loadData(tableName, partition, rows);
-
- GiraphConfiguration conf = new GiraphConfiguration();
- HIVE_EDGE_INPUT.setTable(conf, tableName);
- HIVE_EDGE_INPUT.setPartition(conf, partition);
- HIVE_EDGE_INPUT.setClass(conf, HiveIntNullEdge.class);
- conf.setComputationClass(ComputationCountEdges.class);
- conf.setEdgeInputFormatClass(HiveEdgeInputFormat.class);
- conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
- Iterable<String> output = InternalVertexRunner.run(conf, new String[0], new String[0]);
-
- Map<Integer, Integer> data = Helpers.parseIntIntResults(output);
- assertEquals(3, data.size());
- assertEquals(1, (int) data.get(1));
- assertEquals(2, (int) data.get(2));
- assertEquals(1, (int) data.get(4));
- }
-
- @Test
- public void testEdgeInputWithValues() throws Exception {
- String tableName = "test1";
- hiveServer.createTable("CREATE TABLE " + tableName +
- " (i1 INT, i2 INT, d3 DOUBLE) " +
- " ROW FORMAT DELIMITED " +
- " FIELDS TERMINATED BY '\t' " +
- " COLLECTION ITEMS TERMINATED BY ',' ");
- String[] rows = {
- "1\t2\t0.22",
- "2\t3\t0.33",
- "2\t4\t0.44",
- "4\t1\t0.11",
- };
- hiveServer.loadData(tableName, rows);
-
- GiraphConfiguration conf = new GiraphConfiguration();
- HIVE_EDGE_INPUT.setTable(conf, tableName);
- HIVE_EDGE_INPUT.setClass(conf, HiveIntDoubleEdge.class);
- conf.setComputationClass(ComputationSumEdges.class);
- conf.setEdgeInputFormatClass(HiveEdgeInputFormat.class);
- conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
- Iterable<String> output = InternalVertexRunner.run(conf, new String[0], new String[0]);
-
- Map<Integer, Double> data = Helpers.parseIntDoubleResults(output);
- assertEquals(3, data.size());
- assertEquals(0.22, data.get(1));
- assertEquals(0.77, data.get(2));
- assertEquals(0.11, data.get(4));
- }
-}
-
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/test/java/org/apache/giraph/hive/input/HiveVertexInputTest.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/input/HiveVertexInputTest.java b/giraph-hive/src/test/java/org/apache/giraph/hive/input/HiveVertexInputTest.java
deleted file mode 100644
index 98ba014..0000000
--- a/giraph-hive/src/test/java/org/apache/giraph/hive/input/HiveVertexInputTest.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.giraph.hive.input;
-
-import com.facebook.hiveio.common.HiveMetastores;
-import com.facebook.hiveio.testing.LocalHiveServer;
-import org.apache.giraph.conf.GiraphConfiguration;
-import org.apache.giraph.hive.GiraphHiveTestBase;
-import org.apache.giraph.hive.Helpers;
-import org.apache.giraph.hive.computations.ComputationCountEdges;
-import org.apache.giraph.hive.computations.ComputationSumEdges;
-import org.apache.giraph.hive.input.vertex.HiveVertexInputFormat;
-import org.apache.giraph.hive.input.vertex.examples.HiveIntDoubleDoubleVertex;
-import org.apache.giraph.hive.input.vertex.examples.HiveIntIntNullVertex;
-import org.apache.giraph.io.formats.IdWithValueTextOutputFormat;
-import org.apache.giraph.utils.InternalVertexRunner;
-import org.apache.thrift.TException;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Map;
-
-import static junit.framework.Assert.assertEquals;
-import static org.apache.giraph.hive.common.GiraphHiveConstants.HIVE_VERTEX_INPUT;
-
-public class HiveVertexInputTest extends GiraphHiveTestBase {
- private LocalHiveServer hiveServer = new LocalHiveServer("giraph-hive");
-
- @Before
- public void setUp() throws IOException, TException {
- hiveServer.init();
- HiveMetastores.setTestClient(hiveServer.getClient());
- }
-
- @Test
- public void testVertexInput() throws Exception {
- String tableName = "test1";
- hiveServer.createTable("CREATE TABLE " + tableName +
- " (i1 INT, i2 ARRAY<BIGINT>) " +
- " ROW FORMAT DELIMITED " +
- " FIELDS TERMINATED BY '\t' " +
- " COLLECTION ITEMS TERMINATED BY ','");
- String[] rows = {
- "1\t2",
- "2\t3,4",
- "4\t1",
- };
- hiveServer.loadData(tableName, rows);
-
- GiraphConfiguration conf = new GiraphConfiguration();
- HIVE_VERTEX_INPUT.setTable(conf, tableName);
- HIVE_VERTEX_INPUT.setClass(conf, HiveIntIntNullVertex.class);
- conf.setComputationClass(ComputationCountEdges.class);
- conf.setVertexInputFormatClass(HiveVertexInputFormat.class);
- conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
- Iterable<String> output = InternalVertexRunner.run(conf, new String[0], new String[0]);
-
- Map<Integer, Integer> data = Helpers.parseIntIntResults(output);
- assertEquals(3, data.size());
- assertEquals(1, (int) data.get(1));
- assertEquals(2, (int) data.get(2));
- assertEquals(1, (int) data.get(4));
- }
-
- @Test
- public void testVertexInputWithPartitions() throws Exception {
- String tableName = "test1";
- String partition = "ds='foobar'";
- hiveServer.createTable("CREATE TABLE " + tableName +
- " (i1 INT, i2 ARRAY<BIGINT>) " +
- " PARTITIONED BY (ds STRING) " +
- " ROW FORMAT DELIMITED " +
- " FIELDS TERMINATED BY '\t' " +
- " COLLECTION ITEMS TERMINATED BY ','");
- String[] rows = {
- "1\t2",
- "2\t3,4",
- "4\t1",
- };
- hiveServer.loadData(tableName, partition, rows);
-
- GiraphConfiguration conf = new GiraphConfiguration();
- HIVE_VERTEX_INPUT.setTable(conf, tableName);
- HIVE_VERTEX_INPUT.setPartition(conf, partition);
- HIVE_VERTEX_INPUT.setClass(conf, HiveIntIntNullVertex.class);
- conf.setComputationClass(ComputationCountEdges.class);
- conf.setVertexInputFormatClass(HiveVertexInputFormat.class);
- conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
- Iterable<String> output = InternalVertexRunner.run(conf, new String[0], new String[0]);
-
- Map<Integer, Integer> data = Helpers.parseIntIntResults(output);
- assertEquals(3, data.size());
- assertEquals(1, (int) data.get(1));
- assertEquals(2, (int) data.get(2));
- assertEquals(1, (int) data.get(4));
- }
-
- @Test
- public void testValues() throws Exception {
- String tableName = "test1";
- hiveServer.createTable("CREATE TABLE " + tableName +
- " (i1 INT, d2 DOUBLE, m3 MAP<BIGINT,DOUBLE>) " +
- " ROW FORMAT DELIMITED " +
- " FIELDS TERMINATED BY '\t' " +
- " COLLECTION ITEMS TERMINATED BY ',' " +
- " MAP KEYS TERMINATED BY ':' ");
- String[] rows = {
- "1\t1.11\t2:0.22",
- "2\t2.22\t3:0.33,4:0.44",
- "4\t4.44\t1:0.11",
- };
- hiveServer.loadData(tableName, rows);
-
- GiraphConfiguration conf = new GiraphConfiguration();
- HIVE_VERTEX_INPUT.setTable(conf, tableName);
- HIVE_VERTEX_INPUT.setClass(conf, HiveIntDoubleDoubleVertex.class);
- conf.setComputationClass(ComputationSumEdges.class);
- conf.setVertexInputFormatClass(HiveVertexInputFormat.class);
- conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
- Iterable<String> output = InternalVertexRunner.run(conf, new String[0], new String[0]);
-
- Map<Integer, Double> data = Helpers.parseIntDoubleResults(output);
- assertEquals(3, data.size());
- assertEquals(0.22, data.get(1));
- assertEquals(0.77, data.get(2));
- assertEquals(0.11, data.get(4));
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/test/java/org/apache/giraph/hive/input/package-info.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/input/package-info.java b/giraph-hive/src/test/java/org/apache/giraph/hive/input/package-info.java
deleted file mode 100644
index c2327ca..0000000
--- a/giraph-hive/src/test/java/org/apache/giraph/hive/input/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Hive Input test.
- */
-package org.apache.giraph.hive.input;
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/test/java/org/apache/giraph/hive/jython/TestHiveJythonComplexTypes.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/jython/TestHiveJythonComplexTypes.java b/giraph-hive/src/test/java/org/apache/giraph/hive/jython/TestHiveJythonComplexTypes.java
deleted file mode 100644
index 3a07dc4..0000000
--- a/giraph-hive/src/test/java/org/apache/giraph/hive/jython/TestHiveJythonComplexTypes.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.hive.jython;
-
-import org.apache.giraph.conf.GiraphConfiguration;
-import org.apache.giraph.graph.Language;
-import org.apache.giraph.hive.GiraphHiveTestBase;
-import org.apache.giraph.hive.Helpers;
-import org.apache.giraph.jython.JythonJob;
-import org.apache.giraph.scripting.DeployType;
-import org.apache.giraph.scripting.ScriptLoader;
-import org.apache.giraph.utils.InternalVertexRunner;
-import org.apache.thrift.TException;
-import org.junit.Before;
-import org.junit.Test;
-import org.python.util.PythonInterpreter;
-
-import com.facebook.hiveio.common.HiveMetastores;
-import com.facebook.hiveio.input.HiveInput;
-import com.facebook.hiveio.input.HiveInputDescription;
-import com.facebook.hiveio.record.HiveReadableRecord;
-import com.facebook.hiveio.testing.LocalHiveServer;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Sets;
-import junit.framework.Assert;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Iterator;
-import java.util.Set;
-
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.assertFalse;
-import static junit.framework.Assert.assertNotNull;
-import static junit.framework.Assert.assertTrue;
-import static org.apache.giraph.hive.Helpers.getResource;
-
-public class TestHiveJythonComplexTypes extends GiraphHiveTestBase {
- private LocalHiveServer hiveServer = new LocalHiveServer("jython-test");
-
- @Before
- public void setUp() throws IOException, TException {
- hiveServer.init();
- HiveMetastores.setTestClient(hiveServer.getClient());
- }
-
- @Test
- public void testFakeLabelPropagation() throws Exception {
- String edgesTable = "flp_edges";
- hiveServer.createTable("CREATE TABLE " + edgesTable +
- " (source_id INT, " +
- " target_id INT," +
- " value FLOAT) " +
- " ROW FORMAT DELIMITED " +
- " FIELDS TERMINATED BY '\t'");
- String[] edges = new String[] {
- "1\t2\t0.2",
- "2\t3\t0.3",
- "3\t4\t0.4",
- "4\t1\t0.1",
- };
- hiveServer.loadData(edgesTable, edges);
-
- String vertexesTable = "flp_vertexes";
- hiveServer.createTable("CREATE TABLE " + vertexesTable +
- " (id INT, " +
- " value MAP<INT,FLOAT>) " +
- " ROW FORMAT DELIMITED " +
- " FIELDS TERMINATED BY '\t' " +
- " COLLECTION ITEMS TERMINATED BY ',' " +
- " MAP KEYS TERMINATED BY ':' ");
- String[] vertexes = new String[] {
- "1\t11:0.8,12:0.1",
- "2\t13:0.3,14:0.2",
- "3\t15:0.4,16:0.7",
- "4\t17:0.1,18:0.6",
- };
- hiveServer.loadData(vertexesTable, vertexes);
-
- String outputTable = "flp_output";
- hiveServer.createTable("CREATE TABLE " + outputTable +
- " (id INT," +
- " value MAP<INT,DOUBLE>) " +
- " ROW FORMAT DELIMITED " +
- " FIELDS TERMINATED BY '\t'");
-
- String workerJythonPath =
- "org/apache/giraph/jython/fake-label-propagation-worker.py";
-
- InputStream launcher = getResource(
- "org/apache/giraph/jython/fake-label-propagation-launcher.py");
- assertNotNull(launcher);
- InputStream worker = getResource(workerJythonPath);
- assertNotNull(worker);
-
- PythonInterpreter interpreter = new PythonInterpreter();
-
- JythonJob jythonJob =
- HiveJythonUtils.parseJythonStreams(interpreter, launcher, worker);
-
- GiraphConfiguration conf = new GiraphConfiguration();
-
- ScriptLoader.setScriptsToLoad(conf, workerJythonPath, DeployType.RESOURCE,
- Language.JYTHON);
-
- HiveJythonUtils.writeJythonJobToConf(jythonJob, conf, interpreter);
-
- InternalVertexRunner.run(conf, new String[0], new String[0]);
-
- Helpers.commitJob(conf);
-
- HiveInputDescription inputDesc = new HiveInputDescription();
- inputDesc.getTableDesc().setTableName(outputTable);
-
- Iterator<HiveReadableRecord> records = HiveInput.readTable(inputDesc).iterator();
-
- printRecords(HiveInput.readTable(inputDesc));
-
- final int rows = 4;
-
- Set<Integer>[] expected = new Set[rows+1];
- expected[1] = ImmutableSet.of(11,12,15,16,17,18);
- expected[2] = ImmutableSet.of(13,14,17,18,11,12);
- expected[3] = ImmutableSet.of(15,16,11,12,13,14);
- expected[4] = ImmutableSet.of(17,18,13,14,15,16);
-
- for (int i = 0; i < rows; ++i) {
- assertTrue(records.hasNext());
- HiveReadableRecord record = records.next();
- assertEquals(expected[record.getInt(0)], record.getMap(1).keySet());
- }
-
- assertFalse(records.hasNext());
- }
-
- private void printRecords(Iterable<HiveReadableRecord> records) {
- for (HiveReadableRecord record : records) {
- System.out.println("record: " + record);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/test/java/org/apache/giraph/hive/jython/TestHiveJythonPrimitives.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/jython/TestHiveJythonPrimitives.java b/giraph-hive/src/test/java/org/apache/giraph/hive/jython/TestHiveJythonPrimitives.java
deleted file mode 100644
index d7f6c30..0000000
--- a/giraph-hive/src/test/java/org/apache/giraph/hive/jython/TestHiveJythonPrimitives.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.hive.jython;
-
-import org.apache.giraph.conf.GiraphConfiguration;
-import org.apache.giraph.graph.Language;
-import org.apache.giraph.hive.GiraphHiveTestBase;
-import org.apache.giraph.hive.Helpers;
-import org.apache.giraph.jython.JythonJob;
-import org.apache.giraph.scripting.DeployType;
-import org.apache.giraph.scripting.ScriptLoader;
-import org.apache.giraph.utils.InternalVertexRunner;
-import org.apache.thrift.TException;
-import org.junit.Before;
-import org.junit.Test;
-import org.python.util.PythonInterpreter;
-
-import com.facebook.hiveio.common.HiveMetastores;
-import com.facebook.hiveio.input.HiveInput;
-import com.facebook.hiveio.input.HiveInputDescription;
-import com.facebook.hiveio.record.HiveReadableRecord;
-import com.facebook.hiveio.testing.LocalHiveServer;
-import junit.framework.Assert;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Iterator;
-
-import static junit.framework.Assert.assertFalse;
-import static junit.framework.Assert.assertNotNull;
-import static junit.framework.Assert.assertTrue;
-import static org.apache.giraph.hive.Helpers.getResource;
-import static org.junit.Assert.assertEquals;
-
-public class TestHiveJythonPrimitives extends GiraphHiveTestBase {
- private LocalHiveServer hiveServer = new LocalHiveServer("jython-test");
-
- @Before
- public void setUp() throws IOException, TException {
- hiveServer.init();
- HiveMetastores.setTestClient(hiveServer.getClient());
- }
-
- @Test
- public void testCountEdges() throws Exception {
- String edgesTable = "count_edges_edge_input";
- hiveServer.createTable("CREATE TABLE " + edgesTable +
- " (source_edge_id INT, " +
- " target_edge_id INT) " +
- " ROW FORMAT DELIMITED " +
- " FIELDS TERMINATED BY '\t'");
- String[] edges = new String[] {
- "1\t2",
- "2\t3",
- "2\t4",
- "4\t1"
- };
- hiveServer.loadData(edgesTable, edges);
-
- String outputTable = "count_edges_output";
- hiveServer.createTable("CREATE TABLE " + outputTable +
- " (vertex_id INT," +
- " num_edges INT) " +
- " ROW FORMAT DELIMITED " +
- " FIELDS TERMINATED BY '\t'");
-
- String workerJythonPath = "org/apache/giraph/jython/count-edges.py";
-
- InputStream launcher = getResource(
- "org/apache/giraph/jython/count-edges-launcher.py");
- assertNotNull(launcher);
- InputStream worker = getResource(workerJythonPath);
- assertNotNull(worker);
-
- PythonInterpreter interpreter = new PythonInterpreter();
-
- JythonJob jythonJob =
- HiveJythonUtils.parseJythonStreams(interpreter, launcher, worker);
-
- GiraphConfiguration conf = new GiraphConfiguration();
-
- ScriptLoader.setScriptsToLoad(conf, workerJythonPath,
- DeployType.RESOURCE, Language.JYTHON);
-
- HiveJythonUtils.writeJythonJobToConf(jythonJob, conf, interpreter);
-
- InternalVertexRunner.run(conf, new String[0], new String[0]);
-
- Helpers.commitJob(conf);
-
- HiveInputDescription inputDesc = new HiveInputDescription();
- inputDesc.getTableDesc().setTableName(outputTable);
-
- Iterator<HiveReadableRecord> records = HiveInput.readTable(inputDesc).iterator();
-
- int expected[] = { -1, 1, 2, -1, 1 };
-
- assertTrue(records.hasNext());
- HiveReadableRecord record = records.next();
- Assert.assertEquals(expected[record.getInt(0)], record.getInt(1));
-
- assertTrue(records.hasNext());
- record = records.next();
- Assert.assertEquals(expected[record.getInt(0)], record.getInt(1));
-
- assertTrue(records.hasNext());
- record = records.next();
- Assert.assertEquals(expected[record.getInt(0)], record.getInt(1));
-
- assertFalse(records.hasNext());
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad27a291/giraph-hive/src/test/java/org/apache/giraph/hive/output/CheckOutputTest.java
----------------------------------------------------------------------
diff --git a/giraph-hive/src/test/java/org/apache/giraph/hive/output/CheckOutputTest.java b/giraph-hive/src/test/java/org/apache/giraph/hive/output/CheckOutputTest.java
deleted file mode 100644
index ab020a1..0000000
--- a/giraph-hive/src/test/java/org/apache/giraph/hive/output/CheckOutputTest.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.hive.output;
-
-import org.apache.giraph.conf.GiraphConfiguration;
-import org.apache.giraph.edge.ByteArrayEdges;
-import org.apache.giraph.hive.GiraphHiveTestBase;
-import org.apache.giraph.hive.common.GiraphHiveConstants;
-import org.apache.giraph.hive.computations.ComputationCountEdges;
-import org.apache.giraph.hive.output.examples.HiveOutputIntIntVertex;
-import org.apache.giraph.io.formats.IntNullTextEdgeInputFormat;
-import org.apache.giraph.utils.InternalVertexRunner;
-import org.apache.thrift.TException;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.facebook.hiveio.common.HiveMetastores;
-import com.facebook.hiveio.common.HiveType;
-import com.facebook.hiveio.output.HiveOutputDescription;
-import com.facebook.hiveio.schema.HiveTableSchema;
-import com.facebook.hiveio.schema.TestSchema;
-import com.facebook.hiveio.testing.LocalHiveServer;
-
-import java.io.IOException;
-
-import static com.facebook.hiveio.record.HiveRecordFactory.newWritableRecord;
-import static org.junit.Assert.assertNull;
-
-public class CheckOutputTest extends GiraphHiveTestBase {
- private LocalHiveServer hiveServer = new LocalHiveServer("giraph-hive");
-
- @Before
- public void setUp() throws IOException, TException {
- hiveServer.init();
- HiveMetastores.setTestClient(hiveServer.getClient());
- }
-
- @Test
- public void testCheck() throws Exception {
- VertexToHive vertexToHive = new HiveOutputIntIntVertex();
- HiveOutputDescription outputDesc = new HiveOutputDescription();
- HiveTableSchema schema = TestSchema.builder()
- .addColumn("foo", HiveType.LONG)
- .addColumn("bar", HiveType.LONG)
- .build();
- vertexToHive.checkOutput(outputDesc, schema, newWritableRecord(schema));
-
- schema = TestSchema.builder()
- .addColumn("foo", HiveType.INT)
- .addColumn("bar", HiveType.LONG)
- .build();
- checkThrows(vertexToHive, outputDesc, schema);
- }
-
- private void checkThrows(VertexToHive vertexToHive,
- HiveOutputDescription outputDesc, HiveTableSchema schema) {
- try {
- vertexToHive.checkOutput(outputDesc, schema, newWritableRecord(schema));
- } catch (IllegalArgumentException e) {
- return;
- }
- Assert.fail();
- }
-
- @Test
- public void testCheckFailsJob() throws Exception {
- String tableName = "test1";
- hiveServer.createTable("CREATE TABLE " + tableName +
- " (i1 INT, i2 BIGINT) ");
-
- GiraphConfiguration conf = new GiraphConfiguration();
- String[] edges = new String[] {
- "1 2",
- "2 3",
- "2 4",
- "4 1"
- };
-
- GiraphHiveConstants.HIVE_VERTEX_OUTPUT_TABLE.set(conf, tableName);
- GiraphHiveConstants.VERTEX_TO_HIVE_CLASS.set(conf, HiveOutputIntIntVertex.class);
-
- conf.setComputationClass(ComputationCountEdges.class);
- conf.setOutEdgesClass(ByteArrayEdges.class);
- conf.setEdgeInputFormatClass(IntNullTextEdgeInputFormat.class);
- conf.setVertexOutputFormatClass(HiveVertexOutputFormat.class);
- try {
- Iterable<String> result = InternalVertexRunner.run(conf, null, edges);
- assertNull(result);
- } catch (IllegalArgumentException e) { }
- }
-}