You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by ap...@apache.org on 2012/10/10 03:08:10 UTC
svn commit: r1396441 - in /giraph/trunk: CHANGELOG
giraph-formats-contrib/pom.xml
giraph-formats-contrib/src/main/java/org/apache/giraph/io/hcatalog/HiveGiraphRunner.java
Author: apresta
Date: Wed Oct 10 01:08:10 2012
New Revision: 1396441
URL: http://svn.apache.org/viewvc?rev=1396441&view=rev
Log:
GIRAPH-361: Hive output partition parsing is broken (nitay via apresta)
Modified:
giraph/trunk/CHANGELOG
giraph/trunk/giraph-formats-contrib/pom.xml
giraph/trunk/giraph-formats-contrib/src/main/java/org/apache/giraph/io/hcatalog/HiveGiraphRunner.java
Modified: giraph/trunk/CHANGELOG
URL: http://svn.apache.org/viewvc/giraph/trunk/CHANGELOG?rev=1396441&r1=1396440&r2=1396441&view=diff
==============================================================================
--- giraph/trunk/CHANGELOG (original)
+++ giraph/trunk/CHANGELOG Wed Oct 10 01:08:10 2012
@@ -2,6 +2,8 @@ Giraph Change Log
Release 0.2.0 - unreleased
+ GIRAPH-361: Hive output partition parsing is broken (nitay via apresta)
+
GIRAPH-360: Keep track of the task id in ChannelRotater to send
requests without knowing the worker id upfront (aching via
ekoontz)
Modified: giraph/trunk/giraph-formats-contrib/pom.xml
URL: http://svn.apache.org/viewvc/giraph/trunk/giraph-formats-contrib/pom.xml?rev=1396441&r1=1396440&r2=1396441&view=diff
==============================================================================
--- giraph/trunk/giraph-formats-contrib/pom.xml (original)
+++ giraph/trunk/giraph-formats-contrib/pom.xml Wed Oct 10 01:08:10 2012
@@ -130,6 +130,11 @@ under the License.
</repositories>
<dependencies>
<dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>12.0</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.hcatalog</groupId>
<artifactId>hcatalog-core</artifactId>
<!-- TODO: Use 0.5.0 stable release -->
Modified: giraph/trunk/giraph-formats-contrib/src/main/java/org/apache/giraph/io/hcatalog/HiveGiraphRunner.java
URL: http://svn.apache.org/viewvc/giraph/trunk/giraph-formats-contrib/src/main/java/org/apache/giraph/io/hcatalog/HiveGiraphRunner.java?rev=1396441&r1=1396440&r2=1396441&view=diff
==============================================================================
--- giraph/trunk/giraph-formats-contrib/src/main/java/org/apache/giraph/io/hcatalog/HiveGiraphRunner.java (original)
+++ giraph/trunk/giraph-formats-contrib/src/main/java/org/apache/giraph/io/hcatalog/HiveGiraphRunner.java Wed Oct 10 01:08:10 2012
@@ -23,7 +23,6 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
@@ -43,7 +42,6 @@ import org.apache.hcatalog.mapreduce.Inp
import org.apache.hcatalog.mapreduce.OutputJobInfo;
import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
import org.apache.log4j.Logger;
/**
@@ -292,7 +290,7 @@ public class HiveGiraphRunner implements
inputTableName = cmdln.getOptionValue("inputTable");
inputTableFilterExpr = cmdln.getOptionValue("inputFilter");
outputTableName = cmdln.getOptionValue("outputTable");
- outputTablePartitionValues = parsePartitionValues(cmdln
+ outputTablePartitionValues = HiveUtils.parsePartitionValues(cmdln
.getOptionValue("outputPartition"));
workers = Integer.parseInt(cmdln.getOptionValue("workers"));
isVerbose = cmdln.hasOption("verbose");
@@ -362,50 +360,6 @@ public class HiveGiraphRunner implements
}
}
- // TODO use Hive util class if this is already provided by it
-
- /**
- * @param outputTablePartitionString table partition string
- * @return Map
- */
- public static Map<String, String> parsePartitionValues(
- String outputTablePartitionString) {
- if (outputTablePartitionString != null) {
- Map<String, String> partitionValues = Maps.newHashMap();
- for (String partkeyval : outputTablePartitionString.split(" *, *")) {
- String[] keyval = partkeyval.split(" *= *", 2);
- if (keyval.length < 2) {
- throw new IllegalArgumentException(
- "Unrecognized partition value format: " +
- outputTablePartitionString);
- }
- partitionValues.put(keyval[0], keyval[1]);
- return partitionValues;
- }
- }
- return null;
- }
-
- /**
- *
- * @param outputTablePartitionValues output table partitions
- * @return String partition values
- */
- private static String serializePartitionValues(
- Map<String, String> outputTablePartitionValues) {
- StringBuilder outputTablePartitionValuesString = new StringBuilder();
- for (Entry<String, String> partitionValueEntry : outputTablePartitionValues
- .entrySet()) {
- if (outputTablePartitionValuesString.length() != 0) {
- outputTablePartitionValuesString.append(",");
- outputTablePartitionValuesString
- .append(partitionValueEntry.getKey()).append("=")
- .append(partitionValueEntry.getValue());
- }
- }
- return outputTablePartitionValuesString.toString();
- }
-
@Override
public final Configuration getConf() {
return conf;
@@ -458,9 +412,8 @@ public class HiveGiraphRunner implements
}
LOG.info(prefix + "-outputTable=" + outputTableName);
if (outputTablePartitionValues != null) {
- LOG.info(prefix + "-outputPartition=\"" +
- serializePartitionValues(outputTablePartitionValues) +
- "\"");
+ LOG.info(prefix + "-outputPartition=\"" +
+ outputTablePartitionValues + "\"");
}
LOG.info(prefix + "-workers=" + workers);
}