You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/04/27 22:30:59 UTC
incubator-tinkerpop git commit: fixed a bug in
XXXGlobalStep.MapReduce where if the reduction yields multiple outputs. Fixed
a bug in HadoopLoader.head() around int vs. long.
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/master 1b3348347 -> bef41c74b
fixed a bug in XXXGlobalStep.MapReduce where if the reduction yields multiple outputs. Fixed a bug in HadoopLoader.head() around int vs. long.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/bef41c74
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/bef41c74
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/bef41c74
Branch: refs/heads/master
Commit: bef41c74bb11f0e49c2446fb1e7eac9cd032df57
Parents: 1b33483
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Apr 27 14:30:54 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon Apr 27 14:30:54 2015 -0600
----------------------------------------------------------------------
.../traversal/step/map/CountGlobalStep.java | 12 +++++++----
.../traversal/step/map/MaxGlobalStep.java | 9 ++++++++-
.../traversal/step/map/MinGlobalStep.java | 15 ++++++++++----
.../traversal/step/map/SumGlobalStep.java | 6 +++++-
.../hadoop/groovy/plugin/HadoopLoader.groovy | 16 +++++++--------
.../io/gryo/GryoRecordReaderWriterTest.java | 21 ++++++++++++--------
6 files changed, 53 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bef41c74/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/CountGlobalStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/CountGlobalStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/CountGlobalStep.java
index cb7ede4..843de07 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/CountGlobalStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/CountGlobalStep.java
@@ -18,14 +18,14 @@
*/
package org.apache.tinkerpop.gremlin.process.traversal.step.map;
-import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.computer.KeyValue;
import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.util.StaticMapReduce;
-import org.apache.tinkerpop.gremlin.process.traversal.step.util.ReducingBarrierStep;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.step.MapReducer;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.ReducingBarrierStep;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.TraverserRequirement;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
import org.apache.tinkerpop.gremlin.structure.Vertex;
@@ -122,7 +122,11 @@ public final class CountGlobalStep<S> extends ReducingBarrierStep<S, Long> imple
@Override
public Long generateFinalResult(final Iterator<KeyValue<NullObject, Long>> keyValues) {
- return keyValues.hasNext() ? keyValues.next().getValue() : 0L;
+ long count = 0l;
+ while (keyValues.hasNext()) {
+ count = count + keyValues.next().getValue();
+ }
+ return count;
}
public static final CountGlobalMapReduce instance() {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bef41c74/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MaxGlobalStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MaxGlobalStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MaxGlobalStep.java
index c59ef33..8646ca6 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MaxGlobalStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MaxGlobalStep.java
@@ -123,7 +123,14 @@ public final class MaxGlobalStep<S extends Number> extends ReducingBarrierStep<S
@Override
public Number generateFinalResult(final Iterator<KeyValue<NullObject, Number>> keyValues) {
- return keyValues.hasNext() ? keyValues.next().getValue() : Double.NaN;
+ if(!keyValues.hasNext()) return Double.NaN;
+ Number max = -Double.MAX_VALUE;
+ while (keyValues.hasNext()) {
+ final Number value = keyValues.next().getValue();
+ if (value.doubleValue() > max.doubleValue())
+ max = value;
+ }
+ return max;
}
public static final MaxGlobalMapReduce instance() {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bef41c74/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MinGlobalStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MinGlobalStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MinGlobalStep.java
index ef28d79..a8b1ee3 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MinGlobalStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MinGlobalStep.java
@@ -18,14 +18,14 @@
*/
package org.apache.tinkerpop.gremlin.process.traversal.step.map;
-import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.computer.KeyValue;
import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.util.StaticMapReduce;
-import org.apache.tinkerpop.gremlin.process.traversal.step.util.ReducingBarrierStep;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.step.MapReducer;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.ReducingBarrierStep;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.TraverserRequirement;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
import org.apache.tinkerpop.gremlin.structure.Vertex;
@@ -123,7 +123,14 @@ public final class MinGlobalStep<S extends Number> extends ReducingBarrierStep<S
@Override
public Number generateFinalResult(final Iterator<KeyValue<NullObject, Number>> keyValues) {
- return keyValues.hasNext() ? keyValues.next().getValue() : Double.NaN;
+ if(!keyValues.hasNext()) return Double.NaN;
+ Number min = Double.MAX_VALUE;
+ while (keyValues.hasNext()) {
+ final Number value = keyValues.next().getValue();
+ if (value.doubleValue() < min.doubleValue())
+ min = value;
+ }
+ return min;
}
public static final MinGlobalMapReduce instance() {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bef41c74/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/SumGlobalStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/SumGlobalStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/SumGlobalStep.java
index 99ccd68..109e812 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/SumGlobalStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/SumGlobalStep.java
@@ -127,7 +127,11 @@ public final class SumGlobalStep extends ReducingBarrierStep<Number, Double> imp
@Override
public Number generateFinalResult(final Iterator<KeyValue<NullObject, Number>> keyValues) {
- return keyValues.hasNext() ? keyValues.next().getValue() : 0.0d;
+ double sum = 0.0d;
+ while (keyValues.hasNext()) {
+ sum = sum + keyValues.next().getValue().doubleValue();
+ }
+ return sum;
}
public static final SumGlobalMapReduce instance() {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bef41c74/hadoop-gremlin/src/main/groovy/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopLoader.groovy
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/groovy/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopLoader.groovy b/hadoop-gremlin/src/main/groovy/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopLoader.groovy
index 962fc0d..616c2f0 100644
--- a/hadoop-gremlin/src/main/groovy/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopLoader.groovy
+++ b/hadoop-gremlin/src/main/groovy/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopLoader.groovy
@@ -101,21 +101,21 @@ class HadoopLoader {
outA.close();
}
- FileSystem.metaClass.head = { final String path, final long totalLines ->
+ FileSystem.metaClass.head = { final String path, final int totalLines ->
return headMaker((FileSystem) delegate, path, totalLines, Text.class);
}
FileSystem.metaClass.head = { final String path ->
- return headMaker((FileSystem) delegate, path, Long.MAX_VALUE, Text.class);
+ return headMaker((FileSystem) delegate, path, Integer.MAX_VALUE, Text.class);
}
FileSystem.metaClass.head = {
final String path, final Class<org.apache.hadoop.io.Writable> writableClass ->
- return headMaker((FileSystem) delegate, path, Long.MAX_VALUE, writableClass);
+ return headMaker((FileSystem) delegate, path, Integer.MAX_VALUE, writableClass);
}
FileSystem.metaClass.head = {
- final String path, final long totalLines, final Class<org.apache.hadoop.io.Writable> writableClass ->
+ final String path, final int totalLines, final Class<org.apache.hadoop.io.Writable> writableClass ->
return headMaker((FileSystem) delegate, path, totalLines, writableClass);
}
@@ -127,12 +127,12 @@ class HadoopLoader {
private static Iterator headMaker(
final FileSystem fs,
- final String path, final long totalLines, final Class<org.apache.hadoop.io.Writable> writableClass) {
+ final String path, final int totalLines, final Class<org.apache.hadoop.io.Writable> writableClass) {
if (writableClass.equals(ObjectWritable.class))
- return IteratorUtils.limit(new ObjectWritableIterator(fs.getConf(), new Path(path)), totalLines.intValue());
+ return IteratorUtils.limit(new ObjectWritableIterator(fs.getConf(), new Path(path)), totalLines);
else if (writableClass.equals(VertexWritable.class))
- return IteratorUtils.limit(new VertexWritableIterator(fs.getConf(), new Path(path)), totalLines.intValue());
+ return IteratorUtils.limit(new VertexWritableIterator(fs.getConf(), new Path(path)), totalLines);
else
- return IteratorUtils.limit(new TextIterator(fs.getConf(), new Path(path)), totalLines.intValue());
+ return IteratorUtils.limit(new TextIterator(fs.getConf(), new Path(path)), totalLines);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bef41c74/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordReaderWriterTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordReaderWriterTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordReaderWriterTest.java
index 55fe900..03c6f47 100644
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordReaderWriterTest.java
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordReaderWriterTest.java
@@ -60,8 +60,8 @@ public class GryoRecordReaderWriterTest {
final int numberOfSplits = 4;
final long testFileSize = testFile.length();
final long splitLength = (long) ((double) testFileSize / (double) numberOfSplits);
- System.out.println("Test file size: " + testFileSize);
- System.out.println("Test file split length: " + splitLength);
+ //System.out.println("Test file size: " + testFileSize);
+ //System.out.println("Test file split length: " + splitLength);
final List<FileSplit> splits = new ArrayList<>();
for (int i = 0; i < testFileSize; i = i + (int) splitLength + 1) {
splits.add(new FileSplit(new Path(testFile.getAbsoluteFile().toURI().toString()), i, splitLength, null));
@@ -71,10 +71,12 @@ public class GryoRecordReaderWriterTest {
final List<String> writeLines = new ArrayList<>();
final GryoInputFormat inputFormat = ReflectionUtils.newInstance(GryoInputFormat.class, configuration);
final TaskAttemptContext job = new TaskAttemptContext(configuration, new TaskAttemptID());
- int count = 0;
+ int vertexCount = 0;
+ int outEdgeCount = 0;
+ int inEdgeCount = 0;
boolean foundKeyValue = false;
for (final FileSplit split : splits) {
- System.out.println("reading Gryo file " + testFile.getAbsolutePath() + " (" + split.getStart() + "--to-->" + (split.getStart() + split.getLength()) + " bytes)");
+ System.out.println("reading Gryo file split " + testFile.getAbsolutePath() + " (" + split.getStart() + "--to-->" + (split.getStart() + split.getLength()) + " bytes)");
final RecordReader reader = inputFormat.createRecordReader(split, job);
final ByteArrayOutputStream bos = new ByteArrayOutputStream();
try (final DataOutputStream dos = new DataOutputStream(bos)) {
@@ -83,12 +85,14 @@ public class GryoRecordReaderWriterTest {
float lastProgress = -1f;
while (reader.nextKeyValue()) {
//System.out.println("" + reader.getProgress() + "> " + reader.getCurrentKey() + ": " + reader.getCurrentValue());
- count++;
final float progress = reader.getProgress();
assertTrue(progress >= lastProgress);
assertEquals(NullWritable.class, reader.getCurrentKey().getClass());
final VertexWritable v = (VertexWritable) reader.getCurrentValue();
writer.write(NullWritable.get(), v);
+ vertexCount++;
+ outEdgeCount = outEdgeCount + (int) IteratorUtils.count(v.get().edges(Direction.OUT));
+ inEdgeCount = inEdgeCount + (int) IteratorUtils.count(v.get().edges(Direction.IN));
final Vertex vertex = v.get();
assertEquals(Integer.class, vertex.id().getClass());
@@ -105,10 +109,11 @@ public class GryoRecordReaderWriterTest {
writeLines.addAll(Arrays.asList(new String(bos.toByteArray()).split("\\x3a\\x15.\\x11\\x70...")));
}
}
- assertEquals(808, count);
+ assertEquals(8049,outEdgeCount);
+ assertEquals(8049,inEdgeCount);
+ assertEquals(outEdgeCount,inEdgeCount);
+ assertEquals(808, vertexCount);
assertTrue(foundKeyValue);
-
-
assertEquals(808, writeLines.size());
final String line42 = writeLines.get(41);
assertTrue(line42.contains("ITS ALL OVER NO"));