You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by ed...@apache.org on 2015/02/04 19:05:15 UTC
git commit: updated refs/heads/trunk to 63b8c41
Repository: giraph
Updated Branches:
refs/heads/trunk 931569d58 -> 63b8c412f
GIRAPH-991 Update versions of guava and swift
Summary: Changes in WorkerProgress are needed because newer version of thrift requires thrift classes to be final.
Test Plan: mvn clean verify
+ run a bunch of test jobs
Reviewers: maja.kabiljo
Differential Revision: https://reviews.facebook.net/D32757
Project: http://git-wip-us.apache.org/repos/asf/giraph/repo
Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/63b8c412
Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/63b8c412
Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/63b8c412
Branch: refs/heads/trunk
Commit: 63b8c412fbe731c7fc0a9d7dceb92ae955b553ae
Parents: 931569d
Author: Sergey Edunov <ed...@fb.com>
Authored: Tue Feb 3 14:47:15 2015 -0800
Committer: Sergey Edunov <ed...@fb.com>
Committed: Wed Feb 4 10:04:42 2015 -0800
----------------------------------------------------------------------
CHANGELOG | 2 +
.../giraph/job/CombinedWorkerProgress.java | 9 +-
.../apache/giraph/worker/WorkerProgress.java | 57 ++-----------
.../giraph/worker/WorkerProgressStats.java | 86 ++++++++++++++++++++
pom.xml | 2 +-
5 files changed, 101 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/giraph/blob/63b8c412/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index 2e8e8bf..8da1efe 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,8 @@
Giraph Change Log
Release 1.2.0 - unreleased
+ GIRAPH-991: Update versions of guava and swift (edunov)
+
GIRAPH-987: Improve naming for ReduceOperation (ikabiljo via majakabiljo)
GIRAPH-986: Add no-arg constructor to BasicSet (ikabiljo via edunov)
http://git-wip-us.apache.org/repos/asf/giraph/blob/63b8c412/giraph-core/src/main/java/org/apache/giraph/job/CombinedWorkerProgress.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/job/CombinedWorkerProgress.java b/giraph-core/src/main/java/org/apache/giraph/job/CombinedWorkerProgress.java
index e5fef8a..e931a99 100644
--- a/giraph-core/src/main/java/org/apache/giraph/job/CombinedWorkerProgress.java
+++ b/giraph-core/src/main/java/org/apache/giraph/job/CombinedWorkerProgress.java
@@ -18,22 +18,21 @@
package org.apache.giraph.job;
+import com.google.common.collect.Iterables;
import org.apache.giraph.conf.FloatConfOption;
import org.apache.giraph.worker.WorkerProgress;
+import org.apache.giraph.worker.WorkerProgressStats;
import org.apache.hadoop.conf.Configuration;
-import com.google.common.collect.Iterables;
-
-import java.text.DecimalFormat;
-
import javax.annotation.concurrent.NotThreadSafe;
+import java.text.DecimalFormat;
/**
* Class which combines multiple workers' progresses to get overall
* application progress
*/
@NotThreadSafe
-public class CombinedWorkerProgress extends WorkerProgress {
+public class CombinedWorkerProgress extends WorkerProgressStats {
/** Decimal format which rounds numbers to two decimal places */
public static final DecimalFormat DECIMAL_FORMAT = new DecimalFormat("#.##");
/**
http://git-wip-us.apache.org/repos/asf/giraph/blob/63b8c412/giraph-core/src/main/java/org/apache/giraph/worker/WorkerProgress.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/worker/WorkerProgress.java b/giraph-core/src/main/java/org/apache/giraph/worker/WorkerProgress.java
index 3c25cfe..eb543cd 100644
--- a/giraph-core/src/main/java/org/apache/giraph/worker/WorkerProgress.java
+++ b/giraph-core/src/main/java/org/apache/giraph/worker/WorkerProgress.java
@@ -18,10 +18,9 @@
package org.apache.giraph.worker;
-import org.apache.giraph.utils.MemoryUtils;
-
import com.facebook.swift.codec.ThriftField;
import com.facebook.swift.codec.ThriftStruct;
+import org.apache.giraph.utils.MemoryUtils;
import javax.annotation.concurrent.ThreadSafe;
@@ -31,56 +30,16 @@ import javax.annotation.concurrent.ThreadSafe;
*/
@ThreadSafe
@ThriftStruct
-public class WorkerProgress {
+public final class WorkerProgress extends WorkerProgressStats {
/** Singleton instance for everyone to use */
private static final WorkerProgress INSTANCE = new WorkerProgress();
- /** Superstep which worker is executing, Long.MAX_VALUE if it's output */
- protected long currentSuperstep = -1;
-
- /** How many vertices were loaded until now */
- protected long verticesLoaded = 0;
- /** How many vertex input splits were loaded until now */
- protected int vertexInputSplitsLoaded = 0;
- /** Whether worker finished loading vertices */
- protected boolean loadingVerticesDone = false;
- /** How many edges were loaded */
- protected long edgesLoaded = 0;
- /** How many edge input splits were loaded until now */
- protected int edgeInputSplitsLoaded = 0;
- /** Whether worker finished loading edges until now */
- protected boolean loadingEdgesDone = false;
-
- /** How many vertices are there to compute in current superstep */
- protected long verticesToCompute = 0;
- /** How many vertices were computed in current superstep until now */
- protected long verticesComputed = 0;
- /** How many partitions are there to compute in current superstep */
- protected int partitionsToCompute = 0;
- /** How many partitions were computed in current superstep until now */
- protected int partitionsComputed = 0;
-
- /** Whether all compute supersteps are done */
- protected boolean computationDone = false;
-
- /** How many vertices are there to store */
- protected long verticesToStore = 0;
- /** How many vertices were stored until now */
- protected long verticesStored = 0;
- /** How many partitions are there to store */
- protected int partitionsToStore = 0;
- /** How many partitions were stored until now */
- protected int partitionsStored = 0;
- /** Whether worker finished storing data */
- protected boolean storingDone = false;
-
- /** Id of the mapper */
- protected int taskId;
-
- /** Free memory */
- protected double freeMemoryMB;
- /** Fraction of memory that's free */
- protected double freeMemoryFraction;
+ /**
+ * Public constructor for thrift to create us.
+ * Please use WorkerProgress.get() to get the static instance.
+ */
+ public WorkerProgress() {
+ }
/**
* Get singleton instance of WorkerProgress.
http://git-wip-us.apache.org/repos/asf/giraph/blob/63b8c412/giraph-core/src/main/java/org/apache/giraph/worker/WorkerProgressStats.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/worker/WorkerProgressStats.java b/giraph-core/src/main/java/org/apache/giraph/worker/WorkerProgressStats.java
new file mode 100644
index 0000000..04ed2ea
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/worker/WorkerProgressStats.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.worker;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+/**
+ * Stats about a worker's progress
+ */
+@NotThreadSafe
+public class WorkerProgressStats {
+ /** Superstep which worker is executing, Long.MAX_VALUE if it's output */
+ protected long currentSuperstep = -1;
+
+ /** How many vertices were loaded until now */
+ protected long verticesLoaded = 0;
+ /** How many vertex input splits were loaded until now */
+ protected int vertexInputSplitsLoaded = 0;
+ /** Whether worker finished loading vertices */
+ protected boolean loadingVerticesDone = false;
+ /** How many edges were loaded */
+ protected long edgesLoaded = 0;
+ /** How many edge input splits were loaded until now */
+ protected int edgeInputSplitsLoaded = 0;
+ /** Whether worker finished loading edges until now */
+ protected boolean loadingEdgesDone = false;
+
+ /** How many vertices are there to compute in current superstep */
+ protected long verticesToCompute = 0;
+ /** How many vertices were computed in current superstep until now */
+ protected long verticesComputed = 0;
+ /** How many partitions are there to compute in current superstep */
+ protected int partitionsToCompute = 0;
+ /** How many partitions were computed in current superstep until now */
+ protected int partitionsComputed = 0;
+
+ /** Whether all compute supersteps are done */
+ protected boolean computationDone = false;
+
+ /** How many vertices are there to store */
+ protected long verticesToStore = 0;
+ /** How many vertices were stored until now */
+ protected long verticesStored = 0;
+ /** How many partitions are there to store */
+ protected int partitionsToStore = 0;
+ /** How many partitions were stored until now */
+ protected int partitionsStored = 0;
+ /** Whether worker finished storing data */
+ protected boolean storingDone = false;
+
+ /** Id of the mapper */
+ protected int taskId;
+
+ /** Free memory */
+ protected double freeMemoryMB;
+ /** Fraction of memory that's free */
+ protected double freeMemoryFraction;
+
+ public boolean isInputSuperstep() {
+ return currentSuperstep == -1;
+ }
+
+ public boolean isComputeSuperstep() {
+ return currentSuperstep >= 0 && currentSuperstep < Long.MAX_VALUE;
+ }
+
+ public boolean isOutputSuperstep() {
+ return currentSuperstep == Long.MAX_VALUE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/63b8c412/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index cf0e1f9..87792a3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -291,7 +291,7 @@ under the License.
<dep.commons-logging.version>1.1.1</dep.commons-logging.version>
<dep.commons-io.version>2.1</dep.commons-io.version>
<dep.commons-net.version>3.1</dep.commons-net.version>
- <dep.facebook-swift.version>0.13.1</dep.facebook-swift.version>
+ <dep.facebook-swift.version>0.14.0</dep.facebook-swift.version>
<dep.fasterxml-jackson.version>2.1.2</dep.fasterxml-jackson.version>
<dep.fastutil.version>6.5.4</dep.fastutil.version>
<dep.google.findbugs.version>2.0.2</dep.google.findbugs.version>