You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by he...@apache.org on 2017/03/30 16:20:04 UTC
git commit: updated refs/heads/trunk to 824b34c
Repository: giraph
Updated Branches:
refs/heads/trunk dbce6582a -> 824b34cac
closes #28
Project: http://git-wip-us.apache.org/repos/asf/giraph/repo
Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/824b34ca
Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/824b34ca
Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/824b34ca
Branch: refs/heads/trunk
Commit: 824b34cac1d05917eb2bc3dc8c62c9772d0c6e4d
Parents: dbce658
Author: Dionysios Logothetis <di...@fb.com>
Authored: Thu Mar 30 11:19:41 2017 -0500
Committer: Hassan Eslami <ha...@wirelessprv-10-193-225-240.near.illinois.edu>
Committed: Thu Mar 30 11:19:41 2017 -0500
----------------------------------------------------------------------
.../framework/output/BlockOutputFormat.java | 4 +-
.../giraph/ooc/OutOfCoreIOStatistics.java | 12 ++++-
.../ooc/policy/MemoryEstimatorOracle.java | 55 +++++++++++++-------
3 files changed, 50 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/giraph/blob/824b34ca/giraph-block-app/src/main/java/org/apache/giraph/block_app/framework/output/BlockOutputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-block-app/src/main/java/org/apache/giraph/block_app/framework/output/BlockOutputFormat.java b/giraph-block-app/src/main/java/org/apache/giraph/block_app/framework/output/BlockOutputFormat.java
index 6c1e960..2c9af55 100644
--- a/giraph-block-app/src/main/java/org/apache/giraph/block_app/framework/output/BlockOutputFormat.java
+++ b/giraph-block-app/src/main/java/org/apache/giraph/block_app/framework/output/BlockOutputFormat.java
@@ -54,7 +54,9 @@ public class BlockOutputFormat extends BspOutputFormat {
}
/**
- * Returns an array of output configuration options set in the input configuration.
+ * Returns an array of output configuration options set in the input
+ * configuration.
+ *
* @param conf Configuration
* @return Array of options
*/
http://git-wip-us.apache.org/repos/asf/giraph/blob/824b34ca/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreIOStatistics.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreIOStatistics.java b/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreIOStatistics.java
index 44a0d2f..cc406f8 100644
--- a/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreIOStatistics.java
+++ b/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreIOStatistics.java
@@ -54,6 +54,13 @@ public class OutOfCoreIOStatistics {
"Number of most recent IO operations to consider for reporting the" +
"statistics.");
+ /**
+ * Use this option to control how frequently to print OOC statistics.
+ */
+ public static final IntConfOption STATS_PRINT_FREQUENCY =
+ new IntConfOption("giraph.oocStatPrintFrequency", 200,
+ "Number of updates before stats are printed.");
+
/** Class logger */
private static final Logger LOG =
Logger.getLogger(OutOfCoreIOStatistics.class);
@@ -79,6 +86,8 @@ public class OutOfCoreIOStatistics {
private final Map<IOCommandType, StatisticsEntry> aggregateStats;
/** How many IO command completed? */
private int numUpdates = 0;
+ /** Cached value for {@link #STATS_PRINT_FREQUENCY} */
+ private int statsPrintFrequency = 0;
/**
* Constructor
@@ -101,6 +110,7 @@ public class OutOfCoreIOStatistics {
for (IOCommandType type : IOCommandType.values()) {
aggregateStats.put(type, new StatisticsEntry(type, 0, 0, 0));
}
+ this.statsPrintFrequency = STATS_PRINT_FREQUENCY.get(conf);
}
/**
@@ -145,7 +155,7 @@ public class OutOfCoreIOStatistics {
}
numUpdates++;
// Outputting log every so many commands
- if (numUpdates % 10 == 0) {
+ if (numUpdates % statsPrintFrequency == 0) {
if (LOG.isInfoEnabled()) {
LOG.info(this);
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/824b34ca/giraph-core/src/main/java/org/apache/giraph/ooc/policy/MemoryEstimatorOracle.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/policy/MemoryEstimatorOracle.java b/giraph-core/src/main/java/org/apache/giraph/ooc/policy/MemoryEstimatorOracle.java
index 871ef6c..1233183 100644
--- a/giraph-core/src/main/java/org/apache/giraph/ooc/policy/MemoryEstimatorOracle.java
+++ b/giraph-core/src/main/java/org/apache/giraph/ooc/policy/MemoryEstimatorOracle.java
@@ -35,6 +35,7 @@ import org.apache.giraph.worker.VertexInputSplitsCallable;
import org.apache.giraph.worker.WorkerProgress;
import org.apache.log4j.Logger;
+import javax.annotation.Nullable;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryPoolMXBean;
import java.lang.management.MemoryUsage;
@@ -609,7 +610,12 @@ public class MemoryEstimatorOracle implements OutOfCoreOracle {
double[] yValues =
memorySamples.toDoubleArray(new double[memorySamples.size()]);
mlr.newSampleData(yValues, xValues);
- calculateRegression(coefficient, validColumnIndices, mlr);
+ boolean isRegressionValid =
+ calculateRegression(coefficient, validColumnIndices, mlr);
+
+ if (!isRegressionValid) { // invalid regression result
+ return; // The finally-block at the end will release any locks.
+ }
// After the computation of the regression, some coefficients may have
// values outside the valid value range. In this case, we set the
@@ -650,8 +656,19 @@ public class MemoryEstimatorOracle implements OutOfCoreOracle {
extreme[3] = -1;
extreme[4] = -1;
do {
- changed = refineCoefficient(4, 1, 2, xValues, yValues);
- changed |= refineCoefficient(3, 0, 2, xValues, yValues);
+ Boolean result = null;
+
+ result = refineCoefficient(4, 1, 2, xValues, yValues);
+ if (result == null) { // invalid regression result
+ return; // finally-block will release lock
+ }
+ changed = result;
+
+ result = refineCoefficient(3, 0, 2, xValues, yValues);
+ if (result == null) { // invalid regression result
+ return; // finally-block will release lock
+ }
+ changed |= result;
} while (changed);
if (extreme[3] != -1) {
coefficient[3] = extreme[3];
@@ -662,10 +679,6 @@ public class MemoryEstimatorOracle implements OutOfCoreOracle {
setIsValid = true;
return; // the finally-block will execute before return
}
- // CHECKSTYLE: stop IllegalCatch
- } catch (Exception e) {
- // CHECKSTYLE: resume IllegalCatch
- LOG.warn("addRecord: exception occurred!", e);
} finally {
// This inner try-finally block is necessary to ensure that the
// lock is always released.
@@ -703,12 +716,13 @@ public class MemoryEstimatorOracle implements OutOfCoreOracle {
* @param upperBound Upper bound
* @param xValues double[][] matrix with data samples
* @param yValues double[] matrix with y samples
- * @return True if coefficients were out-of-range
- * @throws Exception
+ * @return True if coefficients were out-of-range, false otherwise. A null
+ * value means the regression result was invalid and the result of
+ * this method is invalid too.
*/
- private boolean refineCoefficient(int coefIndex, double lowerBound,
- double upperBound, double[][] xValues, double[] yValues)
- throws Exception {
+ @Nullable
+ private Boolean refineCoefficient(int coefIndex, double lowerBound,
+ double upperBound, double[][] xValues, double[] yValues) {
boolean result = false;
if (coefficient[coefIndex] < lowerBound ||
@@ -747,8 +761,10 @@ public class MemoryEstimatorOracle implements OutOfCoreOracle {
extreme[coefIndex] = value;
// re-run regression
mlr.newSampleData(yValues, xValues);
- calculateRegression(coefficient, validColumnIndices, mlr);
- result = true;
+ result = calculateRegression(coefficient, validColumnIndices, mlr);
+ if (!result) { // invalid regression result
+ return null;
+ }
} else {
if (LOG.isDebugEnabled()) {
LOG.debug(
@@ -767,16 +783,16 @@ public class MemoryEstimatorOracle implements OutOfCoreOracle {
* @param coefficient Array of coefficients
* @param validColumnIndices List of valid columns
* @param mlr {@link OLSMultipleLinearRegression} instance.
- * @throws Exception
+ * @return True if the result is valid, false otherwise.
*/
- private static void calculateRegression(double[] coefficient,
- List<Integer> validColumnIndices, OLSMultipleLinearRegression mlr)
- throws Exception {
+ private static boolean calculateRegression(double[] coefficient,
+ List<Integer> validColumnIndices, OLSMultipleLinearRegression mlr) {
if (coefficient.length != validColumnIndices.size()) {
- throw new Exception("There are " + coefficient.length +
+ LOG.warn("There are " + coefficient.length +
" coefficients, but " + validColumnIndices.size() +
" valid columns in the regression");
+ return false;
}
double[] beta = mlr.estimateRegressionParameters();
@@ -785,6 +801,7 @@ public class MemoryEstimatorOracle implements OutOfCoreOracle {
coefficient[validColumnIndices.get(i)] = beta[i];
}
coefficient[5] = beta[validColumnIndices.size()];
+ return true;
}
/**