You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@wayang.apache.org by be...@apache.org on 2021/09/03 10:45:19 UTC
[incubator-wayang] 01/01: [WAYANG-35] the library Trove4j was
removed It's important to focus on the TODO added because they may affect
the performance of some code
This is an automated email from the ASF dual-hosted git repository.
bertty pushed a commit to branch WAYANG-35
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit 4ca6c3347b3c225e7c78944b9622ad79430bfc22
Author: Bertty Contreras-Rojas <be...@scalytics.io>
AuthorDate: Fri Sep 3 12:45:05 2021 +0200
[WAYANG-35] the library Trove4j was removed
It's important to focus on the TODO added because they may affect the performance of some code
Signed-off-by: bertty <be...@gmail.com>
---
wayang-commons/pom.xml | 8 ---
.../optimizer/costs/SimpleEstimationContext.java | 11 ++--
.../costs/NestableLoadProfileEstimatorTest.java | 11 ++--
.../java/operators/graph/JavaPageRankOperator.java | 67 +++++++++++-----------
.../wayang/profiler/log/GeneticOptimizer.java | 29 +++++++---
5 files changed, 64 insertions(+), 62 deletions(-)
diff --git a/wayang-commons/pom.xml b/wayang-commons/pom.xml
index e355bc9..5bb233b 100644
--- a/wayang-commons/pom.xml
+++ b/wayang-commons/pom.xml
@@ -38,14 +38,6 @@
other modules
</description>
- <dependencies>
- <dependency>
- <groupId>net.sf.trove4j</groupId>
- <artifactId>trove4j</artifactId>
- <version>3.0.3</version>
- </dependency>
- </dependencies>
-
<dependencyManagement>
<dependencies>
<!-- Apache Hadoop -->
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/SimpleEstimationContext.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/SimpleEstimationContext.java
index bd2367b..c0d0d9c 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/SimpleEstimationContext.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/SimpleEstimationContext.java
@@ -18,8 +18,7 @@
package org.apache.wayang.core.optimizer.costs;
-import gnu.trove.map.TObjectDoubleMap;
-import gnu.trove.map.hash.TObjectDoubleHashMap;
+import java.util.HashMap;
import org.json.JSONObject;
import org.apache.wayang.core.optimizer.cardinality.CardinalityEstimate;
import org.apache.wayang.core.util.JsonSerializables;
@@ -34,8 +33,8 @@ import java.util.List;
public class SimpleEstimationContext implements EstimationContext {
private final CardinalityEstimate[] inputCardinalities, outputCardinalities;
-
- private final TObjectDoubleMap<String> doubleProperties;
+ //TODO: change a for efficient Hashmap
+ private final HashMap<String, Double> doubleProperties;
private final int numExecutions;
@@ -44,7 +43,7 @@ public class SimpleEstimationContext implements EstimationContext {
*/
public SimpleEstimationContext(CardinalityEstimate[] inputCardinalities,
CardinalityEstimate[] outputCardinalities,
- TObjectDoubleMap<String> doubleProperties,
+ HashMap<String, Double> doubleProperties,
int numExecutions) {
this.inputCardinalities = inputCardinalities;
this.outputCardinalities = outputCardinalities;
@@ -106,7 +105,7 @@ public class SimpleEstimationContext implements EstimationContext {
CardinalityEstimate.class
);
- final TObjectDoubleHashMap<String> doubleProperties = new TObjectDoubleHashMap<>();
+ final HashMap<String, Double> doubleProperties = new HashMap<String, Double>();
final JSONObject doublePropertiesJson = json.optJSONObject("properties");
if (doublePropertiesJson != null) {
for (String key : doublePropertiesJson.keySet()) {
diff --git a/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/optimizer/costs/NestableLoadProfileEstimatorTest.java b/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/optimizer/costs/NestableLoadProfileEstimatorTest.java
index 707b9e5..6295021 100644
--- a/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/optimizer/costs/NestableLoadProfileEstimatorTest.java
+++ b/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/optimizer/costs/NestableLoadProfileEstimatorTest.java
@@ -18,8 +18,7 @@
package org.apache.wayang.core.optimizer.costs;
-import gnu.trove.map.TObjectDoubleMap;
-import gnu.trove.map.hash.TObjectDoubleHashMap;
+import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
import org.apache.wayang.core.optimizer.OptimizationUtils;
@@ -56,7 +55,7 @@ public class NestableLoadProfileEstimatorTest {
new CardinalityEstimate(10, 10, 1d), new CardinalityEstimate(100, 100, 1d)
},
new CardinalityEstimate[]{new CardinalityEstimate(200, 300, 1d)},
- new TObjectDoubleHashMap<>(),
+ new HashMap<String, Double>(),
1
));
@@ -89,7 +88,7 @@ public class NestableLoadProfileEstimatorTest {
new CardinalityEstimate(10, 10, 1d), new CardinalityEstimate(100, 100, 1d)
},
new CardinalityEstimate[]{new CardinalityEstimate(200, 300, 1d)},
- new TObjectDoubleHashMap<>(),
+ new HashMap<String, Double>(),
1
));
@@ -118,7 +117,7 @@ public class NestableLoadProfileEstimatorTest {
final NestableLoadProfileEstimator estimator =
LoadProfileEstimators.createFromSpecification(null, specification);
SomeExecutionOperator execOp = new SomeExecutionOperator();
- TObjectDoubleMap<String> properties = new TObjectDoubleHashMap<>();
+ HashMap<String, Double> properties = new HashMap<String, Double>();
properties.put("numIterations", 2d);
final LoadProfile estimate = estimator.estimate(new SimpleEstimationContext(
new CardinalityEstimate[]{
@@ -155,7 +154,7 @@ public class NestableLoadProfileEstimatorTest {
final NestableLoadProfileEstimator estimator =
LoadProfileEstimators.createFromSpecification(null, specification);
SomeExecutionOperator execOp = new SomeExecutionOperator();
- TObjectDoubleMap<String> properties = new TObjectDoubleHashMap<>();
+ HashMap<String, Double> properties = new HashMap<String, Double>();
properties.put("numIterations", 2d);
final LoadProfile estimate = estimator.estimate(new SimpleEstimationContext(
new CardinalityEstimate[]{
diff --git a/wayang-platforms/wayang-java/src/main/java/org/apache/wayang/java/operators/graph/JavaPageRankOperator.java b/wayang-platforms/wayang-java/src/main/java/org/apache/wayang/java/operators/graph/JavaPageRankOperator.java
index e4633a5..d06c1b5 100644
--- a/wayang-platforms/wayang-java/src/main/java/org/apache/wayang/java/operators/graph/JavaPageRankOperator.java
+++ b/wayang-platforms/wayang-java/src/main/java/org/apache/wayang/java/operators/graph/JavaPageRankOperator.java
@@ -18,11 +18,9 @@
package org.apache.wayang.java.operators.graph;
-import gnu.trove.iterator.TLongFloatIterator;
-import gnu.trove.map.TLongFloatMap;
-import gnu.trove.map.TLongIntMap;
-import gnu.trove.map.hash.TLongFloatHashMap;
-import gnu.trove.map.hash.TLongIntHashMap;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.BiFunction;
import org.apache.wayang.basic.data.Tuple2;
import org.apache.wayang.basic.operators.PageRankOperator;
import org.apache.wayang.core.optimizer.OptimizationContext;
@@ -67,8 +65,8 @@ public class JavaPageRankOperator extends PageRankOperator implements JavaExecut
StreamChannel.Instance output = (StreamChannel.Instance) outputs[0];
final Collection<Tuple2<Long, Long>> edges = input.provideCollection();
- final TLongFloatMap pageRanks = this.pageRank(edges);
- final Stream<Tuple2<Long, Float>> pageRankStream = this.stream(pageRanks);
+ final Map<Long, Float> pageRanks = this.pageRank(edges);
+ final Stream<Tuple2<Long, Float>> pageRankStream = pageRanks.entrySet().stream().map(entry -> new Tuple2<>(entry.getKey(), entry.getValue()));
output.accept(pageRankStream);
@@ -81,31 +79,33 @@ public class JavaPageRankOperator extends PageRankOperator implements JavaExecut
* @param edgeDataSet edges of a graph
* @return the page ranks
*/
- private TLongFloatMap pageRank(Collection<Tuple2<Long, Long>> edgeDataSet) {
+ //TODO: change for efficient map
+ private Map<Long, Float> pageRank(Collection<Tuple2<Long, Long>> edgeDataSet) {
// Get the degress of all vertices and make sure we collect *all* vertices.
- TLongIntMap degrees = new TLongIntHashMap();
+ //TODO: change for efficient map
+ HashMap<Long, Integer> degrees = new HashMap<>();
for (Tuple2<Long, Long> edge : edgeDataSet) {
- degrees.adjustOrPutValue(edge.field0, 1, 1);
- degrees.adjustOrPutValue(edge.field0, 0, 0);
+ this.adjustOrPutValue(degrees, edge.field0, 1, 1, Integer::sum);
+ this.adjustOrPutValue(degrees, edge.field0, 0, 0, Integer::sum);
}
int numVertices = degrees.size();
float initialRank = 1f / numVertices;
float dampingRank = (1 - this.dampingFactor) / numVertices;
// Initialize the rank map.
- TLongFloatMap initialRanks = new TLongFloatHashMap();
- degrees.forEachKey(k -> {
+ //TODO: change for efficient map
+ HashMap<Long, Float> initialRanks = new HashMap<>();
+ degrees.forEach( (k, v) -> {
initialRanks.putIfAbsent(k, initialRank);
- return true;
});
- TLongFloatMap currentRanks = initialRanks;
+ HashMap<Long, Float> currentRanks = initialRanks;
for (int iteration = 0; iteration < this.getNumIterations(); iteration++) {
// Add the damping first.
- TLongFloatMap newRanks = new TLongFloatHashMap(currentRanks.size());
- degrees.forEachKey(k -> {
+ //TODO: change for efficient map
+ HashMap<Long, Float> newRanks = new HashMap<Long, Float>(currentRanks.size());
+ degrees.forEach( (k, v) -> {
newRanks.putIfAbsent(k, dampingRank);
- return true;
});
// Now add the other ranks.
@@ -115,7 +115,7 @@ public class JavaPageRankOperator extends PageRankOperator implements JavaExecut
final int degree = degrees.get(sourceVertex);
final float currentRank = currentRanks.get(sourceVertex);
final float partialRank = this.dampingFactor * currentRank / degree;
- newRanks.adjustOrPutValue(targetVertex, partialRank, partialRank);
+ this.adjustOrPutValue(newRanks, targetVertex, partialRank, partialRank, Float::sum);
}
currentRanks = newRanks;
@@ -124,24 +124,21 @@ public class JavaPageRankOperator extends PageRankOperator implements JavaExecut
return currentRanks;
}
- private Stream<Tuple2<Long, Float>> stream(TLongFloatMap map) {
- final TLongFloatIterator tLongFloatIterator = map.iterator();
- Iterator<Tuple2<Long, Float>> iterator = new Iterator<Tuple2<Long, Float>>() {
- @Override
- public boolean hasNext() {
- return tLongFloatIterator.hasNext();
- }
-
- @Override
- public Tuple2<Long, Float> next() {
- tLongFloatIterator.advance();
- return new Tuple2<>(tLongFloatIterator.key(), tLongFloatIterator.value());
- }
- };
- return StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, 0), false);
+ /**
+ * simulate the process on the Trove4j library
+ * @param key key to modify on the map
+ * @param default_value default value in the case of not key
+ * @param correction element to add the array in the case of the key exist
+ */
+ private <T> void adjustOrPutValue(Map<Long, T> map, Long key, T default_value, T correction, BiFunction<T, T, T> update){
+ if(map.containsKey(key)){
+ T value = map.get(key);
+ map.replace(key, update.apply(value, correction) );
+ }else{
+ map.put(key, default_value);
+ }
}
-
@Override
public String getLoadProfileEstimatorConfigurationKey() {
return "wayang.java.pagerank.load";
diff --git a/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/GeneticOptimizer.java b/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/GeneticOptimizer.java
index 33149fd..03645f9 100644
--- a/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/GeneticOptimizer.java
+++ b/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/GeneticOptimizer.java
@@ -18,8 +18,7 @@
package org.apache.wayang.profiler.log;
-import gnu.trove.map.TObjectIntMap;
-import gnu.trove.map.hash.TObjectIntHashMap;
+import java.util.HashMap;
import org.apache.wayang.core.api.Configuration;
import org.apache.wayang.core.optimizer.costs.LoadProfileEstimator;
import org.apache.wayang.core.platform.AtomicExecution;
@@ -61,7 +60,8 @@ public class GeneticOptimizer {
/**
* Counts observation instances, such as an operator or a platform initialization, in the training data.
*/
- private final TObjectIntMap<Object> numObservations;
+ //TODO: change for efficient map
+ private final HashMap<Object, Integer> numObservations;
/**
* {@link Variable}s to learn the overhead of {@link Platform} initialization.
@@ -164,20 +164,35 @@ public class GeneticOptimizer {
}
// Count the distinct elements in the PartialExecutions.
- this.numObservations = new TObjectIntHashMap<>();
+ this.numObservations = new HashMap<>();
this.runtimeSum = 0L;
for (PartialExecution observation : this.observations) {
for (String key : getLoadProfileEstimatorKeys(observation)) {
- this.numObservations.adjustOrPutValue(key, 1, 1);
+ this.adjustOrPutValue(key, 1, 1);
}
for (Platform platform : observation.getInitializedPlatforms()) {
- this.numObservations.adjustOrPutValue(platform, 1, 1);
+ this.adjustOrPutValue(platform, 1, 1);
}
this.runtimeSum += observation.getMeasuredExecutionTime();
}
}
/**
+ * simulate the process on the Trove4j library
+ * @param key key to modify on the map
+ * @param default_value default value in the case of not key
+ * @param correction element to add the array in the case of the key exist
+ */
+ private void adjustOrPutValue(Object key, int default_value, int correction){
+ if(this.numObservations.containsKey(key)){
+ Integer value = this.numObservations.get(key);
+ this.numObservations.replace(key, value + correction );
+ }else{
+ this.numObservations.put(key, default_value);
+ }
+ }
+
+ /**
* Creates a population of random {@link Individual}s.
*
* @return the {@link Individual}s ordered by their fitness
@@ -283,7 +298,7 @@ public class GeneticOptimizer {
return observations;
}
- public TObjectIntMap<Object> getNumObservations() {
+ public HashMap<Object, Integer> getNumObservations() {
return numObservations;
}