You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by mb...@apache.org on 2020/07/10 22:26:43 UTC
[systemds] branch master updated: [MINOR] Cleanups (lineage,
java warnings, tasks, formatting, seeds)
This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/master by this push:
new 4d1e900 [MINOR] Cleanups (lineage, java warnings, tasks, formatting, seeds)
4d1e900 is described below
commit 4d1e90079e118cea1ec27789d2cd273f650d3e41
Author: Matthias Boehm <mb...@gmail.com>
AuthorDate: Fri Jul 10 22:26:49 2020 +0200
[MINOR] Cleanups (lineage, java warnings, tasks, formatting, seeds)
---
dev/Tasks.txt | 7 +++++--
scripts/builtin/executePipeline.dml | 16 ++++++----------
scripts/staging/pipelines/enumerator.dml | 8 +-------
.../controlprogram/federated/FederatedResponse.java | 2 +-
.../controlprogram/federated/FederatedWorkerHandler.java | 2 +-
.../org/apache/sysds/runtime/lineage/LineageCache.java | 4 ++--
.../org/apache/sysds/runtime/lineage/LineageItem.java | 3 +++
.../sysds/runtime/privacy/CheckedConstraintsLog.java | 4 ++--
.../org/apache/sysds/runtime/privacy/PrivacyMonitor.java | 2 +-
.../test/functions/builtin/BuiltinMulticlassSVMTest.java | 4 ++--
.../test/functions/frame/FrameDropInvalidLengthTest.java | 7 ++++---
.../functions/privacy/CheckedConstraintsLogTest.java | 2 +-
12 files changed, 29 insertions(+), 32 deletions(-)
diff --git a/dev/Tasks.txt b/dev/Tasks.txt
index 9f9bbec..da1436b 100644
--- a/dev/Tasks.txt
+++ b/dev/Tasks.txt
@@ -357,8 +357,11 @@ Others:
* Break append instruction to cbind and rbind
SYSTEMDS-510 IO formats
- * 511 Add protobuf support to write and read FrameBlocks to HDFS OKSYSTEMDS-520 Lineage Tracing, Reuse and Integration III
+ * 511 Add protobuf support to write and read FrameBlocks to HDFS OK
+
+SYSTEMDS-520 Lineage Tracing, Reuse and Integration III
* 521 New lineage option exposing cache policies OK
SYSTEMDS-610 Cleaning Pipelines
- * 611 Initial Brute force execution OK
\ No newline at end of file
+ * 611 Initial Brute force execution OK
+
diff --git a/scripts/builtin/executePipeline.dml b/scripts/builtin/executePipeline.dml
index 9ef42ff..dbbbe44 100644
--- a/scripts/builtin/executePipeline.dml
+++ b/scripts/builtin/executePipeline.dml
@@ -19,29 +19,25 @@
#
#-------------------------------------------------------------
-s_executePipeline = function(Frame[String] pipeline, Matrix[Double] X, List[Unknown] hyperParameters, Boolean verbose)
-return (Matrix[Double] X)
+s_executePipeline = function(Frame[String] pipeline, Matrix[Double] X,
+ List[Unknown] hyperParameters, Boolean verbose)
+ return (Matrix[Double] X)
{
for(i in 1: ncol(pipeline)) {
hp = matrixToList(X, as.matrix(hyperParameters[i]))
X = eval(as.scalar(pipeline[1,i]), hp)
}
-
}
-
-
+
# This function will convert the matrix row-vector into list
matrixToList = function(Matrix[Double] X, Matrix[Double] p)
-return (List[Unknown] l)
+ return (List[Unknown] l)
{
l = list()
l = append(l, X)
- if(sum(p) != -1)
- {
+ if(sum(p) != -1) {
for(i in 1:ncol(p))
l = append(l, as.scalar(p[1,i]))
}
l = append(l, FALSE) #verbose parameter value
}
-
-
diff --git a/scripts/staging/pipelines/enumerator.dml b/scripts/staging/pipelines/enumerator.dml
index 4303b52..d70bf9a 100644
--- a/scripts/staging/pipelines/enumerator.dml
+++ b/scripts/staging/pipelines/enumerator.dml
@@ -258,8 +258,7 @@ return (Double psum)
psum = sum(as.matrix(paraList[i]))
else
psum = 0.0
- }
-
+ }
}
# This function will compute the top k pipelines from the results of all executions
@@ -323,7 +322,6 @@ return(Matrix[Double] X)
fclassify = function(Matrix[Double] X)
return (Double accuracy)
{
-
if(min(X[,1]) < 1)
stop("Y should contain value greater than zero")
@@ -344,7 +342,6 @@ return (Double accuracy)
nTest = nrow(testSet)
dTest = ncol(testSet)
-
train_X = trainSet[, 2:dTrain]
train_Y = trainSet[, 1]
@@ -353,12 +350,9 @@ return (Double accuracy)
betas = multiLogReg(X=train_X, Y=train_Y, icpt=2, tol=1e-9, reg=1.2, maxi=100, maxii=0, verbose=FALSE)
[prob, yhat, accuracy] = multiLogRegPredict(test_X, betas, test_Y, FALSE)
-
}
-
-
##########################################
## Call the function Enumerator
#########################################
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedResponse.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedResponse.java
index 6c79457..3335aae 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedResponse.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedResponse.java
@@ -101,7 +101,7 @@ public class FederatedResponse implements Serializable {
*/
public void setCheckedConstraints(Map<PrivacyLevel,LongAdder> checkedConstraints){
if ( checkedConstraints != null && !checkedConstraints.isEmpty() ){
- this.checkedConstraints = new EnumMap<PrivacyLevel, LongAdder>(PrivacyLevel.class);
+ this.checkedConstraints = new EnumMap<>(PrivacyLevel.class);
this.checkedConstraints.putAll(checkedConstraints);
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java
index 3c688f5..1e2e6ea 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java
@@ -92,7 +92,7 @@ public class FederatedWorkerHandler extends ChannelInboundHandlerAdapter {
}
}
- private void conditionalAddCheckedConstraints(FederatedRequest request, FederatedResponse response){
+ private static void conditionalAddCheckedConstraints(FederatedRequest request, FederatedResponse response){
if ( request.checkPrivacy() )
response.setCheckedConstraints(PrivacyMonitor.getCheckedConstraints());
}
diff --git a/src/main/java/org/apache/sysds/runtime/lineage/LineageCache.java b/src/main/java/org/apache/sysds/runtime/lineage/LineageCache.java
index 7f53038..be58d79 100644
--- a/src/main/java/org/apache/sysds/runtime/lineage/LineageCache.java
+++ b/src/main/java/org/apache/sysds/runtime/lineage/LineageCache.java
@@ -371,8 +371,8 @@ public class LineageCache
private static LineageCacheEntry getIntern(LineageItem key) {
// This method is called only when entry is present either in cache or in local FS.
- if (_cache.containsKey(key) && _cache.get(key).getCacheStatus() != LineageCacheStatus.SPILLED) {
- LineageCacheEntry e = _cache.get(key);
+ LineageCacheEntry e = _cache.get(key);
+ if (e != null && e.getCacheStatus() != LineageCacheStatus.SPILLED) {
// Maintain order for eviction
LineageCacheEviction.getEntry(e);
if (DMLScript.STATISTICS)
diff --git a/src/main/java/org/apache/sysds/runtime/lineage/LineageItem.java b/src/main/java/org/apache/sysds/runtime/lineage/LineageItem.java
index ec86a5e..2a16baa 100644
--- a/src/main/java/org/apache/sysds/runtime/lineage/LineageItem.java
+++ b/src/main/java/org/apache/sysds/runtime/lineage/LineageItem.java
@@ -77,6 +77,9 @@ public class LineageItem {
_opcode = opcode;
_data = data;
_inputs = inputs;
+ // materialize hash on construction
+ // (constant time operation if input hashes constructed)
+ _hash = hashCode();
}
public LineageItem[] getInputs() {
diff --git a/src/main/java/org/apache/sysds/runtime/privacy/CheckedConstraintsLog.java b/src/main/java/org/apache/sysds/runtime/privacy/CheckedConstraintsLog.java
index 9f11300..7bb5687 100644
--- a/src/main/java/org/apache/sysds/runtime/privacy/CheckedConstraintsLog.java
+++ b/src/main/java/org/apache/sysds/runtime/privacy/CheckedConstraintsLog.java
@@ -30,12 +30,12 @@ import org.apache.sysds.runtime.privacy.PrivacyConstraint.PrivacyLevel;
* Class counting the checked privacy constraints and the loaded privacy constraints.
*/
public class CheckedConstraintsLog {
- private static Map<PrivacyLevel,LongAdder> loadedConstraintsTotal = new EnumMap<PrivacyLevel,LongAdder>(PrivacyLevel.class);
+ private static Map<PrivacyLevel,LongAdder> loadedConstraintsTotal = new EnumMap<>(PrivacyLevel.class);
static {
for ( PrivacyLevel level : PrivacyLevel.values() )
loadedConstraintsTotal.put(level, new LongAdder());
}
- private static Map<PrivacyLevel,LongAdder> checkedConstraintsTotal = new EnumMap<PrivacyLevel,LongAdder>(PrivacyLevel.class);
+ private static Map<PrivacyLevel,LongAdder> checkedConstraintsTotal = new EnumMap<>(PrivacyLevel.class);
private static BiFunction<LongAdder, LongAdder, LongAdder> mergeLongAdders = (v1, v2) -> {
v1.add(v2.longValue() );
return v1;
diff --git a/src/main/java/org/apache/sysds/runtime/privacy/PrivacyMonitor.java b/src/main/java/org/apache/sysds/runtime/privacy/PrivacyMonitor.java
index 779f464..0eeeb4e 100644
--- a/src/main/java/org/apache/sysds/runtime/privacy/PrivacyMonitor.java
+++ b/src/main/java/org/apache/sysds/runtime/privacy/PrivacyMonitor.java
@@ -34,7 +34,7 @@ public class PrivacyMonitor
private static EnumMap<PrivacyLevel,LongAdder> checkedConstraints;
static {
- checkedConstraints = new EnumMap<PrivacyLevel,LongAdder>(PrivacyLevel.class);
+ checkedConstraints = new EnumMap<>(PrivacyLevel.class);
for ( PrivacyLevel level : PrivacyLevel.values() ){
checkedConstraints.put(level, new LongAdder());
}
diff --git a/src/test/java/org/apache/sysds/test/functions/builtin/BuiltinMulticlassSVMTest.java b/src/test/java/org/apache/sysds/test/functions/builtin/BuiltinMulticlassSVMTest.java
index b5b21a5..37c4014 100644
--- a/src/test/java/org/apache/sysds/test/functions/builtin/BuiltinMulticlassSVMTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/builtin/BuiltinMulticlassSVMTest.java
@@ -113,8 +113,8 @@ public class BuiltinMulticlassSVMTest extends AutomatedTestBase {
Integer.toString(run),
expectedDir());
- double[][] X = getRandomMatrix(rows, colsX, 0, 1, sparsity, -1);
- double[][] Y = getRandomMatrix(rows, 1, 0, 10, 1, -1);
+ double[][] X = getRandomMatrix(rows, colsX, 0, 1, sparsity, 2);
+ double[][] Y = getRandomMatrix(rows, 1, 0, 10, 1, 3);
Y = TestUtils.round(Y);
writeInputMatrixWithMTD("X", X, true);
diff --git a/src/test/java/org/apache/sysds/test/functions/frame/FrameDropInvalidLengthTest.java b/src/test/java/org/apache/sysds/test/functions/frame/FrameDropInvalidLengthTest.java
index c938b28..cd2ee71 100644
--- a/src/test/java/org/apache/sysds/test/functions/frame/FrameDropInvalidLengthTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/frame/FrameDropInvalidLengthTest.java
@@ -186,14 +186,15 @@ public class FrameDropInvalidLengthTest extends AutomatedTestBase {
}
}
- private ArrayList<Integer> getBadIndexes(int length) {
- ArrayList<Integer> list = new ArrayList();
+ private static ArrayList<Integer> getBadIndexes(int length) {
+ ArrayList<Integer> list = new ArrayList<>();
for(int i =0; i<length; i++)
{
int r = ThreadLocalRandom.current().nextInt(0, rows);
list.add(r);
}
- return (ArrayList) list.stream().distinct().collect(Collectors.toList());
+ return (ArrayList<Integer>) list.stream()
+ .distinct().collect(Collectors.toList());
}
public static void initFrameDataString(FrameBlock frame1, double[][] data, Types.ValueType[] lschema) {
diff --git a/src/test/java/org/apache/sysds/test/functions/privacy/CheckedConstraintsLogTest.java b/src/test/java/org/apache/sysds/test/functions/privacy/CheckedConstraintsLogTest.java
index ae0a5f9..1a76019 100644
--- a/src/test/java/org/apache/sysds/test/functions/privacy/CheckedConstraintsLogTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/privacy/CheckedConstraintsLogTest.java
@@ -79,7 +79,7 @@ public class CheckedConstraintsLogTest extends AutomatedTestBase {
&& CheckedConstraintsLog.getCheckedConstraints().get(PrivacyLevel.PrivateAggregation).longValue() == 150);
}
- private EnumMap<PrivacyLevel,LongAdder> getMap(PrivacyLevel level, long value){
+ private static EnumMap<PrivacyLevel,LongAdder> getMap(PrivacyLevel level, long value){
EnumMap<PrivacyLevel,LongAdder> checked = new EnumMap<>(PrivacyLevel.class);
LongAdder valueAdder = new LongAdder();
valueAdder.add(value);