You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by se...@apache.org on 2022/08/01 14:55:59 UTC
[systemds] branch main updated: [SYSTEMDS-3018] Privacy Constraint Mock Config
This is an automated email from the ASF dual-hosted git repository.
sebwrede pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 549ae3175d [SYSTEMDS-3018] Privacy Constraint Mock Config
549ae3175d is described below
commit 549ae3175d8da0ee62a72da0be26bb478403ba1c
Author: sebwrede <sw...@know-center.at>
AuthorDate: Tue Jul 26 11:56:00 2022 +0200
[SYSTEMDS-3018] Privacy Constraint Mock Config
Add Privacy Constraint Mock Config.
Use Mock of Privacy Constraints in Loading of Privacy Constraints.
Closes #1673.
---
src/main/java/org/apache/sysds/conf/DMLConfig.java | 2 +
.../hops/fedplanner/PrivacyConstraintLoader.java | 4 +-
.../fedplanner/PrivacyConstraintLoaderMock.java | 62 ++++++++++++++++++++++
.../hops/ipa/IPAPassRewriteFederatedPlan.java | 15 +++++-
4 files changed, 79 insertions(+), 4 deletions(-)
diff --git a/src/main/java/org/apache/sysds/conf/DMLConfig.java b/src/main/java/org/apache/sysds/conf/DMLConfig.java
index 5ab151f7d5..357e43c495 100644
--- a/src/main/java/org/apache/sysds/conf/DMLConfig.java
+++ b/src/main/java/org/apache/sysds/conf/DMLConfig.java
@@ -120,6 +120,7 @@ public class DMLConfig
public static final String FEDERATED_PLANNER = "sysds.federated.planner";
public static final String FEDERATED_PAR_INST = "sysds.federated.par_inst";
public static final String FEDERATED_PAR_CONN = "sysds.federated.par_conn";
+ public static final String PRIVACY_CONSTRAINT_MOCK = "sysds.federated.priv_mock";
public static final int DEFAULT_FEDERATED_PORT = 4040; // borrowed default Spark Port
public static final int DEFAULT_NUMBER_OF_FEDERATED_WORKER_THREADS = 8;
@@ -189,6 +190,7 @@ public class DMLConfig
_defaultVals.put(FEDERATED_PLANNER, FederatedPlanner.RUNTIME.name());
_defaultVals.put(FEDERATED_PAR_CONN, "-1"); // vcores
_defaultVals.put(FEDERATED_PAR_INST, "-1"); // vcores
+ _defaultVals.put(PRIVACY_CONSTRAINT_MOCK, null);
}
public DMLConfig() {
diff --git a/src/main/java/org/apache/sysds/hops/fedplanner/PrivacyConstraintLoader.java b/src/main/java/org/apache/sysds/hops/fedplanner/PrivacyConstraintLoader.java
index 69a2b638ed..bb8d3b7107 100644
--- a/src/main/java/org/apache/sysds/hops/fedplanner/PrivacyConstraintLoader.java
+++ b/src/main/java/org/apache/sysds/hops/fedplanner/PrivacyConstraintLoader.java
@@ -193,9 +193,9 @@ public class PrivacyConstraintLoader {
/**
* Get privacy constraints from federated workers for DataOps.
- * @hop hop for which privacy constraints are loaded
+ * @param hop for which privacy constraints are loaded
*/
- private static void loadFederatedPrivacyConstraints(Hop hop){
+ public void loadFederatedPrivacyConstraints(Hop hop){
try {
PrivacyConstraint.PrivacyLevel constraintLevel = hop.getInput(0).getInput().stream().parallel()
.map( in -> ((LiteralOp)in).getStringValue() )
diff --git a/src/main/java/org/apache/sysds/hops/fedplanner/PrivacyConstraintLoaderMock.java b/src/main/java/org/apache/sysds/hops/fedplanner/PrivacyConstraintLoaderMock.java
new file mode 100644
index 0000000000..5361e84127
--- /dev/null
+++ b/src/main/java/org/apache/sysds/hops/fedplanner/PrivacyConstraintLoaderMock.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.hops.fedplanner;
+
+import org.apache.sysds.api.DMLException;
+import org.apache.sysds.hops.Hop;
+import org.apache.sysds.runtime.privacy.PrivacyConstraint;
+import org.apache.sysds.runtime.privacy.PrivacyConstraint.PrivacyLevel;
+
+/**
+ * This class is a mockup of the PrivacyConstraintLoader which replaces the call to loadFederatedPrivacyConstraints.
+ * This means that instead of loading the privacy constraints from the federated workers,
+ * the constraint returned for each federated DataOp will have the privacy level specified in the constructor,
+ * without sending any federated requests.
+ */
+public class PrivacyConstraintLoaderMock extends PrivacyConstraintLoader {
+
+ private final PrivacyLevel privacyLevel;
+
+ /**
+ * Creates a mock of PrivacyConstraintLoader where the
+ * given privacy level is given to all federated data.
+ * @param mockLevel string representing the privacy level used for the setting of privacy constraints
+ */
+ public PrivacyConstraintLoaderMock(String mockLevel){
+ try{
+ this.privacyLevel = PrivacyLevel.valueOf(mockLevel);
+ } catch(IllegalArgumentException ex){
+ throw new DMLException("Privacy level loaded from config not recognized. Loaded from config: " + mockLevel, ex);
+ }
+ }
+
+ /**
+ * Set privacy constraint of given hop to mocked privacy level.
+ * This mocks the behavior of the privacy constraint loader by
+ * setting the privacy constraint to a specific level for all
+ * federated data objects instead of retrieving the privacy constraints
+ * from the workers.
+ * @param hop for which privacy constraint is set
+ */
+ @Override
+ public void loadFederatedPrivacyConstraints(Hop hop){
+ hop.setPrivacy(new PrivacyConstraint(privacyLevel));
+ }
+}
diff --git a/src/main/java/org/apache/sysds/hops/ipa/IPAPassRewriteFederatedPlan.java b/src/main/java/org/apache/sysds/hops/ipa/IPAPassRewriteFederatedPlan.java
index e6c683eb38..308625cbf6 100644
--- a/src/main/java/org/apache/sysds/hops/ipa/IPAPassRewriteFederatedPlan.java
+++ b/src/main/java/org/apache/sysds/hops/ipa/IPAPassRewriteFederatedPlan.java
@@ -24,6 +24,7 @@ import org.apache.sysds.conf.DMLConfig;
import org.apache.sysds.hops.OptimizerUtils;
import org.apache.sysds.hops.fedplanner.FTypes.FederatedPlanner;
import org.apache.sysds.hops.fedplanner.PrivacyConstraintLoader;
+import org.apache.sysds.hops.fedplanner.PrivacyConstraintLoaderMock;
import org.apache.sysds.parser.DMLProgram;
/**
@@ -67,8 +68,18 @@ public class IPAPassRewriteFederatedPlan extends IPAPass {
}
private void loadPrivacyConstraints(DMLProgram prog, String splanner){
- if (FederatedPlanner.isCompiled(splanner))
- new PrivacyConstraintLoader().loadConstraints(prog);
+ if (FederatedPlanner.isCompiled(splanner)){
+ String privMock = ConfigurationManager.getDMLConfig().getTextValue(DMLConfig.PRIVACY_CONSTRAINT_MOCK);
+ if ( privMock == null )
+ new PrivacyConstraintLoader().loadConstraints(prog);
+ else if ( privMock.equals("mock_all") )
+ LOG.trace("Privacy Constraint retrieval mocked. " +
+ "Ignoring retrieval and propagation of constraints during compilation.");
+ else {
+ LOG.trace("Mocking privacy constraints with privacy level " + privMock);
+ new PrivacyConstraintLoaderMock(privMock).loadConstraints(prog);
+ }
+ }
}
private void generatePlan(DMLProgram prog, FunctionCallGraph fgraph, FunctionCallSizeInfo fcallSizes, String splanner){