You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by mb...@apache.org on 2025/08/09 10:35:47 UTC
(systemds) branch main updated: [MINOR] Fix various warnings and formatting issues
This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 44cd867d5d [MINOR] Fix various warnings and formatting issues
44cd867d5d is described below
commit 44cd867d5d049a4da47e8515855b7f4df1b32a76
Author: Matthias Boehm <mb...@gmail.com>
AuthorDate: Sat Aug 9 12:35:24 2025 +0200
[MINOR] Fix various warnings and formatting issues
---
.../fedplanner/FederatedPlanCostEnumerator.java | 3 +
.../fedplanner/FederatedPlanRewireTransTable.java | 2802 ++++++++++----------
.../fedplanner/FederatedPlannerFedCostBased.java | 1 -
.../controlprogram/federated/FederatedData.java | 43 +-
.../instructions/gpu/context/CSRPointer.java | 5 -
.../data/DoublePrecisionCudaSupportFunctions.java | 4 -
.../sysds/runtime/matrix/data/LibMatrixCuDNN.java | 9 +
.../data/LibMatrixCuDNNConvolutionAlgorithm.java | 6 +
.../data/LibMatrixCuDNNPoolingDescriptors.java | 2 +
.../matrix/data/LibMatrixCuDNNRnnAlgorithm.java | 2 +
.../runtime/matrix/data/LibMatrixCuMatMult.java | 1 -
.../data/SinglePrecisionCudaSupportFunctions.java | 5 -
.../fedplanning/FederatedL2SVMPlanningTest.java | 4 +-
.../apache/sysds/test/gpu/nn/ResNet18GPUTest.java | 2 -
.../sysds/test/usertest/pythonapi/StartupTest.java | 1 +
15 files changed, 1447 insertions(+), 1443 deletions(-)
diff --git a/src/main/java/org/apache/sysds/hops/fedplanner/FederatedPlanCostEnumerator.java b/src/main/java/org/apache/sysds/hops/fedplanner/FederatedPlanCostEnumerator.java
index 389d08c3e9..76e3eca3ae 100644
--- a/src/main/java/org/apache/sysds/hops/fedplanner/FederatedPlanCostEnumerator.java
+++ b/src/main/java/org/apache/sysds/hops/fedplanner/FederatedPlanCostEnumerator.java
@@ -282,6 +282,7 @@ public class FederatedPlanCostEnumerator {
* generates federated plan variants for both LOUT and FOUT output types,
* and prunes redundant plans before adding them to the memo table.
*/
+ @SuppressWarnings("unused")
private static void enumerateHop(Hop hop, FederatedMemoTable memoTable, Map<Long, HopCommon> hopCommonTable,
Map<Long, List<Hop>> rewireTable, Map<Long, Privacy> privacyConstraintMap,
Map<Long, FType> fTypeMap, Set<Long> unRefTwriteSet, int numOfWorkers) {
@@ -510,6 +511,7 @@ public class FederatedPlanCostEnumerator {
* it generates only
* a single plan for each output type
*/
+ @SuppressWarnings("unused")
private static void enumerateTransChildFedPlan(FedPlanVariants lOutFedPlanVariants,
FedPlanVariants fOutFedPlanVariants,
List<Hop> childHops, double[][] childCumulativeCost,
@@ -647,6 +649,7 @@ public class FederatedPlanCostEnumerator {
* - Re-running BFS with resolved conflicts to ensure all inconsistencies are
* addressed.
*/
+ @SuppressWarnings("unused")
private static double detectAndResolveConflictFedPlan(FedPlan rootPlan, FederatedMemoTable memoTable) {
// Map to track conflicts: maps a plan ID to its federated output type and list
// of parent plans
diff --git a/src/main/java/org/apache/sysds/hops/fedplanner/FederatedPlanRewireTransTable.java b/src/main/java/org/apache/sysds/hops/fedplanner/FederatedPlanRewireTransTable.java
index 9644cb4d57..f41d924999 100644
--- a/src/main/java/org/apache/sysds/hops/fedplanner/FederatedPlanRewireTransTable.java
+++ b/src/main/java/org/apache/sysds/hops/fedplanner/FederatedPlanRewireTransTable.java
@@ -39,7 +39,6 @@ import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.concurrent.Future;
import org.apache.sysds.runtime.controlprogram.federated.FederatedResponse;
-import org.apache.sysds.hops.fedplanner.FederatedPlannerLogger;
import org.apache.sysds.hops.fedplanner.FTypes.Privacy;
import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.hops.fedplanner.FTypes.FType;
@@ -54,1364 +53,1365 @@ import java.util.ArrayList;
import java.util.List;
public class FederatedPlanRewireTransTable {
-
- private static final double DEFAULT_LOOP_WEIGHT = 10.0;
- private static final double DEFAULT_IF_ELSE_WEIGHT = 0.5;
-
- public static final String FED_MATRIX_IDENTIFIER = "matrix";
- public static final String FED_FRAME_IDENTIFIER = "frame";
-
- public static void rewireProgram(DMLProgram prog, Map<Long, List<Hop>> rewireTable,
- Map<Long, HopCommon> hopCommonTable, Map<Long, Privacy> privacyConstraintMap, Map<Long, FType> fTypeMap,
- List<Pair<FederatedRange, FederatedData>> fedMap, Set<Long> unRefTwriteSet, Set<Long> unRefSet,
- Set<Hop> progRootHopSet) {
- // Maps Hop ID and fedOutType pairs to their plan variants
- Set<Long> visitedHops = new HashSet<>();
- Set<String> fnStack = new HashSet<>();
- List<Pair<Long, Double>> loopStack = new ArrayList<>();
-
- List<Map<String, List<Hop>>> outerTransTableList = new ArrayList<>();
- Map<String, List<Hop>> outerTransTable = new HashMap<>();
- outerTransTableList.add(outerTransTable);
-
- for (StatementBlock sb : prog.getStatementBlocks()) {
- Map<String, List<Hop>> innerTransTable = rewireStatementBlock(sb, prog, visitedHops, rewireTable,
- hopCommonTable, outerTransTableList, null, privacyConstraintMap, fTypeMap,
- fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack, 1, 1, loopStack);
- outerTransTableList.get(0).putAll(innerTransTable);
- }
- }
-
- public static void rewireFunctionDynamic(FunctionStatementBlock function, Map<Long, List<Hop>> rewireTable,
- Map<Long, HopCommon> hopCommonTable, Map<Long, Privacy> privacyConstraintMap, Map<Long, FType> fTypeMap,
- List<Pair<FederatedRange, FederatedData>> fedMap, Set<Long> unRefTwriteSet, Set<Long> unRefSet,
- Set<Hop> progRootHopSet) {
- Set<Long> visitedHops = new HashSet<>();
- Set<String> fnStack = new HashSet<>();
- List<Pair<Long, Double>> loopStack = new ArrayList<>();
- List<Map<String, List<Hop>>> outerTransTableList = new ArrayList<>();
- Map<String, List<Hop>> outerTransTable = new HashMap<>();
- outerTransTableList.add(outerTransTable);
- // Todo (Future): not tested & not used
- rewireStatementBlock(function, null, visitedHops, rewireTable, hopCommonTable, outerTransTableList, null,
- privacyConstraintMap, fTypeMap,
- fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack, 1, 1, loopStack);
- }
-
- public static Map<String, List<Hop>> rewireStatementBlock(StatementBlock sb, DMLProgram prog, Set<Long> visitedHops,
- Map<Long, List<Hop>> rewireTable, Map<Long, HopCommon> hopCommonTable,
- List<Map<String, List<Hop>>> outerTransTableList, Map<String, List<Hop>> formerTransTable,
- Map<Long, Privacy> privacyConstraintMap, Map<Long, FType> fTypeMap,
- List<Pair<FederatedRange, FederatedData>> fedMap, Set<Long> unRefTwriteSet, Set<Long> unRefSet,
- Set<Hop> progRootHopSet, Set<String> fnStack,
- double computeWeight, double networkWeight, List<Pair<Long, Double>> parentLoopStack) {
- List<Map<String, List<Hop>>> newOuterTransTableList = new ArrayList<>();
- if (outerTransTableList != null) {
- for (Map<String, List<Hop>> outerTable : outerTransTableList) {
- if (outerTable != null && !outerTable.isEmpty()) {
- newOuterTransTableList.add(outerTable);
- }
- }
- }
- if (formerTransTable != null && !formerTransTable.isEmpty()) {
- newOuterTransTableList.add(formerTransTable);
- }
-
- Map<String, List<Hop>> newFormerTransTable = new HashMap<>();
- Map<String, List<Hop>> innerTransTable = new HashMap<>();
-
- if (sb instanceof IfStatementBlock) {
- IfStatementBlock isb = (IfStatementBlock) sb;
- IfStatement istmt = (IfStatement) isb.getStatement(0);
-
- rewireHopDAG(isb.getPredicateHops(), prog, visitedHops, rewireTable, hopCommonTable, newOuterTransTableList,
- null, innerTransTable,
- privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack, computeWeight,
- networkWeight, parentLoopStack);
-
- newFormerTransTable.putAll(innerTransTable);
- Map<String, List<Hop>> elseFormerTransTable = new HashMap<>();
- elseFormerTransTable.putAll(innerTransTable);
- computeWeight *= DEFAULT_IF_ELSE_WEIGHT;
-
- for (StatementBlock innerIsb : istmt.getIfBody())
- newFormerTransTable.putAll(rewireStatementBlock(innerIsb, prog, visitedHops, rewireTable,
- hopCommonTable, newOuterTransTableList, newFormerTransTable,
- privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack, computeWeight,
- networkWeight, parentLoopStack));
-
- for (StatementBlock innerIsb : istmt.getElseBody())
- elseFormerTransTable.putAll(rewireStatementBlock(innerIsb, prog, visitedHops, rewireTable,
- hopCommonTable, newOuterTransTableList, elseFormerTransTable,
- privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack, computeWeight,
- networkWeight, parentLoopStack));
-
- // If there are common keys: merge elseValue list into ifValue list
- elseFormerTransTable.forEach((key, elseValue) -> {
- newFormerTransTable.merge(key, elseValue, (ifValue, newValue) -> {
- ifValue.addAll(newValue);
- return ifValue;
- });
- });
- } else if (sb instanceof ForStatementBlock) { // incl parfor
- ForStatementBlock fsb = (ForStatementBlock) sb;
- ForStatement fstmt = (ForStatement) fsb.getStatement(0);
-
- // Calculate for-loop iteration count if possible
- double loopWeight = DEFAULT_LOOP_WEIGHT;
- Hop from = fsb.getFromHops().getInput().get(0);
- Hop to = fsb.getToHops().getInput().get(0);
- Hop incr = (fsb.getIncrementHops() != null) ? fsb.getIncrementHops().getInput().get(0) : new LiteralOp(1);
-
- // Calculate for-loop iteration count (weight) if from, to, and incr are literal
- // ops (constant values)
- if (from instanceof LiteralOp && to instanceof LiteralOp && incr instanceof LiteralOp) {
- double dfrom = HopRewriteUtils.getDoubleValue((LiteralOp) from);
- double dto = HopRewriteUtils.getDoubleValue((LiteralOp) to);
- double dincr = HopRewriteUtils.getDoubleValue((LiteralOp) incr);
- if (dfrom > dto && dincr == 1)
- dincr = -1;
- loopWeight = UtilFunctions.getSeqLength(dfrom, dto, dincr, false);
- }
- computeWeight *= loopWeight;
- networkWeight *= loopWeight;
-
- // Create current loop context (copy parent context)
- List<Pair<Long, Double>> currentLoopStack = new ArrayList<>(parentLoopStack);
- currentLoopStack.add(Pair.of(sb.getSBID(), loopWeight));
-
- rewireHopDAG(fsb.getFromHops(), prog, visitedHops, rewireTable, hopCommonTable, newOuterTransTableList,
- null, innerTransTable,
- privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack, computeWeight,
- networkWeight, currentLoopStack);
- rewireHopDAG(fsb.getToHops(), prog, visitedHops, rewireTable, hopCommonTable, newOuterTransTableList, null,
- innerTransTable,
- privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack, computeWeight,
- networkWeight, currentLoopStack);
-
- if (fsb.getIncrementHops() != null) {
- rewireHopDAG(fsb.getIncrementHops(), prog, visitedHops, rewireTable, hopCommonTable,
- newOuterTransTableList, null, innerTransTable,
- privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack, computeWeight,
- networkWeight, currentLoopStack);
- }
- newFormerTransTable.putAll(innerTransTable);
-
- for (StatementBlock innerFsb : fstmt.getBody())
- newFormerTransTable.putAll(rewireStatementBlock(innerFsb, prog, visitedHops, rewireTable,
- hopCommonTable, newOuterTransTableList, newFormerTransTable,
- privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack, computeWeight,
- networkWeight, currentLoopStack));
-
- // Wire UnRefTwrite to liveOutHops
- wireUnRefTwriteToLiveOut(fsb, unRefTwriteSet, hopCommonTable, newFormerTransTable);
- } else if (sb instanceof WhileStatementBlock) {
- WhileStatementBlock wsb = (WhileStatementBlock) sb;
- WhileStatement wstmt = (WhileStatement) wsb.getStatement(0);
-
- computeWeight *= DEFAULT_LOOP_WEIGHT;
- networkWeight *= DEFAULT_LOOP_WEIGHT;
-
- // Create current loop context (copy parent context)
- List<Pair<Long, Double>> currentLoopStack = new ArrayList<>(parentLoopStack);
- currentLoopStack.add(Pair.of(sb.getSBID(), DEFAULT_LOOP_WEIGHT));
-
- rewireHopDAG(wsb.getPredicateHops(), prog, visitedHops, rewireTable, hopCommonTable, newOuterTransTableList,
- null, innerTransTable,
- privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack, computeWeight,
- networkWeight, currentLoopStack);
- newFormerTransTable.putAll(innerTransTable);
-
- for (StatementBlock innerWsb : wstmt.getBody())
- newFormerTransTable.putAll(rewireStatementBlock(innerWsb, prog, visitedHops, rewireTable,
- hopCommonTable, newOuterTransTableList, newFormerTransTable,
- privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack, computeWeight,
- networkWeight, currentLoopStack));
-
- // Wire UnRefTwrite to liveOutHops
- wireUnRefTwriteToLiveOut(wsb, unRefTwriteSet, hopCommonTable, newFormerTransTable);
- } else if (sb instanceof FunctionStatementBlock) {
- FunctionStatementBlock fsb = (FunctionStatementBlock) sb;
- FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0);
-
- for (StatementBlock innerFsb : fstmt.getBody())
- newFormerTransTable.putAll(rewireStatementBlock(innerFsb, prog, visitedHops, rewireTable,
- hopCommonTable, newOuterTransTableList, newFormerTransTable,
- privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack, computeWeight,
- networkWeight, parentLoopStack));
-
- // Wire fcall operation to liveOutHops
- wireUnRefTwriteToLiveOut(fsb, unRefTwriteSet, hopCommonTable, newFormerTransTable);
- } else { // generic (last-level)
- if (sb.getHops() != null) {
- for (Hop c : sb.getHops())
- rewireHopDAG(c, prog, visitedHops, rewireTable, hopCommonTable, newOuterTransTableList, null,
- innerTransTable,
- privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack,
- computeWeight, networkWeight, parentLoopStack);
- }
-
- return innerTransTable;
- }
- return newFormerTransTable;
- }
-
- private static void rewireHopDAG(Hop hop, DMLProgram prog, Set<Long> visitedHops, Map<Long, List<Hop>> rewireTable,
- Map<Long, HopCommon> hopCommonTable, List<Map<String, List<Hop>>> outerTransTableList,
- Map<String, List<Hop>> formerTransTable, Map<String, List<Hop>> innerTransTable,
- Map<Long, Privacy> privacyConstraintMap, Map<Long, FType> fTypeMap,
- List<Pair<FederatedRange, FederatedData>> fedMap, Set<Long> unRefTwriteSet, Set<Long> unRefSet,
- Set<Hop> progRootHopSet,
- Set<String> fnStack, double computeWeight, double networkWeight, List<Pair<Long, Double>> loopStack) {
-
- if (hop.getInput() != null) {
- for (Hop inputHop : hop.getInput()) {
- long inputHopID = inputHop.getHopID();
- if (!visitedHops.contains(inputHopID)) {
- visitedHops.add(inputHopID);
- rewireHopDAG(inputHop, prog, visitedHops, rewireTable, hopCommonTable, outerTransTableList,
- formerTransTable, innerTransTable,
- privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack,
- computeWeight, networkWeight, loopStack);
- }
- }
- }
-
- hopCommonTable.put(hop.getHopID(), new HopCommon(hop, computeWeight, networkWeight, 0, loopStack));
-
- // Identify hops to connect to the root dummy node
- // Connect TWrite pred and u(print) to the root dummy node
- if ((hop instanceof DataOp && (hop.getName().equals("__pred"))) // TWrite "__pred"
- || (hop instanceof UnaryOp && ((UnaryOp) hop).getOp() == Types.OpOp1.PRINT) // u(print)
- || (hop instanceof DataOp && ((DataOp) hop).getOp() == Types.OpOpData.PERSISTENTWRITE)) { // PWrite
- progRootHopSet.add(hop);
- } else if (!(hop instanceof DataOp && ((DataOp) hop).getOp() == Types.OpOpData.TRANSIENTWRITE)
- && hop.getParent().size() == 0) {
- unRefSet.add(hop.getHopID());
- }
-
- if (hop instanceof FunctionOp) {
- // maintain counters and investigate functions if not seen so far
- FunctionOp fop = (FunctionOp) hop;
- unRefTwriteSet.add(fop.getHopID());
-
- if (fop.getFunctionType() == FunctionType.DML) {
- String fkey = fop.getFunctionKey();
-
- if (!fnStack.contains(fkey)) {
- fnStack.add(fkey);
- FunctionStatementBlock fsb = prog.getFunctionStatementBlock(fop.getFunctionNamespace(),
- fop.getFunctionName());
-
- Map<String, List<Hop>> newFormerTransTable = new HashMap<>();
- if (formerTransTable != null) {
- newFormerTransTable.putAll(formerTransTable);
- }
- newFormerTransTable.putAll(innerTransTable);
-
- String[] inputArgs = fop.getInputVariableNames();
- List<Hop> inputHops = fop.getInput();
-
- // Only used outside of functionTransTable.
- for (int i = 0; i < inputHops.size(); i++) {
- newFormerTransTable.computeIfAbsent(inputArgs[i], k -> new ArrayList<>()).add(inputHops.get(i));
- }
-
- Map<String, List<Hop>> functionTransTable = rewireStatementBlock(fsb, prog, visitedHops,
- rewireTable, hopCommonTable, outerTransTableList, newFormerTransTable,
- privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack,
- computeWeight, networkWeight, loopStack);
-
- for (int i = 0; i < fop.getOutputVariableNames().length; i++) {
- String tWriteName = fop.getOutputVariableNames()[i];
- List<Hop> outputHops = functionTransTable.get(fsb.getOutputsofSB().get(i).getName());
- innerTransTable.computeIfAbsent(tWriteName, k -> new ArrayList<>()).addAll(outputHops);
- for (Hop outputHop : outputHops) {
- unRefTwriteSet.add(outputHop.getHopID());
- }
- }
- }
- }
- }
-
- // Propagate Privacy Constraint
- if (!(hop instanceof DataOp) || hop.getName().equals("__pred")
- || (((DataOp) hop).getOp() == Types.OpOpData.PERSISTENTWRITE)) {
- privacyConstraintMap.put(hop.getHopID(),
- getPrivacyConstraint(hop, hop.getInput(), privacyConstraintMap));
- // Todo: Remove this after debugging
- // fTypeMap.put(hop.getHopID(), getFederatedType(hop, fTypeMap));
- fTypeMap.put(hop.getHopID(), getFederatedTypeDebug(hop, fTypeMap));
-
- // Todo: Remove this after debugging
-// FederatedPlannerLogger.logHopInfo(hop, privacyConstraintMap, fTypeMap, "RewireTransHop");
- return;
- }
-
- rewireTransHop(hop, rewireTable, outerTransTableList, formerTransTable, innerTransTable, privacyConstraintMap,
- fTypeMap, fedMap, unRefTwriteSet);
- // Todo: Remove this after debugging
-// FederatedPlannerLogger.logHopInfo(hop, privacyConstraintMap, fTypeMap, "RewireTransHop");
- }
-
- private static void rewireTransHop(Hop hop, Map<Long, List<Hop>> rewireTable,
- List<Map<String, List<Hop>>> outerTransTableList, Map<String, List<Hop>> formerTransTable,
- Map<String, List<Hop>> innerTransTable, Map<Long, Privacy> privacyConstraintMap,
- Map<Long, FType> fTypeMap, List<Pair<FederatedRange, FederatedData>> fedMap, Set<Long> unRefTwriteSet) {
- DataOp dataOp = (DataOp) hop;
- Types.OpOpData opType = dataOp.getOp();
- String hopName = dataOp.getName();
-
- if (opType == Types.OpOpData.FEDERATED) {
- Privacy privacy = getFedWorkerMetaData(fedMap, dataOp);
- privacyConstraintMap.put(hop.getHopID(), privacy);
- FType fType = deriveFType((DataOp)hop);
- fTypeMap.put(hop.getHopID(), fType);
-
- // Debug logging for FEDERATED operation
- FederatedPlannerLogger.logDataOpFTypeDebug(hop, fType, "FEDERATED", "Derived from partition ranges");
- } else if (opType == Types.OpOpData.TRANSIENTWRITE) {
- // Rewire TransWrite
- innerTransTable.computeIfAbsent(hopName, k -> new ArrayList<>()).add(hop);
- unRefTwriteSet.add(hop.getHopID());
- // Propagate Privacy Constraint
- privacyConstraintMap.put(hop.getHopID(),
- getPrivacyConstraint(hop, hop.getInput(), privacyConstraintMap));
- // Propagate FType (TransWrite has only one input)
- FType inputFType = fTypeMap.get(hop.getInput(0).getHopID());
- fTypeMap.put(hop.getHopID(), inputFType);
-
- // Debug logging for TRANSIENTWRITE operation
- FederatedPlannerLogger.logDataOpFTypeDebug(hop, inputFType, "TRANSIENTWRITE",
- "Propagated from single input (HopID: " + hop.getInput(0).getHopID() + ")");
- } else if (opType == Types.OpOpData.TRANSIENTREAD) {
- // Rewire TransRead
- List<Hop> childHops = rewireTransRead(hopName, innerTransTable, formerTransTable, outerTransTableList);
- // Handle rewire table (TransRead -> TransWrite)
- rewireTable.put(hop.getHopID(), childHops);
-
- // Todo: Handle exception when TRead has no Child (check why it's missing)
- if (childHops == null || childHops.isEmpty()) {
- FederatedPlannerLogger.logTransReadRewireDebug(hopName, hop.getHopID(), childHops, true, "RewireTransHop");
- return;
- }
-
- // Remove childHops that have different hopVarName
- List<Hop> filteredChildHops = new ArrayList<>();
- for (Hop childHop : childHops) {
- String hopVarName = hop.getName();
-
- if (hopVarName.equals(childHop.getName())) {
- filteredChildHops.add(childHop);
- }
- }
-
- // Todo: Handle exception when TRead has no Filtered Child (check why it's missing)
- if (filteredChildHops.isEmpty()) {
- FederatedPlannerLogger.logFilteredChildHopsDebug(hopName, hop.getHopID(), filteredChildHops, true, "RewireTransHop");
- return;
- }
-
- FType inputFType = null;
- StringBuilder debugInfo = new StringBuilder();
- for (int i = 0; i < filteredChildHops.size(); i++) {
- Hop filteredChildHop = filteredChildHops.get(i);
- long filteredChildHopID = filteredChildHop.getHopID();
- FType childFType = fTypeMap.get(filteredChildHopID);
-
- // Rewire (TransWrite -> TransRead)
- rewireTable.computeIfAbsent(filteredChildHopID, k -> new ArrayList<>()).add(hop);
- // Remove refTWrite from unRefTwriteSet
- unRefTwriteSet.remove(filteredChildHopID);
-
- // Check FType consistency of childs(TransWrite)
- if ( i==0 ) {
- inputFType = childFType;
- debugInfo.append("First child HopID: ").append(filteredChildHopID).append(" FType: ").append(childFType);
- } else if (inputFType != childFType) {
- // Todo: Handle exception when TRead has different FType
- FType mismatchedFType = childFType;
- FederatedPlannerLogger.logFTypeMismatchError(hop, filteredChildHops, fTypeMap, inputFType, mismatchedFType, i);
-
- debugInfo.append(", Child ").append(i).append(" HopID: ").append(filteredChildHopID)
- .append(" FType: ").append(mismatchedFType).append(" (MISMATCH)");
-
- if (inputFType == null) {
- inputFType = mismatchedFType;
- }
- // throw new DMLRuntimeException("TransRead input FType mismatch: " + inputFType + " != " + mismatchedFType);
- } else {
- debugInfo.append(", Child ").append(i).append(" HopID: ").append(filteredChildHopID)
- .append(" FType: ").append(childFType).append(" (MATCH)");
- }
- }
- // Propagate Privacy Constraint
- privacyConstraintMap.put(hop.getHopID(),
- getPrivacyConstraint(hop, filteredChildHops, privacyConstraintMap));
- // Propagate FType
- fTypeMap.put(hop.getHopID(), inputFType);
-
- // Debug logging for TRANSIENTREAD operation
- FederatedPlannerLogger.logDataOpFTypeDebug(hop, inputFType, "TRANSIENTREAD",
- "Propagated from " + filteredChildHops.size() + " child(s): " + debugInfo.toString());
- } else {
- privacyConstraintMap.put(hop.getHopID(),
- getPrivacyConstraint(hop, hop.getInput(), privacyConstraintMap));
-
- // Todo: Remove this after debugging
- // fTypeMap.put(hop.getHopID(), getFederatedType(hop, fTypeMap));
- fTypeMap.put(hop.getHopID(), getFederatedTypeDebug(hop, fTypeMap));
- }
- }
-
- private static List<Hop> rewireTransRead(String hopName, Map<String, List<Hop>> innerTransTable,
- Map<String, List<Hop>> formerTransTable, List<Map<String, List<Hop>>> outerTransTableList) {
- List<Hop> childHops = new ArrayList<>();
-
- // Read according to priority: inner -> former -> outer
- if (!innerTransTable.isEmpty()) {
- childHops = innerTransTable.get(hopName);
- }
-
- if ((childHops == null || childHops.isEmpty()) && formerTransTable != null) {
- childHops = formerTransTable.get(hopName);
- }
-
- if (childHops == null || childHops.isEmpty()) {
- // Traverse in reverse order from the last inserted outerTransTable
- for (int i = outerTransTableList.size() - 1; i >= 0; i--) {
- Map<String, List<Hop>> outerTransTable = outerTransTableList.get(i);
- childHops = outerTransTable.get(hopName);
- if (childHops != null && !childHops.isEmpty())
- break;
- }
- }
-
- return childHops;
- }
-
- private static Privacy getFedWorkerMetaData(List<Pair<FederatedRange, FederatedData>> fedMap, DataOp initFedOp) {
- // Address
- Hop addressListHop = initFedOp.getInput(initFedOp.getParameterIndex("addresses"));
- List<String> addressList = new ArrayList<>();
- for (Hop addressHop : addressListHop.getInput()) {
- addressList.add(addressHop.getName());
- }
-
- // Range
- Hop rangeListHop = initFedOp.getInput(initFedOp.getParameterIndex("ranges"));
- List<long[]> rangeList = new ArrayList<>();
- for (Hop rangeHop : rangeListHop.getInput()) {
- long beginRange = (long) Double.parseDouble(rangeHop.getInput(0).getName());
- long endRange = (long) Double.parseDouble(rangeHop.getInput(1).getName());
- rangeList.add(new long[] { beginRange, endRange });
- }
-
- // Type
- String type = initFedOp.getInput(initFedOp.getParameterIndex("type")).getName();
- Types.DataType fedDataType;
-
- if (type.equalsIgnoreCase(FED_MATRIX_IDENTIFIER))
- fedDataType = Types.DataType.MATRIX;
- else
- fedDataType = Types.DataType.FRAME;
-
- // Init Fed Data
- for (int i = 0; i < addressList.size(); i++) {
- String address = addressList.get(i);
- // We split address into url/ip, the port and file path of file to read
- String[] parsedValues = InitFEDInstruction.parseURL(address);
- String host = parsedValues[0];
- int port = Integer.parseInt(parsedValues[1]);
- String filePath = parsedValues[2];
-
- long[] beginRange = rangeList.get(2 * i);
- long[] endRange = rangeList.get(2 * i + 1);
-
- try {
- FederatedData federatedData = new FederatedData(fedDataType,
- new InetSocketAddress(InetAddress.getByName(host), port), filePath);
- fedMap.add(new ImmutablePair<>(new FederatedRange(beginRange, endRange), federatedData));
- } catch (UnknownHostException e) {
- throw new RuntimeException("federated host was unknown: " + host, e);
- }
- }
- Privacy privacyConstraint = null;
-
- // Request Privacy Constraints
- for (Pair<FederatedRange, FederatedData> fed : fedMap) {
- FederatedData data = fed.getRight();
- data.initFederatedData(FederationUtils.getNextFedDataID());
-
- Future<FederatedResponse> future = data.requestPrivacyConstraints();
- try {
- FederatedResponse response = future.get(); // Get actual response from Future
-
- if (response.isSuccessful()) {
- Object[] responseData = response.getData();
- String privacyConstraints = (String) responseData[0]; // Cast privacy constraint as string
- String pcLower = privacyConstraints.trim().toLowerCase();
- Privacy tempPrivacy = null;
-
- // Map to appropriate PrivacyConstraint value based on input string
- if (pcLower.equals("private")
- || pcLower.equals(FTypes.Privacy.PRIVATE.toString().toLowerCase())) {
- tempPrivacy = FTypes.Privacy.PRIVATE;
- } else if (pcLower.equals("private-aggregate") || pcLower.equals("private_aggregate") ||
- pcLower.equals(FTypes.Privacy.PRIVATE_AGGREGATE.toString().toLowerCase())) {
- tempPrivacy = FTypes.Privacy.PRIVATE_AGGREGATE;
- } else if (pcLower.equals("public")
- || pcLower.equals(FTypes.Privacy.PUBLIC.toString().toLowerCase())) {
- tempPrivacy = FTypes.Privacy.PUBLIC;
- } else {
- throw new DMLRuntimeException("Invalid privacy constraint: " + privacyConstraints +
- ". Must be one of 'PRIVATE', 'PRIVATE_AGGREGATE', 'PUBLIC'.");
- }
-
- if (privacyConstraint == null) {
- privacyConstraint = tempPrivacy;
- } else {
- if (privacyConstraint != tempPrivacy) {
- throw new DMLRuntimeException("Privacy constraints do not match.");
- }
- }
- } else {
- // Error handling
- String errorMsg = response.getErrorMessage();
- System.err.println("Failed to request privacy constraints: " + errorMsg);
- }
- } catch (Exception e) {
- // Exception handling
- e.printStackTrace();
- }
- }
- return privacyConstraint;
- }
-
- private static Privacy getPrivacyConstraint(Hop hop, List<Hop> inputHops, Map<Long, Privacy> privacyMap) {
- Privacy[] pc = new Privacy[inputHops.size()];
- for (int i = 0; i < inputHops.size(); i++)
- pc[i] = privacyMap.get(inputHops.get(i).getHopID());
-
- boolean hasPrivateAggreate = false;
-
- for (Privacy p : pc) {
- if (p == Privacy.PRIVATE) {
- return Privacy.PRIVATE;
- } else if (p == Privacy.PRIVATE_AGGREGATE) {
- hasPrivateAggreate = true;
- }
- }
-
- if (hasPrivateAggreate) {
- if (hop instanceof AggUnaryOp || hop instanceof AggBinaryOp || hop instanceof QuaternaryOp) {
- return Privacy.PUBLIC;
- } else if (hop instanceof TernaryOp) {
- switch (((TernaryOp) hop).getOp()) {
- case MOMENT:
- case COV:
- case CTABLE:
- case INTERQUANTILE:
- case QUANTILE:
- return Privacy.PUBLIC;
- default:
- return Privacy.PRIVATE_AGGREGATE;
- }
- } else if (hop instanceof ParameterizedBuiltinOp
- && ((ParameterizedBuiltinOp) hop).getOp() == ParamBuiltinOp.GROUPEDAGG) {
- return Privacy.PUBLIC;
- } else {
- return Privacy.PRIVATE_AGGREGATE;
- }
- }
-
- return Privacy.PUBLIC;
- }
-
- /**
- * Determines the federated partition type (FType) for the output of a given hop operation.
- * This method combines the logic of checking federated support and determining output FType.
- *
- * @param hop The hop operation to analyze
- * @param fTypeMap Map containing FType information for all processed hops
- * @return The FType of the output, or null if the operation doesn't support federated execution
- * or produces non-federated output
- */
- private static FType getFederatedType(Hop hop, Map<Long, FType> fTypeMap) {
- // ========================================================================
- // PART 1: Universal constraints - operations that NEVER support federated
- // ========================================================================
-
- // Scalar values don't have FType (no partitioning concept for scalars)
- if (hop.isScalar()) {
- return null;
- }
-
- // Operations architecturally incompatible with federated execution:
- // - DataGenOp: All data generation requires centralized execution (RAND seed sync, SEQ global coords, etc.)
- // - DnnOp: Deep learning operations designed exclusively for CP/GPU (CuDNN dependencies)
- // - FunctionOp: Function calls execute locally on coordinator (no 'fcall' in FEDInstructionParser)
- // - LiteralOp: Constants without computation, created at coordinator only
- // - DataOp: Data operations (FEDERATED, TRANSIENTREAD, TRANSIENTWRITE) are handled separately, others are not supported (PERSISTENTWRITE/READ, FUNCTIONOUTPUT, SQLREAD)
- if (hop instanceof DataGenOp || hop instanceof DnnOp ||
- hop instanceof FunctionOp || hop instanceof LiteralOp ||
- hop instanceof DataOp) {
- return null;
- }
-
- // Extract input FTypes for analysis
- FType[] ft = new FType[hop.getInput().size()];
- for (int i = 0; i < hop.getInput().size(); i++)
- ft[i] = fTypeMap.get(hop.getInput(i).getHopID());
-
- // Handle operations with no inputs
- if (ft.length == 0) {
- return null;
- }
-
- // Common patterns used across multiple operation types
- FType firstFType = ft[0];
- boolean hasFederatedFirstInput = firstFType != null;
-
- // ========================================================================
- // PART 2: Operations NOT requiring federated first input
- // ========================================================================
-
- // NaryOp: N-ary operations with matrix/list support
- if (hop instanceof NaryOp) {
- OpOpN op = ((NaryOp) hop).getOp();
-
- // Unsupported operations:
- // - PRINTF/EVAL: Output operations, execute on coordinator only
- // - LIST: List operations not federated
- // - CBIND/RBIND on lists: Only matrix operations supported
- // - Cell operations on all scalars: No distribution benefit
- if (op == OpOpN.PRINTF || op == OpOpN.EVAL || op == OpOpN.LIST ||
- ((op == OpOpN.CBIND || op == OpOpN.RBIND) &&
- hop.getInput().get(0).getDataType().isList()) ||
- (op.isCellOp() &&
- hop.getInput().stream().allMatch(h -> h.getDataType().isScalar()))) {
- return null;
- }
-
- // Supported matrix operations: CBIND/RBIND (matrix concat), PLUS/MULT (arithmetic), MIN/MAX (comparison)
- if (op == OpOpN.CBIND || op == OpOpN.RBIND ||
- op == OpOpN.PLUS || op == OpOpN.MULT ||
- op == OpOpN.MIN || op == OpOpN.MAX) {
- FType secondFType = ft.length > 1 ? ft[1] : null;
- // Todo: propagate 3rd one if 2nd is null -> N
- return firstFType != null ? firstFType : secondFType;
- }
-
- // Other NaryOp operations not supported
- return null;
- }
-
- // TernaryOp: Three-input operations with complex federation patterns
- if (hop instanceof TernaryOp) {
- // Scalar output operations don't have FType
- if (hop.getDataType().isScalar()) {
- return null;
- }
-
- // Operations that produce scalar output or are unsupported:
- // - MOMENT/COV: Aggregation operations produce scalar
- // - IFELSE/MAP: No federated implementation
- OpOp3 op = ((TernaryOp) hop).getOp();
- if (op == OpOp3.MOMENT || op == OpOp3.COV ||
- op == OpOp3.IFELSE || op == OpOp3.MAP) {
- return null;
- }
-
- // Check if any input is federated
- boolean hasAnyFederatedInput = false;
- boolean hasRowPartition = false;
- for (FType f : ft) {
- if (f == FType.ROW) {
- hasRowPartition = true;
- hasAnyFederatedInput = true;
- break;
- } else if (f != null) {
- hasAnyFederatedInput = true;
- }
- }
-
- // Requires at least one federated input
- // CTABLE: Special ROW partition requirement
- if (!hasAnyFederatedInput || (!hasRowPartition && op == OpOp3.CTABLE)) {
- return null;
- }
-
- // All supported operations propagate first non-null FType
- FType secondFType = ft.length > 1 ? ft[1] : null;
- return firstFType != null ? firstFType :
- secondFType != null ? secondFType :
- ft.length > 2 ? ft[2] : null;
- }
-
- // AggBinaryOp: Matrix multiplication and aggregation operations
- if (hop instanceof AggBinaryOp) {
- FType secondFType = ft.length > 1 ? ft[1] : null;
- boolean hasFederatedSecondInput = secondFType != null;
-
- // Check supported federation patterns
- if(!((hasFederatedFirstInput != hasFederatedSecondInput) || // One federated, one not
- (firstFType != null && firstFType == secondFType) || // Both federated with same type
- (firstFType == FType.COL && secondFType == FType.ROW))) { // Special matrix multiplication patterns
- return null;
- }
-
- // Determine output FType based on operation type
- MMTSJType mmtsj = ((AggBinaryOp) hop).checkTransposeSelf();
-
- // Self-transpose matrix multiplication (X'X or XX') results in BROADCAST
- if (mmtsj != MMTSJType.NONE &&
- ((mmtsj.isLeft() && firstFType == FType.ROW) ||
- (mmtsj.isRight() && firstFType == FType.COL))) {
- return FType.BROADCAST;
- }
- // One federated input: propagate its FType
- else if ((firstFType != null) != (secondFType != null)) {
- return firstFType != null ? firstFType : secondFType;
- }
- // COL x ROW multiplication results in ROW partitioning
- else if (firstFType == FType.COL && secondFType == FType.ROW) {
- return FType.ROW;
- }
- // Same partition type: maintain it
- else if ((firstFType == FType.ROW && secondFType == FType.ROW) ||
- (firstFType == FType.COL && secondFType == FType.COL)) {
- return firstFType;
- }
- return null;
- }
-
- // BinaryOp: Standard binary operations (+, -, *, /, min, max)
- if (hop instanceof BinaryOp) {
- // Scalar operations don't have FType
- if (hop.getDataType().isScalar()) {
- return null;
- }
-
- FType secondFType = ft.length > 1 ? ft[1] : null;
- boolean hasFederatedSecondInput = secondFType != null;
-
- // Unsupported patterns: no federated inputs, or both federated with different types
- if ((!hasFederatedFirstInput && !hasFederatedSecondInput) ||
- (hasFederatedFirstInput && hasFederatedSecondInput && firstFType != secondFType)) {
- return null;
- }
-
- // Propagate first non-null FType
- return firstFType != null ? firstFType : secondFType;
- }
-
- // ========================================================================
- // PART 3: Operations REQUIRING federated first input
- // ========================================================================
-
- // All remaining operations require federated first input
- if (!hasFederatedFirstInput) {
- return null;
- }
-
- // Simple operations that maintain input structure:
- // - IndexingOp: Right indexing X[i:j, k:l] - subset of federated matrix remains federated
- // - LeftIndexingOp: Left-hand side indexing X[i:j, k:l] = Y - updates preserve partitioning
- if (hop instanceof IndexingOp || hop instanceof LeftIndexingOp) {
- return firstFType;
- }
-
- // UnaryOp: Element-wise unary operations
- if (hop instanceof UnaryOp) {
- UnaryOp uop = (UnaryOp) hop;
- OpOp1 op = uop.getOp();
-
- // Unsupported operations:
- // - Output operations (PRINT, ASSERT, STOP): Execute on coordinator
- // - Type/metadata operations (TYPEOF, NROW, NCOL): Return scalars
- // - Complex decompositions (INVERSE, EIGEN, etc.): CP-only algorithms
- // - SQRT_MATRIX_JAVA: Special matrix square root, CP only
- // - List operations: List datatype not federated
- // - Metadata operations: Return scalar metadata
- if (op == OpOp1.PRINT || op == OpOp1.ASSERT || op == OpOp1.STOP ||
- op == OpOp1.TYPEOF || op == OpOp1.INVERSE || op == OpOp1.EIGEN ||
- op == OpOp1.CHOLESKY || op == OpOp1.DET || op == OpOp1.SVD ||
- op == OpOp1.SQRT_MATRIX_JAVA ||
- hop.getInput().get(0).getDataType() == DataType.LIST ||
- uop.isMetadataOperation()) {
- return null;
- }
-
- // Element-wise operations maintain structure
- return firstFType;
- }
-
- // QuaternaryOp: Four-input weighted operations
- if (hop instanceof QuaternaryOp) {
- Types.OpOp4 op = ((QuaternaryOp) hop).getOp();
-
- // Scalar output operations:
- // - WSLOSS: Weighted squared loss (returns scalar loss value)
- // - WCEMM: Weighted cross entropy (returns scalar loss value)
- if (op == Types.OpOp4.WSLOSS || op == Types.OpOp4.WCEMM) {
- return null;
- }
-
- // Operations maintaining first input's structure:
- // - WSIGMOID: Weighted sigmoid
- // - WUMM: Weighted unary matrix multiplication
- if (op == Types.OpOp4.WSIGMOID || op == Types.OpOp4.WUMM) {
- return firstFType;
- }
-
- // WDIVMM: Weighted division matrix multiplication - use first non-null FType
- if (op == Types.OpOp4.WDIVMM) {
- FType firstNonNullFType = null;
- for (FType f : ft) {
- if (f != null) {
- firstNonNullFType = f;
- break;
- }
- }
- return firstNonNullFType;
- }
-
- // Default: maintain first input's FType
- return firstFType;
- }
-
- // AggUnaryOp: Aggregate unary operations with direction awareness
- if (hop instanceof AggUnaryOp) {
- AggOp aggOp = ((AggUnaryOp)hop).getOp();
-
- // Check if aggregation OpCode is supported
- // Supported: SUM, MIN, MAX, SUM_SQ, MEAN, VAR, MAXINDEX, MININDEX
- if (!(aggOp == AggOp.SUM || aggOp == AggOp.MIN || aggOp == AggOp.MAX
- || aggOp == AggOp.SUM_SQ || aggOp == AggOp.MEAN || aggOp == AggOp.VAR
- || aggOp == AggOp.MAXINDEX || aggOp == AggOp.MININDEX)) {
- return null;
- }
-
- // Determine output FType based on aggregation direction
- boolean isColAgg = ((AggUnaryOp) hop).getDirection().isCol();
-
- // Full aggregation produces scalar result:
- // - ROW partition + column aggregation → scalar per row → local result
- // - COL partition + row aggregation → scalar per column → local result
- if ((firstFType == FType.ROW && isColAgg) ||
- (firstFType == FType.COL && !isColAgg)) {
- return null;
- }
-
- // Partial aggregation maintains structure:
- // - ROW partition + row aggregation → maintains ROW
- // - COL partition + column aggregation → maintains COL
- if (firstFType == FType.ROW || firstFType == FType.COL) {
- return firstFType;
- }
-
- // Other FTypes (FULL, BROADCAST) not affected by direction
- return null;
- }
-
- // ReorgOp: Reorganization operations that transform structure
- if (hop instanceof ReorgOp) {
- ReOrgOp op = ((ReorgOp)hop).getOp();
-
- // Unsupported operations:
- // - RESHAPE: Dimension changes break partitioning assumptions
- // - SORT: Requires global ordering across all partitions
- if (op == ReOrgOp.RESHAPE || op == ReOrgOp.SORT) {
- return null;
- }
-
- // TRANS: Transpose swaps ROW↔COL partitioning
- if (op == ReOrgOp.TRANS) {
- if (firstFType == FType.ROW) return FType.COL;
- if (firstFType == FType.COL) return FType.ROW;
- return firstFType; // FULL/BROADCAST unchanged
- }
-
- // Structure-maintaining operations: DIAG, REV, ROLL
- return firstFType;
- }
-
- // ParameterizedBuiltinOp: Builtin operations with parameters
- if (hop instanceof ParameterizedBuiltinOp) {
- ParamBuiltinOp op = ((ParameterizedBuiltinOp) hop).getOp();
-
- // CONTAINS returns scalar boolean result
- if (op == ParamBuiltinOp.CONTAINS) {
- return null;
- }
-
- // Check if operation is supported
- // Supported: REPLACE, RMEMPTY, LOWER_TRI, UPPER_TRI, TRANSFORMDECODE, TRANSFORMAPPLY, TOKENIZE
- if (!(op == ParamBuiltinOp.REPLACE || op == ParamBuiltinOp.RMEMPTY ||
- op == ParamBuiltinOp.LOWER_TRI || op == ParamBuiltinOp.UPPER_TRI ||
- op == ParamBuiltinOp.TRANSFORMDECODE || op == ParamBuiltinOp.TRANSFORMAPPLY ||
- op == ParamBuiltinOp.TOKENIZE)) {
- return null;
- }
-
- // Structure-preserving operations maintain input FType
- return firstFType;
- }
-
- // Default: Unknown operation type or unhandled case
- return null;
- }
-
- /**
- * Debug version of getFederatedType that logs detailed information about the decision process.
- * This method combines the logic of checking federated support and determining output FType,
- * while printing debug information to the terminal.
- *
- * @param hop The hop operation to analyze
- * @param fTypeMap Map containing FType information for all processed hops
- * @return The FType of the output, or null if the operation doesn't support federated execution
- * or produces non-federated output
- */
- private static FType getFederatedTypeDebug(Hop hop, Map<Long, FType> fTypeMap) {
- String reason = "";
- FType returnFType = null;
-
- // ========================================================================
- // PART 1: Universal constraints - operations that NEVER support federated
- // ========================================================================
-
- // Scalar values don't have FType (no partitioning concept for scalars)
- if (hop.isScalar()) {
- reason = "Scalar values don't have FType";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return null;
- }
-
- // Operations architecturally incompatible with federated execution:
- // - DataGenOp: All data generation requires centralized execution (RAND seed sync, SEQ global coords, etc.)
- // - DnnOp: Deep learning operations designed exclusively for CP/GPU (CuDNN dependencies)
- // - FunctionOp: Function calls execute locally on coordinator (no 'fcall' in FEDInstructionParser)
- // - LiteralOp: Constants without computation, created at coordinator only
- // - DataOp: Data operations (FEDERATED, TRANSIENTREAD, TRANSIENTWRITE) are handled separately, others are not supported (PERSISTENTWRITE/READ, FUNCTIONOUTPUT, SQLREAD)
- if (hop instanceof DataGenOp || hop instanceof DnnOp ||
- hop instanceof FunctionOp || hop instanceof LiteralOp ||
- hop instanceof DataOp) {
- if (hop instanceof DataGenOp) reason = "DataGenOp: requires centralized execution";
- else if (hop instanceof DnnOp) reason = "DnnOp: designed for CP/GPU only";
- else if (hop instanceof FunctionOp) reason = "FunctionOp: executes locally on coordinator";
- else if (hop instanceof LiteralOp) reason = "LiteralOp: constants created at coordinator";
- else if (hop instanceof DataOp) reason = "DataOp: handled separately";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return null;
- }
-
- // Extract input FTypes for analysis
- FType[] ft = new FType[hop.getInput().size()];
- for (int i = 0; i < hop.getInput().size(); i++)
- ft[i] = fTypeMap.get(hop.getInput(i).getHopID());
-
- // Handle operations with no inputs
- if (ft.length == 0) {
- reason = "No inputs available";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return null;
- }
-
- // Common patterns used across multiple operation types
- FType firstFType = ft[0];
- boolean hasFederatedFirstInput = firstFType != null;
-
- // ========================================================================
- // PART 2: Operations NOT requiring federated first input
- // ========================================================================
-
- // NaryOp: N-ary operations with matrix/list support
- if (hop instanceof NaryOp) {
- OpOpN op = ((NaryOp) hop).getOp();
-
- // Unsupported operations:
- // - PRINTF/EVAL: Output operations, execute on coordinator only
- // - LIST: List operations not federated
- // - CBIND/RBIND on lists: Only matrix operations supported
- // - Cell operations on all scalars: No distribution benefit
- if (op == OpOpN.PRINTF || op == OpOpN.EVAL || op == OpOpN.LIST ||
- ((op == OpOpN.CBIND || op == OpOpN.RBIND) &&
- hop.getInput().get(0).getDataType().isList()) ||
- (op.isCellOp() &&
- hop.getInput().stream().allMatch(h -> h.getDataType().isScalar()))) {
- if (op == OpOpN.PRINTF || op == OpOpN.EVAL) reason = "NaryOp: PRINTF/EVAL executes on coordinator only";
- else if (op == OpOpN.LIST) reason = "NaryOp: LIST operations not federated";
- else if ((op == OpOpN.CBIND || op == OpOpN.RBIND) && hop.getInput().get(0).getDataType().isList())
- reason = "NaryOp: CBIND/RBIND on lists not supported";
- else if (op.isCellOp() && hop.getInput().stream().allMatch(h -> h.getDataType().isScalar()))
- reason = "NaryOp: Cell operations on all scalars, no distribution benefit";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return null;
- }
-
- // Supported matrix operations: CBIND/RBIND (matrix concat), PLUS/MULT (arithmetic), MIN/MAX (comparison)
- if (op == OpOpN.CBIND || op == OpOpN.RBIND ||
- op == OpOpN.PLUS || op == OpOpN.MULT ||
- op == OpOpN.MIN || op == OpOpN.MAX) {
- FType secondFType = ft.length > 1 ? ft[1] : null;
- // Todo: propagate 3rd one if 2nd is null -> N
- returnFType = firstFType != null ? firstFType : secondFType;
- reason = "NaryOp: " + op + " propagates first non-null FType";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return returnFType;
- }
-
- // Other NaryOp operations not supported
- reason = "NaryOp: " + op + " not supported";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return null;
- }
-
- // TernaryOp: Three-input operations with complex federation patterns
- if (hop instanceof TernaryOp) {
- // Scalar output operations don't have FType
- if (hop.getDataType().isScalar()) {
- reason = "TernaryOp: Scalar output operations don't have FType";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return null;
- }
-
- // Operations that produce scalar output or are unsupported:
- // - MOMENT/COV: Aggregation operations produce scalar
- // - IFELSE/MAP: No federated implementation
- OpOp3 op = ((TernaryOp) hop).getOp();
- if (op == OpOp3.MOMENT || op == OpOp3.COV ||
- op == OpOp3.IFELSE || op == OpOp3.MAP) {
- if (op == OpOp3.MOMENT || op == OpOp3.COV) reason = "TernaryOp: " + op + " produces scalar output";
- else reason = "TernaryOp: " + op + " has no federated implementation";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return null;
- }
-
- // Check if any input is federated
- boolean hasAnyFederatedInput = false;
- boolean hasRowPartition = false;
- for (FType f : ft) {
- if (f == FType.ROW) {
- hasRowPartition = true;
- hasAnyFederatedInput = true;
- break;
- } else if (f != null) {
- hasAnyFederatedInput = true;
- }
- }
-
- // Requires at least one federated input
- // CTABLE: Special ROW partition requirement
- if (!hasAnyFederatedInput || (!hasRowPartition && op == OpOp3.CTABLE)) {
- if (!hasAnyFederatedInput) reason = "TernaryOp: No federated input";
- else reason = "TernaryOp: CTABLE requires ROW partition";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return null;
- }
-
- // All supported operations propagate first non-null FType
- FType secondFType = ft.length > 1 ? ft[1] : null;
- returnFType = firstFType != null ? firstFType :
- secondFType != null ? secondFType :
- ft.length > 2 ? ft[2] : null;
- reason = "TernaryOp: " + op + " propagates first non-null FType";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return returnFType;
- }
-
- // AggBinaryOp: Matrix multiplication and aggregation operations
- if (hop instanceof AggBinaryOp) {
- FType secondFType = ft.length > 1 ? ft[1] : null;
- boolean hasFederatedSecondInput = secondFType != null;
-
- // Check supported federation patterns
- if(!((hasFederatedFirstInput != hasFederatedSecondInput) || // One federated, one not
- (firstFType != null && firstFType == secondFType) || // Both federated with same type
- (firstFType == FType.COL && secondFType == FType.ROW))) { // Special matrix multiplication patterns
- reason = "AggBinaryOp: Unsupported federation pattern";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return null;
- }
-
- // Determine output FType based on operation type
- MMTSJType mmtsj = ((AggBinaryOp) hop).checkTransposeSelf();
-
- // Self-transpose matrix multiplication (X'X or XX') results in BROADCAST
- if (mmtsj != MMTSJType.NONE &&
- ((mmtsj.isLeft() && firstFType == FType.ROW) ||
- (mmtsj.isRight() && firstFType == FType.COL))) {
- returnFType = FType.BROADCAST;
- reason = "AggBinaryOp: Self-transpose multiplication results in BROADCAST";
- }
- // One federated input: propagate its FType
- else if ((firstFType != null) != (secondFType != null)) {
- returnFType = firstFType != null ? firstFType : secondFType;
- reason = "AggBinaryOp: One federated input, propagating its FType";
- }
- // COL x ROW multiplication results in ROW partitioning
- else if (firstFType == FType.COL && secondFType == FType.ROW) {
- returnFType = FType.ROW;
- reason = "AggBinaryOp: COL x ROW multiplication results in ROW";
- }
- // Same partition type: maintain it
- else if ((firstFType == FType.ROW && secondFType == FType.ROW) ||
- (firstFType == FType.COL && secondFType == FType.COL)) {
- returnFType = firstFType;
- reason = "AggBinaryOp: Same partition type maintained";
- }
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return returnFType;
- }
-
- // BinaryOp: Standard binary operations (+, -, *, /, min, max)
- if (hop instanceof BinaryOp) {
- // Scalar operations don't have FType
- if (hop.getDataType().isScalar()) {
- reason = "BinaryOp: Scalar operations don't have FType";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return null;
- }
-
- FType secondFType = ft.length > 1 ? ft[1] : null;
- boolean hasFederatedSecondInput = secondFType != null;
-
- // Unsupported patterns: no federated inputs, or both federated with different types
- if ((!hasFederatedFirstInput && !hasFederatedSecondInput) ||
- (hasFederatedFirstInput && hasFederatedSecondInput && firstFType != secondFType)) {
- if (!hasFederatedFirstInput && !hasFederatedSecondInput)
- reason = "BinaryOp: No federated inputs";
- else
- reason = "BinaryOp: Both inputs federated with different types";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return null;
- }
-
- // Propagate first non-null FType
- returnFType = firstFType != null ? firstFType : secondFType;
- reason = "BinaryOp: Propagating first non-null FType";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return returnFType;
- }
-
- // ========================================================================
- // PART 3: Operations REQUIRING federated first input
- // ========================================================================
-
- // All remaining operations require federated first input
- if (!hasFederatedFirstInput) {
- reason = "Operation requires federated first input but none found";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return null;
- }
-
- // Simple operations that maintain input structure:
- // - IndexingOp: Right indexing X[i:j, k:l] - subset of federated matrix remains federated
- // - LeftIndexingOp: Left-hand side indexing X[i:j, k:l] = Y - updates preserve partitioning
- if (hop instanceof IndexingOp || hop instanceof LeftIndexingOp) {
- returnFType = firstFType;
- reason = hop.getClass().getSimpleName() + ": Maintains input structure";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return returnFType;
- }
-
- // UnaryOp: Element-wise unary operations
- if (hop instanceof UnaryOp) {
- UnaryOp uop = (UnaryOp) hop;
- OpOp1 op = uop.getOp();
-
- // Unsupported operations:
- // - Output operations (PRINT, ASSERT, STOP): Execute on coordinator
- // - Type/metadata operations (TYPEOF, NROW, NCOL): Return scalars
- // - Complex decompositions (INVERSE, EIGEN, etc.): CP-only algorithms
- // - SQRT_MATRIX_JAVA: Special matrix square root, CP only
- // - List operations: List datatype not federated
- // - Metadata operations: Return scalar metadata
- if (op == OpOp1.PRINT || op == OpOp1.ASSERT || op == OpOp1.STOP ||
- op == OpOp1.TYPEOF || op == OpOp1.INVERSE || op == OpOp1.EIGEN ||
- op == OpOp1.CHOLESKY || op == OpOp1.DET || op == OpOp1.SVD ||
- op == OpOp1.SQRT_MATRIX_JAVA ||
- hop.getInput().get(0).getDataType() == DataType.LIST ||
- uop.isMetadataOperation()) {
- if (op == OpOp1.PRINT || op == OpOp1.ASSERT || op == OpOp1.STOP)
- reason = "UnaryOp: " + op + " executes on coordinator";
- else if (op == OpOp1.TYPEOF || uop.isMetadataOperation())
- reason = "UnaryOp: " + op + " returns scalar metadata";
- else if (op == OpOp1.INVERSE || op == OpOp1.EIGEN || op == OpOp1.CHOLESKY ||
- op == OpOp1.DET || op == OpOp1.SVD || op == OpOp1.SQRT_MATRIX_JAVA)
- reason = "UnaryOp: " + op + " requires CP-only algorithms";
- else if (hop.getInput().get(0).getDataType() == DataType.LIST)
- reason = "UnaryOp: List datatype not federated";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return null;
- }
-
- // Element-wise operations maintain structure
- returnFType = firstFType;
- reason = "UnaryOp: " + op + " element-wise operation maintains structure";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return returnFType;
- }
-
- // QuaternaryOp: Four-input weighted operations
- if (hop instanceof QuaternaryOp) {
- Types.OpOp4 op = ((QuaternaryOp) hop).getOp();
-
- // Scalar output operations:
- // - WSLOSS: Weighted squared loss (returns scalar loss value)
- // - WCEMM: Weighted cross entropy (returns scalar loss value)
- if (op == Types.OpOp4.WSLOSS || op == Types.OpOp4.WCEMM) {
- reason = "QuaternaryOp: " + op + " returns scalar loss value";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return null;
- }
-
- // Operations maintaining first input's structure:
- // - WSIGMOID: Weighted sigmoid
- // - WUMM: Weighted unary matrix multiplication
- if (op == Types.OpOp4.WSIGMOID || op == Types.OpOp4.WUMM) {
- returnFType = firstFType;
- reason = "QuaternaryOp: " + op + " maintains first input's structure";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return returnFType;
- }
-
- // WDIVMM: Weighted division matrix multiplication - use first non-null FType
- if (op == Types.OpOp4.WDIVMM) {
- FType firstNonNullFType = null;
- for (FType f : ft) {
- if (f != null) {
- firstNonNullFType = f;
- break;
- }
- }
- returnFType = firstNonNullFType;
- reason = "QuaternaryOp: WDIVMM uses first non-null FType";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return returnFType;
- }
-
- // Default: maintain first input's FType
- returnFType = firstFType;
- reason = "QuaternaryOp: " + op + " default behavior, maintains first input's FType";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return returnFType;
- }
-
- // AggUnaryOp: Aggregate unary operations with direction awareness
- if (hop instanceof AggUnaryOp) {
- AggOp aggOp = ((AggUnaryOp)hop).getOp();
-
- // Check if aggregation OpCode is supported
- // Supported: SUM, MIN, MAX, SUM_SQ, MEAN, VAR, MAXINDEX, MININDEX
- if (!(aggOp == AggOp.SUM || aggOp == AggOp.MIN || aggOp == AggOp.MAX
- || aggOp == AggOp.SUM_SQ || aggOp == AggOp.MEAN || aggOp == AggOp.VAR
- || aggOp == AggOp.MAXINDEX || aggOp == AggOp.MININDEX)) {
- reason = "AggUnaryOp: " + aggOp + " not supported";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return null;
- }
-
- // Determine output FType based on aggregation direction
- boolean isColAgg = ((AggUnaryOp) hop).getDirection().isCol();
-
- // Full aggregation produces scalar result:
- // - ROW partition + column aggregation → scalar per row → local result
- // - COL partition + row aggregation → scalar per column → local result
- if ((firstFType == FType.ROW && isColAgg) ||
- (firstFType == FType.COL && !isColAgg)) {
- reason = "AggUnaryOp: Full aggregation produces scalar result";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return null;
- }
-
- // Partial aggregation maintains structure:
- // - ROW partition + row aggregation → maintains ROW
- // - COL partition + column aggregation → maintains COL
- if (firstFType == FType.ROW || firstFType == FType.COL) {
- returnFType = firstFType;
- reason = "AggUnaryOp: Partial aggregation maintains structure";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return returnFType;
- }
-
- // Other FTypes (FULL, BROADCAST) not affected by direction
- reason = "AggUnaryOp: Other FTypes not affected by direction";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return null;
- }
-
- // ReorgOp: Reorganization operations that transform structure
- if (hop instanceof ReorgOp) {
- ReOrgOp op = ((ReorgOp)hop).getOp();
-
- // Unsupported operations:
- // - RESHAPE: Dimension changes break partitioning assumptions
- // - SORT: Requires global ordering across all partitions
- if (op == ReOrgOp.RESHAPE || op == ReOrgOp.SORT) {
- reason = "ReorgOp: " + op + " breaks partitioning assumptions";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return null;
- }
-
- // TRANS: Transpose swaps ROW↔COL partitioning
- if (op == ReOrgOp.TRANS) {
- if (firstFType == FType.ROW) {
- returnFType = FType.COL;
- reason = "ReorgOp: TRANS swaps ROW to COL";
- } else if (firstFType == FType.COL) {
- returnFType = FType.ROW;
- reason = "ReorgOp: TRANS swaps COL to ROW";
- } else {
- returnFType = firstFType; // FULL/BROADCAST unchanged
- reason = "ReorgOp: TRANS maintains FULL/BROADCAST";
- }
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return returnFType;
- }
-
- // Structure-maintaining operations: DIAG, REV, ROLL
- returnFType = firstFType;
- reason = "ReorgOp: " + op + " maintains structure";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return returnFType;
- }
-
- // ParameterizedBuiltinOp: Builtin operations with parameters
- if (hop instanceof ParameterizedBuiltinOp) {
- ParamBuiltinOp op = ((ParameterizedBuiltinOp) hop).getOp();
-
- // CONTAINS returns scalar boolean result
- if (op == ParamBuiltinOp.CONTAINS) {
- reason = "ParameterizedBuiltinOp: CONTAINS returns scalar boolean";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return null;
- }
-
- // Check if operation is supported
- // Supported: REPLACE, RMEMPTY, LOWER_TRI, UPPER_TRI, TRANSFORMDECODE, TRANSFORMAPPLY, TOKENIZE
- if (!(op == ParamBuiltinOp.REPLACE || op == ParamBuiltinOp.RMEMPTY ||
- op == ParamBuiltinOp.LOWER_TRI || op == ParamBuiltinOp.UPPER_TRI ||
- op == ParamBuiltinOp.TRANSFORMDECODE || op == ParamBuiltinOp.TRANSFORMAPPLY ||
- op == ParamBuiltinOp.TOKENIZE)) {
- reason = "ParameterizedBuiltinOp: " + op + " not supported";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return null;
- }
-
- // Structure-preserving operations maintain input FType
- returnFType = firstFType;
- reason = "ParameterizedBuiltinOp: " + op + " preserves structure";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return returnFType;
- }
-
- // Default: Unknown operation type or unhandled case
- reason = "Unknown operation type or unhandled case";
- FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
- return null;
- }
+
+ private static final double DEFAULT_LOOP_WEIGHT = 10.0;
+ private static final double DEFAULT_IF_ELSE_WEIGHT = 0.5;
+
+ public static final String FED_MATRIX_IDENTIFIER = "matrix";
+ public static final String FED_FRAME_IDENTIFIER = "frame";
+
+ public static void rewireProgram(DMLProgram prog, Map<Long, List<Hop>> rewireTable,
+ Map<Long, HopCommon> hopCommonTable, Map<Long, Privacy> privacyConstraintMap, Map<Long, FType> fTypeMap,
+ List<Pair<FederatedRange, FederatedData>> fedMap, Set<Long> unRefTwriteSet, Set<Long> unRefSet,
+ Set<Hop> progRootHopSet) {
+ // Maps Hop ID and fedOutType pairs to their plan variants
+ Set<Long> visitedHops = new HashSet<>();
+ Set<String> fnStack = new HashSet<>();
+ List<Pair<Long, Double>> loopStack = new ArrayList<>();
+
+ List<Map<String, List<Hop>>> outerTransTableList = new ArrayList<>();
+ Map<String, List<Hop>> outerTransTable = new HashMap<>();
+ outerTransTableList.add(outerTransTable);
+
+ for (StatementBlock sb : prog.getStatementBlocks()) {
+ Map<String, List<Hop>> innerTransTable = rewireStatementBlock(sb, prog, visitedHops, rewireTable,
+ hopCommonTable, outerTransTableList, null, privacyConstraintMap, fTypeMap,
+ fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack, 1, 1, loopStack);
+ outerTransTableList.get(0).putAll(innerTransTable);
+ }
+ }
+
+ public static void rewireFunctionDynamic(FunctionStatementBlock function, Map<Long, List<Hop>> rewireTable,
+ Map<Long, HopCommon> hopCommonTable, Map<Long, Privacy> privacyConstraintMap, Map<Long, FType> fTypeMap,
+ List<Pair<FederatedRange, FederatedData>> fedMap, Set<Long> unRefTwriteSet, Set<Long> unRefSet,
+ Set<Hop> progRootHopSet) {
+ Set<Long> visitedHops = new HashSet<>();
+ Set<String> fnStack = new HashSet<>();
+ List<Pair<Long, Double>> loopStack = new ArrayList<>();
+ List<Map<String, List<Hop>>> outerTransTableList = new ArrayList<>();
+ Map<String, List<Hop>> outerTransTable = new HashMap<>();
+ outerTransTableList.add(outerTransTable);
+ // Todo (Future): not tested & not used
+ rewireStatementBlock(function, null, visitedHops, rewireTable, hopCommonTable, outerTransTableList, null,
+ privacyConstraintMap, fTypeMap,
+ fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack, 1, 1, loopStack);
+ }
+
+ public static Map<String, List<Hop>> rewireStatementBlock(StatementBlock sb, DMLProgram prog, Set<Long> visitedHops,
+ Map<Long, List<Hop>> rewireTable, Map<Long, HopCommon> hopCommonTable,
+ List<Map<String, List<Hop>>> outerTransTableList, Map<String, List<Hop>> formerTransTable,
+ Map<Long, Privacy> privacyConstraintMap, Map<Long, FType> fTypeMap,
+ List<Pair<FederatedRange, FederatedData>> fedMap, Set<Long> unRefTwriteSet, Set<Long> unRefSet,
+ Set<Hop> progRootHopSet, Set<String> fnStack,
+ double computeWeight, double networkWeight, List<Pair<Long, Double>> parentLoopStack) {
+ List<Map<String, List<Hop>>> newOuterTransTableList = new ArrayList<>();
+ if (outerTransTableList != null) {
+ for (Map<String, List<Hop>> outerTable : outerTransTableList) {
+ if (outerTable != null && !outerTable.isEmpty()) {
+ newOuterTransTableList.add(outerTable);
+ }
+ }
+ }
+ if (formerTransTable != null && !formerTransTable.isEmpty()) {
+ newOuterTransTableList.add(formerTransTable);
+ }
+
+ Map<String, List<Hop>> newFormerTransTable = new HashMap<>();
+ Map<String, List<Hop>> innerTransTable = new HashMap<>();
+
+ if (sb instanceof IfStatementBlock) {
+ IfStatementBlock isb = (IfStatementBlock) sb;
+ IfStatement istmt = (IfStatement) isb.getStatement(0);
+
+ rewireHopDAG(isb.getPredicateHops(), prog, visitedHops, rewireTable, hopCommonTable, newOuterTransTableList,
+ null, innerTransTable,
+ privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack, computeWeight,
+ networkWeight, parentLoopStack);
+
+ newFormerTransTable.putAll(innerTransTable);
+ Map<String, List<Hop>> elseFormerTransTable = new HashMap<>();
+ elseFormerTransTable.putAll(innerTransTable);
+ computeWeight *= DEFAULT_IF_ELSE_WEIGHT;
+
+ for (StatementBlock innerIsb : istmt.getIfBody())
+ newFormerTransTable.putAll(rewireStatementBlock(innerIsb, prog, visitedHops, rewireTable,
+ hopCommonTable, newOuterTransTableList, newFormerTransTable,
+ privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack, computeWeight,
+ networkWeight, parentLoopStack));
+
+ for (StatementBlock innerIsb : istmt.getElseBody())
+ elseFormerTransTable.putAll(rewireStatementBlock(innerIsb, prog, visitedHops, rewireTable,
+ hopCommonTable, newOuterTransTableList, elseFormerTransTable,
+ privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack, computeWeight,
+ networkWeight, parentLoopStack));
+
+ // If there are common keys: merge elseValue list into ifValue list
+ elseFormerTransTable.forEach((key, elseValue) -> {
+ newFormerTransTable.merge(key, elseValue, (ifValue, newValue) -> {
+ ifValue.addAll(newValue);
+ return ifValue;
+ });
+ });
+ } else if (sb instanceof ForStatementBlock) { // incl parfor
+ ForStatementBlock fsb = (ForStatementBlock) sb;
+ ForStatement fstmt = (ForStatement) fsb.getStatement(0);
+
+ // Calculate for-loop iteration count if possible
+ double loopWeight = DEFAULT_LOOP_WEIGHT;
+ Hop from = fsb.getFromHops().getInput().get(0);
+ Hop to = fsb.getToHops().getInput().get(0);
+ Hop incr = (fsb.getIncrementHops() != null) ? fsb.getIncrementHops().getInput().get(0) : new LiteralOp(1);
+
+ // Calculate for-loop iteration count (weight) if from, to, and incr are literal
+ // ops (constant values)
+ if (from instanceof LiteralOp && to instanceof LiteralOp && incr instanceof LiteralOp) {
+ double dfrom = HopRewriteUtils.getDoubleValue((LiteralOp) from);
+ double dto = HopRewriteUtils.getDoubleValue((LiteralOp) to);
+ double dincr = HopRewriteUtils.getDoubleValue((LiteralOp) incr);
+ if (dfrom > dto && dincr == 1)
+ dincr = -1;
+ loopWeight = UtilFunctions.getSeqLength(dfrom, dto, dincr, false);
+ }
+ computeWeight *= loopWeight;
+ networkWeight *= loopWeight;
+
+ // Create current loop context (copy parent context)
+ List<Pair<Long, Double>> currentLoopStack = new ArrayList<>(parentLoopStack);
+ currentLoopStack.add(Pair.of(sb.getSBID(), loopWeight));
+
+ rewireHopDAG(fsb.getFromHops(), prog, visitedHops, rewireTable, hopCommonTable, newOuterTransTableList,
+ null, innerTransTable,
+ privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack, computeWeight,
+ networkWeight, currentLoopStack);
+ rewireHopDAG(fsb.getToHops(), prog, visitedHops, rewireTable, hopCommonTable, newOuterTransTableList, null,
+ innerTransTable,
+ privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack, computeWeight,
+ networkWeight, currentLoopStack);
+
+ if (fsb.getIncrementHops() != null) {
+ rewireHopDAG(fsb.getIncrementHops(), prog, visitedHops, rewireTable, hopCommonTable,
+ newOuterTransTableList, null, innerTransTable,
+ privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack, computeWeight,
+ networkWeight, currentLoopStack);
+ }
+ newFormerTransTable.putAll(innerTransTable);
+
+ for (StatementBlock innerFsb : fstmt.getBody())
+ newFormerTransTable.putAll(rewireStatementBlock(innerFsb, prog, visitedHops, rewireTable,
+ hopCommonTable, newOuterTransTableList, newFormerTransTable,
+ privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack, computeWeight,
+ networkWeight, currentLoopStack));
+
+ // Wire UnRefTwrite to liveOutHops
+ wireUnRefTwriteToLiveOut(fsb, unRefTwriteSet, hopCommonTable, newFormerTransTable);
+ } else if (sb instanceof WhileStatementBlock) {
+ WhileStatementBlock wsb = (WhileStatementBlock) sb;
+ WhileStatement wstmt = (WhileStatement) wsb.getStatement(0);
+
+ computeWeight *= DEFAULT_LOOP_WEIGHT;
+ networkWeight *= DEFAULT_LOOP_WEIGHT;
+
+ // Create current loop context (copy parent context)
+ List<Pair<Long, Double>> currentLoopStack = new ArrayList<>(parentLoopStack);
+ currentLoopStack.add(Pair.of(sb.getSBID(), DEFAULT_LOOP_WEIGHT));
+
+ rewireHopDAG(wsb.getPredicateHops(), prog, visitedHops, rewireTable, hopCommonTable, newOuterTransTableList,
+ null, innerTransTable,
+ privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack, computeWeight,
+ networkWeight, currentLoopStack);
+ newFormerTransTable.putAll(innerTransTable);
+
+ for (StatementBlock innerWsb : wstmt.getBody())
+ newFormerTransTable.putAll(rewireStatementBlock(innerWsb, prog, visitedHops, rewireTable,
+ hopCommonTable, newOuterTransTableList, newFormerTransTable,
+ privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack, computeWeight,
+ networkWeight, currentLoopStack));
+
+ // Wire UnRefTwrite to liveOutHops
+ wireUnRefTwriteToLiveOut(wsb, unRefTwriteSet, hopCommonTable, newFormerTransTable);
+ } else if (sb instanceof FunctionStatementBlock) {
+ FunctionStatementBlock fsb = (FunctionStatementBlock) sb;
+ FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0);
+
+ for (StatementBlock innerFsb : fstmt.getBody())
+ newFormerTransTable.putAll(rewireStatementBlock(innerFsb, prog, visitedHops, rewireTable,
+ hopCommonTable, newOuterTransTableList, newFormerTransTable,
+ privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack, computeWeight,
+ networkWeight, parentLoopStack));
+
+ // Wire fcall operation to liveOutHops
+ wireUnRefTwriteToLiveOut(fsb, unRefTwriteSet, hopCommonTable, newFormerTransTable);
+ } else { // generic (last-level)
+ if (sb.getHops() != null) {
+ for (Hop c : sb.getHops())
+ rewireHopDAG(c, prog, visitedHops, rewireTable, hopCommonTable, newOuterTransTableList, null,
+ innerTransTable,
+ privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack,
+ computeWeight, networkWeight, parentLoopStack);
+ }
+
+ return innerTransTable;
+ }
+ return newFormerTransTable;
+ }
+
+ private static void rewireHopDAG(Hop hop, DMLProgram prog, Set<Long> visitedHops, Map<Long, List<Hop>> rewireTable,
+ Map<Long, HopCommon> hopCommonTable, List<Map<String, List<Hop>>> outerTransTableList,
+ Map<String, List<Hop>> formerTransTable, Map<String, List<Hop>> innerTransTable,
+ Map<Long, Privacy> privacyConstraintMap, Map<Long, FType> fTypeMap,
+ List<Pair<FederatedRange, FederatedData>> fedMap, Set<Long> unRefTwriteSet, Set<Long> unRefSet,
+ Set<Hop> progRootHopSet,
+ Set<String> fnStack, double computeWeight, double networkWeight, List<Pair<Long, Double>> loopStack) {
+
+ if (hop.getInput() != null) {
+ for (Hop inputHop : hop.getInput()) {
+ long inputHopID = inputHop.getHopID();
+ if (!visitedHops.contains(inputHopID)) {
+ visitedHops.add(inputHopID);
+ rewireHopDAG(inputHop, prog, visitedHops, rewireTable, hopCommonTable, outerTransTableList,
+ formerTransTable, innerTransTable,
+ privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack,
+ computeWeight, networkWeight, loopStack);
+ }
+ }
+ }
+
+ hopCommonTable.put(hop.getHopID(), new HopCommon(hop, computeWeight, networkWeight, 0, loopStack));
+
+ // Identify hops to connect to the root dummy node
+ // Connect TWrite pred and u(print) to the root dummy node
+ if ((hop instanceof DataOp && (hop.getName().equals("__pred"))) // TWrite "__pred"
+ || (hop instanceof UnaryOp && ((UnaryOp) hop).getOp() == Types.OpOp1.PRINT) // u(print)
+ || (hop instanceof DataOp && ((DataOp) hop).getOp() == Types.OpOpData.PERSISTENTWRITE)) { // PWrite
+ progRootHopSet.add(hop);
+ } else if (!(hop instanceof DataOp && ((DataOp) hop).getOp() == Types.OpOpData.TRANSIENTWRITE)
+ && hop.getParent().size() == 0) {
+ unRefSet.add(hop.getHopID());
+ }
+
+ if (hop instanceof FunctionOp) {
+ // maintain counters and investigate functions if not seen so far
+ FunctionOp fop = (FunctionOp) hop;
+ unRefTwriteSet.add(fop.getHopID());
+
+ if (fop.getFunctionType() == FunctionType.DML) {
+ String fkey = fop.getFunctionKey();
+
+ if (!fnStack.contains(fkey)) {
+ fnStack.add(fkey);
+ FunctionStatementBlock fsb = prog.getFunctionStatementBlock(fop.getFunctionNamespace(),
+ fop.getFunctionName());
+
+ Map<String, List<Hop>> newFormerTransTable = new HashMap<>();
+ if (formerTransTable != null) {
+ newFormerTransTable.putAll(formerTransTable);
+ }
+ newFormerTransTable.putAll(innerTransTable);
+
+ String[] inputArgs = fop.getInputVariableNames();
+ List<Hop> inputHops = fop.getInput();
+
+ // Only used outside of functionTransTable.
+ for (int i = 0; i < inputHops.size(); i++) {
+ newFormerTransTable.computeIfAbsent(inputArgs[i], k -> new ArrayList<>()).add(inputHops.get(i));
+ }
+
+ Map<String, List<Hop>> functionTransTable = rewireStatementBlock(fsb, prog, visitedHops,
+ rewireTable, hopCommonTable, outerTransTableList, newFormerTransTable,
+ privacyConstraintMap, fTypeMap, fedMap, unRefTwriteSet, unRefSet, progRootHopSet, fnStack,
+ computeWeight, networkWeight, loopStack);
+
+ for (int i = 0; i < fop.getOutputVariableNames().length; i++) {
+ String tWriteName = fop.getOutputVariableNames()[i];
+ List<Hop> outputHops = functionTransTable.get(fsb.getOutputsofSB().get(i).getName());
+ innerTransTable.computeIfAbsent(tWriteName, k -> new ArrayList<>()).addAll(outputHops);
+ for (Hop outputHop : outputHops) {
+ unRefTwriteSet.add(outputHop.getHopID());
+ }
+ }
+ }
+ }
+ }
+
+ // Propagate Privacy Constraint
+ if (!(hop instanceof DataOp) || hop.getName().equals("__pred")
+ || (((DataOp) hop).getOp() == Types.OpOpData.PERSISTENTWRITE)) {
+ privacyConstraintMap.put(hop.getHopID(),
+ getPrivacyConstraint(hop, hop.getInput(), privacyConstraintMap));
+ // Todo: Remove this after debugging
+ // fTypeMap.put(hop.getHopID(), getFederatedType(hop, fTypeMap));
+ fTypeMap.put(hop.getHopID(), getFederatedTypeDebug(hop, fTypeMap));
+
+ // Todo: Remove this after debugging
+// FederatedPlannerLogger.logHopInfo(hop, privacyConstraintMap, fTypeMap, "RewireTransHop");
+ return;
+ }
+
+ rewireTransHop(hop, rewireTable, outerTransTableList, formerTransTable, innerTransTable, privacyConstraintMap,
+ fTypeMap, fedMap, unRefTwriteSet);
+ // Todo: Remove this after debugging
+// FederatedPlannerLogger.logHopInfo(hop, privacyConstraintMap, fTypeMap, "RewireTransHop");
+ }
+
+ private static void rewireTransHop(Hop hop, Map<Long, List<Hop>> rewireTable,
+ List<Map<String, List<Hop>>> outerTransTableList, Map<String, List<Hop>> formerTransTable,
+ Map<String, List<Hop>> innerTransTable, Map<Long, Privacy> privacyConstraintMap,
+ Map<Long, FType> fTypeMap, List<Pair<FederatedRange, FederatedData>> fedMap, Set<Long> unRefTwriteSet) {
+ DataOp dataOp = (DataOp) hop;
+ Types.OpOpData opType = dataOp.getOp();
+ String hopName = dataOp.getName();
+
+ if (opType == Types.OpOpData.FEDERATED) {
+ Privacy privacy = getFedWorkerMetaData(fedMap, dataOp);
+ privacyConstraintMap.put(hop.getHopID(), privacy);
+ FType fType = deriveFType((DataOp)hop);
+ fTypeMap.put(hop.getHopID(), fType);
+
+ // Debug logging for FEDERATED operation
+ FederatedPlannerLogger.logDataOpFTypeDebug(hop, fType, "FEDERATED", "Derived from partition ranges");
+ } else if (opType == Types.OpOpData.TRANSIENTWRITE) {
+ // Rewire TransWrite
+ innerTransTable.computeIfAbsent(hopName, k -> new ArrayList<>()).add(hop);
+ unRefTwriteSet.add(hop.getHopID());
+ // Propagate Privacy Constraint
+ privacyConstraintMap.put(hop.getHopID(),
+ getPrivacyConstraint(hop, hop.getInput(), privacyConstraintMap));
+ // Propagate FType (TransWrite has only one input)
+ FType inputFType = fTypeMap.get(hop.getInput(0).getHopID());
+ fTypeMap.put(hop.getHopID(), inputFType);
+
+ // Debug logging for TRANSIENTWRITE operation
+ FederatedPlannerLogger.logDataOpFTypeDebug(hop, inputFType, "TRANSIENTWRITE",
+ "Propagated from single input (HopID: " + hop.getInput(0).getHopID() + ")");
+ } else if (opType == Types.OpOpData.TRANSIENTREAD) {
+ // Rewire TransRead
+ List<Hop> childHops = rewireTransRead(hopName, innerTransTable, formerTransTable, outerTransTableList);
+ // Handle rewire table (TransRead -> TransWrite)
+ rewireTable.put(hop.getHopID(), childHops);
+
+ // Todo: Handle exception when TRead has no Child (check why it's missing)
+ if (childHops == null || childHops.isEmpty()) {
+ FederatedPlannerLogger.logTransReadRewireDebug(hopName, hop.getHopID(), childHops, true, "RewireTransHop");
+ return;
+ }
+
+ // Remove childHops that have different hopVarName
+ List<Hop> filteredChildHops = new ArrayList<>();
+ for (Hop childHop : childHops) {
+ String hopVarName = hop.getName();
+
+ if (hopVarName.equals(childHop.getName())) {
+ filteredChildHops.add(childHop);
+ }
+ }
+
+ // Todo: Handle exception when TRead has no Filtered Child (check why it's missing)
+ if (filteredChildHops.isEmpty()) {
+ FederatedPlannerLogger.logFilteredChildHopsDebug(hopName, hop.getHopID(), filteredChildHops, true, "RewireTransHop");
+ return;
+ }
+
+ FType inputFType = null;
+ StringBuilder debugInfo = new StringBuilder();
+ for (int i = 0; i < filteredChildHops.size(); i++) {
+ Hop filteredChildHop = filteredChildHops.get(i);
+ long filteredChildHopID = filteredChildHop.getHopID();
+ FType childFType = fTypeMap.get(filteredChildHopID);
+
+ // Rewire (TransWrite -> TransRead)
+ rewireTable.computeIfAbsent(filteredChildHopID, k -> new ArrayList<>()).add(hop);
+ // Remove refTWrite from unRefTwriteSet
+ unRefTwriteSet.remove(filteredChildHopID);
+
+ // Check FType consistency of childs(TransWrite)
+ if ( i==0 ) {
+ inputFType = childFType;
+ debugInfo.append("First child HopID: ").append(filteredChildHopID).append(" FType: ").append(childFType);
+ } else if (inputFType != childFType) {
+ // Todo: Handle exception when TRead has different FType
+ FType mismatchedFType = childFType;
+ FederatedPlannerLogger.logFTypeMismatchError(hop, filteredChildHops, fTypeMap, inputFType, mismatchedFType, i);
+
+ debugInfo.append(", Child ").append(i).append(" HopID: ").append(filteredChildHopID)
+ .append(" FType: ").append(mismatchedFType).append(" (MISMATCH)");
+
+ if (inputFType == null) {
+ inputFType = mismatchedFType;
+ }
+ // throw new DMLRuntimeException("TransRead input FType mismatch: " + inputFType + " != " + mismatchedFType);
+ } else {
+ debugInfo.append(", Child ").append(i).append(" HopID: ").append(filteredChildHopID)
+ .append(" FType: ").append(childFType).append(" (MATCH)");
+ }
+ }
+ // Propagate Privacy Constraint
+ privacyConstraintMap.put(hop.getHopID(),
+ getPrivacyConstraint(hop, filteredChildHops, privacyConstraintMap));
+ // Propagate FType
+ fTypeMap.put(hop.getHopID(), inputFType);
+
+ // Debug logging for TRANSIENTREAD operation
+ FederatedPlannerLogger.logDataOpFTypeDebug(hop, inputFType, "TRANSIENTREAD",
+ "Propagated from " + filteredChildHops.size() + " child(s): " + debugInfo.toString());
+ } else {
+ privacyConstraintMap.put(hop.getHopID(),
+ getPrivacyConstraint(hop, hop.getInput(), privacyConstraintMap));
+
+ // Todo: Remove this after debugging
+ // fTypeMap.put(hop.getHopID(), getFederatedType(hop, fTypeMap));
+ fTypeMap.put(hop.getHopID(), getFederatedTypeDebug(hop, fTypeMap));
+ }
+ }
+
+ private static List<Hop> rewireTransRead(String hopName, Map<String, List<Hop>> innerTransTable,
+ Map<String, List<Hop>> formerTransTable, List<Map<String, List<Hop>>> outerTransTableList) {
+ List<Hop> childHops = new ArrayList<>();
+
+ // Read according to priority: inner -> former -> outer
+ if (!innerTransTable.isEmpty()) {
+ childHops = innerTransTable.get(hopName);
+ }
+
+ if ((childHops == null || childHops.isEmpty()) && formerTransTable != null) {
+ childHops = formerTransTable.get(hopName);
+ }
+
+ if (childHops == null || childHops.isEmpty()) {
+ // Traverse in reverse order from the last inserted outerTransTable
+ for (int i = outerTransTableList.size() - 1; i >= 0; i--) {
+ Map<String, List<Hop>> outerTransTable = outerTransTableList.get(i);
+ childHops = outerTransTable.get(hopName);
+ if (childHops != null && !childHops.isEmpty())
+ break;
+ }
+ }
+
+ return childHops;
+ }
+
+ private static Privacy getFedWorkerMetaData(List<Pair<FederatedRange, FederatedData>> fedMap, DataOp initFedOp) {
+ // Address
+ Hop addressListHop = initFedOp.getInput(initFedOp.getParameterIndex("addresses"));
+ List<String> addressList = new ArrayList<>();
+ for (Hop addressHop : addressListHop.getInput()) {
+ addressList.add(addressHop.getName());
+ }
+
+ // Range
+ Hop rangeListHop = initFedOp.getInput(initFedOp.getParameterIndex("ranges"));
+ List<long[]> rangeList = new ArrayList<>();
+ for (Hop rangeHop : rangeListHop.getInput()) {
+ long beginRange = (long) Double.parseDouble(rangeHop.getInput(0).getName());
+ long endRange = (long) Double.parseDouble(rangeHop.getInput(1).getName());
+ rangeList.add(new long[] { beginRange, endRange });
+ }
+
+ // Type
+ String type = initFedOp.getInput(initFedOp.getParameterIndex("type")).getName();
+ Types.DataType fedDataType;
+
+ if (type.equalsIgnoreCase(FED_MATRIX_IDENTIFIER))
+ fedDataType = Types.DataType.MATRIX;
+ else
+ fedDataType = Types.DataType.FRAME;
+
+ // Init Fed Data
+ for (int i = 0; i < addressList.size(); i++) {
+ String address = addressList.get(i);
+ // We split address into url/ip, the port and file path of file to read
+ String[] parsedValues = InitFEDInstruction.parseURL(address);
+ String host = parsedValues[0];
+ int port = Integer.parseInt(parsedValues[1]);
+ String filePath = parsedValues[2];
+
+ long[] beginRange = rangeList.get(2 * i);
+ long[] endRange = rangeList.get(2 * i + 1);
+
+ try {
+ FederatedData federatedData = new FederatedData(fedDataType,
+ new InetSocketAddress(InetAddress.getByName(host), port), filePath);
+ fedMap.add(new ImmutablePair<>(new FederatedRange(beginRange, endRange), federatedData));
+ } catch (UnknownHostException e) {
+ throw new RuntimeException("federated host was unknown: " + host, e);
+ }
+ }
+ Privacy privacyConstraint = null;
+
+ // Request Privacy Constraints
+ for (Pair<FederatedRange, FederatedData> fed : fedMap) {
+ FederatedData data = fed.getRight();
+ data.initFederatedData(FederationUtils.getNextFedDataID());
+
+ Future<FederatedResponse> future = data.requestPrivacyConstraints();
+ try {
+ FederatedResponse response = future.get(); // Get actual response from Future
+
+ if (response.isSuccessful()) {
+ Object[] responseData = response.getData();
+ String privacyConstraints = (String) responseData[0]; // Cast privacy constraint as string
+ String pcLower = privacyConstraints.trim().toLowerCase();
+ Privacy tempPrivacy = null;
+
+ // Map to appropriate PrivacyConstraint value based on input string
+ if (pcLower.equals("private")
+ || pcLower.equals(FTypes.Privacy.PRIVATE.toString().toLowerCase())) {
+ tempPrivacy = FTypes.Privacy.PRIVATE;
+ } else if (pcLower.equals("private-aggregate") || pcLower.equals("private_aggregate") ||
+ pcLower.equals(FTypes.Privacy.PRIVATE_AGGREGATE.toString().toLowerCase())) {
+ tempPrivacy = FTypes.Privacy.PRIVATE_AGGREGATE;
+ } else if (pcLower.equals("public")
+ || pcLower.equals(FTypes.Privacy.PUBLIC.toString().toLowerCase())) {
+ tempPrivacy = FTypes.Privacy.PUBLIC;
+ } else {
+ throw new DMLRuntimeException("Invalid privacy constraint: " + privacyConstraints +
+ ". Must be one of 'PRIVATE', 'PRIVATE_AGGREGATE', 'PUBLIC'.");
+ }
+
+ if (privacyConstraint == null) {
+ privacyConstraint = tempPrivacy;
+ } else {
+ if (privacyConstraint != tempPrivacy) {
+ throw new DMLRuntimeException("Privacy constraints do not match.");
+ }
+ }
+ } else {
+ // Error handling
+ String errorMsg = response.getErrorMessage();
+ System.err.println("Failed to request privacy constraints: " + errorMsg);
+ }
+ } catch (Exception e) {
+ // Exception handling
+ e.printStackTrace();
+ }
+ }
+ return privacyConstraint;
+ }
+
+ private static Privacy getPrivacyConstraint(Hop hop, List<Hop> inputHops, Map<Long, Privacy> privacyMap) {
+ Privacy[] pc = new Privacy[inputHops.size()];
+ for (int i = 0; i < inputHops.size(); i++)
+ pc[i] = privacyMap.get(inputHops.get(i).getHopID());
+
+ boolean hasPrivateAggreate = false;
+
+ for (Privacy p : pc) {
+ if (p == Privacy.PRIVATE) {
+ return Privacy.PRIVATE;
+ } else if (p == Privacy.PRIVATE_AGGREGATE) {
+ hasPrivateAggreate = true;
+ }
+ }
+
+ if (hasPrivateAggreate) {
+ if (hop instanceof AggUnaryOp || hop instanceof AggBinaryOp || hop instanceof QuaternaryOp) {
+ return Privacy.PUBLIC;
+ } else if (hop instanceof TernaryOp) {
+ switch (((TernaryOp) hop).getOp()) {
+ case MOMENT:
+ case COV:
+ case CTABLE:
+ case INTERQUANTILE:
+ case QUANTILE:
+ return Privacy.PUBLIC;
+ default:
+ return Privacy.PRIVATE_AGGREGATE;
+ }
+ } else if (hop instanceof ParameterizedBuiltinOp
+ && ((ParameterizedBuiltinOp) hop).getOp() == ParamBuiltinOp.GROUPEDAGG) {
+ return Privacy.PUBLIC;
+ } else {
+ return Privacy.PRIVATE_AGGREGATE;
+ }
+ }
+
+ return Privacy.PUBLIC;
+ }
+
+ /**
+ * Determines the federated partition type (FType) for the output of a given hop operation.
+ * This method combines the logic of checking federated support and determining output FType.
+ *
+ * @param hop The hop operation to analyze
+ * @param fTypeMap Map containing FType information for all processed hops
+ * @return The FType of the output, or null if the operation doesn't support federated execution
+ * or produces non-federated output
+ */
+ @SuppressWarnings("unused")
+ private static FType getFederatedType(Hop hop, Map<Long, FType> fTypeMap) {
+ // ========================================================================
+ // PART 1: Universal constraints - operations that NEVER support federated
+ // ========================================================================
+
+ // Scalar values don't have FType (no partitioning concept for scalars)
+ if (hop.isScalar()) {
+ return null;
+ }
+
+ // Operations architecturally incompatible with federated execution:
+ // - DataGenOp: All data generation requires centralized execution (RAND seed sync, SEQ global coords, etc.)
+ // - DnnOp: Deep learning operations designed exclusively for CP/GPU (CuDNN dependencies)
+ // - FunctionOp: Function calls execute locally on coordinator (no 'fcall' in FEDInstructionParser)
+ // - LiteralOp: Constants without computation, created at coordinator only
+ // - DataOp: Data operations (FEDERATED, TRANSIENTREAD, TRANSIENTWRITE) are handled separately, others are not supported (PERSISTENTWRITE/READ, FUNCTIONOUTPUT, SQLREAD)
+ if (hop instanceof DataGenOp || hop instanceof DnnOp ||
+ hop instanceof FunctionOp || hop instanceof LiteralOp ||
+ hop instanceof DataOp) {
+ return null;
+ }
+
+ // Extract input FTypes for analysis
+ FType[] ft = new FType[hop.getInput().size()];
+ for (int i = 0; i < hop.getInput().size(); i++)
+ ft[i] = fTypeMap.get(hop.getInput(i).getHopID());
+
+ // Handle operations with no inputs
+ if (ft.length == 0) {
+ return null;
+ }
+
+ // Common patterns used across multiple operation types
+ FType firstFType = ft[0];
+ boolean hasFederatedFirstInput = firstFType != null;
+
+ // ========================================================================
+ // PART 2: Operations NOT requiring federated first input
+ // ========================================================================
+
+ // NaryOp: N-ary operations with matrix/list support
+ if (hop instanceof NaryOp) {
+ OpOpN op = ((NaryOp) hop).getOp();
+
+ // Unsupported operations:
+ // - PRINTF/EVAL: Output operations, execute on coordinator only
+ // - LIST: List operations not federated
+ // - CBIND/RBIND on lists: Only matrix operations supported
+ // - Cell operations on all scalars: No distribution benefit
+ if (op == OpOpN.PRINTF || op == OpOpN.EVAL || op == OpOpN.LIST ||
+ ((op == OpOpN.CBIND || op == OpOpN.RBIND) &&
+ hop.getInput().get(0).getDataType().isList()) ||
+ (op.isCellOp() &&
+ hop.getInput().stream().allMatch(h -> h.getDataType().isScalar()))) {
+ return null;
+ }
+
+ // Supported matrix operations: CBIND/RBIND (matrix concat), PLUS/MULT (arithmetic), MIN/MAX (comparison)
+ if (op == OpOpN.CBIND || op == OpOpN.RBIND ||
+ op == OpOpN.PLUS || op == OpOpN.MULT ||
+ op == OpOpN.MIN || op == OpOpN.MAX) {
+ FType secondFType = ft.length > 1 ? ft[1] : null;
+ // Todo: propagate 3rd one if 2nd is null -> N
+ return firstFType != null ? firstFType : secondFType;
+ }
+
+ // Other NaryOp operations not supported
+ return null;
+ }
+
+ // TernaryOp: Three-input operations with complex federation patterns
+ if (hop instanceof TernaryOp) {
+ // Scalar output operations don't have FType
+ if (hop.getDataType().isScalar()) {
+ return null;
+ }
+
+ // Operations that produce scalar output or are unsupported:
+ // - MOMENT/COV: Aggregation operations produce scalar
+ // - IFELSE/MAP: No federated implementation
+ OpOp3 op = ((TernaryOp) hop).getOp();
+ if (op == OpOp3.MOMENT || op == OpOp3.COV ||
+ op == OpOp3.IFELSE || op == OpOp3.MAP) {
+ return null;
+ }
+
+ // Check if any input is federated
+ boolean hasAnyFederatedInput = false;
+ boolean hasRowPartition = false;
+ for (FType f : ft) {
+ if (f == FType.ROW) {
+ hasRowPartition = true;
+ hasAnyFederatedInput = true;
+ break;
+ } else if (f != null) {
+ hasAnyFederatedInput = true;
+ }
+ }
+
+ // Requires at least one federated input
+ // CTABLE: Special ROW partition requirement
+ if (!hasAnyFederatedInput || (!hasRowPartition && op == OpOp3.CTABLE)) {
+ return null;
+ }
+
+ // All supported operations propagate first non-null FType
+ FType secondFType = ft.length > 1 ? ft[1] : null;
+ return firstFType != null ? firstFType :
+ secondFType != null ? secondFType :
+ ft.length > 2 ? ft[2] : null;
+ }
+
+ // AggBinaryOp: Matrix multiplication and aggregation operations
+ if (hop instanceof AggBinaryOp) {
+ FType secondFType = ft.length > 1 ? ft[1] : null;
+ boolean hasFederatedSecondInput = secondFType != null;
+
+ // Check supported federation patterns
+ if(!((hasFederatedFirstInput != hasFederatedSecondInput) || // One federated, one not
+ (firstFType != null && firstFType == secondFType) || // Both federated with same type
+ (firstFType == FType.COL && secondFType == FType.ROW))) { // Special matrix multiplication patterns
+ return null;
+ }
+
+ // Determine output FType based on operation type
+ MMTSJType mmtsj = ((AggBinaryOp) hop).checkTransposeSelf();
+
+ // Self-transpose matrix multiplication (X'X or XX') results in BROADCAST
+ if (mmtsj != MMTSJType.NONE &&
+ ((mmtsj.isLeft() && firstFType == FType.ROW) ||
+ (mmtsj.isRight() && firstFType == FType.COL))) {
+ return FType.BROADCAST;
+ }
+ // One federated input: propagate its FType
+ else if ((firstFType != null) != (secondFType != null)) {
+ return firstFType != null ? firstFType : secondFType;
+ }
+ // COL x ROW multiplication results in ROW partitioning
+ else if (firstFType == FType.COL && secondFType == FType.ROW) {
+ return FType.ROW;
+ }
+ // Same partition type: maintain it
+ else if ((firstFType == FType.ROW && secondFType == FType.ROW) ||
+ (firstFType == FType.COL && secondFType == FType.COL)) {
+ return firstFType;
+ }
+ return null;
+ }
+
+ // BinaryOp: Standard binary operations (+, -, *, /, min, max)
+ if (hop instanceof BinaryOp) {
+ // Scalar operations don't have FType
+ if (hop.getDataType().isScalar()) {
+ return null;
+ }
+
+ FType secondFType = ft.length > 1 ? ft[1] : null;
+ boolean hasFederatedSecondInput = secondFType != null;
+
+ // Unsupported patterns: no federated inputs, or both federated with different types
+ if ((!hasFederatedFirstInput && !hasFederatedSecondInput) ||
+ (hasFederatedFirstInput && hasFederatedSecondInput && firstFType != secondFType)) {
+ return null;
+ }
+
+ // Propagate first non-null FType
+ return firstFType != null ? firstFType : secondFType;
+ }
+
+ // ========================================================================
+ // PART 3: Operations REQUIRING federated first input
+ // ========================================================================
+
+ // All remaining operations require federated first input
+ if (!hasFederatedFirstInput) {
+ return null;
+ }
+
+ // Simple operations that maintain input structure:
+ // - IndexingOp: Right indexing X[i:j, k:l] - subset of federated matrix remains federated
+ // - LeftIndexingOp: Left-hand side indexing X[i:j, k:l] = Y - updates preserve partitioning
+ if (hop instanceof IndexingOp || hop instanceof LeftIndexingOp) {
+ return firstFType;
+ }
+
+ // UnaryOp: Element-wise unary operations
+ if (hop instanceof UnaryOp) {
+ UnaryOp uop = (UnaryOp) hop;
+ OpOp1 op = uop.getOp();
+
+ // Unsupported operations:
+ // - Output operations (PRINT, ASSERT, STOP): Execute on coordinator
+ // - Type/metadata operations (TYPEOF, NROW, NCOL): Return scalars
+ // - Complex decompositions (INVERSE, EIGEN, etc.): CP-only algorithms
+ // - SQRT_MATRIX_JAVA: Special matrix square root, CP only
+ // - List operations: List datatype not federated
+ // - Metadata operations: Return scalar metadata
+ if (op == OpOp1.PRINT || op == OpOp1.ASSERT || op == OpOp1.STOP ||
+ op == OpOp1.TYPEOF || op == OpOp1.INVERSE || op == OpOp1.EIGEN ||
+ op == OpOp1.CHOLESKY || op == OpOp1.DET || op == OpOp1.SVD ||
+ op == OpOp1.SQRT_MATRIX_JAVA ||
+ hop.getInput().get(0).getDataType() == DataType.LIST ||
+ uop.isMetadataOperation()) {
+ return null;
+ }
+
+ // Element-wise operations maintain structure
+ return firstFType;
+ }
+
+ // QuaternaryOp: Four-input weighted operations
+ if (hop instanceof QuaternaryOp) {
+ Types.OpOp4 op = ((QuaternaryOp) hop).getOp();
+
+ // Scalar output operations:
+ // - WSLOSS: Weighted squared loss (returns scalar loss value)
+ // - WCEMM: Weighted cross entropy (returns scalar loss value)
+ if (op == Types.OpOp4.WSLOSS || op == Types.OpOp4.WCEMM) {
+ return null;
+ }
+
+ // Operations maintaining first input's structure:
+ // - WSIGMOID: Weighted sigmoid
+ // - WUMM: Weighted unary matrix multiplication
+ if (op == Types.OpOp4.WSIGMOID || op == Types.OpOp4.WUMM) {
+ return firstFType;
+ }
+
+ // WDIVMM: Weighted division matrix multiplication - use first non-null FType
+ if (op == Types.OpOp4.WDIVMM) {
+ FType firstNonNullFType = null;
+ for (FType f : ft) {
+ if (f != null) {
+ firstNonNullFType = f;
+ break;
+ }
+ }
+ return firstNonNullFType;
+ }
+
+ // Default: maintain first input's FType
+ return firstFType;
+ }
+
+ // AggUnaryOp: Aggregate unary operations with direction awareness
+ if (hop instanceof AggUnaryOp) {
+ AggOp aggOp = ((AggUnaryOp)hop).getOp();
+
+ // Check if aggregation OpCode is supported
+ // Supported: SUM, MIN, MAX, SUM_SQ, MEAN, VAR, MAXINDEX, MININDEX
+ if (!(aggOp == AggOp.SUM || aggOp == AggOp.MIN || aggOp == AggOp.MAX
+ || aggOp == AggOp.SUM_SQ || aggOp == AggOp.MEAN || aggOp == AggOp.VAR
+ || aggOp == AggOp.MAXINDEX || aggOp == AggOp.MININDEX)) {
+ return null;
+ }
+
+ // Determine output FType based on aggregation direction
+ boolean isColAgg = ((AggUnaryOp) hop).getDirection().isCol();
+
+ // Full aggregation produces scalar result:
+ // - ROW partition + column aggregation → scalar per row → local result
+ // - COL partition + row aggregation → scalar per column → local result
+ if ((firstFType == FType.ROW && isColAgg) ||
+ (firstFType == FType.COL && !isColAgg)) {
+ return null;
+ }
+
+ // Partial aggregation maintains structure:
+ // - ROW partition + row aggregation → maintains ROW
+ // - COL partition + column aggregation → maintains COL
+ if (firstFType == FType.ROW || firstFType == FType.COL) {
+ return firstFType;
+ }
+
+ // Other FTypes (FULL, BROADCAST) not affected by direction
+ return null;
+ }
+
+ // ReorgOp: Reorganization operations that transform structure
+ if (hop instanceof ReorgOp) {
+ ReOrgOp op = ((ReorgOp)hop).getOp();
+
+ // Unsupported operations:
+ // - RESHAPE: Dimension changes break partitioning assumptions
+ // - SORT: Requires global ordering across all partitions
+ if (op == ReOrgOp.RESHAPE || op == ReOrgOp.SORT) {
+ return null;
+ }
+
+ // TRANS: Transpose swaps ROW↔COL partitioning
+ if (op == ReOrgOp.TRANS) {
+ if (firstFType == FType.ROW) return FType.COL;
+ if (firstFType == FType.COL) return FType.ROW;
+ return firstFType; // FULL/BROADCAST unchanged
+ }
+
+ // Structure-maintaining operations: DIAG, REV, ROLL
+ return firstFType;
+ }
+
+ // ParameterizedBuiltinOp: Builtin operations with parameters
+ if (hop instanceof ParameterizedBuiltinOp) {
+ ParamBuiltinOp op = ((ParameterizedBuiltinOp) hop).getOp();
+
+ // CONTAINS returns scalar boolean result
+ if (op == ParamBuiltinOp.CONTAINS) {
+ return null;
+ }
+
+ // Check if operation is supported
+ // Supported: REPLACE, RMEMPTY, LOWER_TRI, UPPER_TRI, TRANSFORMDECODE, TRANSFORMAPPLY, TOKENIZE
+ if (!(op == ParamBuiltinOp.REPLACE || op == ParamBuiltinOp.RMEMPTY ||
+ op == ParamBuiltinOp.LOWER_TRI || op == ParamBuiltinOp.UPPER_TRI ||
+ op == ParamBuiltinOp.TRANSFORMDECODE || op == ParamBuiltinOp.TRANSFORMAPPLY ||
+ op == ParamBuiltinOp.TOKENIZE)) {
+ return null;
+ }
+
+ // Structure-preserving operations maintain input FType
+ return firstFType;
+ }
+
+ // Default: Unknown operation type or unhandled case
+ return null;
+ }
+
+ /**
+ * Debug version of getFederatedType that logs detailed information about the decision process.
+ * This method combines the logic of checking federated support and determining output FType,
+ * while printing debug information to the terminal.
+ *
+ * @param hop The hop operation to analyze
+ * @param fTypeMap Map containing FType information for all processed hops
+ * @return The FType of the output, or null if the operation doesn't support federated execution
+ * or produces non-federated output
+ */
+ private static FType getFederatedTypeDebug(Hop hop, Map<Long, FType> fTypeMap) {
+ String reason = "";
+ FType returnFType = null;
+
+ // ========================================================================
+ // PART 1: Universal constraints - operations that NEVER support federated
+ // ========================================================================
+
+ // Scalar values don't have FType (no partitioning concept for scalars)
+ if (hop.isScalar()) {
+ reason = "Scalar values don't have FType";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return null;
+ }
+
+ // Operations architecturally incompatible with federated execution:
+ // - DataGenOp: All data generation requires centralized execution (RAND seed sync, SEQ global coords, etc.)
+ // - DnnOp: Deep learning operations designed exclusively for CP/GPU (CuDNN dependencies)
+ // - FunctionOp: Function calls execute locally on coordinator (no 'fcall' in FEDInstructionParser)
+ // - LiteralOp: Constants without computation, created at coordinator only
+ // - DataOp: Data operations (FEDERATED, TRANSIENTREAD, TRANSIENTWRITE) are handled separately, others are not supported (PERSISTENTWRITE/READ, FUNCTIONOUTPUT, SQLREAD)
+ if (hop instanceof DataGenOp || hop instanceof DnnOp ||
+ hop instanceof FunctionOp || hop instanceof LiteralOp ||
+ hop instanceof DataOp) {
+ if (hop instanceof DataGenOp) reason = "DataGenOp: requires centralized execution";
+ else if (hop instanceof DnnOp) reason = "DnnOp: designed for CP/GPU only";
+ else if (hop instanceof FunctionOp) reason = "FunctionOp: executes locally on coordinator";
+ else if (hop instanceof LiteralOp) reason = "LiteralOp: constants created at coordinator";
+ else if (hop instanceof DataOp) reason = "DataOp: handled separately";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return null;
+ }
+
+ // Extract input FTypes for analysis
+ FType[] ft = new FType[hop.getInput().size()];
+ for (int i = 0; i < hop.getInput().size(); i++)
+ ft[i] = fTypeMap.get(hop.getInput(i).getHopID());
+
+ // Handle operations with no inputs
+ if (ft.length == 0) {
+ reason = "No inputs available";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return null;
+ }
+
+ // Common patterns used across multiple operation types
+ FType firstFType = ft[0];
+ boolean hasFederatedFirstInput = firstFType != null;
+
+ // ========================================================================
+ // PART 2: Operations NOT requiring federated first input
+ // ========================================================================
+
+ // NaryOp: N-ary operations with matrix/list support
+ if (hop instanceof NaryOp) {
+ OpOpN op = ((NaryOp) hop).getOp();
+
+ // Unsupported operations:
+ // - PRINTF/EVAL: Output operations, execute on coordinator only
+ // - LIST: List operations not federated
+ // - CBIND/RBIND on lists: Only matrix operations supported
+ // - Cell operations on all scalars: No distribution benefit
+ if (op == OpOpN.PRINTF || op == OpOpN.EVAL || op == OpOpN.LIST ||
+ ((op == OpOpN.CBIND || op == OpOpN.RBIND) &&
+ hop.getInput().get(0).getDataType().isList()) ||
+ (op.isCellOp() &&
+ hop.getInput().stream().allMatch(h -> h.getDataType().isScalar()))) {
+ if (op == OpOpN.PRINTF || op == OpOpN.EVAL) reason = "NaryOp: PRINTF/EVAL executes on coordinator only";
+ else if (op == OpOpN.LIST) reason = "NaryOp: LIST operations not federated";
+ else if ((op == OpOpN.CBIND || op == OpOpN.RBIND) && hop.getInput().get(0).getDataType().isList())
+ reason = "NaryOp: CBIND/RBIND on lists not supported";
+ else if (op.isCellOp() && hop.getInput().stream().allMatch(h -> h.getDataType().isScalar()))
+ reason = "NaryOp: Cell operations on all scalars, no distribution benefit";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return null;
+ }
+
+ // Supported matrix operations: CBIND/RBIND (matrix concat), PLUS/MULT (arithmetic), MIN/MAX (comparison)
+ if (op == OpOpN.CBIND || op == OpOpN.RBIND ||
+ op == OpOpN.PLUS || op == OpOpN.MULT ||
+ op == OpOpN.MIN || op == OpOpN.MAX) {
+ FType secondFType = ft.length > 1 ? ft[1] : null;
+ // Todo: propagate 3rd one if 2nd is null -> N
+ returnFType = firstFType != null ? firstFType : secondFType;
+ reason = "NaryOp: " + op + " propagates first non-null FType";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return returnFType;
+ }
+
+ // Other NaryOp operations not supported
+ reason = "NaryOp: " + op + " not supported";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return null;
+ }
+
+ // TernaryOp: Three-input operations with complex federation patterns
+ if (hop instanceof TernaryOp) {
+ // Scalar output operations don't have FType
+ if (hop.getDataType().isScalar()) {
+ reason = "TernaryOp: Scalar output operations don't have FType";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return null;
+ }
+
+ // Operations that produce scalar output or are unsupported:
+ // - MOMENT/COV: Aggregation operations produce scalar
+ // - IFELSE/MAP: No federated implementation
+ OpOp3 op = ((TernaryOp) hop).getOp();
+ if (op == OpOp3.MOMENT || op == OpOp3.COV ||
+ op == OpOp3.IFELSE || op == OpOp3.MAP) {
+ if (op == OpOp3.MOMENT || op == OpOp3.COV) reason = "TernaryOp: " + op + " produces scalar output";
+ else reason = "TernaryOp: " + op + " has no federated implementation";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return null;
+ }
+
+ // Check if any input is federated
+ boolean hasAnyFederatedInput = false;
+ boolean hasRowPartition = false;
+ for (FType f : ft) {
+ if (f == FType.ROW) {
+ hasRowPartition = true;
+ hasAnyFederatedInput = true;
+ break;
+ } else if (f != null) {
+ hasAnyFederatedInput = true;
+ }
+ }
+
+ // Requires at least one federated input
+ // CTABLE: Special ROW partition requirement
+ if (!hasAnyFederatedInput || (!hasRowPartition && op == OpOp3.CTABLE)) {
+ if (!hasAnyFederatedInput) reason = "TernaryOp: No federated input";
+ else reason = "TernaryOp: CTABLE requires ROW partition";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return null;
+ }
+
+ // All supported operations propagate first non-null FType
+ FType secondFType = ft.length > 1 ? ft[1] : null;
+ returnFType = firstFType != null ? firstFType :
+ secondFType != null ? secondFType :
+ ft.length > 2 ? ft[2] : null;
+ reason = "TernaryOp: " + op + " propagates first non-null FType";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return returnFType;
+ }
+
+ // AggBinaryOp: Matrix multiplication and aggregation operations
+ if (hop instanceof AggBinaryOp) {
+ FType secondFType = ft.length > 1 ? ft[1] : null;
+ boolean hasFederatedSecondInput = secondFType != null;
+
+ // Check supported federation patterns
+ if(!((hasFederatedFirstInput != hasFederatedSecondInput) || // One federated, one not
+ (firstFType != null && firstFType == secondFType) || // Both federated with same type
+ (firstFType == FType.COL && secondFType == FType.ROW))) { // Special matrix multiplication patterns
+ reason = "AggBinaryOp: Unsupported federation pattern";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return null;
+ }
+
+ // Determine output FType based on operation type
+ MMTSJType mmtsj = ((AggBinaryOp) hop).checkTransposeSelf();
+
+ // Self-transpose matrix multiplication (X'X or XX') results in BROADCAST
+ if (mmtsj != MMTSJType.NONE &&
+ ((mmtsj.isLeft() && firstFType == FType.ROW) ||
+ (mmtsj.isRight() && firstFType == FType.COL))) {
+ returnFType = FType.BROADCAST;
+ reason = "AggBinaryOp: Self-transpose multiplication results in BROADCAST";
+ }
+ // One federated input: propagate its FType
+ else if ((firstFType != null) != (secondFType != null)) {
+ returnFType = firstFType != null ? firstFType : secondFType;
+ reason = "AggBinaryOp: One federated input, propagating its FType";
+ }
+ // COL x ROW multiplication results in ROW partitioning
+ else if (firstFType == FType.COL && secondFType == FType.ROW) {
+ returnFType = FType.ROW;
+ reason = "AggBinaryOp: COL x ROW multiplication results in ROW";
+ }
+ // Same partition type: maintain it
+ else if ((firstFType == FType.ROW && secondFType == FType.ROW) ||
+ (firstFType == FType.COL && secondFType == FType.COL)) {
+ returnFType = firstFType;
+ reason = "AggBinaryOp: Same partition type maintained";
+ }
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return returnFType;
+ }
+
+ // BinaryOp: Standard binary operations (+, -, *, /, min, max)
+ if (hop instanceof BinaryOp) {
+ // Scalar operations don't have FType
+ if (hop.getDataType().isScalar()) {
+ reason = "BinaryOp: Scalar operations don't have FType";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return null;
+ }
+
+ FType secondFType = ft.length > 1 ? ft[1] : null;
+ boolean hasFederatedSecondInput = secondFType != null;
+
+ // Unsupported patterns: no federated inputs, or both federated with different types
+ if ((!hasFederatedFirstInput && !hasFederatedSecondInput) ||
+ (hasFederatedFirstInput && hasFederatedSecondInput && firstFType != secondFType)) {
+ if (!hasFederatedFirstInput && !hasFederatedSecondInput)
+ reason = "BinaryOp: No federated inputs";
+ else
+ reason = "BinaryOp: Both inputs federated with different types";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return null;
+ }
+
+ // Propagate first non-null FType
+ returnFType = firstFType != null ? firstFType : secondFType;
+ reason = "BinaryOp: Propagating first non-null FType";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return returnFType;
+ }
+
+ // ========================================================================
+ // PART 3: Operations REQUIRING federated first input
+ // ========================================================================
+
+ // All remaining operations require federated first input
+ if (!hasFederatedFirstInput) {
+ reason = "Operation requires federated first input but none found";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return null;
+ }
+
+ // Simple operations that maintain input structure:
+ // - IndexingOp: Right indexing X[i:j, k:l] - subset of federated matrix remains federated
+ // - LeftIndexingOp: Left-hand side indexing X[i:j, k:l] = Y - updates preserve partitioning
+ if (hop instanceof IndexingOp || hop instanceof LeftIndexingOp) {
+ returnFType = firstFType;
+ reason = hop.getClass().getSimpleName() + ": Maintains input structure";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return returnFType;
+ }
+
+ // UnaryOp: Element-wise unary operations
+ if (hop instanceof UnaryOp) {
+ UnaryOp uop = (UnaryOp) hop;
+ OpOp1 op = uop.getOp();
+
+ // Unsupported operations:
+ // - Output operations (PRINT, ASSERT, STOP): Execute on coordinator
+ // - Type/metadata operations (TYPEOF, NROW, NCOL): Return scalars
+ // - Complex decompositions (INVERSE, EIGEN, etc.): CP-only algorithms
+ // - SQRT_MATRIX_JAVA: Special matrix square root, CP only
+ // - List operations: List datatype not federated
+ // - Metadata operations: Return scalar metadata
+ if (op == OpOp1.PRINT || op == OpOp1.ASSERT || op == OpOp1.STOP ||
+ op == OpOp1.TYPEOF || op == OpOp1.INVERSE || op == OpOp1.EIGEN ||
+ op == OpOp1.CHOLESKY || op == OpOp1.DET || op == OpOp1.SVD ||
+ op == OpOp1.SQRT_MATRIX_JAVA ||
+ hop.getInput().get(0).getDataType() == DataType.LIST ||
+ uop.isMetadataOperation()) {
+ if (op == OpOp1.PRINT || op == OpOp1.ASSERT || op == OpOp1.STOP)
+ reason = "UnaryOp: " + op + " executes on coordinator";
+ else if (op == OpOp1.TYPEOF || uop.isMetadataOperation())
+ reason = "UnaryOp: " + op + " returns scalar metadata";
+ else if (op == OpOp1.INVERSE || op == OpOp1.EIGEN || op == OpOp1.CHOLESKY ||
+ op == OpOp1.DET || op == OpOp1.SVD || op == OpOp1.SQRT_MATRIX_JAVA)
+ reason = "UnaryOp: " + op + " requires CP-only algorithms";
+ else if (hop.getInput().get(0).getDataType() == DataType.LIST)
+ reason = "UnaryOp: List datatype not federated";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return null;
+ }
+
+ // Element-wise operations maintain structure
+ returnFType = firstFType;
+ reason = "UnaryOp: " + op + " element-wise operation maintains structure";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return returnFType;
+ }
+
+ // QuaternaryOp: Four-input weighted operations
+ if (hop instanceof QuaternaryOp) {
+ Types.OpOp4 op = ((QuaternaryOp) hop).getOp();
+
+ // Scalar output operations:
+ // - WSLOSS: Weighted squared loss (returns scalar loss value)
+ // - WCEMM: Weighted cross entropy (returns scalar loss value)
+ if (op == Types.OpOp4.WSLOSS || op == Types.OpOp4.WCEMM) {
+ reason = "QuaternaryOp: " + op + " returns scalar loss value";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return null;
+ }
+
+ // Operations maintaining first input's structure:
+ // - WSIGMOID: Weighted sigmoid
+ // - WUMM: Weighted unary matrix multiplication
+ if (op == Types.OpOp4.WSIGMOID || op == Types.OpOp4.WUMM) {
+ returnFType = firstFType;
+ reason = "QuaternaryOp: " + op + " maintains first input's structure";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return returnFType;
+ }
+
+ // WDIVMM: Weighted division matrix multiplication - use first non-null FType
+ if (op == Types.OpOp4.WDIVMM) {
+ FType firstNonNullFType = null;
+ for (FType f : ft) {
+ if (f != null) {
+ firstNonNullFType = f;
+ break;
+ }
+ }
+ returnFType = firstNonNullFType;
+ reason = "QuaternaryOp: WDIVMM uses first non-null FType";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return returnFType;
+ }
+
+ // Default: maintain first input's FType
+ returnFType = firstFType;
+ reason = "QuaternaryOp: " + op + " default behavior, maintains first input's FType";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return returnFType;
+ }
+
+ // AggUnaryOp: Aggregate unary operations with direction awareness
+ if (hop instanceof AggUnaryOp) {
+ AggOp aggOp = ((AggUnaryOp)hop).getOp();
+
+ // Check if aggregation OpCode is supported
+ // Supported: SUM, MIN, MAX, SUM_SQ, MEAN, VAR, MAXINDEX, MININDEX
+ if (!(aggOp == AggOp.SUM || aggOp == AggOp.MIN || aggOp == AggOp.MAX
+ || aggOp == AggOp.SUM_SQ || aggOp == AggOp.MEAN || aggOp == AggOp.VAR
+ || aggOp == AggOp.MAXINDEX || aggOp == AggOp.MININDEX)) {
+ reason = "AggUnaryOp: " + aggOp + " not supported";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return null;
+ }
+
+ // Determine output FType based on aggregation direction
+ boolean isColAgg = ((AggUnaryOp) hop).getDirection().isCol();
+
+ // Full aggregation produces scalar result:
+ // - ROW partition + column aggregation → scalar per row → local result
+ // - COL partition + row aggregation → scalar per column → local result
+ if ((firstFType == FType.ROW && isColAgg) ||
+ (firstFType == FType.COL && !isColAgg)) {
+ reason = "AggUnaryOp: Full aggregation produces scalar result";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return null;
+ }
+
+ // Partial aggregation maintains structure:
+ // - ROW partition + row aggregation → maintains ROW
+ // - COL partition + column aggregation → maintains COL
+ if (firstFType == FType.ROW || firstFType == FType.COL) {
+ returnFType = firstFType;
+ reason = "AggUnaryOp: Partial aggregation maintains structure";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return returnFType;
+ }
+
+ // Other FTypes (FULL, BROADCAST) not affected by direction
+ reason = "AggUnaryOp: Other FTypes not affected by direction";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return null;
+ }
+
+ // ReorgOp: Reorganization operations that transform structure
+ if (hop instanceof ReorgOp) {
+ ReOrgOp op = ((ReorgOp)hop).getOp();
+
+ // Unsupported operations:
+ // - RESHAPE: Dimension changes break partitioning assumptions
+ // - SORT: Requires global ordering across all partitions
+ if (op == ReOrgOp.RESHAPE || op == ReOrgOp.SORT) {
+ reason = "ReorgOp: " + op + " breaks partitioning assumptions";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return null;
+ }
+
+ // TRANS: Transpose swaps ROW↔COL partitioning
+ if (op == ReOrgOp.TRANS) {
+ if (firstFType == FType.ROW) {
+ returnFType = FType.COL;
+ reason = "ReorgOp: TRANS swaps ROW to COL";
+ } else if (firstFType == FType.COL) {
+ returnFType = FType.ROW;
+ reason = "ReorgOp: TRANS swaps COL to ROW";
+ } else {
+ returnFType = firstFType; // FULL/BROADCAST unchanged
+ reason = "ReorgOp: TRANS maintains FULL/BROADCAST";
+ }
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return returnFType;
+ }
+
+ // Structure-maintaining operations: DIAG, REV, ROLL
+ returnFType = firstFType;
+ reason = "ReorgOp: " + op + " maintains structure";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return returnFType;
+ }
+
+ // ParameterizedBuiltinOp: Builtin operations with parameters
+ if (hop instanceof ParameterizedBuiltinOp) {
+ ParamBuiltinOp op = ((ParameterizedBuiltinOp) hop).getOp();
+
+ // CONTAINS returns scalar boolean result
+ if (op == ParamBuiltinOp.CONTAINS) {
+ reason = "ParameterizedBuiltinOp: CONTAINS returns scalar boolean";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return null;
+ }
+
+ // Check if operation is supported
+ // Supported: REPLACE, RMEMPTY, LOWER_TRI, UPPER_TRI, TRANSFORMDECODE, TRANSFORMAPPLY, TOKENIZE
+ if (!(op == ParamBuiltinOp.REPLACE || op == ParamBuiltinOp.RMEMPTY ||
+ op == ParamBuiltinOp.LOWER_TRI || op == ParamBuiltinOp.UPPER_TRI ||
+ op == ParamBuiltinOp.TRANSFORMDECODE || op == ParamBuiltinOp.TRANSFORMAPPLY ||
+ op == ParamBuiltinOp.TOKENIZE)) {
+ reason = "ParameterizedBuiltinOp: " + op + " not supported";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return null;
+ }
+
+ // Structure-preserving operations maintain input FType
+ returnFType = firstFType;
+ reason = "ParameterizedBuiltinOp: " + op + " preserves structure";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return returnFType;
+ }
+
+ // Default: Unknown operation type or unhandled case
+ reason = "Unknown operation type or unhandled case";
+ FederatedPlannerLogger.logGetFederatedTypeDebug(hop, returnFType, reason);
+ return null;
+ }
private static FType deriveFType(DataOp fedInit) {
Hop ranges = fedInit.getInput(fedInit.getParameterIndex(DataExpression.FED_RANGES));
@@ -1432,46 +1432,46 @@ public class FederatedPlanRewireTransTable {
colPartitioned ? FType.COL : FType.OTHER;
}
- private static void wireUnRefTwriteToLiveOut(StatementBlock sb, Set<Long> unRefTwriteSet,
- Map<Long, HopCommon> hopCommonTable, Map<String, List<Hop>> newFormerTransTable) {
- if (unRefTwriteSet.isEmpty())
- return;
-
- VariableSet genHops = sb.getGen();
- VariableSet updatedHops = sb.variablesUpdated();
- VariableSet liveOutHops = sb.liveOut();
-
- Iterator<Long> unRefTwriteIterator = unRefTwriteSet.iterator();
- while (unRefTwriteIterator.hasNext()) {
- Long unRefTwriteHopID = unRefTwriteIterator.next();
- Hop unRefTwriteHop = hopCommonTable.get(unRefTwriteHopID).getHopRef();
- String unRefTwriteHopName = unRefTwriteHop.getName();
-
- if (liveOutHops.containsVariable(unRefTwriteHopName)) {
- continue;
- }
-
- if (unRefTwriteHop instanceof FunctionOp || genHops.containsVariable(unRefTwriteHopName) || updatedHops.containsVariable(unRefTwriteHopName)) {
- Iterator<String> liveOutHopsIterator = liveOutHops.getVariableNames().iterator();
-
- boolean isRewired = false;
- while (liveOutHopsIterator.hasNext()) {
- String liveOutHopName = liveOutHopsIterator.next();
- List<Hop> liveOutHopsList = newFormerTransTable.get(liveOutHopName);
-
- if (liveOutHopsList != null && !liveOutHopsList.isEmpty()) {
- List<Hop> copyLiveOutHopsList = new ArrayList<>(liveOutHopsList);
- copyLiveOutHopsList.add(unRefTwriteHop);
- newFormerTransTable.put(liveOutHopName, copyLiveOutHopsList);
- unRefTwriteIterator.remove();
- isRewired = true;
- break;
- }
- }
- if (!isRewired) {
- throw new RuntimeException("No liveOutHops found for " + unRefTwriteHopName);
- }
- }
- }
- }
+ private static void wireUnRefTwriteToLiveOut(StatementBlock sb, Set<Long> unRefTwriteSet,
+ Map<Long, HopCommon> hopCommonTable, Map<String, List<Hop>> newFormerTransTable) {
+ if (unRefTwriteSet.isEmpty())
+ return;
+
+ VariableSet genHops = sb.getGen();
+ VariableSet updatedHops = sb.variablesUpdated();
+ VariableSet liveOutHops = sb.liveOut();
+
+ Iterator<Long> unRefTwriteIterator = unRefTwriteSet.iterator();
+ while (unRefTwriteIterator.hasNext()) {
+ Long unRefTwriteHopID = unRefTwriteIterator.next();
+ Hop unRefTwriteHop = hopCommonTable.get(unRefTwriteHopID).getHopRef();
+ String unRefTwriteHopName = unRefTwriteHop.getName();
+
+ if (liveOutHops.containsVariable(unRefTwriteHopName)) {
+ continue;
+ }
+
+ if (unRefTwriteHop instanceof FunctionOp || genHops.containsVariable(unRefTwriteHopName) || updatedHops.containsVariable(unRefTwriteHopName)) {
+ Iterator<String> liveOutHopsIterator = liveOutHops.getVariableNames().iterator();
+
+ boolean isRewired = false;
+ while (liveOutHopsIterator.hasNext()) {
+ String liveOutHopName = liveOutHopsIterator.next();
+ List<Hop> liveOutHopsList = newFormerTransTable.get(liveOutHopName);
+
+ if (liveOutHopsList != null && !liveOutHopsList.isEmpty()) {
+ List<Hop> copyLiveOutHopsList = new ArrayList<>(liveOutHopsList);
+ copyLiveOutHopsList.add(unRefTwriteHop);
+ newFormerTransTable.put(liveOutHopName, copyLiveOutHopsList);
+ unRefTwriteIterator.remove();
+ isRewired = true;
+ break;
+ }
+ }
+ if (!isRewired) {
+ throw new RuntimeException("No liveOutHops found for " + unRefTwriteHopName);
+ }
+ }
+ }
+ }
}
diff --git a/src/main/java/org/apache/sysds/hops/fedplanner/FederatedPlannerFedCostBased.java b/src/main/java/org/apache/sysds/hops/fedplanner/FederatedPlannerFedCostBased.java
index b0192b9544..99043860bd 100644
--- a/src/main/java/org/apache/sysds/hops/fedplanner/FederatedPlannerFedCostBased.java
+++ b/src/main/java/org/apache/sysds/hops/fedplanner/FederatedPlannerFedCostBased.java
@@ -20,7 +20,6 @@
package org.apache.sysds.hops.fedplanner;
import java.util.Set;
-import java.util.Map;
import org.apache.sysds.common.Types.ExecType;
import org.apache.sysds.hops.ipa.FunctionCallGraph;
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java
index 0c05612b79..98572e2ddd 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java
@@ -374,15 +374,16 @@ public class FederatedData {
}
public static class GetPrivacyConstraints extends FederatedUDF {
+ private static final long serialVersionUID = 1637852940793579590L;
private final String filename;
- public GetPrivacyConstraints(String filename) {
- super(new long[] { }); // Pass empty ID array to parent constructor as this is a static class
+ public GetPrivacyConstraints(String filename) {
+ super(new long[] { }); // Pass empty ID array to parent constructor as this is a static class
this.filename = filename;
- }
-
- @Override
- public FederatedResponse execute(ExecutionContext ec, Data... data) {
+ }
+
+ @Override
+ public FederatedResponse execute(ExecutionContext ec, Data... data) {
String privacyConstraints = null;
FileSystem fs = null;
MetaDataAll mtd = null;
@@ -416,19 +417,19 @@ public class FederatedData {
finally {
IOUtilFunctions.closeSilently(fs);
}
- }
-
- @Override
- public Pair<String, LineageItem> getLineageItem(ExecutionContext ec) {
- String opcode = "fedprivconst"; // Appropriate operation code
-
- // Create input LineageItem for the operation
- LineageItem[] inputs = new LineageItem[] {
- new LineageItem(filename) // Create literal LineageItem by passing only the string
- };
-
- // Create appropriate LineageItem (for read operation)
- return Pair.of(opcode, new LineageItem(opcode, inputs));
- }
- }
+ }
+
+ @Override
+ public Pair<String, LineageItem> getLineageItem(ExecutionContext ec) {
+ String opcode = "fedprivconst"; // Appropriate operation code
+
+ // Create input LineageItem for the operation
+ LineageItem[] inputs = new LineageItem[] {
+ new LineageItem(filename) // Create literal LineageItem by passing only the string
+ };
+
+ // Create appropriate LineageItem (for read operation)
+ return Pair.of(opcode, new LineageItem(opcode, inputs));
+ }
+ }
}
\ No newline at end of file
diff --git a/src/main/java/org/apache/sysds/runtime/instructions/gpu/context/CSRPointer.java b/src/main/java/org/apache/sysds/runtime/instructions/gpu/context/CSRPointer.java
index 9473e93957..f4f09ad119 100644
--- a/src/main/java/org/apache/sysds/runtime/instructions/gpu/context/CSRPointer.java
+++ b/src/main/java/org/apache/sysds/runtime/instructions/gpu/context/CSRPointer.java
@@ -21,7 +21,6 @@ package org.apache.sysds.runtime.instructions.gpu.context;
import jcuda.Pointer;
import jcuda.Sizeof;
-import jcuda.cudaDataType;
import jcuda.jcublas.cublasHandle;
import jcuda.jcusparse.*;
import org.apache.commons.logging.Log;
@@ -37,12 +36,8 @@ import static jcuda.jcusparse.cusparseMatrixType.CUSPARSE_MATRIX_TYPE_GENERAL;
import static jcuda.runtime.JCuda.*;
import static jcuda.runtime.cudaMemcpyKind.*;
import static jcuda.jcusparse.cusparseIndexType.CUSPARSE_INDEX_32I;
-import static jcuda.cudaDataType.CUDA_R_64F;
-import static jcuda.cudaDataType.CUDA_R_32F;
import static jcuda.jcusparse.cusparseSpGEMMAlg.CUSPARSE_SPGEMM_DEFAULT;
import static org.apache.sysds.runtime.matrix.data.LibMatrixCUDA.cudaSupportFunctions;
-import static jcuda.jcusparse.cusparseOperation.CUSPARSE_OPERATION_TRANSPOSE;
-import static jcuda.jcusparse.cusparseOperation.CUSPARSE_OPERATION_NON_TRANSPOSE;
/**
* Compressed Sparse Row (CSR) format for CUDA
diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/DoublePrecisionCudaSupportFunctions.java b/src/main/java/org/apache/sysds/runtime/matrix/data/DoublePrecisionCudaSupportFunctions.java
index 2a5e03593f..c070b41da3 100644
--- a/src/main/java/org/apache/sysds/runtime/matrix/data/DoublePrecisionCudaSupportFunctions.java
+++ b/src/main/java/org/apache/sysds/runtime/matrix/data/DoublePrecisionCudaSupportFunctions.java
@@ -44,17 +44,13 @@ import jcuda.jcusparse.cusparseDnVecDescr;
import jcuda.jcusparse.cusparseDnMatDescr;
import static jcuda.jcusparse.cusparseIndexType.CUSPARSE_INDEX_32I;
-import static jcuda.jcusparse.cusparseIndexBase.CUSPARSE_INDEX_BASE_ZERO;
import static jcuda.cudaDataType.CUDA_R_64F;
-import static jcuda.jcusparse.cusparseSpGEMMAlg.CUSPARSE_SPGEMM_DEFAULT;
import static jcuda.jcusparse.cusparseStatus.CUSPARSE_STATUS_SUCCESS;
import static jcuda.jcusparse.cusparseSpMVAlg.CUSPARSE_SPMV_ALG_DEFAULT;
-import static jcuda.jcusparse.cusparseOperation.CUSPARSE_OPERATION_NON_TRANSPOSE;
import static jcuda.jcusparse.cusparseOrder.CUSPARSE_ORDER_COL;
import static jcuda.jcusparse.cusparseSpMMAlg.CUSPARSE_SPMM_ALG_DEFAULT;
import static jcuda.jcusparse.cusparseCsr2CscAlg.CUSPARSE_CSR2CSC_ALG1;
import static jcuda.jcusparse.cusparseSparseToDenseAlg.CUSPARSE_SPARSETODENSE_ALG_DEFAULT;
-import static jcuda.jcusparse.cusparseIndexBase.CUSPARSE_INDEX_BASE_ONE;
import static jcuda.jcusparse.cusparseDenseToSparseAlg.CUSPARSE_DENSETOSPARSE_ALG_DEFAULT;
public class DoublePrecisionCudaSupportFunctions implements CudaSupportFunctions {
diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixCuDNN.java b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixCuDNN.java
index f32dbd029f..cb78cdf682 100644
--- a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixCuDNN.java
+++ b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixCuDNN.java
@@ -345,6 +345,7 @@ public class LibMatrixCuDNN extends LibMatrixCUDA {
* @param output the output matrix allocated on the GPU
* @param algo cudnn algorithm wrapper
*/
+ @SuppressWarnings("deprecation")
private static void cudnnConv2d(GPUContext gCtx, String instName, Pointer image, Pointer filter, Pointer output,
LibMatrixCuDNNConvolutionAlgorithm algo) {
if(LOG.isTraceEnabled()) {
@@ -458,6 +459,7 @@ public class LibMatrixCuDNN extends LibMatrixCUDA {
* @param dwPointer output errors
* @param algo cudnn algorithm wrapper
*/
+ @SuppressWarnings("deprecation")
private static void cudnnConv2dBackwardFilter(GPUContext gCtx, String instName, Pointer imagePointer, Pointer doutPointer,
Pointer dwPointer, LibMatrixCuDNNConvolutionAlgorithm algo) {
if(LOG.isTraceEnabled()) {
@@ -558,6 +560,7 @@ public class LibMatrixCuDNN extends LibMatrixCUDA {
* @param dx pointer to output errors
* @param algo cudnn algorithm wrapper
*/
+ @SuppressWarnings("deprecation")
private static void cudnnConv2dBackwardData(GPUContext gCtx, String instName, Pointer w, Pointer dy,
Pointer dx, LibMatrixCuDNNConvolutionAlgorithm algo) {
if(LOG.isTraceEnabled()) {
@@ -625,6 +628,7 @@ public class LibMatrixCuDNN extends LibMatrixCUDA {
}
}
+ @SuppressWarnings("deprecation")
private static void cudnnPoolingHelper(GPUContext gCtx, String instName, Pointer x,
Pointer y, int N, int C, int H, int W, int K, int R,
int S, int pad_h, int pad_w, int stride_h, int stride_w, int P,
@@ -714,6 +718,7 @@ public class LibMatrixCuDNN extends LibMatrixCUDA {
}
}
+ @SuppressWarnings("deprecation")
private static void cudnnPoolingBackwardHelper(GPUContext gCtx, String instName,
Pointer x, Pointer dy, Pointer y, Pointer dx,
int N, int C, int H, int W, int K, int R,
@@ -751,6 +756,7 @@ public class LibMatrixCuDNN extends LibMatrixCUDA {
}
}
+ @SuppressWarnings("deprecation")
private static void cudnnReLU(GPUContext gCtx, String instName, MatrixObject in, Pointer dstData, cudnnTensorDescriptor srcTensorDesc) {
try {
if(LOG.isTraceEnabled()) {
@@ -1076,6 +1082,7 @@ public class LibMatrixCuDNN extends LibMatrixCUDA {
* @param resultSaveInvVariance (output) running variance accumulated during training phase: shape [1, C, 1, 1]
* @throws DMLRuntimeException if error occurs
*/
+ @SuppressWarnings("deprecation")
public static void batchNormalizationForwardTraining(GPUContext gCtx, String instName, MatrixObject image,
MatrixObject scale, MatrixObject bias, MatrixObject runningMean, MatrixObject runningVar,
MatrixObject ret, MatrixObject retRunningMean, MatrixObject retRunningVar,
@@ -1132,6 +1139,7 @@ public class LibMatrixCuDNN extends LibMatrixCUDA {
* @param epsilon epsilon value used in the batch normalization formula
* @throws DMLRuntimeException if error occurs
*/
+ @SuppressWarnings("deprecation")
public static void batchNormalizationForwardInference(GPUContext gCtx, String instName, MatrixObject image,
MatrixObject scale, MatrixObject bias, MatrixObject runningMean, MatrixObject runningVar,
MatrixObject ret, double epsilon) throws DMLRuntimeException {
@@ -1179,6 +1187,7 @@ public class LibMatrixCuDNN extends LibMatrixCUDA {
* @param resultSaveInvVariance (input) running variance accumulated during training phase: shape [1, C, 1, 1]
* @throws DMLRuntimeException if error occurs
*/
+ @SuppressWarnings("deprecation")
public static void batchNormalizationBackward(GPUContext gCtx, String instName, MatrixObject image, MatrixObject dout,
MatrixObject scale, MatrixObject dX, MatrixObject dScale, MatrixObject dBias,
double epsilon, MatrixObject resultSaveMean, MatrixObject resultSaveInvVariance) throws DMLRuntimeException {
diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixCuDNNConvolutionAlgorithm.java b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixCuDNNConvolutionAlgorithm.java
index fab29fd1e8..e0021e7117 100644
--- a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixCuDNNConvolutionAlgorithm.java
+++ b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixCuDNNConvolutionAlgorithm.java
@@ -85,6 +85,7 @@ public class LibMatrixCuDNNConvolutionAlgorithm implements java.lang.AutoCloseab
/**
* Deallocates the tensor and filter descriptors as well as allocated workspace
*/
+ @SuppressWarnings("deprecation")
@Override
public void close() {
if(nchwTensorDesc != null)
@@ -125,6 +126,7 @@ public class LibMatrixCuDNNConvolutionAlgorithm implements java.lang.AutoCloseab
* @param workspaceLimit maximum intermediate memory to use
* @return algorithm wrapper
*/
+ @SuppressWarnings("deprecation")
public static LibMatrixCuDNNConvolutionAlgorithm cudnnGetConvolutionForwardAlgorithm(
GPUContext gCtx, String instName, int N, int C, int H, int W, int K, int R, int S,
int pad_h, int pad_w, int stride_h, int stride_w, int P, int Q, long workspaceLimit) {
@@ -168,6 +170,7 @@ public class LibMatrixCuDNNConvolutionAlgorithm implements java.lang.AutoCloseab
* @param workspaceLimit maximum intermediate memory to use
* @return algorithm wrapper
*/
+ @SuppressWarnings("deprecation")
public static LibMatrixCuDNNConvolutionAlgorithm cudnnGetConvolutionBackwardFilterAlgorithm(
GPUContext gCtx, String instName, int N, int C, int H, int W, int K, int R, int S,
int pad_h, int pad_w, int stride_h, int stride_w, int P, int Q, long workspaceLimit) {
@@ -227,6 +230,7 @@ public class LibMatrixCuDNNConvolutionAlgorithm implements java.lang.AutoCloseab
* @param workspaceLimit maximum intermediate memory to use
* @return algorithm wrapper
*/
+ @SuppressWarnings("deprecation")
public static LibMatrixCuDNNConvolutionAlgorithm cudnnGetConvolutionBackwardDataAlgorithm(
GPUContext gCtx, String instName, int N, int C, int H, int W, int K, int R, int S,
int pad_h, int pad_w, int stride_h, int stride_w, int P, int Q, long workspaceLimit) {
@@ -289,6 +293,7 @@ public class LibMatrixCuDNNConvolutionAlgorithm implements java.lang.AutoCloseab
return tensorDescriptor;
}
+ @SuppressWarnings("deprecation")
private static cudnnFilterDescriptor allocateFilterDescriptor(int K, int C, int R, int S) {
cudnnFilterDescriptor filterDesc = new cudnnFilterDescriptor();
cudnnCreateFilterDescriptor(filterDesc);
@@ -296,6 +301,7 @@ public class LibMatrixCuDNNConvolutionAlgorithm implements java.lang.AutoCloseab
return filterDesc;
}
+ @SuppressWarnings("deprecation")
private static cudnnConvolutionDescriptor allocateConvolutionDescriptor(int padding [], int strides []) {
cudnnConvolutionDescriptor convDesc = new cudnnConvolutionDescriptor();
cudnnCreateConvolutionDescriptor(convDesc);
diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixCuDNNPoolingDescriptors.java b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixCuDNNPoolingDescriptors.java
index 111bac53df..b72ed41aef 100644
--- a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixCuDNNPoolingDescriptors.java
+++ b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixCuDNNPoolingDescriptors.java
@@ -49,6 +49,7 @@ public class LibMatrixCuDNNPoolingDescriptors implements java.lang.AutoCloseable
public cudnnTensorDescriptor dyDesc;
public cudnnPoolingDescriptor poolingDesc;
+ @SuppressWarnings("deprecation")
@Override
public void close() {
if(xDesc != null)
@@ -155,6 +156,7 @@ public class LibMatrixCuDNNPoolingDescriptors implements java.lang.AutoCloseable
* @param poolingType type of pooling
* @return cudnn pooling descriptor
*/
+ @SuppressWarnings("deprecation")
private static cudnnPoolingDescriptor allocatePoolingDescriptor(int R, int S, int pad_h, int pad_w, int stride_h, int stride_w, PoolingType poolingType) {
cudnnPoolingDescriptor poolingDesc = new cudnnPoolingDescriptor();
cudnnCreatePoolingDescriptor(poolingDesc);
diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixCuDNNRnnAlgorithm.java b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixCuDNNRnnAlgorithm.java
index 55f47cb948..21d31d2929 100644
--- a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixCuDNNRnnAlgorithm.java
+++ b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixCuDNNRnnAlgorithm.java
@@ -219,6 +219,7 @@ public class LibMatrixCuDNNRnnAlgorithm implements java.lang.AutoCloseable {
return LibMatrixCUDA.toInt(numScalars);
}
+ @SuppressWarnings("deprecation")
private static cudnnFilterDescriptor allocateFilterDescriptor(int numWeights) {
cudnnFilterDescriptor filterDesc = new cudnnFilterDescriptor();
cudnnCreateFilterDescriptor(filterDesc);
@@ -238,6 +239,7 @@ public class LibMatrixCuDNNRnnAlgorithm implements java.lang.AutoCloseable {
}
+ @SuppressWarnings("deprecation")
@Override
public void close() {
if(dropoutDesc != null)
diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixCuMatMult.java b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixCuMatMult.java
index e0f3d12cca..15cd5f0d25 100644
--- a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixCuMatMult.java
+++ b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixCuMatMult.java
@@ -25,7 +25,6 @@ import static jcuda.jcusparse.cusparseOperation.CUSPARSE_OPERATION_TRANSPOSE;
import static jcuda.runtime.JCuda.cudaMemcpy;
import static jcuda.runtime.cudaMemcpyKind.cudaMemcpyHostToDevice;
import static jcuda.jcusparse.cusparseSpMMAlg.CUSPARSE_SPMM_ALG_DEFAULT;
-import static org.apache.sysds.runtime.instructions.gpu.context.CSRPointer.getCSRMatrixInfo;
import static org.apache.sysds.runtime.instructions.gpu.context.CSRPointer.transposeCSR;
import jcuda.Pointer;
diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/SinglePrecisionCudaSupportFunctions.java b/src/main/java/org/apache/sysds/runtime/matrix/data/SinglePrecisionCudaSupportFunctions.java
index ab9f6d82be..a035a34c0d 100644
--- a/src/main/java/org/apache/sysds/runtime/matrix/data/SinglePrecisionCudaSupportFunctions.java
+++ b/src/main/java/org/apache/sysds/runtime/matrix/data/SinglePrecisionCudaSupportFunctions.java
@@ -18,7 +18,6 @@
*/
package org.apache.sysds.runtime.matrix.data;
-import static jcuda.cudaDataType.CUDA_R_64F;
import static jcuda.jcusparse.JCusparse.*;
import static jcuda.runtime.JCuda.cudaMemcpy;
import static jcuda.runtime.cudaMemcpyKind.cudaMemcpyDeviceToHost;
@@ -48,17 +47,13 @@ import jcuda.jcusparse.cusparseSpMatDescr;
import jcuda.jcusparse.cusparseSpGEMMDescr;
import static jcuda.jcusparse.cusparseIndexType.CUSPARSE_INDEX_32I;
-import static jcuda.jcusparse.cusparseIndexBase.CUSPARSE_INDEX_BASE_ZERO;
import static jcuda.cudaDataType.CUDA_R_32F;
-import static jcuda.jcusparse.cusparseSpGEMMAlg.CUSPARSE_SPGEMM_DEFAULT;
import static jcuda.jcusparse.cusparseStatus.CUSPARSE_STATUS_SUCCESS;
import static jcuda.jcusparse.cusparseSpMVAlg.CUSPARSE_SPMV_ALG_DEFAULT;
-import static jcuda.jcusparse.cusparseOperation.CUSPARSE_OPERATION_NON_TRANSPOSE;
import static jcuda.jcusparse.cusparseOrder.CUSPARSE_ORDER_COL;
import static jcuda.jcusparse.cusparseSpMMAlg.CUSPARSE_SPMM_ALG_DEFAULT;
import static jcuda.jcusparse.cusparseCsr2CscAlg.CUSPARSE_CSR2CSC_ALG1;
import static jcuda.jcusparse.cusparseSparseToDenseAlg.CUSPARSE_SPARSETODENSE_ALG_DEFAULT;
-import static jcuda.jcusparse.cusparseIndexBase.CUSPARSE_INDEX_BASE_ONE;
import static jcuda.jcusparse.cusparseDenseToSparseAlg.CUSPARSE_DENSETOSPARSE_ALG_DEFAULT;
import static jcuda.runtime.JCuda.cudaMalloc;
import static jcuda.runtime.JCuda.cudaFree;
diff --git a/src/test/java/org/apache/sysds/test/functions/federated/fedplanning/FederatedL2SVMPlanningTest.java b/src/test/java/org/apache/sysds/test/functions/federated/fedplanning/FederatedL2SVMPlanningTest.java
index 48d578c2ff..9114339620 100644
--- a/src/test/java/org/apache/sysds/test/functions/federated/fedplanning/FederatedL2SVMPlanningTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/federated/fedplanning/FederatedL2SVMPlanningTest.java
@@ -31,9 +31,6 @@ import org.junit.Ignore;
import org.junit.Test;
import java.io.File;
-import java.util.Arrays;
-
-import static org.junit.Assert.fail;
@net.jcip.annotations.NotThreadSafe
public class FederatedL2SVMPlanningTest extends AutomatedTestBase {
@@ -147,6 +144,7 @@ public class FederatedL2SVMPlanningTest extends AutomatedTestBase {
}
+ @SuppressWarnings("unused")
private void writeStandardMatrix(String matrixName, long seed, int numRows, String privacyConstraints){
double[][] matrix = getRandomMatrix(numRows, cols, 0, 1, 1, seed);
writeStandardMatrix(matrixName, numRows, matrix, privacyConstraints);
diff --git a/src/test/java/org/apache/sysds/test/gpu/nn/ResNet18GPUTest.java b/src/test/java/org/apache/sysds/test/gpu/nn/ResNet18GPUTest.java
index d306271e50..859c66cea2 100644
--- a/src/test/java/org/apache/sysds/test/gpu/nn/ResNet18GPUTest.java
+++ b/src/test/java/org/apache/sysds/test/gpu/nn/ResNet18GPUTest.java
@@ -39,8 +39,6 @@ public class ResNet18GPUTest extends AutomatedTestBase {
private static final String TEST_DIR = "gpu/nn/";
private static final String TEST_CLASS_DIR = TEST_DIR + ResNet18GPUTest.class.getSimpleName() + "/";
- private static final double eps = Math.pow(10, -10);
-
@BeforeClass
public static void checkGPU() {
boolean gpuAvailable = false;
diff --git a/src/test/java/org/apache/sysds/test/usertest/pythonapi/StartupTest.java b/src/test/java/org/apache/sysds/test/usertest/pythonapi/StartupTest.java
index b048e66e95..f08dd63c65 100644
--- a/src/test/java/org/apache/sysds/test/usertest/pythonapi/StartupTest.java
+++ b/src/test/java/org/apache/sysds/test/usertest/pythonapi/StartupTest.java
@@ -68,6 +68,7 @@ public class StartupTest {
System.setSecurityManager(sm);
}
+ @SuppressWarnings("unused")
private void assertLogMessages(String... expectedMessages) {
assertLogMessages(true, expectedMessages);
}