You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by mb...@apache.org on 2024/10/24 10:48:47 UTC
(systemds) branch main updated: [SYSTEMDS-3787] Fix missing transformencode update federated backend
This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 6652d43939 [SYSTEMDS-3787] Fix missing transformencode update federated backend
6652d43939 is described below
commit 6652d439393029b84830c792c9cb8da82e49f298
Author: Matthias Boehm <mb...@gmail.com>
AuthorDate: Thu Oct 24 12:48:33 2024 +0200
[SYSTEMDS-3787] Fix missing transformencode update federated backend
This patch adds support for the new transform encode w/o metadata
output also to the federated backend, and fixes the robustness for
handling empty frame blocks.
---
.../sysds/runtime/frame/data/FrameBlock.java | 7 ++++---
...ltiReturnParameterizedBuiltinCPInstruction.java | 4 ++++
...tiReturnParameterizedBuiltinFEDInstruction.java | 24 ++++++++++++++--------
...ltiReturnParameterizedBuiltinSPInstruction.java | 4 ++++
4 files changed, 28 insertions(+), 11 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/FrameBlock.java b/src/main/java/org/apache/sysds/runtime/frame/data/FrameBlock.java
index 4896d82f28..61875d2e14 100644
--- a/src/main/java/org/apache/sysds/runtime/frame/data/FrameBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/frame/data/FrameBlock.java
@@ -925,9 +925,10 @@ public class FrameBlock implements CacheBlock<FrameBlock>, Externalizable {
public boolean isShallowSerialize(boolean inclConvert) {
// shallow serialize if non-string schema because a frame block
// is always dense but strings have large array overhead per cell
- for(int j = 0; j < _schema.length; j++)
- if(!_coldata[j].isShallowSerialize())
- return false;
+ if( _schema != null )
+ for(int j = 0; j < _schema.length; j++)
+ if(!_coldata[j].isShallowSerialize())
+ return false;
return true;
}
diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/MultiReturnParameterizedBuiltinCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/MultiReturnParameterizedBuiltinCPInstruction.java
index f5ba63b0ef..e5e486752d 100644
--- a/src/main/java/org/apache/sysds/runtime/instructions/cp/MultiReturnParameterizedBuiltinCPInstruction.java
+++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/MultiReturnParameterizedBuiltinCPInstruction.java
@@ -123,4 +123,8 @@ public class MultiReturnParameterizedBuiltinCPInstruction extends ComputationCPI
}
return ret;
}
+
+ public boolean getMetaReturn() {
+ return _metaReturn;
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/instructions/fed/MultiReturnParameterizedBuiltinFEDInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/fed/MultiReturnParameterizedBuiltinFEDInstruction.java
index 416c2838af..917226fef7 100644
--- a/src/main/java/org/apache/sysds/runtime/instructions/fed/MultiReturnParameterizedBuiltinFEDInstruction.java
+++ b/src/main/java/org/apache/sysds/runtime/instructions/fed/MultiReturnParameterizedBuiltinFEDInstruction.java
@@ -70,10 +70,12 @@ import org.apache.sysds.runtime.util.IndexRange;
public class MultiReturnParameterizedBuiltinFEDInstruction extends ComputationFEDInstruction {
protected final List<CPOperand> _outputs;
-
+ protected final boolean _metaReturn;
+
private MultiReturnParameterizedBuiltinFEDInstruction(Operator op, CPOperand input1, CPOperand input2,
- List<CPOperand> outputs, String opcode, String istr) {
+ List<CPOperand> outputs, boolean metaReturn, String opcode, String istr) {
super(FEDType.MultiReturnParameterizedBuiltin, op, input1, input2, null, opcode, istr);
+ _metaReturn = metaReturn;
_outputs = outputs;
}
@@ -104,13 +106,13 @@ public class MultiReturnParameterizedBuiltinFEDInstruction extends ComputationFE
private static MultiReturnParameterizedBuiltinFEDInstruction parseInstruction(
MultiReturnParameterizedBuiltinCPInstruction instr) {
return new MultiReturnParameterizedBuiltinFEDInstruction(instr.getOperator(), instr.input1, instr.input2,
- instr.getOutputs(), instr.getOpcode(), instr.getInstructionString());
+ instr.getOutputs(), instr.getMetaReturn(), instr.getOpcode(), instr.getInstructionString());
}
private static MultiReturnParameterizedBuiltinFEDInstruction parseInstruction(
MultiReturnParameterizedBuiltinSPInstruction instr) {
return new MultiReturnParameterizedBuiltinFEDInstruction(instr.getOperator(), instr.input1, instr.input2,
- instr.getOutputs(), instr.getOpcode(), instr.getInstructionString());
+ instr.getOutputs(), instr.getMetaReturn(), instr.getOpcode(), instr.getInstructionString());
}
public static MultiReturnParameterizedBuiltinFEDInstruction parseInstruction(String str) {
@@ -122,9 +124,15 @@ public class MultiReturnParameterizedBuiltinFEDInstruction extends ComputationFE
// one input and two outputs
CPOperand in1 = new CPOperand(parts[1]);
CPOperand in2 = new CPOperand(parts[2]);
- outputs.add(new CPOperand(parts[3], Types.ValueType.FP64, Types.DataType.MATRIX));
- outputs.add(new CPOperand(parts[4], Types.ValueType.STRING, Types.DataType.FRAME));
- return new MultiReturnParameterizedBuiltinFEDInstruction(null, in1, in2, outputs, opcode, str);
+ int pos = 3;
+ boolean metaReturn = true;
+ System.out.println(Arrays.toString(parts));
+ if( parts.length == 7 ) //no need for meta data
+ metaReturn = new CPOperand(parts[pos++]).getLiteral().getBooleanValue();
+ outputs.add(new CPOperand(parts[pos], Types.ValueType.FP64, Types.DataType.MATRIX));
+ outputs.add(new CPOperand(parts[pos+1], Types.ValueType.STRING, Types.DataType.FRAME));
+ return new MultiReturnParameterizedBuiltinFEDInstruction(
+ null, in1, in2, outputs, metaReturn, opcode, str);
}
else {
throw new DMLRuntimeException("Invalid opcode in MultiReturnBuiltin instruction: " + opcode);
@@ -198,7 +206,7 @@ public class MultiReturnParameterizedBuiltinFEDInstruction extends ComputationFE
encodeFederatedFrames(fedMapping, globalEncoder, ec.getMatrixObject(getOutput(0)));
// release input and outputs
- ec.setFrameOutput(getOutput(1).getName(), meta);
+ ec.setFrameOutput(getOutput(1).getName(), _metaReturn ? meta : new FrameBlock());
}
private class EncoderColnames {
diff --git a/src/main/java/org/apache/sysds/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java
index ebd1bf7b44..90683eab29 100644
--- a/src/main/java/org/apache/sysds/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java
+++ b/src/main/java/org/apache/sysds/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java
@@ -468,4 +468,8 @@ public class MultiReturnParameterizedBuiltinSPInstruction extends ComputationSPI
return ret.iterator();
}
}
+
+ public boolean getMetaReturn() {
+ return _metaReturn;
+ }
}