You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@systemds.apache.org by GitBox <gi...@apache.org> on 2021/04/09 00:56:15 UTC
[GitHub] [systemds] OlgaOvcharenko opened a new pull request #1219: [WIP][SYSTEMDS-2925] Federated left indexing
OlgaOvcharenko opened a new pull request #1219:
URL: https://github.com/apache/systemds/pull/1219
This PR adds federated left indexing, the simplest case when rhs matrix or frame is read and broadcast sliced (e.g. A[2:12, 3:4] = B).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [systemds] Baunsgaard commented on a change in pull request #1219: [SYSTEMDS-2925] Federated left indexing
Posted by GitBox <gi...@apache.org>.
Baunsgaard commented on a change in pull request #1219:
URL: https://github.com/apache/systemds/pull/1219#discussion_r613148938
##########
File path: src/main/java/org/apache/sysds/runtime/instructions/fed/IndexingFEDInstruction.java
##########
@@ -171,4 +190,110 @@ private void rightIndexing(ExecutionContext ec)
out.setFedMapping(fedMap.copyWithNewID(fr1[0].getID()));
}
}
+
+ private void leftIndexing(ExecutionContext ec)
+ {
+ //get input and requested index range
+ CacheableData<?> in1 = ec.getCacheableData(input1);
+ CacheableData<?> in2 = ec.getCacheableData(input2);
+ IndexRange ixrange = getIndexRange(ec);
+
+ //check bounds
+ if( ixrange.rowStart < 0 || ixrange.rowStart >= in1.getNumRows() || ixrange.rowEnd >= in1.getNumRows()
+ || ixrange.colStart < 0 || ixrange.colStart >= in1.getNumColumns() || ixrange.colEnd >= in1.getNumColumns() ) {
+ throw new DMLRuntimeException("Invalid values for matrix indexing: ["+(ixrange.rowStart+1)+":"+(ixrange.rowEnd+1)+","
+ + (ixrange.colStart+1)+":"+(ixrange.colEnd+1)+"] " + "must be within matrix dimensions ["+in1.getNumRows()+","+in1.getNumColumns()+"].");
+ }
+ if( (ixrange.rowEnd-ixrange.rowStart+1) != in2.getNumRows() || (ixrange.colEnd-ixrange.colStart+1) != in2.getNumColumns()) {
+ throw new DMLRuntimeException("Invalid values for matrix indexing: " +
+ "dimensions of the source matrix ["+in2.getNumRows()+"x" + in2.getNumColumns() + "] " +
+ "do not match the shape of the matrix specified by indices [" +
+ (ixrange.rowStart+1) +":" + (ixrange.rowEnd+1) + ", " + (ixrange.colStart+1) + ":" + (ixrange.colEnd+1) + "].");
+ }
+
+ FederationMap fedMap = in1.getFedMapping();
+
+ String[] instStrings = new String[fedMap.getSize()];
+ int[][] sliceIxs = new int[fedMap.getSize()][];
+ FederatedRange[] ranges = new FederatedRange[fedMap.getSize()];
+
+ // replace old reshape values for each worker
+ int i = 0, prev = 0, from = fedMap.getSize();
+ for(FederatedRange range : fedMap.getMap().keySet()) {
+ long rs = range.getBeginDims()[0], re = range.getEndDims()[0],
+ cs = range.getBeginDims()[1], ce = range.getEndDims()[1];
+ long rsn = (ixrange.rowStart >= rs) ? (ixrange.rowStart - rs) : 0;
+ long ren = (ixrange.rowEnd >= rs && ixrange.rowEnd < re) ? (ixrange.rowEnd - rs) : (re - rs - 1);
+ long csn = (ixrange.colStart >= cs) ? (ixrange.colStart - cs) : 0;
+ long cen = (ixrange.colEnd >= cs && ixrange.colEnd < ce) ? (ixrange.colEnd - cs) : (ce - cs - 1);
+
+ long[] newIx = new long[]{(int) rsn, (int) ren, (int) csn, (int) cen};
+
+ // find ranges where to apply leftIndex
+ long to;
+ if(in1.isFederated(FederationMap.FType.ROW) && (to = (prev + ren - rsn)) >= 0 &&
+ to < in2.getNumRows() && ixrange.rowStart <= re) {
+ sliceIxs[i] = new int[] { prev, (int) to, 0, (int) in2.getNumColumns()-1};
+ prev = (int) (to + 1);
Review comment:
something has gone wrong with the indentation
##########
File path: src/main/java/org/apache/sysds/runtime/instructions/fed/IndexingFEDInstruction.java
##########
@@ -171,4 +190,110 @@ private void rightIndexing(ExecutionContext ec)
out.setFedMapping(fedMap.copyWithNewID(fr1[0].getID()));
}
}
+
+ private void leftIndexing(ExecutionContext ec)
+ {
+ //get input and requested index range
+ CacheableData<?> in1 = ec.getCacheableData(input1);
+ CacheableData<?> in2 = ec.getCacheableData(input2);
+ IndexRange ixrange = getIndexRange(ec);
+
+ //check bounds
+ if( ixrange.rowStart < 0 || ixrange.rowStart >= in1.getNumRows() || ixrange.rowEnd >= in1.getNumRows()
+ || ixrange.colStart < 0 || ixrange.colStart >= in1.getNumColumns() || ixrange.colEnd >= in1.getNumColumns() ) {
+ throw new DMLRuntimeException("Invalid values for matrix indexing: ["+(ixrange.rowStart+1)+":"+(ixrange.rowEnd+1)+","
+ + (ixrange.colStart+1)+":"+(ixrange.colEnd+1)+"] " + "must be within matrix dimensions ["+in1.getNumRows()+","+in1.getNumColumns()+"].");
+ }
+ if( (ixrange.rowEnd-ixrange.rowStart+1) != in2.getNumRows() || (ixrange.colEnd-ixrange.colStart+1) != in2.getNumColumns()) {
+ throw new DMLRuntimeException("Invalid values for matrix indexing: " +
+ "dimensions of the source matrix ["+in2.getNumRows()+"x" + in2.getNumColumns() + "] " +
+ "do not match the shape of the matrix specified by indices [" +
+ (ixrange.rowStart+1) +":" + (ixrange.rowEnd+1) + ", " + (ixrange.colStart+1) + ":" + (ixrange.colEnd+1) + "].");
+ }
+
+ FederationMap fedMap = in1.getFedMapping();
+
+ String[] instStrings = new String[fedMap.getSize()];
+ int[][] sliceIxs = new int[fedMap.getSize()][];
+ FederatedRange[] ranges = new FederatedRange[fedMap.getSize()];
+
+ // replace old reshape values for each worker
+ int i = 0, prev = 0, from = fedMap.getSize();
+ for(FederatedRange range : fedMap.getMap().keySet()) {
+ long rs = range.getBeginDims()[0], re = range.getEndDims()[0],
+ cs = range.getBeginDims()[1], ce = range.getEndDims()[1];
+ long rsn = (ixrange.rowStart >= rs) ? (ixrange.rowStart - rs) : 0;
+ long ren = (ixrange.rowEnd >= rs && ixrange.rowEnd < re) ? (ixrange.rowEnd - rs) : (re - rs - 1);
+ long csn = (ixrange.colStart >= cs) ? (ixrange.colStart - cs) : 0;
+ long cen = (ixrange.colEnd >= cs && ixrange.colEnd < ce) ? (ixrange.colEnd - cs) : (ce - cs - 1);
+
+ long[] newIx = new long[]{(int) rsn, (int) ren, (int) csn, (int) cen};
+
+ // find ranges where to apply leftIndex
+ long to;
+ if(in1.isFederated(FederationMap.FType.ROW) && (to = (prev + ren - rsn)) >= 0 &&
+ to < in2.getNumRows() && ixrange.rowStart <= re) {
+ sliceIxs[i] = new int[] { prev, (int) to, 0, (int) in2.getNumColumns()-1};
+ prev = (int) (to + 1);
+
+ instStrings[i] = modifyIndices(newIx, 4, 8);
+ ranges[i] = range;
+ from = Math.min(i, from);
+ } else if(in1.isFederated(FederationMap.FType.COL) && (to = (prev + cen - csn)) >= 0 &&
+ to < in2.getNumColumns() && ixrange.colStart <= ce) {
+ sliceIxs[i] = new int[] {0, (int) in2.getNumRows() - 1, prev, (int) to};
+ prev = (int) (to + 1);
+
+ instStrings[i] = modifyIndices(newIx, 4, 8);
+ ranges[i] = range;
+ from = Math.min(i, from);
+ } else
Review comment:
code style specifies newline before else after a }
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [systemds] Baunsgaard closed pull request #1219: [SYSTEMDS-2925] Federated left indexing
Posted by GitBox <gi...@apache.org>.
Baunsgaard closed pull request #1219:
URL: https://github.com/apache/systemds/pull/1219
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [systemds] OlgaOvcharenko commented on a change in pull request #1219: [SYSTEMDS-2925] Federated left indexing
Posted by GitBox <gi...@apache.org>.
OlgaOvcharenko commented on a change in pull request #1219:
URL: https://github.com/apache/systemds/pull/1219#discussion_r613154819
##########
File path: src/main/java/org/apache/sysds/runtime/instructions/fed/IndexingFEDInstruction.java
##########
@@ -171,4 +190,110 @@ private void rightIndexing(ExecutionContext ec)
out.setFedMapping(fedMap.copyWithNewID(fr1[0].getID()));
}
}
+
+ private void leftIndexing(ExecutionContext ec)
+ {
+ //get input and requested index range
+ CacheableData<?> in1 = ec.getCacheableData(input1);
+ CacheableData<?> in2 = ec.getCacheableData(input2);
+ IndexRange ixrange = getIndexRange(ec);
+
+ //check bounds
+ if( ixrange.rowStart < 0 || ixrange.rowStart >= in1.getNumRows() || ixrange.rowEnd >= in1.getNumRows()
+ || ixrange.colStart < 0 || ixrange.colStart >= in1.getNumColumns() || ixrange.colEnd >= in1.getNumColumns() ) {
+ throw new DMLRuntimeException("Invalid values for matrix indexing: ["+(ixrange.rowStart+1)+":"+(ixrange.rowEnd+1)+","
+ + (ixrange.colStart+1)+":"+(ixrange.colEnd+1)+"] " + "must be within matrix dimensions ["+in1.getNumRows()+","+in1.getNumColumns()+"].");
+ }
+ if( (ixrange.rowEnd-ixrange.rowStart+1) != in2.getNumRows() || (ixrange.colEnd-ixrange.colStart+1) != in2.getNumColumns()) {
+ throw new DMLRuntimeException("Invalid values for matrix indexing: " +
+ "dimensions of the source matrix ["+in2.getNumRows()+"x" + in2.getNumColumns() + "] " +
+ "do not match the shape of the matrix specified by indices [" +
+ (ixrange.rowStart+1) +":" + (ixrange.rowEnd+1) + ", " + (ixrange.colStart+1) + ":" + (ixrange.colEnd+1) + "].");
+ }
+
+ FederationMap fedMap = in1.getFedMapping();
+
+ String[] instStrings = new String[fedMap.getSize()];
+ int[][] sliceIxs = new int[fedMap.getSize()][];
+ FederatedRange[] ranges = new FederatedRange[fedMap.getSize()];
+
+ // replace old reshape values for each worker
+ int i = 0, prev = 0, from = fedMap.getSize();
+ for(FederatedRange range : fedMap.getMap().keySet()) {
+ long rs = range.getBeginDims()[0], re = range.getEndDims()[0],
+ cs = range.getBeginDims()[1], ce = range.getEndDims()[1];
+ long rsn = (ixrange.rowStart >= rs) ? (ixrange.rowStart - rs) : 0;
+ long ren = (ixrange.rowEnd >= rs && ixrange.rowEnd < re) ? (ixrange.rowEnd - rs) : (re - rs - 1);
+ long csn = (ixrange.colStart >= cs) ? (ixrange.colStart - cs) : 0;
+ long cen = (ixrange.colEnd >= cs && ixrange.colEnd < ce) ? (ixrange.colEnd - cs) : (ce - cs - 1);
+
+ long[] newIx = new long[]{(int) rsn, (int) ren, (int) csn, (int) cen};
+
+ // find ranges where to apply leftIndex
+ long to;
+ if(in1.isFederated(FederationMap.FType.ROW) && (to = (prev + ren - rsn)) >= 0 &&
+ to < in2.getNumRows() && ixrange.rowStart <= re) {
+ sliceIxs[i] = new int[] { prev, (int) to, 0, (int) in2.getNumColumns()-1};
+ prev = (int) (to + 1);
+
+ instStrings[i] = modifyIndices(newIx, 4, 8);
+ ranges[i] = range;
+ from = Math.min(i, from);
+ } else if(in1.isFederated(FederationMap.FType.COL) && (to = (prev + cen - csn)) >= 0 &&
+ to < in2.getNumColumns() && ixrange.colStart <= ce) {
+ sliceIxs[i] = new int[] {0, (int) in2.getNumRows() - 1, prev, (int) to};
+ prev = (int) (to + 1);
+
+ instStrings[i] = modifyIndices(newIx, 4, 8);
+ ranges[i] = range;
+ from = Math.min(i, from);
+ } else
Review comment:
Thanks for the review, fixed the formatting now
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org