You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by de...@apache.org on 2017/06/06 17:44:04 UTC
[1/2] incubator-systemml git commit: [MINOR] Code format/trailing
whitespace for mlcontext package
Repository: incubator-systemml
Updated Branches:
refs/heads/master 882fa843e -> a712dad87
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a712dad8/src/main/java/org/apache/sysml/api/mlcontext/MatrixMetadata.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MatrixMetadata.java b/src/main/java/org/apache/sysml/api/mlcontext/MatrixMetadata.java
index a583e90..8f4cad7 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/MatrixMetadata.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/MatrixMetadata.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -42,7 +42,7 @@ public class MatrixMetadata extends Metadata {
/**
* Constructor to create a MatrixMetadata object based on matrix format.
- *
+ *
* @param matrixFormat
* The matrix format.
*/
@@ -53,7 +53,7 @@ public class MatrixMetadata extends Metadata {
/**
* Constructor to create a MatrixMetadata object based on matrix format, the
* number of rows, and the number of columns in a matrix.
- *
+ *
* @param matrixFormat
* The matrix format.
* @param numRows
@@ -70,7 +70,7 @@ public class MatrixMetadata extends Metadata {
/**
* Constructor to create a MatrixMetadata object based on matrix format, the
* number of rows, and the number of columns in a matrix.
- *
+ *
* @param matrixFormat
* The matrix format.
* @param numRows
@@ -88,7 +88,7 @@ public class MatrixMetadata extends Metadata {
* Constructor to create a MatrixMetadata object based on matrix format, the
* number of rows, the number of columns, and the number of non-zero values
* in a matrix.
- *
+ *
* @param matrixFormat
* The matrix format.
* @param numRows
@@ -109,7 +109,7 @@ public class MatrixMetadata extends Metadata {
* Constructor to create a MatrixMetadata object based on matrix format, the
* number of rows, the number of columns, and the number of non-zero values
* in a matrix.
- *
+ *
* @param matrixFormat
* The matrix format.
* @param numRows
@@ -131,7 +131,7 @@ public class MatrixMetadata extends Metadata {
* number of rows, the number of columns, the number of non-zero values, the
* number of rows per block, and the number of columns per block in a
* matrix.
- *
+ *
* @param matrixFormat
* The matrix format.
* @param numRows
@@ -160,7 +160,7 @@ public class MatrixMetadata extends Metadata {
* number of rows, the number of columns, the number of non-zero values, the
* number of rows per block, and the number of columns per block in a
* matrix.
- *
+ *
* @param matrixFormat
* The matrix format.
* @param numRows
@@ -187,7 +187,7 @@ public class MatrixMetadata extends Metadata {
/**
* Constructor to create a MatrixMetadata object based on the number of rows
* and the number of columns in a matrix.
- *
+ *
* @param numRows
* The number of rows in the matrix.
* @param numColumns
@@ -201,7 +201,7 @@ public class MatrixMetadata extends Metadata {
/**
* Constructor to create a MatrixMetadata object based on the number of rows
* and the number of columns in a matrix.
- *
+ *
* @param numRows
* The number of rows in the matrix.
* @param numColumns
@@ -216,7 +216,7 @@ public class MatrixMetadata extends Metadata {
* Constructor to create a MatrixMetadata object based on the number of
* rows, the number of columns, and the number of non-zero values in a
* matrix.
- *
+ *
* @param numRows
* The number of rows in the matrix.
* @param numColumns
@@ -234,7 +234,7 @@ public class MatrixMetadata extends Metadata {
* Constructor to create a MatrixMetadata object based on the number of
* rows, the number of columns, and the number of non-zero values in a
* matrix.
- *
+ *
* @param numRows
* The number of rows in the matrix.
* @param numColumns
@@ -252,7 +252,7 @@ public class MatrixMetadata extends Metadata {
* Constructor to create a MatrixMetadata object based on the number of
* rows, the number of columns, the number of rows per block, and the number
* of columns per block in a matrix.
- *
+ *
* @param numRows
* The number of rows in the matrix.
* @param numColumns
@@ -273,7 +273,7 @@ public class MatrixMetadata extends Metadata {
* Constructor to create a MatrixMetadata object based on the number of
* rows, the number of columns, the number of rows per block, and the number
* of columns per block in a matrix.
- *
+ *
* @param numRows
* The number of rows in the matrix.
* @param numColumns
@@ -294,7 +294,7 @@ public class MatrixMetadata extends Metadata {
* Constructor to create a MatrixMetadata object based on the number of
* rows, the number of columns, the number of non-zero values, the number of
* rows per block, and the number of columns per block in a matrix.
- *
+ *
* @param numRows
* The number of rows in the matrix.
* @param numColumns
@@ -319,7 +319,7 @@ public class MatrixMetadata extends Metadata {
* Constructor to create a MatrixMetadata object based on the number of
* rows, the number of columns, the number of non-zero values, the number of
* rows per block, and the number of columns per block in a matrix.
- *
+ *
* @param numRows
* The number of rows in the matrix.
* @param numColumns
@@ -342,7 +342,7 @@ public class MatrixMetadata extends Metadata {
/**
* Constructor to create a MatrixMetadata object based on a
* MatrixCharacteristics object.
- *
+ *
* @param matrixCharacteristics
* the matrix metadata as a MatrixCharacteristics object
*/
@@ -356,7 +356,7 @@ public class MatrixMetadata extends Metadata {
/**
* Set the MatrixMetadata fields based on a MatrixCharacteristics object.
- *
+ *
* @param matrixCharacteristics
* the matrix metadata as a MatrixCharacteristics object
*/
@@ -370,7 +370,7 @@ public class MatrixMetadata extends Metadata {
/**
* Obtain the number of rows
- *
+ *
* @return the number of rows
*/
public Long getNumRows() {
@@ -379,7 +379,7 @@ public class MatrixMetadata extends Metadata {
/**
* Set the number of rows
- *
+ *
* @param numRows
* the number of rows
*/
@@ -389,7 +389,7 @@ public class MatrixMetadata extends Metadata {
/**
* Obtain the number of columns
- *
+ *
* @return the number of columns
*/
public Long getNumColumns() {
@@ -398,7 +398,7 @@ public class MatrixMetadata extends Metadata {
/**
* Set the number of columns
- *
+ *
* @param numColumns
* the number of columns
*/
@@ -408,7 +408,7 @@ public class MatrixMetadata extends Metadata {
/**
* Obtain the number of non-zero values
- *
+ *
* @return the number of non-zero values
*/
public Long getNumNonZeros() {
@@ -417,7 +417,7 @@ public class MatrixMetadata extends Metadata {
/**
* Set the number of non-zero values
- *
+ *
* @param numNonZeros
* the number of non-zero values
*/
@@ -427,7 +427,7 @@ public class MatrixMetadata extends Metadata {
/**
* Obtain the number of rows per block
- *
+ *
* @return the number of rows per block
*/
public Integer getNumRowsPerBlock() {
@@ -436,7 +436,7 @@ public class MatrixMetadata extends Metadata {
/**
* Set the number of rows per block
- *
+ *
* @param numRowsPerBlock
* the number of rows per block
*/
@@ -446,7 +446,7 @@ public class MatrixMetadata extends Metadata {
/**
* Obtain the number of columns per block
- *
+ *
* @return the number of columns per block
*/
public Integer getNumColumnsPerBlock() {
@@ -455,7 +455,7 @@ public class MatrixMetadata extends Metadata {
/**
* Set the number of columns per block
- *
+ *
* @param numColumnsPerBlock
* the number of columns per block
*/
@@ -466,7 +466,7 @@ public class MatrixMetadata extends Metadata {
/**
* Convert the matrix metadata to a MatrixCharacteristics object. If all
* field values are {@code null}, {@code null} is returned.
- *
+ *
* @return the matrix metadata as a MatrixCharacteristics object, or
* {@code null} if all field values are null
*/
@@ -502,7 +502,7 @@ public class MatrixMetadata extends Metadata {
/**
* Obtain the matrix format
- *
+ *
* @return the matrix format
*/
public MatrixFormat getMatrixFormat() {
@@ -511,7 +511,7 @@ public class MatrixMetadata extends Metadata {
/**
* Set the matrix format
- *
+ *
* @param matrixFormat
* the matrix format
*/
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a712dad8/src/main/java/org/apache/sysml/api/mlcontext/Metadata.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/Metadata.java b/src/main/java/org/apache/sysml/api/mlcontext/Metadata.java
index c1c0a36..4061b5c 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/Metadata.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/Metadata.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a712dad8/src/main/java/org/apache/sysml/api/mlcontext/ProjectInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ProjectInfo.java b/src/main/java/org/apache/sysml/api/mlcontext/ProjectInfo.java
index 7d6163b..9af6696 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/ProjectInfo.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/ProjectInfo.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -40,7 +40,7 @@ public class ProjectInfo {
/**
* Return a ProjectInfo singleton instance.
- *
+ *
* @return the ProjectInfo singleton instance
*/
public static ProjectInfo getProjectInfo() {
@@ -87,7 +87,7 @@ public class ProjectInfo {
/**
* Obtain a manifest property value based on the key.
- *
+ *
* @param key
* the property key
* @return the property value
@@ -98,7 +98,7 @@ public class ProjectInfo {
/**
* Obtain the project version from the manifest.
- *
+ *
* @return the project version
*/
public String version() {
@@ -107,7 +107,7 @@ public class ProjectInfo {
/**
* Object the artifact build time from the manifest.
- *
+ *
* @return the artifact build time
*/
public String buildTime() {
@@ -116,7 +116,7 @@ public class ProjectInfo {
/**
* Obtain the minimum recommended Spark version from the manifest.
- *
+ *
* @return the minimum recommended Spark version
*/
public String minimumRecommendedSparkVersion() {
@@ -125,7 +125,7 @@ public class ProjectInfo {
/**
* Obtain all the properties from the manifest as a sorted map.
- *
+ *
* @return the manifest properties as a sorted map
*/
public SortedMap<String, String> properties() {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a712dad8/src/main/java/org/apache/sysml/api/mlcontext/Script.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/Script.java b/src/main/java/org/apache/sysml/api/mlcontext/Script.java
index 7b2b475..fb7fc41 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/Script.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/Script.java
@@ -342,19 +342,18 @@ public class Script {
inputParameters = new LinkedHashMap<String, Object>();
}
inputParameters.put(name, value);
- }
- else {
+ } else {
Data data = MLContextUtil.convertInputType(name, value, metadata);
if (data != null) {
- //store input variable name and data
+ // store input variable name and data
symbolTable.put(name, data);
inputVariables.add(name);
-
- //store matrix/frame meta data and disable variable cleanup
- if( data instanceof CacheableData ) {
- if( metadata != null )
+
+ // store matrix/frame meta data and disable variable cleanup
+ if (data instanceof CacheableData) {
+ if (metadata != null)
inputMetadata.put(name, metadata);
- ((CacheableData<?>)data).enableCleanup(false);
+ ((CacheableData<?>) data).enableCleanup(false);
}
}
}
@@ -387,7 +386,7 @@ public class Script {
/**
* Register output variables.
- *
+ *
* @param outputNames
* names of the output variables
* @return {@code this} Script object to allow chaining of methods
@@ -400,7 +399,7 @@ public class Script {
/**
* Register output variables.
- *
+ *
* @param outputNames
* names of the output variables
* @return {@code this} Script object to allow chaining of methods
@@ -542,7 +541,8 @@ public class Script {
String quotedString = MLContextUtil.quotedString((String) inValue);
sb.append(" = " + quotedString + ";\n");
} else if (MLContextUtil.isBasicType(inValue)) {
- sb.append(" = read('', data_type='scalar', value_type='" + MLContextUtil.getBasicTypeString(inValue) + "');\n");
+ sb.append(" = read('', data_type='scalar', value_type='" + MLContextUtil.getBasicTypeString(inValue)
+ + "');\n");
} else if (MLContextUtil.doesSymbolTableContainFrameObject(symbolTable, in)) {
sb.append(" = read('', data_type='frame');\n");
} else {
@@ -553,7 +553,8 @@ public class Script {
String quotedString = MLContextUtil.quotedString((String) inValue);
sb.append(" = " + quotedString + "\n");
} else if (MLContextUtil.isBasicType(inValue)) {
- sb.append(" = load('', data_type='scalar', value_type='" + MLContextUtil.getBasicTypeString(inValue) + "')\n");
+ sb.append(" = load('', data_type='scalar', value_type='" + MLContextUtil.getBasicTypeString(inValue)
+ + "')\n");
} else if (MLContextUtil.doesSymbolTableContainFrameObject(symbolTable, in)) {
sb.append(" = load('', data_type='frame')\n");
} else {
@@ -705,7 +706,7 @@ public class Script {
/**
* Execute the script and return the results as an MLResults object.
- *
+ *
* @return results as an MLResults object
*/
public MLResults execute() {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a712dad8/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
index 8fa4345..cd884f2 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -94,7 +94,7 @@ import org.apache.sysml.utils.Statistics;
* flow optimization check by subclassing ScriptExecutor and overriding the
* globalDataFlowOptimization method.
* </p>
- *
+ *
* <code>ScriptExecutor scriptExecutor = new ScriptExecutor() {
* <br> // turn off global data flow optimization check
* <br> @Override
@@ -104,7 +104,7 @@ import org.apache.sysml.utils.Statistics;
* <br>};
* <br>ml.execute(script, scriptExecutor);</code>
* <p>
- *
+ *
* For more information, please see the {@link #execute} method.
*/
public class ScriptExecutor {
@@ -137,7 +137,7 @@ public class ScriptExecutor {
/**
* ScriptExecutor constructor, where the configuration properties are passed
* in.
- *
+ *
* @param config
* the configuration properties to use by the ScriptExecutor
*/
@@ -181,16 +181,15 @@ public class ScriptExecutor {
* Output a description of the program to standard output.
*/
protected void showExplanation() {
- if( !explain ) return;
-
+ if (!explain)
+ return;
+
try {
- ExplainType explainType = (explainLevel != null) ?
- explainLevel.getExplainType() : ExplainType.RUNTIME;
+ ExplainType explainType = (explainLevel != null) ? explainLevel.getExplainType() : ExplainType.RUNTIME;
System.out.println(Explain.explain(dmlProgram, runtimeProgram, explainType));
- }
- catch (Exception e) {
+ } catch (Exception e) {
throw new MLContextException("Exception occurred while explaining dml program", e);
- }
+ }
}
/**
@@ -249,9 +248,9 @@ public class ScriptExecutor {
if (symbolTable != null) {
executionContext.setVariables(symbolTable);
}
-
+
}
-
+
/**
* Set the global flags (for example: statistics, gpu, etc).
*/
@@ -264,9 +263,10 @@ public class ScriptExecutor {
DMLScript.USE_ACCELERATOR = gpu;
DMLScript.STATISTICS_COUNT = statisticsMaxHeavyHitters;
}
-
+
/**
- * Reset the global flags (for example: statistics, gpu, etc) post-execution.
+ * Reset the global flags (for example: statistics, gpu, etc)
+ * post-execution.
*/
protected void resetGlobalFlags() {
DMLScript.STATISTICS = oldStatistics;
@@ -278,7 +278,7 @@ public class ScriptExecutor {
/**
* Execute a DML or PYDML script. This is broken down into the following
* primary methods:
- *
+ *
* <ol>
* <li>{@link #setup(Script)}</li>
* <li>{@link #parseScript()}</li>
@@ -298,7 +298,7 @@ public class ScriptExecutor {
* <li>{@link #executeRuntimeProgram()}</li>
* <li>{@link #cleanupAfterExecution()}</li>
* </ol>
- *
+ *
* @param script
* the DML or PYDML script to execute
* @return the results as a MLResults object
@@ -320,15 +320,14 @@ public class ScriptExecutor {
countCompiledMRJobsAndSparkInstructions();
initializeCachingAndScratchSpace();
cleanupRuntimeProgram();
-
+
try {
createAndInitializeExecutionContext();
executeRuntimeProgram();
- }
- finally {
+ } finally {
cleanupAfterExecution();
}
-
+
// add symbol table to MLResults
MLResults mlResults = new MLResults(script);
script.setResults(mlResults);
@@ -341,7 +340,7 @@ public class ScriptExecutor {
* and string, sets the ScriptExecutor in the script, sets the script string
* in the Spark Monitor, globally sets the script type, sets global flags,
* and resets statistics if needed.
- *
+ *
* @param script
* the DML or PYDML script to execute
*/
@@ -416,8 +415,9 @@ public class ScriptExecutor {
* initialize caching, and reset statistics.
*/
protected void initializeCachingAndScratchSpace() {
- if( !init ) return;
-
+ if (!init)
+ return;
+
try {
DMLScript.initHadoopExecution(config);
} catch (ParseException e) {
@@ -454,8 +454,8 @@ public class ScriptExecutor {
try {
ParserWrapper parser = ParserFactory.createParser(script.getScriptType());
Map<String, Object> inputParameters = script.getInputParameters();
- Map<String, String> inputParametersStringMaps = MLContextUtil.convertInputParametersForParser(
- inputParameters, script.getScriptType());
+ Map<String, String> inputParametersStringMaps = MLContextUtil
+ .convertInputParametersForParser(inputParameters, script.getScriptType());
String scriptExecutionString = script.getScriptExecutionString();
dmlProgram = parser.parse(null, scriptExecutionString, inputParametersStringMaps);
@@ -471,11 +471,12 @@ public class ScriptExecutor {
protected void rewritePersistentReadsAndWrites() {
LocalVariableMap symbolTable = script.getSymbolTable();
if (symbolTable != null) {
- String[] inputs = (script.getInputVariables() == null) ? new String[0] : script.getInputVariables()
- .toArray(new String[0]);
+ String[] inputs = (script.getInputVariables() == null) ? new String[0]
+ : script.getInputVariables().toArray(new String[0]);
String[] outputs = (script.getOutputVariables() == null) ? new String[0]
: script.getOutputVariables().toArray(new String[0]);
- RewriteRemovePersistentReadWrite rewrite = new RewriteRemovePersistentReadWrite(inputs, outputs, script.getSymbolTable());
+ RewriteRemovePersistentReadWrite rewrite = new RewriteRemovePersistentReadWrite(inputs, outputs,
+ script.getSymbolTable());
ProgramRewriter programRewriter = new ProgramRewriter(rewrite);
try {
programRewriter.rewriteProgramHopDAGs(dmlProgram);
@@ -490,7 +491,7 @@ public class ScriptExecutor {
/**
* Set the SystemML configuration properties.
- *
+ *
* @param config
* The configuration properties
*/
@@ -549,7 +550,7 @@ public class ScriptExecutor {
/**
* Obtain the program
- *
+ *
* @return the program
*/
public DMLProgram getDmlProgram() {
@@ -558,7 +559,7 @@ public class ScriptExecutor {
/**
* Obtain the translator
- *
+ *
* @return the translator
*/
public DMLTranslator getDmlTranslator() {
@@ -567,7 +568,7 @@ public class ScriptExecutor {
/**
* Obtain the runtime program
- *
+ *
* @return the runtime program
*/
public Program getRuntimeProgram() {
@@ -576,7 +577,7 @@ public class ScriptExecutor {
/**
* Obtain the execution context
- *
+ *
* @return the execution context
*/
public ExecutionContext getExecutionContext() {
@@ -585,7 +586,7 @@ public class ScriptExecutor {
/**
* Obtain the Script object associated with this ScriptExecutor
- *
+ *
* @return the Script object associated with this ScriptExecutor
*/
public Script getScript() {
@@ -595,7 +596,7 @@ public class ScriptExecutor {
/**
* Whether or not an explanation of the DML/PYDML program should be output
* to standard output.
- *
+ *
* @param explain
* {@code true} if explanation should be output, {@code false}
* otherwise
@@ -607,7 +608,7 @@ public class ScriptExecutor {
/**
* Whether or not statistics about the DML/PYDML program should be output to
* standard output.
- *
+ *
* @param statistics
* {@code true} if statistics should be output, {@code false}
* otherwise
@@ -618,8 +619,9 @@ public class ScriptExecutor {
/**
* Set the maximum number of heavy hitters to display with statistics.
- *
- * @param maxHeavyHitters the maximum number of heavy hitters
+ *
+ * @param maxHeavyHitters
+ * the maximum number of heavy hitters
*/
public void setStatisticsMaxHeavyHitters(int maxHeavyHitters) {
this.statisticsMaxHeavyHitters = maxHeavyHitters;
@@ -628,7 +630,7 @@ public class ScriptExecutor {
/**
* Obtain whether or not all values should be maintained in the symbol table
* after execution.
- *
+ *
* @return {@code true} if all values should be maintained in the symbol
* table, {@code false} otherwise
*/
@@ -639,7 +641,7 @@ public class ScriptExecutor {
/**
* Set whether or not all values should be maintained in the symbol table
* after execution.
- *
+ *
* @param maintainSymbolTable
* {@code true} if all values should be maintained in the symbol
* table, {@code false} otherwise
@@ -649,11 +651,12 @@ public class ScriptExecutor {
}
/**
- * Whether or not to initialize the scratch_space, bufferpool, etc. Note that any
- * redundant initialize (e.g., multiple scripts from one MLContext) clears existing
- * files from the scratch space and buffer pool.
- *
- * @param init {@code true} if should initialize, {@code false} otherwise
+ * Whether or not to initialize the scratch_space, bufferpool, etc. Note
+ * that any redundant initialize (e.g., multiple scripts from one MLContext)
+ * clears existing files from the scratch space and buffer pool.
+ *
+ * @param init
+ * {@code true} if should initialize, {@code false} otherwise
*/
public void setInit(boolean init) {
this.init = init;
@@ -662,7 +665,7 @@ public class ScriptExecutor {
/**
* Set the level of program explanation that should be displayed if explain
* is set to true.
- *
+ *
* @param explainLevel
* the level of program explanation
*/
@@ -678,27 +681,27 @@ public class ScriptExecutor {
/**
* Whether or not to enable GPU usage.
- *
+ *
* @param enabled
- * {@code true} if enabled, {@code false} otherwise
+ * {@code true} if enabled, {@code false} otherwise
*/
- public void setGPU(boolean enabled) {
- this.gpu = enabled;
- }
-
+ public void setGPU(boolean enabled) {
+ this.gpu = enabled;
+ }
+
/**
* Whether or not to force GPU usage.
- *
+ *
* @param enabled
- * {@code true} if enabled, {@code false} otherwise
+ * {@code true} if enabled, {@code false} otherwise
*/
- public void setForceGPU(boolean enabled) {
- this.forceGPU = enabled;
- }
+ public void setForceGPU(boolean enabled) {
+ this.forceGPU = enabled;
+ }
/**
* Obtain the SystemML configuration properties.
- *
+ *
* @return the configuration properties
*/
public DMLConfig getConfig() {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a712dad8/src/main/java/org/apache/sysml/api/mlcontext/ScriptType.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptType.java b/src/main/java/org/apache/sysml/api/mlcontext/ScriptType.java
index 94c9057..ee24636 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptType.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/ScriptType.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -37,7 +37,7 @@ public enum ScriptType {
/**
* Obtain script type as a lowercase string ("dml" or "pydml").
- *
+ *
* @return lowercase string representing the script type
*/
public String lowerCase() {
@@ -46,7 +46,7 @@ public enum ScriptType {
/**
* Is the script type DML?
- *
+ *
* @return {@code true} if the script type is DML, {@code false} otherwise
*/
public boolean isDML() {
@@ -55,7 +55,7 @@ public enum ScriptType {
/**
* Is the script type PYDML?
- *
+ *
* @return {@code true} if the script type is PYDML, {@code false} otherwise
*/
public boolean isPYDML() {
[2/2] incubator-systemml git commit: [MINOR] Code format/trailing
whitespace for mlcontext package
Posted by de...@apache.org.
[MINOR] Code format/trailing whitespace for mlcontext package
Apply dev/code-style/systemml-style-eclipse.xml format and remove
trailing whitespace in api.mlcontext package.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/a712dad8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/a712dad8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/a712dad8
Branch: refs/heads/master
Commit: a712dad8792157c56d0292ef4826ea70ef9f3849
Parents: 882fa84
Author: Deron Eriksson <de...@us.ibm.com>
Authored: Tue Jun 6 10:38:20 2017 -0700
Committer: Deron Eriksson <de...@us.ibm.com>
Committed: Tue Jun 6 10:38:20 2017 -0700
----------------------------------------------------------------------
.../sysml/api/mlcontext/BinaryBlockFrame.java | 30 +-
.../sysml/api/mlcontext/BinaryBlockMatrix.java | 31 +-
.../org/apache/sysml/api/mlcontext/Frame.java | 22 +-
.../apache/sysml/api/mlcontext/FrameFormat.java | 4 +-
.../sysml/api/mlcontext/FrameMetadata.java | 84 +++---
.../apache/sysml/api/mlcontext/FrameSchema.java | 18 +-
.../apache/sysml/api/mlcontext/MLContext.java | 35 ++-
.../api/mlcontext/MLContextConversionUtil.java | 287 +++++++++----------
.../sysml/api/mlcontext/MLContextException.java | 6 +-
.../sysml/api/mlcontext/MLContextUtil.java | 74 ++---
.../apache/sysml/api/mlcontext/MLResults.java | 48 ++--
.../org/apache/sysml/api/mlcontext/Matrix.java | 30 +-
.../sysml/api/mlcontext/MatrixFormat.java | 14 +-
.../sysml/api/mlcontext/MatrixMetadata.java | 64 ++---
.../apache/sysml/api/mlcontext/Metadata.java | 4 +-
.../apache/sysml/api/mlcontext/ProjectInfo.java | 16 +-
.../org/apache/sysml/api/mlcontext/Script.java | 27 +-
.../sysml/api/mlcontext/ScriptExecutor.java | 123 ++++----
.../apache/sysml/api/mlcontext/ScriptType.java | 10 +-
19 files changed, 472 insertions(+), 455 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a712dad8/src/main/java/org/apache/sysml/api/mlcontext/BinaryBlockFrame.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/BinaryBlockFrame.java b/src/main/java/org/apache/sysml/api/mlcontext/BinaryBlockFrame.java
index e36dfb4..8c58315 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/BinaryBlockFrame.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/BinaryBlockFrame.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -40,7 +40,7 @@ public class BinaryBlockFrame {
/**
* Convert a Spark DataFrame to a SystemML binary-block representation.
- *
+ *
* @param dataFrame
* the Spark DataFrame
* @param frameMetadata
@@ -54,7 +54,7 @@ public class BinaryBlockFrame {
/**
* Convert a Spark DataFrame to a SystemML binary-block representation,
* specifying the number of rows and columns.
- *
+ *
* @param dataFrame
* the Spark DataFrame
* @param numRows
@@ -63,13 +63,13 @@ public class BinaryBlockFrame {
* the number of columns
*/
public BinaryBlockFrame(Dataset<Row> dataFrame, long numRows, long numCols) {
- this(dataFrame, new FrameMetadata(numRows, numCols,
- ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize()));
+ this(dataFrame, new FrameMetadata(numRows, numCols, ConfigurationManager.getBlocksize(),
+ ConfigurationManager.getBlocksize()));
}
/**
* Convert a Spark DataFrame to a SystemML binary-block representation.
- *
+ *
* @param dataFrame
* the Spark DataFrame
*/
@@ -80,7 +80,7 @@ public class BinaryBlockFrame {
/**
* Create a BinaryBlockFrame, specifying the SystemML binary-block frame and
* its metadata.
- *
+ *
* @param binaryBlocks
* the {@code JavaPairRDD<Long, FrameBlock>} frame
* @param matrixCharacteristics
@@ -94,7 +94,7 @@ public class BinaryBlockFrame {
/**
* Create a BinaryBlockFrame, specifying the SystemML binary-block frame and
* its metadata.
- *
+ *
* @param binaryBlocks
* the {@code JavaPairRDD<Long, FrameBlock>} frame
* @param frameMetadata
@@ -108,7 +108,7 @@ public class BinaryBlockFrame {
/**
* Obtain a SystemML binary-block frame as a
* {@code JavaPairRDD<Long, FrameBlock>}
- *
+ *
* @return the SystemML binary-block frame
*/
public JavaPairRDD<Long, FrameBlock> getBinaryBlocks() {
@@ -117,7 +117,7 @@ public class BinaryBlockFrame {
/**
* Obtain a SystemML binary-block frame as a {@code FrameBlock}
- *
+ *
* @return the SystemML binary-block frame as a {@code FrameBlock}
*/
public FrameBlock getFrameBlock() {
@@ -133,7 +133,7 @@ public class BinaryBlockFrame {
/**
* Obtain the SystemML binary-block frame characteristics
- *
+ *
* @return the frame metadata as {@code MatrixCharacteristics}
*/
public MatrixCharacteristics getMatrixCharacteristics() {
@@ -142,7 +142,7 @@ public class BinaryBlockFrame {
/**
* Obtain the SystemML binary-block frame metadata
- *
+ *
* @return the frame metadata as {@code FrameMetadata}
*/
public FrameMetadata getFrameMetadata() {
@@ -151,7 +151,7 @@ public class BinaryBlockFrame {
/**
* Set the SystemML binary-block frame metadata
- *
+ *
* @param frameMetadata
* the frame metadata
*/
@@ -162,7 +162,7 @@ public class BinaryBlockFrame {
/**
* Set the SystemML binary-block frame as a
* {@code JavaPairRDD<Long, FrameBlock>}
- *
+ *
* @param binaryBlocks
* the SystemML binary-block frame
*/
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a712dad8/src/main/java/org/apache/sysml/api/mlcontext/BinaryBlockMatrix.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/BinaryBlockMatrix.java b/src/main/java/org/apache/sysml/api/mlcontext/BinaryBlockMatrix.java
index abfad09..c0f46be 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/BinaryBlockMatrix.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/BinaryBlockMatrix.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -30,7 +30,8 @@ import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
/**
- * BinaryBlockMatrix stores data as a SystemML binary-block matrix representation.
+ * BinaryBlockMatrix stores data as a SystemML binary-block matrix
+ * representation.
*
*/
public class BinaryBlockMatrix {
@@ -40,7 +41,7 @@ public class BinaryBlockMatrix {
/**
* Convert a Spark DataFrame to a SystemML binary-block representation.
- *
+ *
* @param dataFrame
* the Spark DataFrame
* @param matrixMetadata
@@ -54,7 +55,7 @@ public class BinaryBlockMatrix {
/**
* Convert a Spark DataFrame to a SystemML binary-block representation,
* specifying the number of rows and columns.
- *
+ *
* @param dataFrame
* the Spark DataFrame
* @param numRows
@@ -63,13 +64,13 @@ public class BinaryBlockMatrix {
* the number of columns
*/
public BinaryBlockMatrix(Dataset<Row> dataFrame, long numRows, long numCols) {
- this(dataFrame, new MatrixMetadata(numRows, numCols,
- ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize()));
+ this(dataFrame, new MatrixMetadata(numRows, numCols, ConfigurationManager.getBlocksize(),
+ ConfigurationManager.getBlocksize()));
}
/**
* Convert a Spark DataFrame to a SystemML binary-block representation.
- *
+ *
* @param dataFrame
* the Spark DataFrame
*/
@@ -80,7 +81,7 @@ public class BinaryBlockMatrix {
/**
* Create a BinaryBlockMatrix, specifying the SystemML binary-block matrix
* and its metadata.
- *
+ *
* @param binaryBlocks
* the {@code JavaPairRDD<MatrixIndexes, MatrixBlock>} matrix
* @param matrixCharacteristics
@@ -95,7 +96,7 @@ public class BinaryBlockMatrix {
/**
* Obtain a SystemML binary-block matrix as a
* {@code JavaPairRDD<MatrixIndexes, MatrixBlock>}
- *
+ *
* @return the SystemML binary-block matrix
*/
public JavaPairRDD<MatrixIndexes, MatrixBlock> getBinaryBlocks() {
@@ -104,7 +105,7 @@ public class BinaryBlockMatrix {
/**
* Obtain a SystemML binary-block matrix as a {@code MatrixBlock}
- *
+ *
* @return the SystemML binary-block matrix as a {@code MatrixBlock}
*/
public MatrixBlock getMatrixBlock() {
@@ -119,7 +120,7 @@ public class BinaryBlockMatrix {
/**
* Obtain the SystemML binary-block matrix characteristics
- *
+ *
* @return the matrix metadata as {@code MatrixCharacteristics}
*/
public MatrixCharacteristics getMatrixCharacteristics() {
@@ -128,7 +129,7 @@ public class BinaryBlockMatrix {
/**
* Obtain the SystemML binary-block matrix metadata
- *
+ *
* @return the matrix metadata as {@code MatrixMetadata}
*/
public MatrixMetadata getMatrixMetadata() {
@@ -137,7 +138,7 @@ public class BinaryBlockMatrix {
/**
* Set the SystemML binary-block matrix metadata
- *
+ *
* @param matrixMetadata
* the matrix metadata
*/
@@ -148,7 +149,7 @@ public class BinaryBlockMatrix {
/**
* Set the SystemML binary-block matrix as a
* {@code JavaPairRDD<MatrixIndexes, MatrixBlock>}
- *
+ *
* @param binaryBlocks
* the SystemML binary-block matrix
*/
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a712dad8/src/main/java/org/apache/sysml/api/mlcontext/Frame.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/Frame.java b/src/main/java/org/apache/sysml/api/mlcontext/Frame.java
index 6ca911d..9d3bb2c 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/Frame.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/Frame.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -42,7 +42,7 @@ public class Frame {
/**
* Obtain the frame as a SystemML FrameObject.
- *
+ *
* @return the frame as a SystemML FrameObject
*/
public FrameObject toFrameObject() {
@@ -51,7 +51,7 @@ public class Frame {
/**
* Obtain the frame as a two-dimensional String array
- *
+ *
* @return the frame as a two-dimensional String array
*/
public String[][] to2DStringArray() {
@@ -60,7 +60,7 @@ public class Frame {
/**
* Obtain the frame as a {@code JavaRDD<String>} in IJV format
- *
+ *
* @return the frame as a {@code JavaRDD<String>} in IJV format
*/
public JavaRDD<String> toJavaRDDStringIJV() {
@@ -69,7 +69,7 @@ public class Frame {
/**
* Obtain the frame as a {@code JavaRDD<String>} in CSV format
- *
+ *
* @return the frame as a {@code JavaRDD<String>} in CSV format
*/
public JavaRDD<String> toJavaRDDStringCSV() {
@@ -78,7 +78,7 @@ public class Frame {
/**
* Obtain the frame as a {@code RDD<String>} in CSV format
- *
+ *
* @return the frame as a {@code RDD<String>} in CSV format
*/
public RDD<String> toRDDStringCSV() {
@@ -87,7 +87,7 @@ public class Frame {
/**
* Obtain the frame as a {@code RDD<String>} in IJV format
- *
+ *
* @return the frame as a {@code RDD<String>} in IJV format
*/
public RDD<String> toRDDStringIJV() {
@@ -96,7 +96,7 @@ public class Frame {
/**
* Obtain the frame as a {@code DataFrame}
- *
+ *
* @return the frame as a {@code DataFrame}
*/
public Dataset<Row> toDF() {
@@ -105,7 +105,7 @@ public class Frame {
/**
* Obtain the matrix as a {@code BinaryBlockFrame}
- *
+ *
* @return the matrix as a {@code BinaryBlockFrame}
*/
public BinaryBlockFrame toBinaryBlockFrame() {
@@ -114,7 +114,7 @@ public class Frame {
/**
* Obtain the frame metadata
- *
+ *
* @return the frame metadata
*/
public FrameMetadata getFrameMetadata() {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a712dad8/src/main/java/org/apache/sysml/api/mlcontext/FrameFormat.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/FrameFormat.java b/src/main/java/org/apache/sysml/api/mlcontext/FrameFormat.java
index f339eda..edea831 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/FrameFormat.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/FrameFormat.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a712dad8/src/main/java/org/apache/sysml/api/mlcontext/FrameMetadata.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/FrameMetadata.java b/src/main/java/org/apache/sysml/api/mlcontext/FrameMetadata.java
index 7dd20f1..6b0e68c 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/FrameMetadata.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/FrameMetadata.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -44,7 +44,7 @@ public class FrameMetadata extends Metadata {
/**
* Constructor to create a FrameMetadata object based on a string
* representation of a frame schema.
- *
+ *
* @param schema
* String representation of the frame schema.
*/
@@ -54,7 +54,7 @@ public class FrameMetadata extends Metadata {
/**
* Constructor to create a FrameMetadata object based on frame format.
- *
+ *
* @param frameFormat
* The frame format.
*/
@@ -64,7 +64,7 @@ public class FrameMetadata extends Metadata {
/**
* Constructor to create a FrameMetadata object based on frame schema.
- *
+ *
* @param frameSchema
* The frame schema.
*/
@@ -75,7 +75,7 @@ public class FrameMetadata extends Metadata {
/**
* Constructor to create a FrameMetadata object based on frame format and
* frame schema.
- *
+ *
* @param frameFormat
* The frame format.
* @param frameSchema
@@ -89,7 +89,7 @@ public class FrameMetadata extends Metadata {
/**
* Constructor to create a FrameMetadata object based on frame format, frame
* schema, the number of rows, and the number of columns in a frame.
- *
+ *
* @param frameFormat
* The frame format.
* @param frameSchema
@@ -109,7 +109,7 @@ public class FrameMetadata extends Metadata {
/**
* Constructor to create a FrameMetadata object based on frame format, frame
* schema, the number of rows, and the number of columns in a frame.
- *
+ *
* @param frameFormat
* The frame format.
* @param frameSchema
@@ -131,7 +131,7 @@ public class FrameMetadata extends Metadata {
* schema, the number of rows, the number of columns, the number of non-zero
* values, the number of rows per block, and the number of columns per block
* in a frame.
- *
+ *
* @param frameFormat
* The frame format.
* @param frameSchema
@@ -163,7 +163,7 @@ public class FrameMetadata extends Metadata {
* schema, the number of rows, the number of columns, the number of non-zero
* values, the number of rows per block, and the number of columns per block
* in a frame.
- *
+ *
* @param frameFormat
* The frame format.
* @param frameSchema
@@ -193,7 +193,7 @@ public class FrameMetadata extends Metadata {
/**
* Constructor to create a FrameMetadata object based on frame format, the
* number of rows, and the number of columns in a frame.
- *
+ *
* @param frameFormat
* The frame format.
* @param numRows
@@ -210,7 +210,7 @@ public class FrameMetadata extends Metadata {
/**
* Constructor to create a FrameMetadata object based on frame format, the
* number of rows, and the number of columns in a frame.
- *
+ *
* @param frameFormat
* The frame format.
* @param numRows
@@ -228,7 +228,7 @@ public class FrameMetadata extends Metadata {
* Constructor to create a FrameMetadata object based on frame format, the
* number of rows, the number of columns, and the number of non-zero values
* in a frame.
- *
+ *
* @param frameFormat
* The frame format.
* @param numRows
@@ -249,7 +249,7 @@ public class FrameMetadata extends Metadata {
* Constructor to create a FrameMetadata object based on frame format, the
* number of rows, the number of columns, and the number of non-zero values
* in a frame.
- *
+ *
* @param frameFormat
* The frame format.
* @param numRows
@@ -270,7 +270,7 @@ public class FrameMetadata extends Metadata {
* Constructor to create a FrameMetadata object based on frame format, the
* number of rows, the number of columns, the number of non-zero values, the
* number of rows per block, and the number of columns per block in a frame.
- *
+ *
* @param frameFormat
* The frame format.
* @param numRows
@@ -298,7 +298,7 @@ public class FrameMetadata extends Metadata {
* Constructor to create a FrameMetadata object based on frame format, the
* number of rows, the number of columns, the number of non-zero values, the
* number of rows per block, and the number of columns per block in a frame.
- *
+ *
* @param frameFormat
* The frame format.
* @param numRows
@@ -325,7 +325,7 @@ public class FrameMetadata extends Metadata {
/**
* Constructor to create a FrameMetadata object based on the number of rows
* and the number of columns in a frame.
- *
+ *
* @param numRows
* The number of rows in the frame.
* @param numColumns
@@ -339,7 +339,7 @@ public class FrameMetadata extends Metadata {
/**
* Constructor to create a FrameMetadata object based on the number of rows
* and the number of columns in a frame.
- *
+ *
* @param numRows
* The number of rows in the frame.
* @param numColumns
@@ -353,7 +353,7 @@ public class FrameMetadata extends Metadata {
/**
* Constructor to create a FrameMetadata object based on the number of rows,
* the number of columns, and the number of non-zero values in a frame.
- *
+ *
* @param numRows
* The number of rows in the frame.
* @param numColumns
@@ -370,7 +370,7 @@ public class FrameMetadata extends Metadata {
/**
* Constructor to create a FrameMetadata object based on the number of rows,
* the number of columns, and the number of non-zero values in a frame.
- *
+ *
* @param numRows
* The number of rows in the frame.
* @param numColumns
@@ -388,7 +388,7 @@ public class FrameMetadata extends Metadata {
* Constructor to create a FrameMetadata object based on the number of rows,
* the number of columns, the number of rows per block, and the number of
* columns per block in a frame.
- *
+ *
* @param numRows
* The number of rows in the frame.
* @param numColumns
@@ -409,7 +409,7 @@ public class FrameMetadata extends Metadata {
* Constructor to create a FrameMetadata object based on the number of rows,
* the number of columns, the number of rows per block, and the number of
* columns per block in a frame.
- *
+ *
* @param numRows
* The number of rows in the frame.
* @param numColumns
@@ -430,7 +430,7 @@ public class FrameMetadata extends Metadata {
* Constructor to create a FrameMetadata object based on the number of rows,
* the number of columns, the number of non-zero values, the number of rows
* per block, and the number of columns per block in a frame.
- *
+ *
* @param numRows
* The number of rows in the frame.
* @param numColumns
@@ -455,7 +455,7 @@ public class FrameMetadata extends Metadata {
* Constructor to create a FrameMetadata object based on the number of rows,
* the number of columns, the number of non-zero values, the number of rows
* per block, and the number of columns per block in a frame.
- *
+ *
* @param numRows
* The number of rows in the frame.
* @param numColumns
@@ -478,7 +478,7 @@ public class FrameMetadata extends Metadata {
/**
* Constructor to create a FrameMetadata object based on a
* MatrixCharacteristics object.
- *
+ *
* @param matrixCharacteristics
* the frame metadata as a MatrixCharacteristics object
*/
@@ -493,7 +493,7 @@ public class FrameMetadata extends Metadata {
/**
* Constructor to create a FrameMetadata object based on the frame schema
* and a MatrixCharacteristics object.
- *
+ *
* @param frameSchema
* The frame schema.
* @param matrixCharacteristics
@@ -510,7 +510,7 @@ public class FrameMetadata extends Metadata {
/**
* Set the FrameMetadata fields based on a MatrixCharacteristics object.
- *
+ *
* @param matrixCharacteristics
* the frame metadata as a MatrixCharacteristics object
*/
@@ -524,7 +524,7 @@ public class FrameMetadata extends Metadata {
/**
* Obtain the number of rows
- *
+ *
* @return the number of rows
*/
public Long getNumRows() {
@@ -533,7 +533,7 @@ public class FrameMetadata extends Metadata {
/**
* Set the number of rows
- *
+ *
* @param numRows
* the number of rows
*/
@@ -543,7 +543,7 @@ public class FrameMetadata extends Metadata {
/**
* Obtain the number of columns
- *
+ *
* @return the number of columns
*/
public Long getNumColumns() {
@@ -552,7 +552,7 @@ public class FrameMetadata extends Metadata {
/**
* Set the number of columns
- *
+ *
* @param numColumns
* the number of columns
*/
@@ -562,7 +562,7 @@ public class FrameMetadata extends Metadata {
/**
* Obtain the number of non-zero values
- *
+ *
* @return the number of non-zero values
*/
public Long getNumNonZeros() {
@@ -571,7 +571,7 @@ public class FrameMetadata extends Metadata {
/**
* Set the number of non-zero values
- *
+ *
* @param numNonZeros
* the number of non-zero values
*/
@@ -581,7 +581,7 @@ public class FrameMetadata extends Metadata {
/**
* Obtain the number of rows per block
- *
+ *
* @return the number of rows per block
*/
public Integer getNumRowsPerBlock() {
@@ -590,7 +590,7 @@ public class FrameMetadata extends Metadata {
/**
* Set the number of rows per block
- *
+ *
* @param numRowsPerBlock
* the number of rows per block
*/
@@ -600,7 +600,7 @@ public class FrameMetadata extends Metadata {
/**
* Obtain the number of columns per block
- *
+ *
* @return the number of columns per block
*/
public Integer getNumColumnsPerBlock() {
@@ -609,7 +609,7 @@ public class FrameMetadata extends Metadata {
/**
* Set the number of columns per block
- *
+ *
* @param numColumnsPerBlock
* the number of columns per block
*/
@@ -620,7 +620,7 @@ public class FrameMetadata extends Metadata {
/**
* Convert the frame metadata to a MatrixCharacteristics object. If all
* field values are {@code null}, {@code null} is returned.
- *
+ *
* @return the frame metadata as a MatrixCharacteristics object, or
* {@code null} if all field values are null
*/
@@ -657,7 +657,7 @@ public class FrameMetadata extends Metadata {
/**
* Obtain the frame format
- *
+ *
* @return the frame format
*/
public FrameFormat getFrameFormat() {
@@ -666,7 +666,7 @@ public class FrameMetadata extends Metadata {
/**
* Set the frame format
- *
+ *
* @param frameFormat
* the frame format
*/
@@ -676,7 +676,7 @@ public class FrameMetadata extends Metadata {
/**
* Obtain the frame schema
- *
+ *
* @return the frame schema
*/
public FrameSchema getFrameSchema() {
@@ -685,7 +685,7 @@ public class FrameMetadata extends Metadata {
/**
* Set the frame schema
- *
+ *
* @param frameSchema
* the frame schema
*/
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a712dad8/src/main/java/org/apache/sysml/api/mlcontext/FrameSchema.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/FrameSchema.java b/src/main/java/org/apache/sysml/api/mlcontext/FrameSchema.java
index 040d77b..c1f7466 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/FrameSchema.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/FrameSchema.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -39,7 +39,7 @@ public class FrameSchema {
/**
* Constructor that specifies the schema as a list of {@code ValueType}
* values.
- *
+ *
* @param schema
* the frame schema
*/
@@ -49,7 +49,7 @@ public class FrameSchema {
/**
* Constructor that specifies the schema as a comma-separated string.
- *
+ *
* @param schema
* the frame schema as a string
*/
@@ -59,7 +59,7 @@ public class FrameSchema {
/**
* Obtain the frame schema
- *
+ *
* @return the frame schema as a list of {@code ValueType} values
*/
public List<ValueType> getSchema() {
@@ -68,7 +68,7 @@ public class FrameSchema {
/**
* Set the frame schema
- *
+ *
* @param schema
* the frame schema
*/
@@ -79,7 +79,7 @@ public class FrameSchema {
/**
* Set the frame schema, specifying the frame schema as a comma-separated
* string
- *
+ *
* @param schema
* the frame schema as a string
*/
@@ -89,7 +89,7 @@ public class FrameSchema {
/**
* Convert a schema string to a list of {@code ValueType} values
- *
+ *
* @param schemaString
* the frame schema as a string
* @return the frame schema as a list of {@code ValueType} values
@@ -108,7 +108,7 @@ public class FrameSchema {
/**
* Obtain the schema as a comma-separated string
- *
+ *
* @return the frame schema as a string
*/
public String getSchemaAsString() {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a712dad8/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
index f4cce99..a9ab393 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
@@ -114,8 +114,8 @@ public class MLContext {
private ExplainLevel explainLevel = null;
/**
- * Whether or not all values should be maintained in the symbol table
- * after execution.
+ * Whether or not all values should be maintained in the symbol table after
+ * execution.
*/
private boolean maintainSymbolTable = false;
@@ -173,7 +173,8 @@ public class MLContext {
* Create an MLContext based on a SparkSession for interaction with SystemML
* on Spark.
*
- * @param spark SparkSession
+ * @param spark
+ * SparkSession
*/
public MLContext(SparkSession spark) {
initMLContext(spark);
@@ -215,13 +216,16 @@ public class MLContext {
MLContextUtil.verifySparkVersionSupported(spark);
} catch (MLContextException e) {
if (info() != null) {
- log.warn("Apache Spark " + this.info().minimumRecommendedSparkVersion() + " or above is recommended for SystemML " + this.info().version());
+ log.warn("Apache Spark " + this.info().minimumRecommendedSparkVersion()
+ + " or above is recommended for SystemML " + this.info().version());
} else {
try {
String minSparkVersion = MLContextUtil.getMinimumRecommendedSparkVersionFromPom();
- log.warn("Apache Spark " + minSparkVersion + " or above is recommended for this version of SystemML.");
+ log.warn("Apache Spark " + minSparkVersion
+ + " or above is recommended for this version of SystemML.");
} catch (MLContextException e1) {
- log.error("Minimum recommended Spark version could not be determined from SystemML jar file manifest or pom.xml");
+ log.error(
+ "Minimum recommended Spark version could not be determined from SystemML jar file manifest or pom.xml");
}
}
}
@@ -359,7 +363,6 @@ public class MLContext {
this.explain = explain;
}
-
/**
* Obtain whether or not all values should be maintained in the symbol table
* after execution.
@@ -416,20 +419,24 @@ public class MLContext {
}
/**
- * Whether or not to use (an available) GPU on the driver node.
- * If a GPU is not available, and the GPU mode is set, SystemML will crash when the program is run.
+ * Whether or not to use (an available) GPU on the driver node. If a GPU is
+ * not available, and the GPU mode is set, SystemML will crash when the
+ * program is run.
+ *
* @param enable
- * true if needs to be enabled, false otherwise
+ * true if needs to be enabled, false otherwise
*/
public void setGPU(boolean enable) {
this.gpu = enable;
}
/**
- * Whether or not to explicitly "force" the usage of GPU.
- * If a GPU is not available, and the GPU mode is set or if available memory on GPU is less, SystemML will crash when the program is run.
+ * Whether or not to explicitly "force" the usage of GPU. If a GPU is not
+ * available, and the GPU mode is set or if available memory on GPU is less,
+ * SystemML will crash when the program is run.
+ *
* @param enable
- * true if needs to be enabled, false otherwise
+ * true if needs to be enabled, false otherwise
*/
public void setForceGPU(boolean enable) {
this.forceGPU = enable;
@@ -437,13 +444,13 @@ public class MLContext {
/**
* Whether or not the GPU mode is enabled.
+ *
* @return true if enabled, false otherwise
*/
public boolean isGPU() {
return this.gpu;
}
-
/**
* Used internally by MLContextProxy.
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a712dad8/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java
index e392a5f..149b541 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -78,7 +78,7 @@ public class MLContextConversionUtil {
/**
* Convert a two-dimensional double array to a {@code MatrixObject}.
- *
+ *
* @param variableName
* name of the variable associated with the matrix
* @param doubleMatrix
@@ -92,7 +92,7 @@ public class MLContextConversionUtil {
/**
* Convert a two-dimensional double array to a {@code MatrixObject}.
- *
+ *
* @param variableName
* name of the variable associated with the matrix
* @param doubleMatrix
@@ -106,13 +106,13 @@ public class MLContextConversionUtil {
MatrixMetadata matrixMetadata) {
try {
MatrixBlock matrixBlock = DataConverter.convertToMatrixBlock(doubleMatrix);
- MatrixCharacteristics mc = (matrixMetadata != null) ?
- matrixMetadata.asMatrixCharacteristics() : new MatrixCharacteristics(matrixBlock.getNumRows(),
- matrixBlock.getNumColumns(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
+ MatrixCharacteristics mc = (matrixMetadata != null) ? matrixMetadata.asMatrixCharacteristics()
+ : new MatrixCharacteristics(matrixBlock.getNumRows(), matrixBlock.getNumColumns(),
+ ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
- MatrixObject matrixObject = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(),
+ MatrixObject matrixObject = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(),
new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo));
-
+
matrixObject.acquireModify(matrixBlock);
matrixObject.release();
return matrixObject;
@@ -123,7 +123,7 @@ public class MLContextConversionUtil {
/**
* Convert a matrix at a URL to a {@code MatrixObject}.
- *
+ *
* @param variableName
* name of the variable associated with the matrix
* @param url
@@ -150,7 +150,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code MatrixBlock} to a {@code MatrixObject}.
- *
+ *
* @param variableName
* name of the variable associated with the matrix
* @param matrixBlock
@@ -162,9 +162,9 @@ public class MLContextConversionUtil {
public static MatrixObject matrixBlockToMatrixObject(String variableName, MatrixBlock matrixBlock,
MatrixMetadata matrixMetadata) {
try {
- MatrixCharacteristics mc = (matrixMetadata != null) ?
- matrixMetadata.asMatrixCharacteristics() : new MatrixCharacteristics();
- MatrixObject matrixObject = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(),
+ MatrixCharacteristics mc = (matrixMetadata != null) ? matrixMetadata.asMatrixCharacteristics()
+ : new MatrixCharacteristics();
+ MatrixObject matrixObject = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(),
new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo));
matrixObject.acquireModify(matrixBlock);
matrixObject.release();
@@ -176,7 +176,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code FrameBlock} to a {@code FrameObject}.
- *
+ *
* @param variableName
* name of the variable associated with the frame
* @param frameBlock
@@ -188,11 +188,11 @@ public class MLContextConversionUtil {
public static FrameObject frameBlockToFrameObject(String variableName, FrameBlock frameBlock,
FrameMetadata frameMetadata) {
try {
- MatrixCharacteristics mc = (frameMetadata != null) ?
- frameMetadata.asMatrixCharacteristics() : new MatrixCharacteristics();
- MatrixFormatMetaData mtd = new MatrixFormatMetaData(mc,
- OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
- FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(), mtd,
+ MatrixCharacteristics mc = (frameMetadata != null) ? frameMetadata.asMatrixCharacteristics()
+ : new MatrixCharacteristics();
+ MatrixFormatMetaData mtd = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo,
+ InputInfo.BinaryBlockInputInfo);
+ FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(), mtd,
frameMetadata.getFrameSchema().getSchema().toArray(new ValueType[0]));
frameObject.acquireModify(frameBlock);
frameObject.release();
@@ -205,7 +205,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code JavaPairRDD<MatrixIndexes, MatrixBlock>} to a
* {@code MatrixObject}.
- *
+ *
* @param variableName
* name of the variable associated with the matrix
* @param binaryBlocks
@@ -222,7 +222,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code JavaPairRDD<MatrixIndexes, MatrixBlock>} to a
* {@code MatrixObject}.
- *
+ *
* @param variableName
* name of the variable associated with the matrix
* @param binaryBlocks
@@ -237,14 +237,14 @@ public class MLContextConversionUtil {
JavaPairRDD<MatrixIndexes, MatrixBlock> binaryBlocks, MatrixMetadata matrixMetadata) {
return binaryBlocksToMatrixObject(variableName, binaryBlocks, matrixMetadata, true);
}
-
+
private static MatrixObject binaryBlocksToMatrixObject(String variableName,
JavaPairRDD<MatrixIndexes, MatrixBlock> binaryBlocks, MatrixMetadata matrixMetadata, boolean copy) {
- MatrixCharacteristics mc = (matrixMetadata != null) ?
- matrixMetadata.asMatrixCharacteristics() : new MatrixCharacteristics();
+ MatrixCharacteristics mc = (matrixMetadata != null) ? matrixMetadata.asMatrixCharacteristics()
+ : new MatrixCharacteristics();
JavaPairRDD<MatrixIndexes, MatrixBlock> javaPairRdd = SparkUtils.copyBinaryBlockMatrix(binaryBlocks, copy);
-
+
MatrixObject matrixObject = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(),
new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo));
matrixObject.setRDDHandle(new RDDObject(javaPairRdd, variableName));
@@ -253,7 +253,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code JavaPairRDD<Long, FrameBlock>} to a {@code FrameObject}.
- *
+ *
* @param variableName
* name of the variable associated with the frame
* @param binaryBlocks
@@ -269,7 +269,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code JavaPairRDD<Long, FrameBlock>} to a {@code FrameObject}.
- *
+ *
* @param variableName
* name of the variable associated with the frame
* @param binaryBlocks
@@ -283,11 +283,11 @@ public class MLContextConversionUtil {
public static FrameObject binaryBlocksToFrameObject(String variableName, JavaPairRDD<Long, FrameBlock> binaryBlocks,
FrameMetadata frameMetadata) {
- MatrixCharacteristics mc = (frameMetadata != null) ?
- frameMetadata.asMatrixCharacteristics() : new MatrixCharacteristics();
+ MatrixCharacteristics mc = (frameMetadata != null) ? frameMetadata.asMatrixCharacteristics()
+ : new MatrixCharacteristics();
- FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(),
- new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo),
+ FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(),
+ new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo),
frameMetadata.getFrameSchema().getSchema().toArray(new ValueType[0]));
frameObject.setRDDHandle(new RDDObject(binaryBlocks, variableName));
return frameObject;
@@ -295,7 +295,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code DataFrame} to a {@code MatrixObject}.
- *
+ *
* @param variableName
* name of the variable associated with the matrix
* @param dataFrame
@@ -309,7 +309,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code DataFrame} to a {@code MatrixObject}.
- *
+ *
* @param variableName
* name of the variable associated with the matrix
* @param dataFrame
@@ -320,20 +320,20 @@ public class MLContextConversionUtil {
* {@code MatrixObject}
*/
public static MatrixObject dataFrameToMatrixObject(String variableName, Dataset<Row> dataFrame,
- MatrixMetadata matrixMetadata)
- {
- matrixMetadata = (matrixMetadata!=null) ? matrixMetadata : new MatrixMetadata();
+ MatrixMetadata matrixMetadata) {
+ matrixMetadata = (matrixMetadata != null) ? matrixMetadata : new MatrixMetadata();
JavaPairRDD<MatrixIndexes, MatrixBlock> binaryBlock = dataFrameToMatrixBinaryBlocks(dataFrame, matrixMetadata);
MatrixObject mo = binaryBlocksToMatrixObject(variableName, binaryBlock, matrixMetadata, false);
- //keep lineage of original dataset to allow bypassing binary block conversion if possible
- mo.getRDDHandle().addLineageChild(new DatasetObject(dataFrame, variableName,
- isDataFrameWithIDColumn(matrixMetadata),isVectorBasedDataFrame(matrixMetadata)));
+ // keep lineage of original dataset to allow bypassing binary block
+ // conversion if possible
+ mo.getRDDHandle().addLineageChild(new DatasetObject(dataFrame, variableName,
+ isDataFrameWithIDColumn(matrixMetadata), isVectorBasedDataFrame(matrixMetadata)));
return mo;
}
/**
* Convert a {@code DataFrame} to a {@code FrameObject}.
- *
+ *
* @param variableName
* name of the variable associated with the frame
* @param dataFrame
@@ -347,7 +347,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code DataFrame} to a {@code FrameObject}.
- *
+ *
* @param variableName
* name of the variable associated with the frame
* @param dataFrame
@@ -357,29 +357,29 @@ public class MLContextConversionUtil {
* @return the {@code DataFrame} frame converted to a converted to a
* {@code FrameObject}
*/
- public static FrameObject dataFrameToFrameObject(String variableName, Dataset<Row> dataFrame, FrameMetadata frameMetadata)
- {
+ public static FrameObject dataFrameToFrameObject(String variableName, Dataset<Row> dataFrame,
+ FrameMetadata frameMetadata) {
try {
- //setup meta data and java spark context
+ // setup meta data and java spark context
if (frameMetadata == null)
frameMetadata = new FrameMetadata();
determineFrameFormatIfNeeded(dataFrame, frameMetadata);
boolean containsID = isDataFrameWithIDColumn(frameMetadata);
MatrixCharacteristics mc = frameMetadata.asMatrixCharacteristics();
- if( mc == null )
+ if (mc == null)
mc = new MatrixCharacteristics();
-
- //convert data frame and obtain column names / schema
- //TODO extend frame schema by column names (right now dropped)
- Pair<String[], ValueType[]> ret = new Pair<String[], ValueType[]>();
- JavaPairRDD<Long, FrameBlock> binaryBlock = FrameRDDConverterUtils
- .dataFrameToBinaryBlock(jsc(), dataFrame, mc, containsID, ret);
+
+ // convert data frame and obtain column names / schema
+ // TODO extend frame schema by column names (right now dropped)
+ Pair<String[], ValueType[]> ret = new Pair<String[], ValueType[]>();
+ JavaPairRDD<Long, FrameBlock> binaryBlock = FrameRDDConverterUtils.dataFrameToBinaryBlock(jsc(), dataFrame,
+ mc, containsID, ret);
frameMetadata.setFrameSchema(new FrameSchema(Arrays.asList(ret.getValue())));
- frameMetadata.setMatrixCharacteristics(mc); //required due to meta data copy
-
+ frameMetadata.setMatrixCharacteristics(mc); // required due to meta
+ // data copy
+
return MLContextConversionUtil.binaryBlocksToFrameObject(variableName, binaryBlock, frameMetadata);
- }
- catch (DMLRuntimeException e) {
+ } catch (DMLRuntimeException e) {
throw new MLContextException("Exception converting DataFrame to FrameObject", e);
}
}
@@ -387,7 +387,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code DataFrame} to a
* {@code JavaPairRDD<MatrixIndexes, MatrixBlock>} binary-block matrix.
- *
+ *
* @param dataFrame
* the Spark {@code DataFrame}
* @return the {@code DataFrame} matrix converted to a
@@ -401,7 +401,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code DataFrame} to a
* {@code JavaPairRDD<MatrixIndexes, MatrixBlock>} binary-block matrix.
- *
+ *
* @param dataFrame
* the Spark {@code DataFrame}
* @param matrixMetadata
@@ -410,31 +410,30 @@ public class MLContextConversionUtil {
* {@code JavaPairRDD<MatrixIndexes,
* MatrixBlock>} binary-block matrix
*/
- public static JavaPairRDD<MatrixIndexes, MatrixBlock> dataFrameToMatrixBinaryBlocks(
- Dataset<Row> dataFrame, MatrixMetadata matrixMetadata)
- {
- //handle meta data
+ public static JavaPairRDD<MatrixIndexes, MatrixBlock> dataFrameToMatrixBinaryBlocks(Dataset<Row> dataFrame,
+ MatrixMetadata matrixMetadata) {
+ // handle meta data
determineMatrixFormatIfNeeded(dataFrame, matrixMetadata);
- MatrixCharacteristics mc = (matrixMetadata != null && matrixMetadata.asMatrixCharacteristics()!=null) ?
- matrixMetadata.asMatrixCharacteristics() : new MatrixCharacteristics();
+ MatrixCharacteristics mc = (matrixMetadata != null && matrixMetadata.asMatrixCharacteristics() != null)
+ ? matrixMetadata.asMatrixCharacteristics() : new MatrixCharacteristics();
boolean containsID = isDataFrameWithIDColumn(matrixMetadata);
boolean isVector = isVectorBasedDataFrame(matrixMetadata);
- //convert data frame to binary block matrix
- JavaPairRDD<MatrixIndexes,MatrixBlock> out = RDDConverterUtils
- .dataFrameToBinaryBlock(jsc(), dataFrame, mc, containsID, isVector);
-
- //update determined matrix characteristics
- if( matrixMetadata != null )
+ // convert data frame to binary block matrix
+ JavaPairRDD<MatrixIndexes, MatrixBlock> out = RDDConverterUtils.dataFrameToBinaryBlock(jsc(), dataFrame, mc,
+ containsID, isVector);
+
+ // update determined matrix characteristics
+ if (matrixMetadata != null)
matrixMetadata.setMatrixCharacteristics(mc);
-
+
return out;
}
/**
* Convert a {@code DataFrame} to a {@code JavaPairRDD<Long, FrameBlock>}
* binary-block frame.
- *
+ *
* @param dataFrame
* the Spark {@code DataFrame}
* @param frameMetadata
@@ -451,7 +450,7 @@ public class MLContextConversionUtil {
/**
* If the MatrixFormat of the DataFrame has not been explicitly specified,
* attempt to determine the proper MatrixFormat.
- *
+ *
* @param dataFrame
* the Spark {@code DataFrame}
* @param matrixMetadata
@@ -521,49 +520,46 @@ public class MLContextConversionUtil {
/**
* Return whether or not the DataFrame has an ID column.
- *
+ *
* @param matrixMetadata
* the matrix metadata
* @return {@code true} if the DataFrame has an ID column, {@code false}
* otherwise.
*/
public static boolean isDataFrameWithIDColumn(MatrixMetadata matrixMetadata) {
- return (matrixMetadata != null
- && matrixMetadata.getMatrixFormat() != null
- && matrixMetadata.getMatrixFormat().hasIDColumn());
+ return (matrixMetadata != null && matrixMetadata.getMatrixFormat() != null
+ && matrixMetadata.getMatrixFormat().hasIDColumn());
}
/**
* Return whether or not the DataFrame has an ID column.
- *
+ *
* @param frameMetadata
* the frame metadata
* @return {@code true} if the DataFrame has an ID column, {@code false}
* otherwise.
*/
public static boolean isDataFrameWithIDColumn(FrameMetadata frameMetadata) {
- return (frameMetadata != null
- && frameMetadata.getFrameFormat() != null
- && frameMetadata.getFrameFormat().hasIDColumn());
+ return (frameMetadata != null && frameMetadata.getFrameFormat() != null
+ && frameMetadata.getFrameFormat().hasIDColumn());
}
/**
* Return whether or not the DataFrame is vector-based.
- *
+ *
* @param matrixMetadata
* the matrix metadata
* @return {@code true} if the DataFrame is vector-based, {@code false}
* otherwise.
*/
public static boolean isVectorBasedDataFrame(MatrixMetadata matrixMetadata) {
- return (matrixMetadata != null
- && matrixMetadata.getMatrixFormat() != null
- && matrixMetadata.getMatrixFormat().isVectorBased());
+ return (matrixMetadata != null && matrixMetadata.getMatrixFormat() != null
+ && matrixMetadata.getMatrixFormat().isVectorBased());
}
/**
* Convert a {@code JavaRDD<String>} in CSV format to a {@code MatrixObject}
- *
+ *
* @param variableName
* name of the variable associated with the matrix
* @param javaRDD
@@ -576,7 +572,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code JavaRDD<String>} in CSV format to a {@code MatrixObject}
- *
+ *
* @param variableName
* name of the variable associated with the matrix
* @param javaRDD
@@ -588,8 +584,8 @@ public class MLContextConversionUtil {
public static MatrixObject javaRDDStringCSVToMatrixObject(String variableName, JavaRDD<String> javaRDD,
MatrixMetadata matrixMetadata) {
JavaPairRDD<LongWritable, Text> javaPairRDD = javaRDD.mapToPair(new ConvertStringToLongTextPair());
- MatrixCharacteristics mc = (matrixMetadata != null) ?
- matrixMetadata.asMatrixCharacteristics() : new MatrixCharacteristics();
+ MatrixCharacteristics mc = (matrixMetadata != null) ? matrixMetadata.asMatrixCharacteristics()
+ : new MatrixCharacteristics();
MatrixObject matrixObject = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(),
new MatrixFormatMetaData(mc, OutputInfo.CSVOutputInfo, InputInfo.CSVInputInfo));
@@ -600,7 +596,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code JavaRDD<String>} in CSV format to a {@code FrameObject}
- *
+ *
* @param variableName
* name of the variable associated with the frame
* @param javaRDD
@@ -613,7 +609,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code JavaRDD<String>} in CSV format to a {@code FrameObject}
- *
+ *
* @param variableName
* name of the variable associated with the frame
* @param javaRDD
@@ -625,17 +621,17 @@ public class MLContextConversionUtil {
public static FrameObject javaRDDStringCSVToFrameObject(String variableName, JavaRDD<String> javaRDD,
FrameMetadata frameMetadata) {
JavaPairRDD<LongWritable, Text> javaPairRDD = javaRDD.mapToPair(new ConvertStringToLongTextPair());
- MatrixCharacteristics mc = (frameMetadata != null) ?
- frameMetadata.asMatrixCharacteristics() : new MatrixCharacteristics();
+ MatrixCharacteristics mc = (frameMetadata != null) ? frameMetadata.asMatrixCharacteristics()
+ : new MatrixCharacteristics();
JavaPairRDD<LongWritable, Text> javaPairRDDText = javaPairRDD.mapToPair(new CopyTextInputFunction());
- FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(),
- new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo),
+ FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(),
+ new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo),
frameMetadata.getFrameSchema().getSchema().toArray(new ValueType[0]));
JavaPairRDD<Long, FrameBlock> rdd;
try {
- rdd = FrameRDDConverterUtils.csvToBinaryBlock(jsc(), javaPairRDDText, mc,
- frameObject.getSchema(), false, ",", false, -1);
+ rdd = FrameRDDConverterUtils.csvToBinaryBlock(jsc(), javaPairRDDText, mc, frameObject.getSchema(), false,
+ ",", false, -1);
} catch (DMLRuntimeException e) {
e.printStackTrace();
return null;
@@ -647,7 +643,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code JavaRDD<String>} in IJV format to a {@code MatrixObject}
* . Note that metadata is required for IJV format.
- *
+ *
* @param variableName
* name of the variable associated with the matrix
* @param javaRDD
@@ -659,10 +655,10 @@ public class MLContextConversionUtil {
public static MatrixObject javaRDDStringIJVToMatrixObject(String variableName, JavaRDD<String> javaRDD,
MatrixMetadata matrixMetadata) {
JavaPairRDD<LongWritable, Text> javaPairRDD = javaRDD.mapToPair(new ConvertStringToLongTextPair());
- MatrixCharacteristics mc = (matrixMetadata != null) ?
- matrixMetadata.asMatrixCharacteristics() : new MatrixCharacteristics();
+ MatrixCharacteristics mc = (matrixMetadata != null) ? matrixMetadata.asMatrixCharacteristics()
+ : new MatrixCharacteristics();
- MatrixObject matrixObject = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(),
+ MatrixObject matrixObject = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(),
new MatrixFormatMetaData(mc, OutputInfo.TextCellOutputInfo, InputInfo.TextCellInputInfo));
JavaPairRDD<LongWritable, Text> javaPairRDD2 = javaPairRDD.mapToPair(new CopyTextInputFunction());
matrixObject.setRDDHandle(new RDDObject(javaPairRDD2, variableName));
@@ -672,7 +668,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code JavaRDD<String>} in IJV format to a {@code FrameObject}
* . Note that metadata is required for IJV format.
- *
+ *
* @param variableName
* name of the variable associated with the frame
* @param javaRDD
@@ -684,13 +680,13 @@ public class MLContextConversionUtil {
public static FrameObject javaRDDStringIJVToFrameObject(String variableName, JavaRDD<String> javaRDD,
FrameMetadata frameMetadata) {
JavaPairRDD<LongWritable, Text> javaPairRDD = javaRDD.mapToPair(new ConvertStringToLongTextPair());
- MatrixCharacteristics mc = (frameMetadata != null) ?
- frameMetadata.asMatrixCharacteristics() : new MatrixCharacteristics();
+ MatrixCharacteristics mc = (frameMetadata != null) ? frameMetadata.asMatrixCharacteristics()
+ : new MatrixCharacteristics();
JavaPairRDD<LongWritable, Text> javaPairRDDText = javaPairRDD.mapToPair(new CopyTextInputFunction());
- FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(),
- new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo),
+ FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(),
+ new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo),
frameMetadata.getFrameSchema().getSchema().toArray(new ValueType[0]));
JavaPairRDD<Long, FrameBlock> rdd;
try {
@@ -708,7 +704,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code RDD<String>} in CSV format to a {@code MatrixObject}
- *
+ *
* @param variableName
* name of the variable associated with the matrix
* @param rdd
@@ -721,7 +717,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code RDD<String>} in CSV format to a {@code MatrixObject}
- *
+ *
* @param variableName
* name of the variable associated with the matrix
* @param rdd
@@ -739,7 +735,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code RDD<String>} in CSV format to a {@code FrameObject}
- *
+ *
* @param variableName
* name of the variable associated with the frame
* @param rdd
@@ -752,7 +748,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code RDD<String>} in CSV format to a {@code FrameObject}
- *
+ *
* @param variableName
* name of the variable associated with the frame
* @param rdd
@@ -771,7 +767,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code RDD<String>} in IJV format to a {@code MatrixObject}.
* Note that metadata is required for IJV format.
- *
+ *
* @param variableName
* name of the variable associated with the matrix
* @param rdd
@@ -790,7 +786,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code RDD<String>} in IJV format to a {@code FrameObject}.
* Note that metadata is required for IJV format.
- *
+ *
* @param variableName
* name of the variable associated with the frame
* @param rdd
@@ -809,7 +805,7 @@ public class MLContextConversionUtil {
/**
* Convert an {@code BinaryBlockMatrix} to a {@code JavaRDD<String>} in IVJ
* format.
- *
+ *
* @param binaryBlockMatrix
* the {@code BinaryBlockMatrix}
* @return the {@code BinaryBlockMatrix} converted to a
@@ -824,7 +820,7 @@ public class MLContextConversionUtil {
/**
* Convert an {@code BinaryBlockMatrix} to a {@code RDD<String>} in IVJ
* format.
- *
+ *
* @param binaryBlockMatrix
* the {@code BinaryBlockMatrix}
* @return the {@code BinaryBlockMatrix} converted to a {@code RDD<String>}
@@ -837,7 +833,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code MatrixObject} to a {@code JavaRDD<String>} in CSV
* format.
- *
+ *
* @param matrixObject
* the {@code MatrixObject}
* @return the {@code MatrixObject} converted to a {@code JavaRDD<String>}
@@ -850,10 +846,11 @@ public class MLContextConversionUtil {
/**
* Convert a {@code FrameObject} to a {@code JavaRDD<String>} in CSV format.
- *
+ *
* @param frameObject
* the {@code FrameObject}
- * @param delimiter the delimiter
+ * @param delimiter
+ * the delimiter
* @return the {@code FrameObject} converted to a {@code JavaRDD<String>}
*/
public static JavaRDD<String> frameObjectToJavaRDDStringCSV(FrameObject frameObject, String delimiter) {
@@ -865,7 +862,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code MatrixObject} to a {@code JavaRDD<String>} in IJV
* format.
- *
+ *
* @param matrixObject
* the {@code MatrixObject}
* @return the {@code MatrixObject} converted to a {@code JavaRDD<String>}
@@ -878,7 +875,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code FrameObject} to a {@code JavaRDD<String>} in IJV format.
- *
+ *
* @param frameObject
* the {@code FrameObject}
* @return the {@code FrameObject} converted to a {@code JavaRDD<String>}
@@ -891,7 +888,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code MatrixObject} to a {@code RDD<String>} in IJV format.
- *
+ *
* @param matrixObject
* the {@code MatrixObject}
* @return the {@code MatrixObject} converted to a {@code RDD<String>}
@@ -916,7 +913,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code FrameObject} to a {@code RDD<String>} in IJV format.
- *
+ *
* @param frameObject
* the {@code FrameObject}
* @return the {@code FrameObject} converted to a {@code RDD<String>}
@@ -941,7 +938,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code MatrixObject} to a {@code RDD<String>} in CSV format.
- *
+ *
* @param matrixObject
* the {@code MatrixObject}
* @return the {@code MatrixObject} converted to a {@code RDD<String>}
@@ -966,10 +963,11 @@ public class MLContextConversionUtil {
/**
* Convert a {@code FrameObject} to a {@code RDD<String>} in CSV format.
- *
+ *
* @param frameObject
* the {@code FrameObject}
- * @param delimiter the delimiter
+ * @param delimiter
+ * the delimiter
* @return the {@code FrameObject} converted to a {@code RDD<String>}
*/
public static RDD<String> frameObjectToRDDStringCSV(FrameObject frameObject, String delimiter) {
@@ -992,7 +990,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code MatrixObject} to a {@code List<String>} in CSV format.
- *
+ *
* @param matrixObject
* the {@code MatrixObject}
* @return the {@code MatrixObject} converted to a {@code List<String>}
@@ -1055,10 +1053,11 @@ public class MLContextConversionUtil {
/**
* Convert a {@code FrameObject} to a {@code List<String>} in CSV format.
- *
+ *
* @param frameObject
* the {@code FrameObject}
- * @param delimiter the delimiter
+ * @param delimiter
+ * the delimiter
* @return the {@code FrameObject} converted to a {@code List<String>}
*/
public static List<String> frameObjectToListStringCSV(FrameObject frameObject, String delimiter) {
@@ -1091,7 +1090,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code MatrixObject} to a {@code List<String>} in IJV format.
- *
+ *
* @param matrixObject
* the {@code MatrixObject}
* @return the {@code MatrixObject} converted to a {@code List<String>}
@@ -1145,7 +1144,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code FrameObject} to a {@code List<String>} in IJV format.
- *
+ *
* @param frameObject
* the {@code FrameObject}
* @return the {@code FrameObject} converted to a {@code List<String>}
@@ -1183,7 +1182,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code MatrixObject} to a two-dimensional double array.
- *
+ *
* @param matrixObject
* the {@code MatrixObject}
* @return the {@code MatrixObject} converted to a {@code double[][]}
@@ -1201,12 +1200,13 @@ public class MLContextConversionUtil {
/**
* Convert a {@code MatrixObject} to a {@code DataFrame}.
- *
+ *
* @param matrixObject
* the {@code MatrixObject}
* @param sparkExecutionContext
* the Spark execution context
- * @param isVectorDF is the DataFrame a vector DataFrame?
+ * @param isVectorDF
+ * is the DataFrame a vector DataFrame?
* @return the {@code MatrixObject} converted to a {@code DataFrame}
*/
public static Dataset<Row> matrixObjectToDataFrame(MatrixObject matrixObject,
@@ -1218,15 +1218,14 @@ public class MLContextConversionUtil {
MatrixCharacteristics mc = matrixObject.getMatrixCharacteristics();
return RDDConverterUtils.binaryBlockToDataFrame(spark(), binaryBlockMatrix, mc, isVectorDF);
- }
- catch (DMLRuntimeException e) {
+ } catch (DMLRuntimeException e) {
throw new MLContextException("DMLRuntimeException while converting matrix object to DataFrame", e);
}
}
/**
* Convert a {@code FrameObject} to a {@code DataFrame}.
- *
+ *
* @param frameObject
* the {@code FrameObject}
* @param sparkExecutionContext
@@ -1241,16 +1240,16 @@ public class MLContextConversionUtil {
.getRDDHandleForFrameObject(frameObject, InputInfo.BinaryBlockInputInfo);
MatrixCharacteristics mc = frameObject.getMatrixCharacteristics();
- return FrameRDDConverterUtils.binaryBlockToDataFrame(spark(), binaryBlockFrame, mc, frameObject.getSchema());
- }
- catch (DMLRuntimeException e) {
+ return FrameRDDConverterUtils.binaryBlockToDataFrame(spark(), binaryBlockFrame, mc,
+ frameObject.getSchema());
+ } catch (DMLRuntimeException e) {
throw new MLContextException("DMLRuntimeException while converting frame object to DataFrame", e);
}
}
/**
* Convert a {@code MatrixObject} to a {@code BinaryBlockMatrix}.
- *
+ *
* @param matrixObject
* the {@code MatrixObject}
* @param sparkExecutionContext
@@ -1272,7 +1271,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code FrameObject} to a {@code BinaryBlockFrame}.
- *
+ *
* @param frameObject
* the {@code FrameObject}
* @param sparkExecutionContext
@@ -1296,7 +1295,7 @@ public class MLContextConversionUtil {
/**
* Convert a {@code FrameObject} to a two-dimensional string array.
- *
+ *
* @param frameObject
* the {@code FrameObject}
* @return the {@code FrameObject} converted to a {@code String[][]}
@@ -1314,7 +1313,7 @@ public class MLContextConversionUtil {
/**
* Obtain JavaSparkContext from MLContextProxy.
- *
+ *
* @return the Java Spark Context
*/
public static JavaSparkContext jsc() {
@@ -1323,7 +1322,7 @@ public class MLContextConversionUtil {
/**
* Obtain SparkContext from MLContextProxy.
- *
+ *
* @return the Spark Context
*/
public static SparkContext sc() {
@@ -1332,7 +1331,7 @@ public class MLContextConversionUtil {
/**
* Obtain SparkSession from MLContextProxy.
- *
+ *
* @return the Spark Session
*/
public static SparkSession spark() {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a712dad8/src/main/java/org/apache/sysml/api/mlcontext/MLContextException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContextException.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContextException.java
index ca98a89..7bd8aa9 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/MLContextException.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContextException.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -49,7 +49,7 @@ public class MLContextException extends RuntimeException {
* Generate an exception and optionally suppress the stacktrace. This can be
* useful in an environment such as a Spark Shell in certain situations
* where a stacktrace may be extraneous.
- *
+ *
* @param message
* the exception message
* @param suppressStacktrace
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a712dad8/src/main/java/org/apache/sysml/api/mlcontext/MLContextUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContextUtil.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContextUtil.java
index bc55e41..50defcc 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/MLContextUtil.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContextUtil.java
@@ -94,9 +94,9 @@ public final class MLContextUtil {
* Complex data types supported by the MLContext API
*/
@SuppressWarnings("rawtypes")
- public static final Class[] COMPLEX_DATA_TYPES = { JavaRDD.class, RDD.class, Dataset.class,
- BinaryBlockMatrix.class, BinaryBlockFrame.class, Matrix.class, Frame.class, (new double[][] {}).getClass(),
- MatrixBlock.class, URL.class };
+ public static final Class[] COMPLEX_DATA_TYPES = { JavaRDD.class, RDD.class, Dataset.class, BinaryBlockMatrix.class,
+ BinaryBlockFrame.class, Matrix.class, Frame.class, (new double[][] {}).getClass(), MatrixBlock.class,
+ URL.class };
/**
* All data types supported by the MLContext API
@@ -182,12 +182,15 @@ public final class MLContextUtil {
minimumRecommendedSparkVersion = projectInfo.minimumRecommendedSparkVersion();
} catch (MLContextException e) {
try {
- // During development (such as in an IDE), there is no jar file typically
- // built, so attempt to obtain the minimum recommended Spark version from
+ // During development (such as in an IDE), there is no jar file
+ // typically
+ // built, so attempt to obtain the minimum recommended Spark
+ // version from
// the pom.xml file
minimumRecommendedSparkVersion = getMinimumRecommendedSparkVersionFromPom();
} catch (MLContextException e1) {
- throw new MLContextException("Minimum recommended Spark version could not be determined from SystemML jar file manifest or pom.xml");
+ throw new MLContextException(
+ "Minimum recommended Spark version could not be determined from SystemML jar file manifest or pom.xml");
}
}
String sparkVersion = spark.version();
@@ -200,17 +203,20 @@ public final class MLContextUtil {
/**
* Obtain minimum recommended Spark version from the pom.xml file.
*
- * @return the minimum recommended Spark version from XML parsing of the pom file (during development).
+ * @return the minimum recommended Spark version from XML parsing of the pom
+ * file (during development).
*/
static String getMinimumRecommendedSparkVersionFromPom() {
return getUniquePomProperty("spark.version");
}
/**
- * Obtain the text associated with an XML element from the pom.xml file. In this implementation,
- * the element should be uniquely named, or results will be unpredicable.
+ * Obtain the text associated with an XML element from the pom.xml file. In
+ * this implementation, the element should be uniquely named, or results
+ * will be unpredicable.
*
- * @param property unique property (element) from the pom.xml file
+ * @param property
+ * unique property (element) from the pom.xml file
* @return the text value associated with the given property
*/
static String getUniquePomProperty(String property) {
@@ -663,7 +669,7 @@ public final class MLContextUtil {
DataType dataType = field.dataType();
if ((dataType != DataTypes.DoubleType) && (dataType != DataTypes.IntegerType)
&& (dataType != DataTypes.LongType) && (!(dataType instanceof org.apache.spark.ml.linalg.VectorUDT))
- && (!(dataType instanceof org.apache.spark.mllib.linalg.VectorUDT)) ) {
+ && (!(dataType instanceof org.apache.spark.mllib.linalg.VectorUDT))) {
// uncomment if we support arrays of doubles for matrices
// if (dataType instanceof ArrayType) {
// ArrayType arrayType = (ArrayType) dataType;
@@ -851,7 +857,8 @@ public final class MLContextUtil {
* the title to display for the inputs
* @param map
* the map of inputs
- * @param symbolTable the symbol table
+ * @param symbolTable
+ * the symbol table
* @return the script inputs represented as a String
*/
public static String displayInputs(String name, Map<String, Object> map, LocalVariableMap symbolTable) {
@@ -887,17 +894,18 @@ public final class MLContextUtil {
}
sb.append(") ");
- sb.append(key);
- sb.append(": ");
- String str = null;
- if(object instanceof MatrixBlock) {
- MatrixBlock mb = (MatrixBlock) object;
- str = "MatrixBlock [sparse? = " + mb.isInSparseFormat() + ", nonzeros = " + mb.getNonZeros() + ", size: " + mb.getNumRows() + " X " + mb.getNumColumns() + "]";
- }
- else
- str = object.toString(); // TODO: Deal with OOM for other objects such as Frame, etc
- str = StringUtils.abbreviate(str, 100);
- sb.append(str);
+ sb.append(key);
+ sb.append(": ");
+ String str = null;
+ if (object instanceof MatrixBlock) {
+ MatrixBlock mb = (MatrixBlock) object;
+ str = "MatrixBlock [sparse? = " + mb.isInSparseFormat() + ", nonzeros = " + mb.getNonZeros()
+ + ", size: " + mb.getNumRows() + " X " + mb.getNumColumns() + "]";
+ } else
+ str = object.toString(); // TODO: Deal with OOM for other
+ // objects such as Frame, etc
+ str = StringUtils.abbreviate(str, 100);
+ sb.append(str);
sb.append("\n");
}
}
@@ -970,16 +978,16 @@ public final class MLContextUtil {
* @return the SystemML welcome message
*/
public static String welcomeMessage() {
- StringBuilder sb = new StringBuilder();
- sb.append("\nWelcome to Apache SystemML!\n");
- try {
- ProjectInfo info = ProjectInfo.getProjectInfo();
- if (info.version() != null) {
- sb.append("Version ");
- sb.append(info.version());
- }
- } catch (MLContextException e) {
- }
+ StringBuilder sb = new StringBuilder();
+ sb.append("\nWelcome to Apache SystemML!\n");
+ try {
+ ProjectInfo info = ProjectInfo.getProjectInfo();
+ if (info.version() != null) {
+ sb.append("Version ");
+ sb.append(info.version());
+ }
+ } catch (MLContextException e) {
+ }
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a712dad8/src/main/java/org/apache/sysml/api/mlcontext/MLResults.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLResults.java b/src/main/java/org/apache/sysml/api/mlcontext/MLResults.java
index 17cac1f..ebde918 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/MLResults.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/MLResults.java
@@ -297,7 +297,7 @@ public class MLResults {
/**
* Is the output a MatrixObject?
- *
+ *
* @param outputName
* the name of the output
* @return {@code true} if the output is a MatrixObject, {@code false}
@@ -310,7 +310,7 @@ public class MLResults {
/**
* Is the output a FrameObject?
- *
+ *
* @param outputName
* the name of the output
* @return {@code true} if the output is a FrameObject, {@code false}
@@ -608,7 +608,7 @@ public class MLResults {
/**
* Obtain a Scala tuple.
- *
+ *
* @param <T>
* the type of the first output
* @param outputName1
@@ -622,7 +622,7 @@ public class MLResults {
/**
* Obtain a Scala tuple.
- *
+ *
* @param <T1>
* the type of the first output
* @param <T2>
@@ -640,7 +640,7 @@ public class MLResults {
/**
* Obtain a Scala tuple.
- *
+ *
* @param <T1>
* the type of the first output
* @param <T2>
@@ -663,7 +663,7 @@ public class MLResults {
/**
* Obtain a Scala tuple.
- *
+ *
* @param <T1>
* the type of the first output
* @param <T2>
@@ -691,7 +691,7 @@ public class MLResults {
/**
* Obtain a Scala tuple.
- *
+ *
* @param <T1>
* the type of the first output
* @param <T2>
@@ -723,7 +723,7 @@ public class MLResults {
/**
* Obtain a Scala tuple.
- *
+ *
* @param <T1>
* the type of the first output
* @param <T2>
@@ -760,7 +760,7 @@ public class MLResults {
/**
* Obtain a Scala tuple.
- *
+ *
* @param <T1>
* the type of the first output
* @param <T2>
@@ -802,7 +802,7 @@ public class MLResults {
/**
* Obtain a Scala tuple.
- *
+ *
* @param <T1>
* the type of the first output
* @param <T2>
@@ -848,7 +848,7 @@ public class MLResults {
/**
* Obtain a Scala tuple.
- *
+ *
* @param <T1>
* the type of the first output
* @param <T2>
@@ -899,7 +899,7 @@ public class MLResults {
/**
* Obtain a Scala tuple.
- *
+ *
* @param <T1>
* the type of the first output
* @param <T2>
@@ -954,7 +954,7 @@ public class MLResults {
/**
* Obtain a Scala tuple.
- *
+ *
* @param <T1>
* the type of the first output
* @param <T2>
@@ -1015,7 +1015,7 @@ public class MLResults {
/**
* Obtain a Scala tuple.
- *
+ *
* @param <T1>
* the type of the first output
* @param <T2>
@@ -1080,7 +1080,7 @@ public class MLResults {
/**
* Obtain a Scala tuple.
- *
+ *
* @param <T1>
* the type of the first output
* @param <T2>
@@ -1149,7 +1149,7 @@ public class MLResults {
/**
* Obtain a Scala tuple.
- *
+ *
* @param <T1>
* the type of the first output
* @param <T2>
@@ -1223,7 +1223,7 @@ public class MLResults {
/**
* Obtain a Scala tuple.
- *
+ *
* @param <T1>
* the type of the first output
* @param <T2>
@@ -1301,7 +1301,7 @@ public class MLResults {
/**
* Obtain a Scala tuple.
- *
+ *
* @param <T1>
* the type of the first output
* @param <T2>
@@ -1385,7 +1385,7 @@ public class MLResults {
/**
* Obtain a Scala tuple.
- *
+ *
* @param <T1>
* the type of the first output
* @param <T2>
@@ -1473,7 +1473,7 @@ public class MLResults {
/**
* Obtain a Scala tuple.
- *
+ *
* @param <T1>
* the type of the first output
* @param <T2>
@@ -1565,7 +1565,7 @@ public class MLResults {
/**
* Obtain a Scala tuple.
- *
+ *
* @param <T1>
* the type of the first output
* @param <T2>
@@ -1662,7 +1662,7 @@ public class MLResults {
/**
* Obtain a Scala tuple.
- *
+ *
* @param <T1>
* the type of the first output
* @param <T2>
@@ -1763,7 +1763,7 @@ public class MLResults {
/**
* Obtain a Scala tuple.
- *
+ *
* @param <T1>
* the type of the first output
* @param <T2>
@@ -1869,7 +1869,7 @@ public class MLResults {
/**
* Obtain a Scala tuple.
- *
+ *
* @param <T1>
* the type of the first output
* @param <T2>
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a712dad8/src/main/java/org/apache/sysml/api/mlcontext/Matrix.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/Matrix.java b/src/main/java/org/apache/sysml/api/mlcontext/Matrix.java
index 9952350..07ed516 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/Matrix.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/Matrix.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -46,7 +46,7 @@ public class Matrix {
/**
* Obtain the matrix as a SystemML MatrixObject.
- *
+ *
* @return the matrix as a SystemML MatrixObject
*/
public MatrixObject toMatrixObject() {
@@ -55,7 +55,7 @@ public class Matrix {
/**
* Obtain the matrix as a two-dimensional double array
- *
+ *
* @return the matrix as a two-dimensional double array
*/
public double[][] to2DDoubleArray() {
@@ -64,7 +64,7 @@ public class Matrix {
/**
* Obtain the matrix as a {@code JavaRDD<String>} in IJV format
- *
+ *
* @return the matrix as a {@code JavaRDD<String>} in IJV format
*/
public JavaRDD<String> toJavaRDDStringIJV() {
@@ -73,7 +73,7 @@ public class Matrix {
/**
* Obtain the matrix as a {@code JavaRDD<String>} in CSV format
- *
+ *
* @return the matrix as a {@code JavaRDD<String>} in CSV format
*/
public JavaRDD<String> toJavaRDDStringCSV() {
@@ -82,7 +82,7 @@ public class Matrix {
/**
* Obtain the matrix as a {@code RDD<String>} in CSV format
- *
+ *
* @return the matrix as a {@code RDD<String>} in CSV format
*/
public RDD<String> toRDDStringCSV() {
@@ -91,7 +91,7 @@ public class Matrix {
/**
* Obtain the matrix as a {@code RDD<String>} in IJV format
- *
+ *
* @return the matrix as a {@code RDD<String>} in IJV format
*/
public RDD<String> toRDDStringIJV() {
@@ -100,7 +100,7 @@ public class Matrix {
/**
* Obtain the matrix as a {@code DataFrame} of doubles with an ID column
- *
+ *
* @return the matrix as a {@code DataFrame} of doubles with an ID column
*/
public Dataset<Row> toDF() {
@@ -109,7 +109,7 @@ public class Matrix {
/**
* Obtain the matrix as a {@code DataFrame} of doubles with an ID column
- *
+ *
* @return the matrix as a {@code DataFrame} of doubles with an ID column
*/
public Dataset<Row> toDFDoubleWithIDColumn() {
@@ -118,7 +118,7 @@ public class Matrix {
/**
* Obtain the matrix as a {@code DataFrame} of doubles with no ID column
- *
+ *
* @return the matrix as a {@code DataFrame} of doubles with no ID column
*/
public Dataset<Row> toDFDoubleNoIDColumn() {
@@ -128,7 +128,7 @@ public class Matrix {
/**
* Obtain the matrix as a {@code DataFrame} of vectors with an ID column
- *
+ *
* @return the matrix as a {@code DataFrame} of vectors with an ID column
*/
public Dataset<Row> toDFVectorWithIDColumn() {
@@ -137,7 +137,7 @@ public class Matrix {
/**
* Obtain the matrix as a {@code DataFrame} of vectors with no ID column
- *
+ *
* @return the matrix as a {@code DataFrame} of vectors with no ID column
*/
public Dataset<Row> toDFVectorNoIDColumn() {
@@ -147,7 +147,7 @@ public class Matrix {
/**
* Obtain the matrix as a {@code BinaryBlockMatrix}
- *
+ *
* @return the matrix as a {@code BinaryBlockMatrix}
*/
public BinaryBlockMatrix toBinaryBlockMatrix() {
@@ -156,7 +156,7 @@ public class Matrix {
/**
* Obtain the matrix metadata
- *
+ *
* @return the matrix metadata
*/
public MatrixMetadata getMatrixMetadata() {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a712dad8/src/main/java/org/apache/sysml/api/mlcontext/MatrixFormat.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MatrixFormat.java b/src/main/java/org/apache/sysml/api/mlcontext/MatrixFormat.java
index 0c07dd2..46709a5 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/MatrixFormat.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/MatrixFormat.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -58,24 +58,22 @@ public enum MatrixFormat {
/**
* Is the matrix format vector-based?
- *
+ *
* @return {@code true} if matrix is a vector-based DataFrame, {@code false}
* otherwise.
*/
public boolean isVectorBased() {
- return (this == DF_VECTOR_WITH_INDEX
- || this == DF_VECTOR);
+ return (this == DF_VECTOR_WITH_INDEX || this == DF_VECTOR);
}
/**
* Does the DataFrame have a row index?
- *
+ *
* @return {@code true} if the DataFrame has a row index, {@code false}
* otherwise.
*/
public boolean hasIDColumn() {
- return (this == DF_DOUBLES_WITH_INDEX
- || this == DF_VECTOR_WITH_INDEX);
+ return (this == DF_DOUBLES_WITH_INDEX || this == DF_VECTOR_WITH_INDEX);
}
}