You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by ba...@apache.org on 2023/05/15 16:37:44 UTC
[systemds] branch main updated: [SYSTEMDS-3570] Error statistics builtins
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 85479037dd [SYSTEMDS-3570] Error statistics builtins
85479037dd is described below
commit 85479037dd7295ece53447f14daf1128119846bc
Author: baunsgaard <ba...@tu-berlin.de>
AuthorDate: Thu Apr 27 15:36:37 2023 +0200
[SYSTEMDS-3570] Error statistics builtins
This commit adds new builtins for error statistics.
The main parts is the differenceStatistics builtin that calls most of
the other statistics to extract the difference between two matrices.
Close #1818
---
scripts/builtin/.imputeByMode.dml.swp | Bin 12288 -> 0 bytes
scripts/builtin/differenceStatistics.dml | 91 +++++++++++++++++
scripts/builtin/flattenQuantile.dml | 44 ++++++++
scripts/builtin/mae.dml | 42 ++++++++
scripts/builtin/mape.dml | 48 +++++++++
scripts/builtin/mse.dml | 42 ++++++++
scripts/builtin/msmape.dml | 49 +++++++++
scripts/builtin/nrmse.dml | 42 ++++++++
scripts/builtin/psnr.dml | 40 ++++++++
scripts/builtin/rmse.dml | 44 ++++++++
scripts/builtin/skewness.dml | 37 +++++++
scripts/builtin/smape.dml | 49 +++++++++
.../java/org/apache/sysds/common/Builtins.java | 11 ++
.../sysds/parser/dml/DmlSyntacticValidator.java | 11 +-
.../builtin/part2/BuiltinDifferenceStatistics.java | 113 +++++++++++++++++++++
.../functions/builtin/differenceStatistics.dml | 25 +++++
16 files changed, 684 insertions(+), 4 deletions(-)
diff --git a/scripts/builtin/.imputeByMode.dml.swp b/scripts/builtin/.imputeByMode.dml.swp
deleted file mode 100644
index 9c85094594..0000000000
Binary files a/scripts/builtin/.imputeByMode.dml.swp and /dev/null differ
diff --git a/scripts/builtin/differenceStatistics.dml b/scripts/builtin/differenceStatistics.dml
new file mode 100644
index 0000000000..0e9019f096
--- /dev/null
+++ b/scripts/builtin/differenceStatistics.dml
@@ -0,0 +1,91 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+# Prints the difference statistics of two matrices given, to indicate how
+# they are different. This can be used for instance in comparison of lossy
+# compression techniques, that reduce the fidelity of the data.
+#
+# INPUT:
+# --------------------------------------------------------------------------------
+# X First Matrix to compare
+# Y Second Matrix to compare
+# --------------------------------------------------------------------------------
+
+m_differenceStatistics = function(Matrix[Double] X, Matrix[Double] Y) {
+
+ P = matrix("0.0 0.01 0.1 0.25 0.5 0.75 0.90 0.99 1.0", rows= 9, cols=1)
+ print("quantiles are: " + toString(target=P, linesep=" "))
+
+ # Mean Square Error
+ [MSE, SEQ] = mse(X=X, Y=Y, P=P)
+ print(" Mean Square Error: " + toString(as.scalar(MSE)))
+ print(" Quantile Square Error: " + toString(target=SEQ , linesep=" "))
+
+ # Root Mean Square Error
+ [RMSE, RSEQ] = rmse(X=X, Y=Y, P=P)
+ print(" Root Mean Square Error: " + toString(as.scalar(RMSE)))
+ print(" Quantile Root Square Error: " + toString(target=RSEQ , linesep=" "))
+
+ # Normalized Root Mean Square Error
+ [NRMSE, NRSEQ] = nrmse(X=X, Y=Y, P=P)
+ print("Normalized Root Mean Square Error: " + toString(as.scalar(NRMSE)))
+ print(" Quantile Norm Root Square Error: " + toString(target=NRSEQ , linesep=" "))
+
+ # Mean Absolute Error
+ [MAE, AEQ] = mae(X=X, Y=Y, P=P)
+ print(" Mean Absolute Error: " + toString(as.scalar(MAE)))
+ print(" Quantile Absolute Error: " + toString(target=AEQ , linesep=" "))
+
+ # Mean Absolute Percentage Error
+ [MAPE, APEQ] = mape(X=X, Y=Y, P=P)
+ print(" Mean Absolute percentage Error: " + toString(as.scalar(MAPE)))
+ print(" Quantile APE: " + toString(target=APEQ , linesep=" "))
+
+ # Symmetric Mean Absolute Percentage Error
+ [sMAPE, sAPEQ] = smape(X=X, Y=Y, P=P)
+ print("symmetric Mean Absolute per Error: " + toString(as.scalar(sMAPE)))
+ print(" Quantile symmetric MAPE: " + toString(target=sAPEQ , linesep=" "))
+
+ # Modified Symmetric Mean Absolute Percentage Error
+ [msMAPE, msAPEQ] = msmape(X=X, Y=Y, P=P)
+ print(" modified symmetric MAPE: " + toString(as.scalar(msMAPE)))
+ print(" Quantile modified symmetric MAPE: " + toString(target=msAPEQ , linesep=" "))
+
+
+ # Peak Signal to Noise Ratio
+ PSNR = psnr(X=X, Y=Y)
+ print(" Peak Signal to Noise Ratio: " + toString(target=PSNR , linesep=" "))
+
+
+ # Standard deviation
+ sd_Y = sd(Y)
+ sd_X = sd(X)
+ sd_diff = abs(sd_Y - sd_X)
+ print(" Standard Deviation Values (X,Y): (" + toString(sd_X) + ", " + toString(sd_Y) + ")")
+ print(" Standard Deviation Difference: " + toString(sd_diff))
+
+ skew_x = skewness(X)
+ skew_y = skewness(Y)
+ skew_diff = abs(skew_y - skew_x)
+ print(" Skew Values (X,Y): (" + toString(target=skew_x, linesep=" ") + ", " + toString(target=skew_y, linesep=" ") + ")")
+ print(" Skew Difference: " + toString(skew_diff))
+
+}
diff --git a/scripts/builtin/flattenQuantile.dml b/scripts/builtin/flattenQuantile.dml
new file mode 100644
index 0000000000..4dc92846a2
--- /dev/null
+++ b/scripts/builtin/flattenQuantile.dml
@@ -0,0 +1,44 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+# Returns the quantiles requested, but treating the input matrix X as a flattened matrix
+# to return quantiles of all cells as if it was a continuous allocation.
+#
+# INPUT:
+# --------------------------------------------------------------------------------
+# X Matrix with values to extract quantiles from.
+# P Quantiles to extract as well if empty matrix not calculated
+# --------------------------------------------------------------------------------
+#
+# OUTPUT:
+# -----------------------------------------------------------------------------------------------
+# Q Quantiles calculated
+# -----------------------------------------------------------------------------------------------
+
+m_flattenQuantile = function(Matrix[Double] X, Matrix[Double] P)
+ return (Matrix[Double] Q){
+ Q = matrix(0, rows=0, cols=0)
+ if(nrow(P) > 0){ # calculate the quantiles
+ flatten_X = matrix(X, nrow(X) * ncol(X), 1)
+ quantile_X = quantile(flatten_X, P)
+ Q = t(quantile_X)
+ }
+}
diff --git a/scripts/builtin/mae.dml b/scripts/builtin/mae.dml
new file mode 100644
index 0000000000..f9465ae5d6
--- /dev/null
+++ b/scripts/builtin/mae.dml
@@ -0,0 +1,42 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+# Returns the means absolute error between the two inputs
+#
+# INPUT:
+# --------------------------------------------------------------------------------
+# X First Matrix to compare
+# Y Second Matrix to compare
+# P Quantiles to extract as well if empty matrix not calculated
+# --------------------------------------------------------------------------------
+#
+# OUTPUT:
+# -----------------------------------------------------------------------------------------------
+# Z Mean absolute error
+# Q Quantiles calculated
+# -----------------------------------------------------------------------------------------------
+
+m_mae = function(Matrix[Double] X, Matrix[Double] Y, Matrix[Double] P = matrix(0, rows=0, cols=0))
+ return (Matrix[Double] Z, Matrix[Double] Q) {
+ AE = abs(X - Y)
+ Z = as.matrix(mean(AE))
+ Q = flattenQuantile(AE, P)
+}
diff --git a/scripts/builtin/mape.dml b/scripts/builtin/mape.dml
new file mode 100644
index 0000000000..2005e15a16
--- /dev/null
+++ b/scripts/builtin/mape.dml
@@ -0,0 +1,48 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+# Returns the means absolute percentage error between the two inputs
+#
+#
+# Monash Time Series Forecasting Archive
+# Rakshitha Godahewaa,∗, Christoph Bergmeira , Geoffrey I. Webba , Rob J. Hyndmanb ,
+# Pablo Montero-Mansoc
+#
+#
+# INPUT:
+# --------------------------------------------------------------------------------
+# X First Matrix to compare
+# Y Second Matrix to compare
+# P Quantiles to extract as well if empty matrix not calculated
+# --------------------------------------------------------------------------------
+#
+# OUTPUT:
+# -----------------------------------------------------------------------------------------------
+# Z Mean absolute percentage error
+# Q Quantiles calculated
+# -----------------------------------------------------------------------------------------------
+
+m_mape = function(Matrix[Double] X, Matrix[Double] Y, Matrix[Double] P = matrix(0, rows=0, cols=0))
+ return (Matrix[Double] Z, Matrix[Double] Q) {
+ APE = abs(X - Y) / abs(X)
+ Z = as.matrix(mean(APE))
+ Q = flattenQuantile(APE, P)
+}
diff --git a/scripts/builtin/mse.dml b/scripts/builtin/mse.dml
new file mode 100644
index 0000000000..52dcfb4327
--- /dev/null
+++ b/scripts/builtin/mse.dml
@@ -0,0 +1,42 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+# Returns the means square error between the two inputs
+#
+# INPUT:
+# --------------------------------------------------------------------------------
+# X First Matrix to compare
+# Y Second Matrix to compare
+# P Quantiles to extract as well if empty matrix not calculated
+# --------------------------------------------------------------------------------
+#
+# OUTPUT:
+# -----------------------------------------------------------------------------------------------
+# Z Mean Square error
+# Q Quantiles calculated
+# -----------------------------------------------------------------------------------------------
+
+m_mse = function(Matrix[Double] X, Matrix[Double] Y, Matrix[Double] P = matrix(0, rows=0, cols=0))
+ return (Matrix[Double] Z, Matrix[Double] Q) {
+ SE = (X - Y)^2
+ Z = as.matrix(mean(SE))
+ Q = flattenQuantile(SE, P)
+}
diff --git a/scripts/builtin/msmape.dml b/scripts/builtin/msmape.dml
new file mode 100644
index 0000000000..8eb155810b
--- /dev/null
+++ b/scripts/builtin/msmape.dml
@@ -0,0 +1,49 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+# Returns the modified symmetric means absolute percentage error between the two inputs
+#
+#
+# Monash Time Series Forecasting Archive
+# Rakshitha Godahewaa,∗, Christoph Bergmeira , Geoffrey I. Webba , Rob J. Hyndmanb ,
+# Pablo Montero-Mansoc
+#
+# INPUT:
+# --------------------------------------------------------------------------------
+# X First Matrix to compare
+# Y Second Matrix to compare
+# --------------------------------------------------------------------------------
+#
+# OUTPUT:
+# -----------------------------------------------------------------------------------------------
+# Z The modified symmetric mean absolute percentage error
+# -----------------------------------------------------------------------------------------------
+
+m_msmape = function(Matrix[Double] X, Matrix[Double] Y, Matrix[Double] P = matrix(0, rows=0, cols=0),
+ Double eps = 1e-10) return (Matrix[Double] Z, Matrix[Double] Q) {
+ top = abs(X - Y)
+ denom_1 = abs(X) + abs(Y) + eps
+ denom_2 = 0.5 + eps
+ denom = max(denom_1, denom_2) / 2
+ MSAPE = top / denom
+ Z = as.matrix(mean(MSAPE))
+ Q = flattenQuantile(MSAPE, P)
+}
diff --git a/scripts/builtin/nrmse.dml b/scripts/builtin/nrmse.dml
new file mode 100644
index 0000000000..73e513b01c
--- /dev/null
+++ b/scripts/builtin/nrmse.dml
@@ -0,0 +1,42 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+# Returns the normalized root means square error between the two inputs
+#
+# INPUT:
+# --------------------------------------------------------------------------------
+# X First Matrix to compare
+# Y Second Matrix to compare
+# P Quantiles to extract as well if empty matrix not calculated
+# --------------------------------------------------------------------------------
+#
+# OUTPUT:
+# -----------------------------------------------------------------------------------------------
+# Z The normalized root means square error
+# Q Quantiles calculated
+# -----------------------------------------------------------------------------------------------
+
+m_nrmse = function(Matrix[Double] X, Matrix[Double] Y, Matrix[Double] P = matrix(0, rows=0, cols=0))
+ return (Matrix[Double] Z, Matrix[Double] Q) {
+ [RMSE, RSEQ] = rmse(X=X, Y=Y, P=P)
+ Z = RMSE / (max(X) - min(X))
+ Q = RSEQ / (max(X) - min(X))
+}
diff --git a/scripts/builtin/psnr.dml b/scripts/builtin/psnr.dml
new file mode 100644
index 0000000000..0c9c0fca5c
--- /dev/null
+++ b/scripts/builtin/psnr.dml
@@ -0,0 +1,40 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+# Returns the peak signal to noise ratio
+#
+# https://en.wikipedia.org/wiki/Peak_signal-to-noise_ratio
+#
+# INPUT:
+# --------------------------------------------------------------------------------
+# X First Matrix to compare
+# Y Second Matrix to compare
+# --------------------------------------------------------------------------------
+#
+# OUTPUT:
+# -----------------------------------------------------------------------------------------------
+# Z The peak signal to noise ratio
+# -----------------------------------------------------------------------------------------------
+
+m_psnr = function(Matrix[Double] X, Matrix[Double] Y) return (Matrix[Double] Z) {
+ [MSE, NN] = mse(X=X, Y=Y)
+ Z = 10 * log(max(X)^2 / MSE, 10)
+}
diff --git a/scripts/builtin/rmse.dml b/scripts/builtin/rmse.dml
new file mode 100644
index 0000000000..ac46b88be0
--- /dev/null
+++ b/scripts/builtin/rmse.dml
@@ -0,0 +1,44 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+# Returns the root means square error between the two inputs
+#
+# INPUT:
+# --------------------------------------------------------------------------------
+# X First Matrix to compare
+# Y Second Matrix to compare
+# P Quantiles to extract as well if empty matrix not calculated
+# --------------------------------------------------------------------------------
+#
+# OUTPUT:
+# -----------------------------------------------------------------------------------------------
+# Z The root means square error
+# Q Quantiles calculated
+# -----------------------------------------------------------------------------------------------
+
+m_rmse = function(Matrix[Double] X, Matrix[Double] Y, Matrix[Double] P = matrix(0, rows=0, cols=0))
+ return (Matrix[Double] Z, Matrix[Double] Q) {
+
+ [MSE, SEQ] = mse(X=X, Y=Y, P=P)
+
+ Z = sqrt(MSE)
+ Q = sqrt(SEQ)
+}
diff --git a/scripts/builtin/skewness.dml b/scripts/builtin/skewness.dml
new file mode 100644
index 0000000000..65c914f5ba
--- /dev/null
+++ b/scripts/builtin/skewness.dml
@@ -0,0 +1,37 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+# Returns the skewness of the matrix input
+#
+# INPUT:
+# ------------------------------------------------------------
+# X The matrix input
+# ------------------------------------------------------------
+#
+# OUTPUT:
+# ------------------------------------------------------------
+# Y The skewness of the input matrix
+# ------------------------------------------------------------
+
+m_skewness = function(Matrix[Double] X) return (Matrix[Double] Y) {
+ individual = ((X-mean(X))/sd(X))^3
+ Y = as.matrix(mean(individual))
+}
diff --git a/scripts/builtin/smape.dml b/scripts/builtin/smape.dml
new file mode 100644
index 0000000000..4396d62df7
--- /dev/null
+++ b/scripts/builtin/smape.dml
@@ -0,0 +1,49 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+# Returns the symmetric means absolute percentage error between the two inputs
+#
+# Monash Time Series Forecasting Archive
+# Rakshitha Godahewaa, Christoph Bergmeira, Geoffrey I. Webba, Rob J. Hyndmanb,
+# Pablo Montero-Mansoc
+#
+# Another Look at Measures of Forecast Accuracy, R. J. Hyndman and A. B. Koehler, 2006.
+#
+# INPUT:
+# --------------------------------------------------------------------------------
+# X First Matrix to compare
+# Y Second Matrix to compare
+# P Quantiles to extract as well if empty matrix not calculated
+# --------------------------------------------------------------------------------
+#
+# OUTPUT:
+# -----------------------------------------------------------------------------------------------
+# Z The symmetric mean absolute percentage error
+# Q Quantiles calculated
+# -----------------------------------------------------------------------------------------------
+
+m_smape = function(Matrix[Double] X, Matrix[Double] Y, Matrix[Double] P = matrix(0, rows=0, cols=0))
+ return (Matrix[Double] Z, Matrix[Double] Q) {
+ denom = (abs(X) + abs(Y)) / 2
+ SAPE = abs(X - Y) / denom
+ Z = as.matrix( mean(SAPE))
+ Q = flattenQuantile(SAPE, P)
+}
diff --git a/src/main/java/org/apache/sysds/common/Builtins.java b/src/main/java/org/apache/sysds/common/Builtins.java
index 424a01438e..efb260360e 100644
--- a/src/main/java/org/apache/sysds/common/Builtins.java
+++ b/src/main/java/org/apache/sysds/common/Builtins.java
@@ -114,6 +114,7 @@ public enum Builtins {
DEEPWALK("deepWalk", true),
DETECTSCHEMA("detectSchema", false),
DENIALCONSTRAINTS("denialConstraints", true),
+ DIFFERENCESTATISTICS("differenceStatistics", true),
DIAG("diag", false),
DISCOVER_FD("discoverFD", true),
DISCOVER_MD("mdedup", true),
@@ -135,6 +136,7 @@ public enum Builtins {
FF_TRAIN("ffTrain", true),
FF_PREDICT("ffPredict", true),
FLOOR("floor", false),
+ FLATTENQUANTILE("flattenQuantile", true),
FRAME_SORT("frameSort", true),
FRAME_ROW_REPLICATE("freplicate", false),
FREQUENCYENCODE("frequencyEncode", true),
@@ -210,12 +212,16 @@ public enum Builtins {
MAX_POOL("max_pool", false),
MAX_POOL_BACKWARD("max_pool_backward", false),
MCC("mcc", true),
+ MAE("mae", true),
+ MAPE("mape", true),
MEAN("mean", "avg", false),
MEDIAN("median", false),
MICE("mice", true),
MICE_APPLY("miceApply", true),
MIN("min", "pmin", false),
MOMENT("moment", "centralMoment", false),
+ MSE("mse", true),
+ MSMAPE("msmape", true),
MSVM("msvm", true),
MSVMPREDICT("msvmPredict", true),
MULTILOGREG("multiLogReg", true),
@@ -227,6 +233,7 @@ public enum Builtins {
NORMALIZE("normalize", true),
NORMALIZEAPPLY("normalizeApply", true),
NROW("nrow", false),
+ NRMSE("nrmse", true),
OUTER("outer", false),
OUTLIER("outlier", true, false), //TODO parameterize opposite
OUTLIER_ARIMA("outlierByArima",true),
@@ -241,6 +248,7 @@ public enum Builtins {
PPCA("ppca", true),
PPRED("ppred", false),
PROD("prod", false),
+ PSNR("psnr", true),
QR("qr", false, ReturnType.MULTI_RETURN),
QUANTILE("quantile", false),
RANDOM_FOREST("randomForest", true),
@@ -260,6 +268,7 @@ public enum Builtins {
ROWSD("rowSds", false),
ROWSUM("rowSums", false),
ROWVAR("rowVars", false),
+ RMSE("rmse", true),
SAMPLE("sample", false),
SD("sd", false),
SELVARTHRESH("selectByVarThresh", true),
@@ -273,6 +282,8 @@ public enum Builtins {
SIN("sin", false),
SINH("sinh", false),
SLICEFINDER("slicefinder", true),
+ SKEWNESS("skewness", true),
+ SMAPE("smape", true),
SMOTE("smote", true),
SOFTMAX("softmax", true),
SOLVE("solve", false),
diff --git a/src/main/java/org/apache/sysds/parser/dml/DmlSyntacticValidator.java b/src/main/java/org/apache/sysds/parser/dml/DmlSyntacticValidator.java
index cbf61fc480..6387cfcaf4 100644
--- a/src/main/java/org/apache/sysds/parser/dml/DmlSyntacticValidator.java
+++ b/src/main/java/org/apache/sysds/parser/dml/DmlSyntacticValidator.java
@@ -25,8 +25,8 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.Map.Entry;
+import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -34,12 +34,14 @@ import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.tree.ErrorNode;
import org.antlr.v4.runtime.tree.TerminalNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.sysds.api.DMLScript;
import org.apache.sysds.common.Builtins;
import org.apache.sysds.common.Types.DataType;
import org.apache.sysds.common.Types.ValueType;
-import org.apache.sysds.conf.ConfigurationManager;
import org.apache.sysds.conf.CompilerConfig.ConfigType;
+import org.apache.sysds.conf.ConfigurationManager;
import org.apache.sysds.parser.AssignmentStatement;
import org.apache.sysds.parser.BinaryExpression;
import org.apache.sysds.parser.BooleanExpression;
@@ -53,6 +55,7 @@ import org.apache.sysds.parser.DataExpression;
import org.apache.sysds.parser.DataIdentifier;
import org.apache.sysds.parser.DoubleIdentifier;
import org.apache.sysds.parser.Expression;
+import org.apache.sysds.parser.Expression.DataOp;
import org.apache.sysds.parser.ExpressionList;
import org.apache.sysds.parser.ForStatement;
import org.apache.sysds.parser.FunctionCallIdentifier;
@@ -79,7 +82,6 @@ import org.apache.sysds.parser.Statement;
import org.apache.sysds.parser.StatementBlock;
import org.apache.sysds.parser.StringIdentifier;
import org.apache.sysds.parser.WhileStatement;
-import org.apache.sysds.parser.Expression.DataOp;
import org.apache.sysds.parser.dml.DmlParser.AccumulatorAssignmentStatementContext;
import org.apache.sysds.parser.dml.DmlParser.AddSubExpressionContext;
import org.apache.sysds.parser.dml.DmlParser.AssignmentStatementContext;
@@ -135,6 +137,7 @@ import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.util.UtilFunctions;
public class DmlSyntacticValidator implements DmlListener {
+ protected static final Log LOG = LogFactory.getLog(DmlSyntacticValidator.class.getName());
private static final String DEF_WORK_DIR = ".";
@@ -654,7 +657,7 @@ public class DmlSyntacticValidator implements DmlListener {
//construct output map of all functions
if(dict == null)
- throw new RuntimeException("Failed function load: "+name+" "+namespace);
+ throw new RuntimeException("Failed function load: " + name + " " + namespace + "\n" + filePath);
return dict.getFunctions();
}
diff --git a/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinDifferenceStatistics.java b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinDifferenceStatistics.java
new file mode 100644
index 0000000000..e488296dd1
--- /dev/null
+++ b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinDifferenceStatistics.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.functions.builtin.part2;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.sysds.common.Types.ExecMode;
+import org.apache.sysds.common.Types.ExecType;
+import org.apache.sysds.runtime.functionobjects.Multiply;
+import org.apache.sysds.runtime.matrix.data.LibMatrixBincell;
+import org.apache.sysds.runtime.matrix.data.MatrixBlock;
+import org.apache.sysds.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysds.test.AutomatedTestBase;
+import org.apache.sysds.test.TestConfiguration;
+import org.apache.sysds.test.TestUtils;
+import org.junit.Test;
+
+public class BuiltinDifferenceStatistics extends AutomatedTestBase {
+ protected static final Log LOG = LogFactory.getLog(BuiltinDifferenceStatistics.class.getName());
+
+ private final static String TEST_NAME = "differenceStatistics";
+ private final static String TEST_DIR = "functions/builtin/";
+ private static final String TEST_CLASS_DIR = TEST_DIR + BuiltinDifferenceStatistics.class.getSimpleName() + "/";
+
+ @Override
+ public void setUp() {
+ addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"B"}));
+ }
+
+ @Test
+ public void testDifferenceStatistics() {
+ run(ExecType.CP, 0.1);
+ }
+
+ @Test
+ public void testDifferenceStatisticsV2() {
+ run(ExecType.CP, 0.2);
+ }
+
+ @Test
+ public void testDifferenceStatisticsV3() {
+ run(ExecType.CP, 0.01);
+ }
+
+ @Test
+ public void testDifferenceStatisticsV4() {
+ run(ExecType.CP, 0.001);
+ }
+
+ @Test
+ public void testDifferenceStatisticsV5() {
+ run(ExecType.CP, 0.4);
+ }
+
+ private void run(ExecType instType, double error) {
+
+ ExecMode platformOld = setExecMode(instType);
+
+ try {
+
+ loadTestConfiguration(getTestConfiguration(TEST_NAME));
+ String HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = HOME + TEST_NAME + ".dml";
+ programArgs = new String[] {"-args", input("A"), input("B")};
+
+ MatrixBlock A = TestUtils.generateTestMatrixBlock(100, 5, -3, 3, 1.0, 1342);
+ writeInputMatrixWithMTD("A", A, false);
+ MatrixBlock C = TestUtils.generateTestMatrixBlock(1, 5, 1 - error, 1 + error, 1.0, 1342);
+ MatrixBlock B = new MatrixBlock(100, 5, false);
+ LibMatrixBincell.bincellOp(A, C, B, new BinaryOperator(Multiply.getMultiplyFnObject()));
+ writeInputMatrixWithMTD("B", B, true);
+ String log = runTest(null).toString();
+ // LOG.error(log);
+ assertTrue(log.contains("Quantile Root Square Error"));
+ double rmse = extractRootMeanSquareError(log);
+
+ assertEquals(error, rmse, error * 0.01);
+ }
+ finally {
+ rtplatform = platformOld;
+ }
+ }
+
+ private double extractRootMeanSquareError(String log) {
+ String[] lines = log.split("\n");
+ for(String l : lines) {
+ if(l.contains("Root Mean Square Error:")) {
+ return Double.parseDouble(l.substring(35, l.length()));
+ }
+ }
+ return 1;
+ }
+}
diff --git a/src/test/scripts/functions/builtin/differenceStatistics.dml b/src/test/scripts/functions/builtin/differenceStatistics.dml
new file mode 100644
index 0000000000..92337e4381
--- /dev/null
+++ b/src/test/scripts/functions/builtin/differenceStatistics.dml
@@ -0,0 +1,25 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+A = read($1);
+B = read($2);
+
+differenceStatistics(A, B)