You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@madlib.apache.org by nj...@apache.org on 2019/05/14 23:26:48 UTC
[madlib] branch master updated (a16dffd -> b9453b4)
This is an automated email from the ASF dual-hosted git repository.
njayaram pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/madlib.git.
from a16dffd DL: Improve minibatch performance by using array_cat
new a569221 DL: Add new param metrics_compute_frequency to madlib_keras_fit()
new 8bd5fea DL: Add dev-check and unit tests for metrics_compute_frequency
new b9453b4 DL: Change how we create tables in dev-check
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../modules/deep_learning/madlib_keras.py_in | 320 +++++++++++++--------
.../modules/deep_learning/madlib_keras.sql_in | 40 ++-
.../deep_learning/madlib_keras_validator.py_in | 32 ++-
.../modules/deep_learning/test/madlib_keras.sql_in | 163 +++++++----
.../test/unit_tests/test_madlib_keras.py_in | 46 +++
5 files changed, 412 insertions(+), 189 deletions(-)
[madlib] 02/03: DL: Add dev-check and unit tests for
metrics_compute_frequency
Posted by nj...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
njayaram pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/madlib.git
commit 8bd5fea5af1402d9de46fd67d5aea63f2a094229
Author: Nandish Jayaram <nj...@apache.org>
AuthorDate: Wed May 8 15:17:08 2019 -0700
DL: Add dev-check and unit tests for metrics_compute_frequency
JIRA: MADLIB-1335
Add asserts in dev-check and new tests cases in unit-tests for
associated functions.
Closes #388
Co-authored-by: Ekta Khanna <ek...@pivotal.io>
---
.../modules/deep_learning/test/madlib_keras.sql_in | 37 +++++++++++++++--
.../test/unit_tests/test_madlib_keras.py_in | 46 ++++++++++++++++++++++
2 files changed, 79 insertions(+), 4 deletions(-)
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras.sql_in
index 8567637..5ed3014 100644
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras.sql_in
+++ b/src/ports/postgres/modules/deep_learning/test/madlib_keras.sql_in
@@ -135,6 +135,7 @@ SELECT assert(
compile_params = $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text AND
fit_params = $$ batch_size=2, epochs=1, verbose=0 $$::text AND
num_iterations = 3 AND
+ metrics_compute_frequency = 3 AND
num_classes = 2 AND
class_values = '{0,1}' AND
training_metrics_final >= 0 AND
@@ -151,6 +152,35 @@ FROM (SELECT * FROM keras_saved_out_summary) summary;
SELECT assert(model_data IS NOT NULL , 'Keras model output validation failed') FROM (SELECT * FROM keras_saved_out) k;
+
+-- Verify number of iterations for which metrics and loss are computed
+DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary;
+SELECT madlib_keras_fit(
+ 'cifar_10_sample_batched',
+ 'keras_saved_out',
+ 'model_arch',
+ 1,
+ $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text,
+ $$ batch_size=2, epochs=1, verbose=0 $$::text,
+ 7,
+ NULL,
+ 'cifar_10_sample_val',
+ 4);
+SELECT assert(
+ num_iterations = 7 AND
+ metrics_compute_frequency = 4 AND
+ training_metrics_final >= 0 AND
+ training_loss_final >= 0 AND
+ -- TODO: Uncomment this after MADLIB-1332 is merged to master
+ -- array_upper(training_metrics, 1) = 2 AND
+ -- array_upper(training_loss, 1) = 2 AND
+ -- array_upper(time_iter, 1) = 2 AND
+ validation_metrics_final >= 0 AND
+ validation_loss_final >= 0 AND
+ array_upper(validation_metrics, 1) = 2 AND
+ array_upper(validation_loss, 1) = 2 ,
+ 'Keras model output Summary Validation failed. Actual:' || __to_char(summary))
+FROM (SELECT * FROM keras_saved_out_summary) summary;
-- Fit with gpus_per_host set to 2 must error out on machines
-- that don't have GPUs. Since Jenkins builds are run on docker containers
-- that don't have GPUs, these queries must error out.
@@ -202,8 +232,7 @@ SELECT madlib_keras_fit(
NULL,
NULL,
'model name', 'model desc');
-\x on;
-select * from keras_out_summary;
+
SELECT assert(
source_table = 'cifar_10_sample_batched' AND
model = 'keras_out' AND
@@ -234,8 +263,8 @@ SELECT assert(
array_upper(training_loss, 1) = 2 AND
validation_metrics_final is NULL AND
validation_loss_final is NULL AND
- array_upper(validation_metrics, 1) = 0 AND
- array_upper(validation_loss, 1) = 0 ,
+ validation_metrics is NULL AND
+ validation_loss is NULL,
'Keras model output Summary Validation failed. Actual:' || __to_char(summary))
FROM (SELECT * FROM keras_out_summary) summary;
diff --git a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in
index 61f7f0a..26fde8c 100644
--- a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in
+++ b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in
@@ -386,6 +386,24 @@ class MadlibKerasFitTestCase(unittest.TestCase):
def test_fit_final_none(self):
result = self.subject.fit_final(None)
self.assertEqual(result, None)
+ def test_should_compute_metrics_this_iter_last_iter_case1(self):
+ res = self.subject.should_compute_metrics_this_iter(5, 5, 5)
+ self.assertEqual(True, res)
+ def test_should_compute_metrics_this_iter_last_iter_case2(self):
+ res = self.subject.should_compute_metrics_this_iter(5, 3, 5)
+ self.assertEqual(True, res)
+ def test_should_compute_metrics_this_iter_before_interval(self):
+ res = self.subject.should_compute_metrics_this_iter(1, 3, 5)
+ self.assertEqual(False, res)
+ def test_should_compute_metrics_this_iter_after_interval(self):
+ res = self.subject.should_compute_metrics_this_iter(4, 3, 5)
+ self.assertEqual(False, res)
+ def test_should_compute_metrics_this_iter_at_interval(self):
+ res = self.subject.should_compute_metrics_this_iter(3, 3, 5)
+ self.assertEqual(True, res)
+ def test_should_compute_metrics_this_iter_every_iter(self):
+ res = self.subject.should_compute_metrics_this_iter(2, 1, 5)
+ self.assertEqual(True, res)
def test_fit_final_image_count_zero(self):
input_state = [0, 0, 0]
@@ -733,6 +751,34 @@ class MadlibKerasValidatorTestCase(unittest.TestCase):
self.subject._validate_input_shapes(
'dummy_tbl', 'dummy_col', [32,32,3], 1)
+ def test_is_valid_metrics_compute_frequency_True_None(self):
+ self.subject.FitInputValidator._validate_input_args = Mock()
+ obj = self.subject.FitInputValidator(
+ 'test_table', 'val_table', 'model_table', 'model_arch_table',
+ 'dep_varname', 'independent_varname', 5, None)
+ self.assertEqual(True, obj._is_valid_metrics_compute_frequency())
+
+ def test_is_valid_metrics_compute_frequency_True_num(self):
+ self.subject.FitInputValidator._validate_input_args = Mock()
+ obj = self.subject.FitInputValidator(
+ 'test_table', 'val_table', 'model_table', 'model_arch_table',
+ 'dep_varname', 'independent_varname', 5, 3)
+ self.assertEqual(True, obj._is_valid_metrics_compute_frequency())
+
+ def test_is_valid_metrics_compute_frequency_False_zero(self):
+ self.subject.FitInputValidator._validate_input_args = Mock()
+ obj = self.subject.FitInputValidator(
+ 'test_table', 'val_table', 'model_table', 'model_arch_table',
+ 'dep_varname', 'independent_varname', 5, 0)
+ self.assertEqual(False, obj._is_valid_metrics_compute_frequency())
+
+ def test_is_valid_metrics_compute_frequency_False_greater(self):
+ self.subject.FitInputValidator._validate_input_args = Mock()
+ obj = self.subject.FitInputValidator(
+ 'test_table', 'val_table', 'model_table', 'model_arch_table',
+ 'dep_varname', 'independent_varname', 5, 6)
+ self.assertEqual(False, obj._is_valid_metrics_compute_frequency())
+
class MadlibSerializerTestCase(unittest.TestCase):
def setUp(self):
self.plpy_mock = Mock(spec='error')
[madlib] 03/03: DL: Change how we create tables in dev-check
Posted by nj...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
njayaram pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/madlib.git
commit b9453b4c8ab04a47ca5e25f341e7682a82d5bf8e
Author: Nandish Jayaram <nj...@apache.org>
AuthorDate: Tue May 14 15:10:00 2019 -0700
DL: Change how we create tables in dev-check
Due to the issues mentioned in MADLIB-1326, running
training_preprocessor_dl in deep learning dev-check results in flaky
errors. This commit manually creates one such table.
Co-authored-by: Jingyi Mei <jm...@pivotal.io>
---
.../modules/deep_learning/test/madlib_keras.sql_in | 71 ++++++++++++++--------
1 file changed, 46 insertions(+), 25 deletions(-)
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras.sql_in
index 5ed3014..b5aaa6d 100644
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras.sql_in
+++ b/src/ports/postgres/modules/deep_learning/test/madlib_keras.sql_in
@@ -19,10 +19,10 @@
*
*//* ---------------------------------------------------------------------*/
drop table if exists cifar_10_sample;
-create table cifar_10_sample(id INTEGER, y SMALLINT, imgpath TEXT, x REAL[]);
+create table cifar_10_sample(id INTEGER, y SMALLINT, y_text TEXT, imgpath TEXT, x REAL[]);
copy cifar_10_sample from stdin delimiter '|';
-1|0|'0/img0.jpg'|{{{202,204,199},{202,204,199},{204,206,201},{206,208,203},{208,210,205},{209,211,206},{210,212,207},{212,214,210},{213,215,212},{215,217,214},{216,218,215},{216,218,215},{215,217,214},{216,218,215},{216,218,215},{216,218,214},{217,219,214},{217,219,214},{218,220,215},{218,219,214},{216,217,212},{217,218,213},{218,219,214},{214,215,209},{213,214,207},{212,213,206},{211,212,205},{209,210,203},{208,209,202},{207,208,200},{205,206,199},{203,204,198}},{{206,208,203},{206,208, [...]
-2|1|'0/img2.jpg'|{{{126,118,110},{122,115,108},{126,119,111},{127,119,109},{130,122,111},{130,122,111},{132,124,113},{133,125,114},{130,122,111},{132,124,113},{134,126,115},{131,123,112},{131,123,112},{134,126,115},{133,125,114},{136,128,117},{137,129,118},{137,129,118},{136,128,117},{131,123,112},{130,122,111},{132,124,113},{132,124,113},{132,124,113},{129,122,110},{127,121,109},{127,121,109},{125,119,107},{124,118,106},{124,118,106},{120,114,102},{117,111,99}},{{122,115,107},{119,112,1 [...]
+1|0|'cat'|'0/img0.jpg'|{{{202,204,199},{202,204,199},{204,206,201},{206,208,203},{208,210,205},{209,211,206},{210,212,207},{212,214,210},{213,215,212},{215,217,214},{216,218,215},{216,218,215},{215,217,214},{216,218,215},{216,218,215},{216,218,214},{217,219,214},{217,219,214},{218,220,215},{218,219,214},{216,217,212},{217,218,213},{218,219,214},{214,215,209},{213,214,207},{212,213,206},{211,212,205},{209,210,203},{208,209,202},{207,208,200},{205,206,199},{203,204,198}},{{206,208,203},{20 [...]
+2|1|'dog'|'0/img2.jpg'|{{{126,118,110},{122,115,108},{126,119,111},{127,119,109},{130,122,111},{130,122,111},{132,124,113},{133,125,114},{130,122,111},{132,124,113},{134,126,115},{131,123,112},{131,123,112},{134,126,115},{133,125,114},{136,128,117},{137,129,118},{137,129,118},{136,128,117},{131,123,112},{130,122,111},{132,124,113},{132,124,113},{132,124,113},{129,122,110},{127,121,109},{127,121,109},{125,119,107},{124,118,106},{124,118,106},{120,114,102},{117,111,99}},{{122,115,107},{119 [...]
\.
drop table if exists cifar_10_sample_val;
@@ -43,12 +43,13 @@ copy cifar_10_sample_val from stdin delimiter '|';
DROP TABLE IF EXISTS cifar_10_sample_batched;
CREATE TABLE cifar_10_sample_batched(
- independent_var real[],
+ buffer_id smallint,
dependent_var integer[],
- buffer_id smallint);
+ dependent_var_text_with_null integer[],
+ independent_var real[]);
copy cifar_10_sample_batched from stdin delimiter '|';
-{{{{0.494118,0.462745,0.431373},{0.478431,0.45098,0.423529},{0.494118,0.466667,0.435294},{0.498039,0.466667,0.427451},{0.509804,0.478431,0.435294},{0.509804,0.478431,0.435294},{0.517647,0.486275,0.443137},{0.521569,0.490196,0.447059},{0.509804,0.478431,0.435294},{0.517647,0.486275,0.443137},{0.52549,0.494118,0.45098},{0.513726,0.482353,0.439216},{0.513726,0.482353,0.439216},{0.52549,0.494118,0.45098},{0.521569,0.490196,0.447059},{0.533333,0.501961,0.458824},{0.537255,0.505882,0.462745},{ [...]
-{{{{0.792157,0.8,0.780392},{0.792157,0.8,0.780392},{0.8,0.807843,0.788235},{0.807843,0.815686,0.796079},{0.815686,0.823529,0.803922},{0.819608,0.827451,0.807843},{0.823529,0.831373,0.811765},{0.831373,0.839216,0.823529},{0.835294,0.843137,0.831373},{0.843137,0.85098,0.839216},{0.847059,0.854902,0.843137},{0.847059,0.854902,0.843137},{0.843137,0.85098,0.839216},{0.847059,0.854902,0.843137},{0.847059,0.854902,0.843137},{0.847059,0.854902,0.839216},{0.85098,0.858824,0.839216},{0.85098,0.858 [...]
+0|{{0,1}}|{{0,0,1,0,0}}|{{{{0.494118,0.462745,0.431373},{0.478431,0.45098,0.423529},{0.494118,0.466667,0.435294},{0.498039,0.466667,0.427451},{0.509804,0.478431,0.435294},{0.509804,0.478431,0.435294},{0.517647,0.486275,0.443137},{0.521569,0.490196,0.447059},{0.509804,0.478431,0.435294},{0.517647,0.486275,0.443137},{0.52549,0.494118,0.45098},{0.513726,0.482353,0.439216},{0.513726,0.482353,0.439216},{0.52549,0.494118,0.45098},{0.521569,0.490196,0.447059},{0.533333,0.501961,0.458824},{0.537 [...]
+1|{{1,0}}|{{0,1,0,0,0}}|{{{{0.792157,0.8,0.780392},{0.792157,0.8,0.780392},{0.8,0.807843,0.788235},{0.807843,0.815686,0.796079},{0.815686,0.823529,0.803922},{0.819608,0.827451,0.807843},{0.823529,0.831373,0.811765},{0.831373,0.839216,0.823529},{0.835294,0.843137,0.831373},{0.843137,0.85098,0.839216},{0.847059,0.854902,0.843137},{0.847059,0.854902,0.843137},{0.843137,0.85098,0.839216},{0.847059,0.854902,0.843137},{0.847059,0.854902,0.843137},{0.847059,0.854902,0.839216},{0.85098,0.858824, [...]
\.
DROP TABLE IF EXISTS cifar_10_sample_batched_summary;
@@ -346,7 +347,6 @@ SELECT madlib_keras_fit(
1,
$$ optimizer=Adam(epsilon=None), loss='categorical_crossentropy', metrics=['accuracy']$$::text,
$$ batch_size=2, epochs=1, verbose=0 $$::text,
-
1,
0,
NULL,
@@ -378,15 +378,17 @@ SELECT madlib_keras_fit(
1,
NULL,
NULL,
+ NULL,
'model name', 'model desc');
-- -- negative test case for passing non numeric y to fit
-- induce failure by passing a non numeric column
-create table cifar_10_sample_val_failure as select * from cifar_10_sample_val;
-alter table cifar_10_sample_val_failure rename dependent_var to dependent_var_original;
-alter table cifar_10_sample_val_failure rename buffer_id to dependent_var;
+DROP TABLE IF EXISTS cifar_10_sample_val_failure;
+CREATE TABLE cifar_10_sample_val_failure AS SELECT * FROM cifar_10_sample_val;
+ALTER TABLE cifar_10_sample_val_failure rename dependent_var to dependent_var_original;
+ALTER TABLE cifar_10_sample_val_failure rename buffer_id to dependent_var;
DROP TABLE IF EXISTS keras_out, keras_out_summary;
-select assert(trap_error($TRAP$madlib_keras_fit(
+SELECT assert(trap_error($TRAP$madlib_keras_fit(
'cifar_10_sample_batched',
'keras_out',
'model_arch',
@@ -425,21 +427,36 @@ WHERE attrelid='cifar10_predict'::regclass AND attnum>0;
-- Tests with text class values:
-- Modify input data to have text classes, and mini-batch it.
-CREATE TABLE cifar_10_sample_text AS
-SELECT * FROM cifar_10_sample;
-
-ALTER TABLE cifar_10_sample_text ALTER COLUMN y type TEXT;
-UPDATE cifar_10_sample_text SET y='cat' where y='0';
-UPDATE cifar_10_sample_text SET y='dog' where y='1';
--- Add a new image with NULL class value
-INSERT INTO cifar_10_sample_text(id, x, y, imgpath)
-SELECT 3, x, NULL, '0/img3.jpg' FROM cifar_10_sample_text
-WHERE y='cat';
-
-
DROP TABLE IF EXISTS cifar_10_sample_text_batched;
+-- Create a new table using the text based column for dep var.
+CREATE TABLE cifar_10_sample_text_batched AS
+ SELECT buffer_id, independent_var, dependent_var_text_with_null AS dependent_var
+ FROM cifar_10_sample_batched;
+-- Insert a new row with NULL as the dependent var (one-hot encoded)
+INSERT INTO cifar_10_sample_text_batched(buffer_id, independent_var, dependent_var)
+ SELECT 2, independent_var, ARRAY[[0,1,0,0,0]]
+ FROM cifar_10_sample_batched
+ WHERE cifar_10_sample_batched.buffer_id=0;
+-- Create the necessary summary table for the batched input.
DROP TABLE IF EXISTS cifar_10_sample_text_batched_summary;
-SELECT training_preprocessor_dl('cifar_10_sample_text','cifar_10_sample_text_batched','y','x', 2, 255, 5);
+CREATE TABLE cifar_10_sample_text_batched_summary(
+ source_table text,
+ output_table text,
+ dependent_varname text,
+ independent_varname text,
+ dependent_vartype text,
+ class_values text[],
+ buffer_size integer,
+ normalizing_const numeric);
+INSERT INTO cifar_10_sample_text_batched_summary values (
+ 'cifar_10_sample',
+ 'cifar_10_sample_text_batched',
+ 'y_text',
+ 'x',
+ 'text',
+ ARRAY[NULL,'cat','dog',NULL,NULL],
+ 1,
+ 255.0);
-- Change model_arch to reflect 5 num_classes
DROP TABLE IF EXISTS model_arch;
@@ -481,6 +498,10 @@ SELECT assert(
FROM (SELECT * FROM keras_saved_out_summary) summary;
-- Predict with pred_type=prob
+DROP TABLE IF EXISTS cifar_10_sample_text;
+CREATE TABLE cifar_10_sample_text AS
+ SELECT id, x, y_text
+ FROM cifar_10_sample;
DROP TABLE IF EXISTS cifar10_predict;
SELECT madlib_keras_predict(
'keras_saved_out',
[madlib] 01/03: DL: Add new param metrics_compute_frequency to
madlib_keras_fit()
Posted by nj...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
njayaram pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/madlib.git
commit a5692219a361fad8b8b09c4c6789dd9830a04e8f
Author: Ekta Khanna <ek...@pivotal.io>
AuthorDate: Wed May 1 17:09:12 2019 -0700
DL: Add new param metrics_compute_frequency to madlib_keras_fit()
JIRA: MADLIB-1335
This commit adds a new optional parameter `metrics_compute_frequency` to
`madlib_keras_fit()` to control how often Keras.evaluate() is run on
training and validation data for computing loss/metrics. We only support
one metric at the moment.
Closes #388
Co-authored-by: Nandish Jayaram <nj...@apache.org>
---
.../modules/deep_learning/madlib_keras.py_in | 320 +++++++++++++--------
.../modules/deep_learning/madlib_keras.sql_in | 40 ++-
.../deep_learning/madlib_keras_validator.py_in | 32 ++-
.../modules/deep_learning/test/madlib_keras.sql_in | 63 ++--
4 files changed, 291 insertions(+), 164 deletions(-)
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras.py_in
index d0dde1d..9cda792 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras.py_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras.py_in
@@ -51,11 +51,34 @@ from utilities.utilities import is_platform_pg
from utilities.utilities import get_segments_per_host
from utilities.utilities import madlib_version
from utilities.validate_args import get_col_value_and_type
+from utilities.validate_args import get_expr_type
from utilities.validate_args import quote_ident
+def get_source_summary_table_dict(fit_validator):
+ source_summary = plpy.execute("""
+ SELECT
+ {class_values} AS class_values,
+ {norm_const} AS norm_const,
+ {dep_vartype} AS dep_vartype,
+ {dep_varname} AS dependent_varname_in_source_table,
+ {indep_varname} AS independent_varname_in_source_table
+ FROM {tbl}
+ """.format(class_values=CLASS_VALUES_COLNAME,
+ norm_const=NORMALIZING_CONST_COLNAME,
+ dep_vartype=DEPENDENT_VARTYPE_COLNAME,
+ dep_varname='dependent_varname',
+ indep_varname='independent_varname',
+ tbl=fit_validator.source_summary_table))[0]
+ source_summary['class_values_type'] = get_expr_type(
+ CLASS_VALUES_COLNAME, fit_validator.source_summary_table)
+ source_summary['norm_const_type'] = get_expr_type(
+ NORMALIZING_CONST_COLNAME, fit_validator.source_summary_table)
+ return source_summary
+
def fit(schema_madlib, source_table, model,model_arch_table,
model_arch_id, compile_params, fit_params, num_iterations,
- gpus_per_host = 0, validation_table=None, name="",
+ gpus_per_host = 0, validation_table=None,
+ metrics_compute_frequency=None, name="",
description="", **kwargs):
source_table = quote_ident(source_table)
dependent_varname = MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL
@@ -65,7 +88,10 @@ def fit(schema_madlib, source_table, model,model_arch_table,
fit_validator = FitInputValidator(
source_table, validation_table, model, model_arch_table,
- dependent_varname, independent_varname, num_iterations)
+ dependent_varname, independent_varname, num_iterations,
+ metrics_compute_frequency)
+ if metrics_compute_frequency is None:
+ metrics_compute_frequency = num_iterations
start_training_time = datetime.datetime.now()
@@ -135,7 +161,7 @@ def fit(schema_madlib, source_table, model,model_arch_table,
# Construct validation dataset if provided
validation_set_provided = bool(validation_table)
- validation_aggregate_accuracy = []; validation_aggregate_loss = []
+ validation_metrics = []; validation_loss = []
# Prepare the SQL for running distributed training via UDA
compile_params_to_pass = "$madlib$" + compile_params + "$madlib$"
@@ -161,143 +187,138 @@ def fit(schema_madlib, source_table, model,model_arch_table,
# Define the state for the model and loss/accuracy storage lists
model_state = madlib_keras_serializer.serialize_weights(
0, 0, 0, model_weights)
- aggregate_loss, aggregate_accuracy, aggregate_runtime = [], [], []
+ training_loss, training_metrics, aggregate_runtime = [], [], []
+ metrics_iters = []
plpy.info("Model architecture size: {}KB".format(len(model_arch)/1024))
plpy.info("Model state (serialized) size: {}MB".format(
len(model_state)/1024/1024))
# Run distributed training for specified number of iterations
- for i in range(num_iterations):
+ for i in range(1, num_iterations+1):
start_iteration = time.time()
- iteration_result = plpy.execute(run_training_iteration, [model_state])[0]['iteration_result']
+ iteration_result = plpy.execute(run_training_iteration,
+ [model_state])[0]['iteration_result']
end_iteration = time.time()
plpy.info("Time for iteration {0}: {1} sec".
- format(i + 1, end_iteration - start_iteration))
+ format(i, end_iteration - start_iteration))
aggregate_runtime.append(datetime.datetime.now())
- avg_loss, avg_accuracy, model_state = madlib_keras_serializer.deserialize_iteration_state(iteration_result)
+ avg_loss, avg_metric, model_state = madlib_keras_serializer.\
+ deserialize_iteration_state(iteration_result)
plpy.info("Average loss after training iteration {0}: {1}".format(
- i + 1, avg_loss))
+ i, avg_loss))
plpy.info("Average accuracy after training iteration {0}: {1}".format(
- i + 1, avg_accuracy))
- if validation_set_provided:
- _, _, _, updated_weights = madlib_keras_serializer.deserialize_weights(
- model_state, model_shapes)
- master_model.set_weights(updated_weights)
- start_val = time.time()
- evaluate_result = get_loss_acc_from_keras_eval(schema_madlib,
- validation_table,
- dependent_varname,
- independent_varname,
- compile_params_to_pass,
- model_arch, model_state,
- gpus_per_host,
- segments_per_host,
- seg_ids_val,
- images_per_seg_val,
- gp_segment_id_col)
- end_val = time.time()
- plpy.info("Time for validation in iteration {0}: {1} sec". format(i + 1, end_val - start_val))
- if len(evaluate_result) < 2:
- plpy.error('Calling evaluate on validation data returned < 2 '
- 'metrics. Expected metrics are loss and accuracy')
- validation_loss = evaluate_result[0]
- validation_accuracy = evaluate_result[1]
- plpy.info("Validation set accuracy after iteration {0}: {1}".
- format(i + 1, validation_accuracy))
- validation_aggregate_accuracy.append(validation_accuracy)
- validation_aggregate_loss.append(validation_loss)
- aggregate_loss.append(avg_loss)
- aggregate_accuracy.append(avg_accuracy)
+ i, avg_metric))
+
+ if should_compute_metrics_this_iter(i, metrics_compute_frequency,
+ num_iterations):
+ # TODO: Do we need this code?
+ # _, _, _, updated_weights = madlib_keras_serializer.deserialize_weights(
+ # model_state, model_shapes)
+ # master_model.set_weights(updated_weights)
+ # Compute loss/accuracy for training data.
+ # TODO: Uncomment this once JIRA MADLIB-1332 is merged to master
+ # compute_loss_and_metrics(
+ # schema_madlib, source_table, dependent_varname,
+ # independent_varname, compile_params_to_pass, model_arch,
+ # model_state, gpus_per_host, segments_per_host, seg_ids_val,
+ # images_per_seg_val, gp_segment_id_col,
+ # training_metrics, training_loss,
+ # i, "Training")
+ metrics_iters.append(i)
+ if validation_set_provided:
+ # Compute loss/accuracy for validation data.
+ compute_loss_and_metrics(
+ schema_madlib, validation_table, dependent_varname,
+ independent_varname, compile_params_to_pass, model_arch,
+ model_state, gpus_per_host, segments_per_host, seg_ids_val,
+ images_per_seg_val, gp_segment_id_col,
+ validation_metrics, validation_loss,
+ i, "Validation")
+ training_loss.append(avg_loss)
+ training_metrics.append(avg_metric)
end_training_time = datetime.datetime.now()
- final_validation_acc = None
- if validation_aggregate_accuracy and len(validation_aggregate_accuracy) > 0:
- final_validation_acc = validation_aggregate_accuracy[-1]
-
- final_validation_loss = None
- if validation_aggregate_loss and len(validation_aggregate_loss) > 0:
- final_validation_loss = validation_aggregate_loss[-1]
version = madlib_version(schema_madlib)
- class_values, class_values_type = get_col_value_and_type(
- fit_validator.source_summary_table, CLASS_VALUES_COLNAME)
- norm_const, norm_const_type = get_col_value_and_type(
- fit_validator.source_summary_table, NORMALIZING_CONST_COLNAME)
- dep_vartype = plpy.execute("SELECT {0} AS dep FROM {1}".format(
- DEPENDENT_VARTYPE_COLNAME, fit_validator.source_summary_table))[0]['dep']
- dependent_varname_in_source_table = quote_ident(plpy.execute("SELECT {0} FROM {1}".format(
- 'dependent_varname', fit_validator.source_summary_table))[0]['dependent_varname'])
- independent_varname_in_source_table = quote_ident(plpy.execute("SELECT {0} FROM {1}".format(
- 'independent_varname', fit_validator.source_summary_table))[0]['independent_varname'])
+ src_summary_dict = get_source_summary_table_dict(fit_validator)
+ class_values = src_summary_dict['class_values']
+ class_values_type = src_summary_dict['class_values_type']
+ norm_const = src_summary_dict['norm_const']
+ norm_const_type = src_summary_dict['norm_const_type']
+ dep_vartype = src_summary_dict['dep_vartype']
+ dependent_varname_in_source_table = src_summary_dict['dependent_varname_in_source_table']
+ independent_varname_in_source_table = src_summary_dict['independent_varname_in_source_table']
+ # Define some constants to be inserted into the summary table.
+ model_type = "madlib_keras"
+ model_size = sys.getsizeof(model)
+ metrics_iters = metrics_iters if metrics_iters else 'NULL'
+ # We always compute the training loss and metrics, at least once.
+ training_metrics_final = training_metrics[-1]
+ training_loss_final = training_loss[-1]
+ training_metrics = training_metrics if training_metrics else 'NULL'
+ training_loss = training_loss if training_loss else 'NULL'
+ # Validation loss and metrics are computed only if validation_table
+ # is provided.
+ if validation_set_provided:
+ validation_metrics_final = validation_metrics[-1]
+ validation_loss_final = validation_loss[-1]
+ validation_metrics = 'ARRAY{0}'.format(validation_metrics)
+ validation_loss = 'ARRAY{0}'.format(validation_loss)
+ # Must quote the string before inserting to table. Explicitly
+ # quoting it here since this can also take a NULL value, done
+ # in the else part.
+ validation_table = "$MAD${0}$MAD$".format(validation_table)
+ else:
+ validation_metrics = validation_loss = 'NULL'
+ validation_metrics_final = validation_loss_final = 'NULL'
+ validation_table = 'NULL'
+
create_output_summary_table = plpy.prepare("""
- CREATE TABLE {0}_summary AS
+ CREATE TABLE {output_summary_model_table} AS
SELECT
- $1 AS model_arch_table,
- $2 AS model_arch_id,
- $3 AS model_type,
- $4 AS start_training_time,
- $5 AS end_training_time,
- $6 AS source_table,
- $7 AS validation_table,
- $8 AS model,
- $9 AS dependent_varname,
- $10 AS independent_varname,
- $11 AS name,
- $12 AS description,
- $13 AS model_size,
- $14 AS madlib_version,
- $15 AS compile_params,
- $16 AS fit_params,
- $17 AS num_iterations,
- $18 AS num_classes,
- $19 AS accuracy,
- $20 AS loss,
- $21 AS accuracy_iter,
- $22 AS loss_iter,
- $23 AS time_iter,
- $24 AS accuracy_validation,
- $25 AS loss_validation,
- $26 AS accuracy_iter_validation,
- $27 AS loss_iter_validation,
- $28 AS {1},
- $29 AS {2},
- $30 AS {3}
- """.format(model, CLASS_VALUES_COLNAME, DEPENDENT_VARTYPE_COLNAME,
- NORMALIZING_CONST_COLNAME),
- ["TEXT", "INTEGER", "TEXT", "TIMESTAMP",
- "TIMESTAMP", "TEXT", "TEXT","TEXT",
- "TEXT", "TEXT", "TEXT", "TEXT", "INTEGER",
- "TEXT", "TEXT", "TEXT", "INTEGER",
- "INTEGER", "DOUBLE PRECISION",
- "DOUBLE PRECISION", "DOUBLE PRECISION[]",
- "DOUBLE PRECISION[]", "TIMESTAMP[]",
- "DOUBLE PRECISION", "DOUBLE PRECISION",
- "DOUBLE PRECISION[]", "DOUBLE PRECISION[]",
- class_values_type, "TEXT", norm_const_type])
- plpy.execute(
- create_output_summary_table,
- [
- model_arch_table, model_arch_id,
- "madlib_keras",
- start_training_time, end_training_time,
- source_table, validation_table,
- model, dependent_varname_in_source_table,
- independent_varname_in_source_table, name, description,
- sys.getsizeof(model), version, compile_params,
- fit_params, num_iterations, num_classes,
- aggregate_accuracy[-1],
- aggregate_loss[-1],
- aggregate_accuracy, aggregate_loss,
- aggregate_runtime, final_validation_acc,
- final_validation_loss,
- validation_aggregate_accuracy,
- validation_aggregate_loss,
- class_values,
- dep_vartype,
- norm_const
- ]
- )
+ $MAD${source_table}$MAD$::TEXT AS source_table,
+ $MAD${model}$MAD$::TEXT AS model,
+ $MAD${dependent_varname_in_source_table}$MAD$::TEXT AS dependent_varname,
+ $MAD${independent_varname_in_source_table}$MAD$::TEXT AS independent_varname,
+ $MAD${model_arch_table}$MAD$::TEXT AS model_arch_table,
+ {model_arch_id} AS model_arch_id,
+ $1 AS compile_params,
+ $2 AS fit_params,
+ {num_iterations} AS num_iterations,
+ {validation_table}::TEXT AS validation_table,
+ {metrics_compute_frequency} AS metrics_compute_frequency,
+ $3 AS name,
+ $4 AS description,
+ '{model_type}'::TEXT AS model_type,
+ {model_size} AS model_size,
+ '{start_training_time}'::TIMESTAMP AS start_training_time,
+ '{end_training_time}'::TIMESTAMP AS end_training_time,
+ $5 AS time_iter,
+ '{version}'::TEXT AS madlib_version,
+ {num_classes} AS num_classes,
+ $6 AS {class_values_colname},
+ '{dep_vartype}' AS {dependent_vartype_colname},
+ {norm_const} AS {normalizing_const_colname},
+ {training_metrics_final} AS training_metrics_final,
+ {training_loss_final} AS training_loss_final,
+ ARRAY{training_metrics}::DOUBLE PRECISION[] AS training_metrics,
+ ARRAY{training_loss}::DOUBLE PRECISION[] AS training_loss,
+ {validation_metrics_final} AS validation_metrics_final,
+ {validation_loss_final} AS validation_loss_final,
+ {validation_metrics}::DOUBLE PRECISION[] AS validation_metrics,
+ {validation_loss}::DOUBLE PRECISION[] AS validation_loss,
+ ARRAY{metrics_iters}::INTEGER[] AS metrics_iters
+ """.format(output_summary_model_table=fit_validator.output_summary_model_table,
+ class_values_colname=CLASS_VALUES_COLNAME,
+ dependent_vartype_colname=DEPENDENT_VARTYPE_COLNAME,
+ normalizing_const_colname=NORMALIZING_CONST_COLNAME,
+ **locals()),
+ ["TEXT", "TEXT", "TEXT","TEXT", "TIMESTAMP[]",
+ class_values_type])
+ plpy.execute(create_output_summary_table,
+ [compile_params, fit_params, name,
+ description, aggregate_runtime, class_values])
create_output_table = plpy.prepare("""
CREATE TABLE {0} AS
@@ -310,6 +331,61 @@ def fit(schema_madlib, source_table, model,model_arch_table,
#TODO add a unit test for this in a future PR
reset_cuda_env(original_cuda_env)
+def compute_loss_and_metrics(schema_madlib, table, dependent_varname,
+ independent_varname, compile_params, model_arch,
+ model_state, gpus_per_host, segments_per_host,
+ seg_ids_val, rows_per_seg_val,
+ gp_segment_id_col, metrics_list, loss_list,
+ curr_iter, dataset_name):
+ """
+ Compute the loss and metric using a given model (model_state) on the
+ given dataset (table.)
+ """
+ start_val = time.time()
+ evaluate_result = get_loss_acc_from_keras_eval(schema_madlib,
+ table,
+ dependent_varname,
+ independent_varname,
+ compile_params,
+ model_arch, model_state,
+ gpus_per_host,
+ segments_per_host,
+ seg_ids_val,
+ rows_per_seg_val,
+ gp_segment_id_col)
+ end_val = time.time()
+ plpy.info("Time for evaluation in iteration {0}: {1} sec.". format(
+ curr_iter, end_val - start_val))
+ if len(evaluate_result) < 2:
+ plpy.error('Calling evaluate on table {0} returned < 2 '
+ 'metrics. Expected both loss and a metric.'.format(
+ table))
+ loss = evaluate_result[0]
+ metric = evaluate_result[1]
+ plpy.info("{0} set metric after iteration {1}: {2}.".
+ format(dataset_name, curr_iter, metric))
+ plpy.info("{0} set loss after iteration {1}: {2}.".
+ format(dataset_name, curr_iter, loss))
+ metrics_list.append(metric)
+ loss_list.append(loss)
+
+def should_compute_metrics_this_iter(curr_iter, metrics_compute_frequency,
+ num_iterations):
+ """
+ Check if we want to compute loss/accuracy for the current iteration
+ :param curr_iter:
+ :param metrics_compute_frequency:
+ :param num_iterations:
+ :return: Returns a boolean
+ return TRUE, if it is the last iteration, or if metrics_compute_frequency
+ iterations have elapsed since the last time it was computed.
+ return FALSE otherwise.
+ """
+ # Compute loss/accuracy every metrics_compute_frequency'th iteration,
+ # and also for the last iteration.
+ return (curr_iter)%metrics_compute_frequency == 0 or \
+ curr_iter == num_iterations
+
def get_images_per_seg(source_table, dependent_varname):
"""
Compute total images in each segment, by querying source_table. For
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in b/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in
index 2d7170b..77222a9 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in
@@ -38,6 +38,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
num_iterations INTEGER,
gpus_per_host INTEGER,
validation_table VARCHAR,
+ metrics_compute_frequency INTEGER,
name VARCHAR,
description VARCHAR
) RETURNS VOID AS $$
@@ -47,6 +48,39 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
$$ LANGUAGE plpythonu VOLATILE
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
+ source_table VARCHAR,
+ model VARCHAR,
+ model_arch_table VARCHAR,
+ model_arch_id INTEGER,
+ compile_params VARCHAR,
+ fit_params VARCHAR,
+ num_iterations INTEGER,
+ gpus_per_host INTEGER,
+ validation_table VARCHAR,
+ metrics_compute_frequency INTEGER,
+ name VARCHAR
+) RETURNS VOID AS $$
+ SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, NULL);
+$$ LANGUAGE sql VOLATILE
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
+ source_table VARCHAR,
+ model VARCHAR,
+ model_arch_table VARCHAR,
+ model_arch_id INTEGER,
+ compile_params VARCHAR,
+ fit_params VARCHAR,
+ num_iterations INTEGER,
+ gpus_per_host INTEGER,
+ validation_table VARCHAR,
+ metrics_compute_frequency INTEGER
+) RETURNS VOID AS $$
+ SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, NULL, NULL);
+$$ LANGUAGE sql VOLATILE
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
+
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
source_table VARCHAR,
@@ -59,7 +93,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
gpus_per_host INTEGER,
validation_table VARCHAR
) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, $8, $9, NULL, NULL);
+ SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, $8, $9, NULL, NULL, NULL);
$$ LANGUAGE sql VOLATILE
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
@@ -73,7 +107,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
num_iterations INTEGER,
gpus_per_host INTEGER
) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, $8, NULL, NULL, NULL);
+ SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, $8, NULL, NULL, NULL, NULL);
$$ LANGUAGE sql VOLATILE
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
@@ -86,7 +120,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
fit_params VARCHAR,
num_iterations INTEGER
) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, 0, NULL, NULL, NULL);
+ SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, 0, NULL, NULL, NULL, NULL);
$$ LANGUAGE sql VOLATILE
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA');
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_validator.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_validator.py_in
index d66219c..9210550 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_validator.py_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras_validator.py_in
@@ -34,6 +34,7 @@ from utilities.utilities import is_var_valid
from utilities.utilities import is_valid_psql_type
from utilities.utilities import NUMERIC
from utilities.utilities import ONLY_ARRAY
+from utilities.validate_args import cols_in_tbl_valid
from utilities.validate_args import columns_exist_in_table
from utilities.validate_args import get_expr_type
from utilities.validate_args import input_tbl_valid
@@ -157,13 +158,14 @@ class PredictInputValidator:
class FitInputValidator:
def __init__(self, source_table, validation_table, output_model_table,
model_arch_table, dependent_varname, independent_varname,
- num_iterations):
+ num_iterations, metrics_compute_frequency):
self.source_table = source_table
self.validation_table = validation_table
self.output_model_table = output_model_table
self.model_arch_table = model_arch_table
self.dependent_varname = dependent_varname
self.independent_varname = independent_varname
+ self.metrics_compute_frequency = metrics_compute_frequency
self.num_iterations = num_iterations
self.source_summary_table = None
if self.source_table:
@@ -172,36 +174,42 @@ class FitInputValidator:
if self.output_model_table:
self.output_summary_model_table = add_postfix(
self.output_model_table, "_summary")
- self.module_name = 'model_keras'
+ self.module_name = 'madlib_keras_fit'
self._validate_input_args()
def _validate_input_table(self, table):
_assert(is_var_valid(table, self.independent_varname),
- "model_keras error: invalid independent_varname "
+ "{module_name}: invalid independent_varname "
"('{independent_varname}') for table "
"({table}).".format(
+ module_name=self.module_name,
independent_varname=self.independent_varname,
table=table))
_assert(is_var_valid(table, self.dependent_varname),
- "model_keras error: invalid dependent_varname "
+ "{module_name}: invalid dependent_varname "
"('{dependent_varname}') for table "
"({table}).".format(
+ module_name=self.module_name,
dependent_varname=self.dependent_varname,
table=table))
+ def _is_valid_metrics_compute_frequency(self):
+ return self.metrics_compute_frequency is None or \
+ (self.metrics_compute_frequency >= 1 and \
+ self.metrics_compute_frequency <= self.num_iterations)
+
def _validate_input_args(self):
_assert(self.num_iterations > 0,
- "model_keras error: Number of iterations cannot be < 1.")
+ "{0}: Number of iterations cannot be < 1.".format(self.module_name))
+ _assert(self._is_valid_metrics_compute_frequency(),
+ "{0}: metrics_compute_frequency must be in the range (1 - {1}).".format(
+ self.module_name, self.num_iterations))
input_tbl_valid(self.source_table, self.module_name)
input_tbl_valid(self.source_summary_table, self.module_name)
- _assert(is_var_valid(
- self.source_summary_table, CLASS_VALUES_COLNAME),
- "model_keras error: invalid class_values varname "
- "('{class_values}') for source_summary_table "
- "({source_summary_table}).".format(
- class_values=CLASS_VALUES_COLNAME,
- source_summary_table=self.source_summary_table))
+ cols_in_tbl_valid(self.source_summary_table, [CLASS_VALUES_COLNAME,
+ NORMALIZING_CONST_COLNAME, DEPENDENT_VARTYPE_COLNAME,
+ 'dependent_varname', 'independent_varname'], self.module_name)
# Source table and validation tables must have the same schema
self._validate_input_table(self.source_table)
validate_dependent_var_for_minibatch(self.source_table,
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras.sql_in
index bc0789b..8567637 100644
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras.sql_in
+++ b/src/ports/postgres/modules/deep_learning/test/madlib_keras.sql_in
@@ -137,15 +137,15 @@ SELECT assert(
num_iterations = 3 AND
num_classes = 2 AND
class_values = '{0,1}' AND
- accuracy >= 0 AND
- loss >= 0 AND
- array_upper(accuracy_iter, 1) = 3 AND
- array_upper(loss_iter, 1) = 3 AND
+ training_metrics_final >= 0 AND
+ training_loss_final >= 0 AND
+ array_upper(training_metrics, 1) = 3 AND
+ array_upper(training_loss, 1) = 3 AND
array_upper(time_iter, 1) = 3 AND
- accuracy_validation >= 0 AND
- loss_validation >= 0 AND
- array_upper(accuracy_iter_validation, 1) = 3 AND
- array_upper(loss_iter_validation, 1) = 3 ,
+ validation_metrics_final >= 0 AND
+ validation_loss_final >= 0 AND
+ array_upper(validation_metrics, 1) = 1 AND
+ array_upper(validation_loss, 1) = 1 ,
'Keras model output Summary Validation failed. Actual:' || __to_char(summary))
FROM (SELECT * FROM keras_saved_out_summary) summary;
@@ -200,37 +200,42 @@ SELECT madlib_keras_fit(
2,
NULL,
NULL,
+ NULL,
'model name', 'model desc');
+\x on;
+select * from keras_out_summary;
SELECT assert(
- model_arch_table = 'model_arch' AND
- model_arch_id = 1 AND
- model_type = 'madlib_keras' AND
- start_training_time < now() AND
- end_training_time > start_training_time AND
source_table = 'cifar_10_sample_batched' AND
- validation_table is NULL AND
model = 'keras_out' AND
dependent_varname = 'y' AND
- dependent_vartype = 'smallint' AND
independent_varname = 'x' AND
+ model_arch_table = 'model_arch' AND
+ model_arch_id = 1 AND
+ compile_params = $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text AND
+ fit_params = $$ batch_size=2, epochs=1, verbose=0 $$::text AND
+ num_iterations = 2 AND
+ validation_table is NULL AND
+ metrics_compute_frequency = 2 AND
name = 'model name' AND
description = 'model desc' AND
+ model_type = 'madlib_keras' AND
model_size > 0 AND
+ start_training_time < now() AND
+ end_training_time > start_training_time AND
+ array_upper(time_iter, 1) = 2 AND
+ dependent_vartype = 'smallint' AND
madlib_version is NOT NULL AND
- compile_params = $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text AND
- fit_params = $$ batch_size=2, epochs=1, verbose=0 $$::text AND
- num_iterations = 2 AND
num_classes = 2 AND
class_values = '{0,1}' AND
- accuracy is not NULL AND
- loss is not NULL AND
- array_upper(accuracy_iter, 1) = 2 AND
- array_upper(loss_iter, 1) = 2 AND
- array_upper(time_iter, 1) = 2 AND
- accuracy_validation is NULL AND
- loss_validation is NULL AND
- array_upper(accuracy_iter_validation,1) = 0 AND
- array_upper(loss_iter_validation,1) = 0 ,
+ normalizing_const = 255.0 AND
+ training_metrics_final is not NULL AND
+ training_loss_final is not NULL AND
+ array_upper(training_metrics, 1) = 2 AND
+ array_upper(training_loss, 1) = 2 AND
+ validation_metrics_final is NULL AND
+ validation_loss_final is NULL AND
+ array_upper(validation_metrics, 1) = 0 AND
+ array_upper(validation_loss, 1) = 0 ,
'Keras model output Summary Validation failed. Actual:' || __to_char(summary))
FROM (SELECT * FROM keras_out_summary) summary;
@@ -287,6 +292,7 @@ SELECT madlib_keras_fit(
1,
NULL,
NULL,
+ NULL,
'model name', 'model desc');
DROP TABLE IF EXISTS keras_out, keras_out_summary;
@@ -300,6 +306,7 @@ SELECT madlib_keras_fit(
1,
NULL,
NULL,
+ NULL,
'model name', 'model desc');
DROP TABLE IF EXISTS keras_out, keras_out_summary;
@@ -314,6 +321,7 @@ SELECT madlib_keras_fit(
1,
0,
NULL,
+ NULL,
'model name', 'model desc');
DROP TABLE IF EXISTS keras_out, keras_out_summary;
@@ -327,6 +335,7 @@ SELECT madlib_keras_fit(
1,
NULL,
NULL,
+ NULL,
'model name', 'model desc');
DROP TABLE IF EXISTS keras_out, keras_out_summary;