You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@madlib.apache.org by kh...@apache.org on 2019/11/12 18:26:58 UTC
[madlib] branch master updated: DL: Make model_id naming consistent
This is an automated email from the ASF dual-hosted git repository.
khannaekta pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/madlib.git
The following commit(s) were added to refs/heads/master by this push:
new 0f50490 DL: Make model_id naming consistent
0f50490 is described below
commit 0f50490e9d7cdf117fefb81bd8efbfee85d95573
Author: Ekta Khanna <ek...@pivotal.io>
AuthorDate: Wed Nov 6 16:42:30 2019 -0800
DL: Make model_id naming consistent
Prior to this commit, `model_id` was referenced with names like
`model_arch_id` and `model_arch_table_id`. This commit ensures, that its
consistent throughout using `model_id`.
---
.../modules/deep_learning/madlib_keras.py_in | 11 +++---
.../modules/deep_learning/madlib_keras.sql_in | 42 +++++++++++-----------
.../deep_learning/madlib_keras_helper.py_in | 10 +++---
.../madlib_keras_model_selection.py_in | 24 ++++++-------
.../madlib_keras_model_selection.sql_in | 18 +++++-----
.../deep_learning/madlib_keras_predict.py_in | 10 +++---
.../deep_learning/madlib_keras_validator.py_in | 40 ++++++++++-----------
.../modules/deep_learning/model_arch_info.py_in | 6 ++--
.../deep_learning/test/madlib_keras_fit.sql_in | 4 +--
.../deep_learning/test/madlib_keras_predict.sql_in | 2 +-
.../test/unit_tests/test_madlib_keras.py_in | 14 ++++----
.../test_madlib_keras_model_selection_table.py_in | 10 +++---
12 files changed, 95 insertions(+), 96 deletions(-)
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras.py_in
index 6f6e8fc..702c288 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras.py_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras.py_in
@@ -75,7 +75,7 @@ def get_init_model_and_sess(SD, device_name, gpus_per_host, segments_per_host,
@MinWarning("warning")
def fit(schema_madlib, source_table, model, model_arch_table,
- model_arch_id, compile_params, fit_params, num_iterations,
+ model_id, compile_params, fit_params, num_iterations,
gpus_per_host=0, validation_table=None,
metrics_compute_frequency=None, warm_start=False, name="",
description="", **kwargs):
@@ -92,7 +92,7 @@ def fit(schema_madlib, source_table, model, model_arch_table,
fit_validator = FitInputValidator(
source_table, validation_table, model, model_arch_table,
- model_arch_id, mb_dep_var_col, mb_indep_var_col,
+ model_id, mb_dep_var_col, mb_indep_var_col,
num_iterations, metrics_compute_frequency, warm_start)
if metrics_compute_frequency is None:
metrics_compute_frequency = num_iterations
@@ -112,7 +112,7 @@ def fit(schema_madlib, source_table, model, model_arch_table,
# Get the serialized master model
start_deserialization = time.time()
- model_arch, model_weights = get_model_arch_weights(model_arch_table, model_arch_id)
+ model_arch, model_weights = get_model_arch_weights(model_arch_table, model_id)
num_classes = get_num_classes(model_arch)
input_shape = get_input_shape(model_arch)
fit_validator.validate_input_shapes(input_shape)
@@ -257,7 +257,7 @@ def fit(schema_madlib, source_table, model, model_arch_table,
$MAD${dependent_varname}$MAD$::TEXT AS dependent_varname,
$MAD${independent_varname}$MAD$::TEXT AS independent_varname,
$MAD${model_arch_table}$MAD$::TEXT AS model_arch_table,
- {model_arch_id}::INTEGER AS model_arch_id,
+ {model_id}::INTEGER AS {model_id_colname},
$1 AS compile_params,
$2 AS fit_params,
{num_iterations}::INTEGER AS num_iterations,
@@ -290,6 +290,7 @@ def fit(schema_madlib, source_table, model, model_arch_table,
dependent_vartype_colname=DEPENDENT_VARTYPE_COLNAME,
normalizing_const_colname=NORMALIZING_CONST_COLNAME,
FLOAT32_SQL_TYPE = FLOAT32_SQL_TYPE,
+ model_id_colname = ModelArchSchema.MODEL_ID,
**locals()),
["TEXT", "TEXT", "TEXT", "TEXT", "DOUBLE PRECISION[]", class_values_type])
plpy.execute(create_output_summary_table,
@@ -814,7 +815,7 @@ For more details on function usage:
the model
model_arch_table, -- Name of the table containing the
model architecture
- model_arch_id, -- This is the id in 'model_arch_table'
+ model_id, -- This is the id in 'model_arch_table'
containing the model architecture
compile_params, -- Parameters passed to the compile
method of the Keras model class
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in b/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in
index a81452c..e081760 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in
@@ -93,7 +93,7 @@ madlib_keras_fit(
source_table,
model,
model_arch_table,
- model_arch_id,
+ model_id,
compile_params,
fit_params,
num_iterations,
@@ -127,7 +127,7 @@ madlib_keras_fit(
training.
</dd>
- <dt>model_arch_id</dt>
+ <dt>model_id</dt>
<dd>INTEGER. This is the id in 'model_arch_table'
containing the model architecture and (optionally)
initial weights to use for training.
@@ -292,7 +292,7 @@ A summary table named \<model\>_summary is also created, which has the following
initial model weights.</td>
</tr>
<tr>
- <th>model_arch_table_id</th>
+ <th>model_id</th>
<td>The id of the model in
the model architecture table used for training.</td>
</tr>
@@ -644,7 +644,7 @@ It has the following format:
<pre class="syntax">
madlib_keras_predict_byom(
model_arch_table,
- model_arch_id,
+ model_id,
test_table,
id_col,
independent_varname,
@@ -667,7 +667,7 @@ madlib_keras_predict_byom(
<a href="group__grp__keras__model__arch.html">load_keras_model</a> function.
</DD>
- <DT>model_arch_id</DT>
+ <DT>model_id</DT>
<DD>INTEGER. This is the id in 'model_arch_table' containing the model
architecture and model weights to use for prediction.
</DD>
@@ -1064,7 +1064,7 @@ model | iris_model
dependent_varname | class_text
independent_varname | attributes
model_arch_table | model_arch_library
-model_arch_id | 1
+model_id | 1
compile_params | loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']
fit_params | batch_size=5, epochs=3
num_iterations | 10
@@ -1306,7 +1306,7 @@ model | iris_model
dependent_varname | class_text
independent_varname | attributes
model_arch_table | model_arch_library
-model_arch_id | 1
+model_id | 1
compile_params | loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']
fit_params | batch_size=5, epochs=3
num_iterations | 10
@@ -1412,7 +1412,7 @@ model | iris_model
dependent_varname | class_text
independent_varname | attributes
model_arch_table | model_arch_library
-model_arch_id | 1
+model_id | 1
compile_params | loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']
fit_params | batch_size=5, epochs=3
num_iterations | 5
@@ -1521,7 +1521,7 @@ model | iris_model
dependent_varname | class_text
independent_varname | attributes
model_arch_table | model_arch_library
-model_arch_id | 2
+model_id | 2
compile_params | loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']
fit_params | batch_size=5, epochs=3
num_iterations | 10
@@ -1606,7 +1606,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
source_table VARCHAR,
model VARCHAR,
model_arch_table VARCHAR,
- model_arch_id INTEGER,
+ model_id INTEGER,
compile_params VARCHAR,
fit_params VARCHAR,
num_iterations INTEGER,
@@ -1627,7 +1627,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
source_table VARCHAR,
model VARCHAR,
model_arch_table VARCHAR,
- model_arch_id INTEGER,
+ model_id INTEGER,
compile_params VARCHAR,
fit_params VARCHAR,
num_iterations INTEGER,
@@ -1645,7 +1645,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
source_table VARCHAR,
model VARCHAR,
model_arch_table VARCHAR,
- model_arch_id INTEGER,
+ model_id INTEGER,
compile_params VARCHAR,
fit_params VARCHAR,
num_iterations INTEGER,
@@ -1663,7 +1663,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
source_table VARCHAR,
model VARCHAR,
model_arch_table VARCHAR,
- model_arch_id INTEGER,
+ model_id INTEGER,
compile_params VARCHAR,
fit_params VARCHAR,
num_iterations INTEGER,
@@ -1680,7 +1680,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
source_table VARCHAR,
model VARCHAR,
model_arch_table VARCHAR,
- model_arch_id INTEGER,
+ model_id INTEGER,
compile_params VARCHAR,
fit_params VARCHAR,
num_iterations INTEGER,
@@ -1695,7 +1695,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
source_table VARCHAR,
model VARCHAR,
model_arch_table VARCHAR,
- model_arch_id INTEGER,
+ model_id INTEGER,
compile_params VARCHAR,
fit_params VARCHAR,
num_iterations INTEGER,
@@ -1709,7 +1709,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit(
source_table VARCHAR,
model VARCHAR,
model_arch_table VARCHAR,
- model_arch_id INTEGER,
+ model_id INTEGER,
compile_params VARCHAR,
fit_params VARCHAR,
num_iterations INTEGER
@@ -1874,7 +1874,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-------------------------------------------------------------------------------
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict_byom(
model_arch_table VARCHAR,
- model_arch_id INTEGER,
+ model_id INTEGER,
test_table VARCHAR,
id_col VARCHAR,
independent_varname VARCHAR,
@@ -1892,7 +1892,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict_byom(
model_arch_table VARCHAR,
- model_arch_id INTEGER,
+ model_id INTEGER,
test_table VARCHAR,
id_col VARCHAR,
independent_varname VARCHAR,
@@ -1908,7 +1908,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict_byom(
model_arch_table VARCHAR,
- model_arch_id INTEGER,
+ model_id INTEGER,
test_table VARCHAR,
id_col VARCHAR,
independent_varname VARCHAR,
@@ -1923,7 +1923,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict_byom(
model_arch_table VARCHAR,
- model_arch_id INTEGER,
+ model_id INTEGER,
test_table VARCHAR,
id_col VARCHAR,
independent_varname VARCHAR,
@@ -1936,7 +1936,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict_byom(
model_arch_table VARCHAR,
- model_arch_id INTEGER,
+ model_id INTEGER,
test_table VARCHAR,
id_col VARCHAR,
independent_varname VARCHAR,
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in
index fe00a2e..2621670 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in
@@ -18,6 +18,7 @@
# under the License.
import numpy as np
+from model_arch_info import ModelArchSchema
from utilities.utilities import add_postfix
from utilities.utilities import unique_string
from utilities.utilities import is_platform_pg
@@ -34,7 +35,7 @@ DEPENDENT_VARNAME_COLNAME = "dependent_varname"
DEPENDENT_VARTYPE_COLNAME = "dependent_vartype"
INDEPENDENT_VARNAME_COLNAME = "independent_varname"
MODEL_ARCH_TABLE_COLNAME = "model_arch_table"
-MODEL_ARCH_ID_COLNAME = "model_arch_id"
+MODEL_ID_COLNAME = ModelArchSchema.MODEL_ID
MODEL_WEIGHTS_COLNAME = "model_weights"
METRIC_TYPE_COLNAME = "metrics_type"
@@ -257,13 +258,10 @@ def create_summary_view(module_name, model_table, mst_key):
plpy.error("{0}: mst_key {1} does not exist in the info table".format(
module_name, mst_key))
- # Since fit multiple does not have a model arch id, we set its value to -1.
- # Otherwise, the model arch validation will fail.
- # This aproach is chosen in case we decide to support model arch id in the future.
plpy.execute("""
CREATE VIEW {tmp_view_summary} AS
- SELECT *, -1::SMALLINT AS {model_arch_id_colname}
+ SELECT *
FROM {model_summary_table}, {model_info_table}
WHERE mst_key = {mst_key}
- """.format(model_arch_id_colname=MODEL_ARCH_ID_COLNAME, **locals()))
+ """.format(**locals()))
return tmp_view_summary
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_model_selection.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_model_selection.py_in
index 5d2cbbf..f3b02c7 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_model_selection.py_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras_model_selection.py_in
@@ -42,7 +42,7 @@ class MstLoader():
Attributes:
compile_params_list (list): The input list of compile params choices.
fit_params_list (list): The input list of fit params choices.
- model_arch_id_list (list): The input list of model id choices.
+ model_id_list (list): The input list of model id choices.
model_arch_table (str): The name of model architechure table.
model_selection_table (str): The name of the output mst table.
msts (list): The list of generated msts.
@@ -52,7 +52,7 @@ class MstLoader():
def __init__(self,
model_arch_table,
model_selection_table,
- model_arch_id_list,
+ model_id_list,
compile_params_list,
fit_params_list,
**kwargs):
@@ -61,12 +61,12 @@ class MstLoader():
self.model_selection_table = model_selection_table
self.model_selection_summary_table = add_postfix(
model_selection_table, "_summary")
- self.model_arch_id_list = sorted(list(set(model_arch_id_list)))
+ self.model_id_list = sorted(list(set(model_id_list)))
MstLoaderInputValidator(
model_arch_table=self.model_arch_table,
model_selection_table=self.model_selection_table,
model_selection_summary_table=self.model_selection_summary_table,
- model_arch_id_list=self.model_arch_id_list,
+ model_id_list=self.model_id_list,
compile_params_list=compile_params_list,
fit_params_list=fit_params_list
)
@@ -109,7 +109,7 @@ class MstLoader():
"""Backtracking helper for generating the combinations.
"""
param_grid = OrderedDict([
- (ModelSelectionSchema.MODEL_ID, self.model_arch_id_list),
+ (ModelSelectionSchema.MODEL_ID, self.model_id_list),
(ModelSelectionSchema.COMPILE_PARAMS, self.compile_params_list),
(ModelSelectionSchema.FIT_PARAMS, self.fit_params_list)
])
@@ -130,14 +130,14 @@ class MstLoader():
create_query = """
CREATE TABLE {self.model_selection_table} (
{mst_key} SERIAL,
- {model_arch_id} INTEGER,
+ {model_id} INTEGER,
{compile_params} VARCHAR,
{fit_params} VARCHAR,
- unique ({model_arch_id}, {compile_params}, {fit_params})
+ unique ({model_id}, {compile_params}, {fit_params})
);
""".format(self=self,
mst_key=ModelSelectionSchema.MST_KEY,
- model_arch_id=ModelSelectionSchema.MODEL_ID,
+ model_id=ModelSelectionSchema.MODEL_ID,
compile_params=ModelSelectionSchema.COMPILE_PARAMS,
fit_params=ModelSelectionSchema.FIT_PARAMS)
with MinWarning('warning'):
@@ -159,22 +159,22 @@ class MstLoader():
"""Insert every thing in self.msts into the mst table.
"""
for mst in self.msts:
- model_arch_id = mst[ModelSelectionSchema.MODEL_ID]
+ model_id = mst[ModelSelectionSchema.MODEL_ID]
compile_params = mst[ModelSelectionSchema.COMPILE_PARAMS]
fit_params = mst[ModelSelectionSchema.FIT_PARAMS]
insert_query = """
INSERT INTO
{self.model_selection_table}(
- {model_arch_id_col},
+ {model_id_col},
{compile_params_col},
{fit_params_col}
)
VALUES (
- {model_arch_id},
+ {model_id},
$${compile_params}$$,
$${fit_params}$$
)
- """.format(model_arch_id_col=ModelSelectionSchema.MODEL_ID,
+ """.format(model_id_col=ModelSelectionSchema.MODEL_ID,
compile_params_col=ModelSelectionSchema.COMPILE_PARAMS,
fit_params_col=ModelSelectionSchema.FIT_PARAMS,
**locals())
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_model_selection.sql_in b/src/ports/postgres/modules/deep_learning/madlib_keras_model_selection.sql_in
index f040045..d776efd 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_model_selection.sql_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras_model_selection.sql_in
@@ -56,7 +56,7 @@ to run on a massively parallel processing database cluster.
load_model_selection_table(
model_arch_table,
model_selection_table,
- model_arch_id_list,
+ model_id_list,
compile_params_list,
fit_params_list
)
@@ -76,7 +76,7 @@ load_model_selection_table(
tables are described below.
</dd>
- <dt>model_arch_id_list</dt>
+ <dt>model_id_list</dt>
<dd>INTEGER[]. Array of model IDs from the 'model_arch_table' to be included
in the run combinations. For hyperparameter search, this will typically be
one model ID. For model architecture search, this will be the different model IDs
@@ -108,7 +108,7 @@ load_model_selection_table(
</td>
</tr>
<tr>
- <th>model_arch_id</th>
+ <th>model_id</th>
<td>VARCHAR. Model architecture ID from the 'model_arch_table'.
</td>
</tr>
@@ -259,7 +259,7 @@ SELECT madlib.load_model_selection_table('model_arch_library', -- model architec
SELECT * FROM mst_table ORDER BY mst_key;
</pre>
<pre class="result">
- mst_key | model_arch_id | compile_params | fit_params
+ mst_key | model_id | compile_params | fit_params
---------+---------------+---------------------------------------------------------------------------------+-----------------------
1 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=4,epochs=1
2 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=8,epochs=1
@@ -297,11 +297,11 @@ but only want batch_size=4 for model_id=1 and batch_size=8 for model_id=2:
DROP TABLE IF EXISTS mst_table_manual;
CREATE TABLE mst_table_manual(
mst_key serial,
- model_arch_id integer,
+ model_id integer,
compile_params varchar,
fit_params varchar
);
-INSERT INTO mst_table_manual(model_arch_id, compile_params, fit_params) VALUES
+INSERT INTO mst_table_manual(model_id, compile_params, fit_params) VALUES
(1, $$loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']$$, 'batch_size=4,epochs=1'),
(1, $$loss='categorical_crossentropy',optimizer='Adam(lr=0.01)',metrics=['accuracy']$$, 'batch_size=4,epochs=1'),
(1, $$loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy']$$, 'batch_size=4,epochs=1'),
@@ -311,7 +311,7 @@ INSERT INTO mst_table_manual(model_arch_id, compile_params, fit_params) VALUES
SELECT * FROM mst_table_manual ORDER BY mst_key;
</pre>
<pre class="result">
- mst_key | model_arch_id | compile_params | fit_params
+ mst_key | model_id | compile_params | fit_params
---------+---------------+---------------------------------------------------------------------------------+-----------------------
1 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=4,epochs=1
2 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1
@@ -377,7 +377,7 @@ conn.commit()
%sql SELECT * FROM mst_table_auto ORDER BY mst_key;
</pre>
<pre class="result">
- mst_key | model_arch_id | compile_params | fit_params
+ mst_key | model_id | compile_params | fit_params
---------+---------------+----------------------------------------------------------------------------------+-----------------------
1 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.0001)',metrics=['accuracy'] | batch_size=4,epochs=1
2 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.0001)',metrics=['accuracy'] | batch_size=8,epochs=1
@@ -414,7 +414,7 @@ See keras_model_arch_table.sql_in
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.load_model_selection_table(
model_arch_table VARCHAR,
model_selection_table VARCHAR,
- model_arch_id_list INTEGER[],
+ model_id_list INTEGER[],
compile_params_list VARCHAR[],
fit_params_list VARCHAR[]
) RETURNS VOID AS $$
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_predict.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_predict.py_in
index 3d8564b..8c756ee 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_predict.py_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras_predict.py_in
@@ -192,14 +192,14 @@ class Predict(BasePredict):
@MinWarning("warning")
class PredictBYOM(BasePredict):
- def __init__(self, schema_madlib, model_arch_table, model_arch_id,
+ def __init__(self, schema_madlib, model_arch_table, model_id,
test_table, id_col, independent_varname, output_table,
pred_type, gpus_per_host, class_values, normalizing_const,
**kwargs):
self.module_name='madlib_keras_predict_byom'
self.model_arch_table = model_arch_table
- self.model_arch_id = model_arch_id
+ self.model_id = model_id
self.class_values = class_values
self.normalizing_const = normalizing_const
self.dependent_varname = 'dependent_var'
@@ -216,7 +216,7 @@ class PredictBYOM(BasePredict):
if self.normalizing_const is None:
self.normalizing_const = DEFAULT_NORMALIZING_CONST
InputValidator.validate_predict_byom_tables(
- self.module_name, self.model_arch_table, self.model_arch_id,
+ self.module_name, self.model_arch_table, self.model_id,
self.test_table, self.id_col, self.output_table,
self.independent_varname)
self.validate_and_set_defaults()
@@ -225,7 +225,7 @@ class PredictBYOM(BasePredict):
def validate_and_set_defaults(self):
# Set some defaults first and then validate and then set some more defaults
self.model_arch, self.model_weights = get_model_arch_weights(
- self.model_arch_table, self.model_arch_id)
+ self.model_arch_table, self.model_id)
# Assert model_weights and model_arch are not empty.
_assert(self.model_weights and self.model_arch,
"{0}: Model weights and architecture should not be NULL.".format(
@@ -393,7 +393,7 @@ For more details on function usage:
SELECT {schema_madlib}.madlib_keras_predict_byom(
model_arch_table, -- Name of the table containing the model architecture
and the pre trained model weights
- model_arch_id, -- This is the id in 'model_arch_table' containing the
+ model_id, -- This is the id in 'model_arch_table' containing the
model architecture
test_table, -- Name of the table containing the evaluation dataset
id_col, -- Name of the id column in the test data table
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_validator.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_validator.py_in
index 015ecfa..e24b8bd 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_validator.py_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras_validator.py_in
@@ -24,7 +24,7 @@ from madlib_keras_helper import CLASS_VALUES_COLNAME
from madlib_keras_helper import COMPILE_PARAMS_COLNAME
from madlib_keras_helper import DEPENDENT_VARNAME_COLNAME
from madlib_keras_helper import DEPENDENT_VARTYPE_COLNAME
-from madlib_keras_helper import MODEL_ARCH_ID_COLNAME
+from madlib_keras_helper import MODEL_ID_COLNAME
from madlib_keras_helper import MODEL_ARCH_TABLE_COLNAME
from madlib_keras_helper import MODEL_WEIGHTS_COLNAME
from madlib_keras_helper import NORMALIZING_CONST_COLNAME
@@ -67,11 +67,11 @@ class InputValidator:
table=test_table))
@staticmethod
- def validate_predict_byom_tables(module_name, model_arch_table, model_arch_id,
+ def validate_predict_byom_tables(module_name, model_arch_table, model_id,
test_table, id_col, output_table,
independent_varname):
InputValidator.validate_model_arch_table(
- module_name, model_arch_table, model_arch_id)
+ module_name, model_arch_table, model_id)
InputValidator._validate_test_tbl(
module_name, test_table, independent_varname)
InputValidator.validate_id_in_test_tbl(module_name, test_table, id_col)
@@ -151,9 +151,9 @@ class InputValidator:
independent_varname, table))
@staticmethod
- def validate_model_arch_table(module_name, model_arch_table, model_arch_id):
+ def validate_model_arch_table(module_name, model_arch_table, model_id):
input_tbl_valid(model_arch_table, module_name)
- _assert(model_arch_id is not None,
+ _assert(model_id is not None,
"{0}: Invalid model architecture ID.".format(module_name))
@@ -214,7 +214,7 @@ class InputValidator:
cols_to_check_for = [CLASS_VALUES_COLNAME,
DEPENDENT_VARNAME_COLNAME,
DEPENDENT_VARTYPE_COLNAME,
- MODEL_ARCH_ID_COLNAME,
+ MODEL_ID_COLNAME,
MODEL_ARCH_TABLE_COLNAME,
NORMALIZING_CONST_COLNAME,
COMPILE_PARAMS_COLNAME,
@@ -228,14 +228,14 @@ class InputValidator:
class FitCommonValidator(object):
def __init__(self, source_table, validation_table, output_model_table,
- model_arch_table, model_arch_id, dependent_varname,
+ model_arch_table, model_id, dependent_varname,
independent_varname, num_iterations,
metrics_compute_frequency, warm_start, module_name):
self.source_table = source_table
self.validation_table = validation_table
self.output_model_table = output_model_table
self.model_arch_table = model_arch_table
- self.model_arch_id = model_arch_id
+ self.model_id = model_id
self.dependent_varname = dependent_varname
self.independent_varname = independent_varname
self.dep_shape_col = add_postfix(dependent_varname, "_shape")
@@ -358,7 +358,7 @@ class FitCommonValidator(object):
class FitInputValidator(FitCommonValidator):
def __init__(self, source_table, validation_table, output_model_table,
- model_arch_table, model_arch_id, dependent_varname,
+ model_arch_table, model_id, dependent_varname,
independent_varname, num_iterations,
metrics_compute_frequency, warm_start):
@@ -367,7 +367,7 @@ class FitInputValidator(FitCommonValidator):
validation_table,
output_model_table,
model_arch_table,
- model_arch_id,
+ model_id,
dependent_varname,
independent_varname,
num_iterations,
@@ -375,7 +375,7 @@ class FitInputValidator(FitCommonValidator):
warm_start,
self.module_name)
InputValidator.validate_model_arch_table(self.module_name, self.model_arch_table,
- self.model_arch_id)
+ self.model_id)
class FitMultipleInputValidator(FitCommonValidator):
def __init__(self, source_table, validation_table, output_model_table,
@@ -416,14 +416,14 @@ class MstLoaderInputValidator():
model_arch_table,
model_selection_table,
model_selection_summary_table,
- model_arch_id_list,
+ model_id_list,
compile_params_list,
fit_params_list
):
self.model_arch_table = model_arch_table
self.model_selection_table = model_selection_table
self.model_selection_summary_table = model_selection_summary_table
- self.model_arch_id_list = model_arch_id_list
+ self.model_id_list = model_id_list
self.compile_params_list = compile_params_list
self.fit_params_list = fit_params_list
self.module_name = 'load_model_selection_table'
@@ -431,23 +431,23 @@ class MstLoaderInputValidator():
def _validate_input_args(self):
self._validate_input_output_tables()
- self._validate_model_arch_ids()
+ self._validate_model_ids()
self._validate_compile_and_fit_params()
- def _validate_model_arch_ids(self):
- model_arch_id_str = '({0})'\
- .format(','.join([str(x) for x in self.model_arch_id_list]))
+ def _validate_model_ids(self):
+ model_id_str = '({0})'\
+ .format(','.join([str(x) for x in self.model_id_list]))
query = """
SELECT count(model_id)
FROM {self.model_arch_table}
- WHERE model_id IN {model_arch_id_str}
+ WHERE model_id IN {model_id_str}
""".format(**locals())
res = int(plpy.execute(query)[0]['count'])
_assert(
- res == len(self.model_arch_id_list),
+ res == len(self.model_id_list),
"{0}: One or more model_id of {1} not found in table {2}".format(
self.module_name,
- model_arch_id_str,
+ model_id_str,
self.model_arch_table
)
)
diff --git a/src/ports/postgres/modules/deep_learning/model_arch_info.py_in b/src/ports/postgres/modules/deep_learning/model_arch_info.py_in
index c749144..3a225cc 100644
--- a/src/ports/postgres/modules/deep_learning/model_arch_info.py_in
+++ b/src/ports/postgres/modules/deep_learning/model_arch_info.py_in
@@ -84,17 +84,17 @@ def get_model_arch_layers_str(model_arch):
layers += "{1}\n".format(class_name)
return layers
-def get_model_arch_weights(model_arch_table, model_arch_id):
+def get_model_arch_weights(model_arch_table, model_id):
#assume validation is already called
model_arch_query = "SELECT {0}, {1} FROM {2} WHERE {3} = {4}".format(
ModelArchSchema.MODEL_ARCH, ModelArchSchema.MODEL_WEIGHTS,
model_arch_table, ModelArchSchema.MODEL_ID,
- model_arch_id)
+ model_id)
model_arch_result = plpy.execute(model_arch_query)
if not model_arch_result:
plpy.error("no model arch found in table {0} with id {1}".format(
- model_arch_table, model_arch_id))
+ model_arch_table, model_id))
model_arch_result = model_arch_result[0]
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_fit.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_fit.sql_in
index 933e5d0..591beec 100644
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_fit.sql_in
+++ b/src/ports/postgres/modules/deep_learning/test/madlib_keras_fit.sql_in
@@ -42,7 +42,7 @@ SELECT madlib_keras_fit(
SELECT assert(
model_arch_table = 'model_arch' AND
- model_arch_id = 1 AND
+ model_id = 1 AND
model_type = 'madlib_keras' AND
start_training_time < now() AND
end_training_time > start_training_time AND
@@ -151,7 +151,7 @@ SELECT assert(
dependent_varname = 'y' AND
independent_varname = 'x' AND
model_arch_table = 'model_arch' AND
- model_arch_id = 1 AND
+ model_id = 1 AND
compile_params = $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text AND
fit_params = $$ batch_size=2, epochs=1, verbose=0 $$::text AND
num_iterations = 2 AND
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_predict.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_predict.sql_in
index 6d74956..ec5a1f2 100644
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_predict.sql_in
+++ b/src/ports/postgres/modules/deep_learning/test/madlib_keras_predict.sql_in
@@ -318,7 +318,7 @@ FROM cifar10_predict;
-- Predict with correctly shaped data, must go thru.
-- Update output_summary table to reflect
--- class_values, num_classes and model_arch_id for shaped data
+-- class_values, num_classes and model_id for shaped data
DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary;
SELECT madlib_keras_fit(
'cifar_10_sample_test_shape_batched',
diff --git a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in
index 3fe852a..901ebbd 100644
--- a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in
+++ b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in
@@ -644,7 +644,7 @@ class MadlibKerasPredictBYOMTestCase(unittest.TestCase):
def test_predictbyom_defaults_1(self):
res = self.module.PredictBYOM('schema_madlib', 'model_arch_table',
- 'model_arch_id', 'test_table', 'id_col',
+ 'model_id', 'test_table', 'id_col',
'independent_varname', 'output_table', None,
None, None, None)
self.assertEqual('response', res.pred_type)
@@ -655,7 +655,7 @@ class MadlibKerasPredictBYOMTestCase(unittest.TestCase):
def test_predictbyom_defaults_2(self):
res = self.module.PredictBYOM('schema_madlib', 'model_arch_table',
- 'model_arch_id', 'test_table', 'id_col',
+ 'model_id', 'test_table', 'id_col',
'independent_varname', 'output_table',
self.pred_type, self.gpus_per_host,
self.class_values, self.normalizing_const)
@@ -668,7 +668,7 @@ class MadlibKerasPredictBYOMTestCase(unittest.TestCase):
def test_predictbyom_exception_invalid_params(self):
with self.assertRaises(plpy.PLPYException) as error:
self.module.PredictBYOM('schema_madlib', 'model_arch_table',
- 'model_arch_id', 'test_table', 'id_col',
+ 'model_id', 'test_table', 'id_col',
'independent_varname', 'output_table',
'invalid_pred_type', self.gpus_per_host,
self.class_values, self.normalizing_const)
@@ -676,7 +676,7 @@ class MadlibKerasPredictBYOMTestCase(unittest.TestCase):
with self.assertRaises(plpy.PLPYException) as error:
self.module.PredictBYOM('schema_madlib', 'model_arch_table',
- 'model_arch_id', 'test_table', 'id_col',
+ 'model_id', 'test_table', 'id_col',
'independent_varname', 'output_table',
self.pred_type, self.gpus_per_host,
["foo", "bar", "baaz"], self.normalizing_const)
@@ -684,7 +684,7 @@ class MadlibKerasPredictBYOMTestCase(unittest.TestCase):
with self.assertRaises(plpy.PLPYException) as error:
self.module.PredictBYOM('schema_madlib', 'model_arch_table',
- 'model_arch_id', 'test_table', 'id_col',
+ 'model_id', 'test_table', 'id_col',
'independent_varname', 'output_table',
self.pred_type, self.gpus_per_host,
self.class_values, 0)
@@ -1085,7 +1085,7 @@ class InputValidatorTestCase(unittest.TestCase):
self.id_col = 'id_col'
self.ind_var = 'ind_var'
self.model_arch_table = 'model_arch_table'
- self.model_arch_id = 2
+ self.model_id = 2
self.num_classes = 1598
self.model = Sequential()
self.model.add(Conv2D(2, kernel_size=(1, 1), activation='relu',
@@ -1150,7 +1150,7 @@ class InputValidatorTestCase(unittest.TestCase):
def test_validate_model_arch_table_none_values(self):
with self.assertRaises(plpy.PLPYException) as error:
obj = self.subject.validate_model_arch_table(
- self.module_name, None, self.model_arch_id)
+ self.module_name, None, self.model_id)
self.assertIn('null', str(error.exception).lower())
self.module.input_tbl_valid = Mock()
diff --git a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras_model_selection_table.py_in b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras_model_selection_table.py_in
index 1579239..57e08a5 100644
--- a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras_model_selection_table.py_in
+++ b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras_model_selection_table.py_in
@@ -53,7 +53,7 @@ class LoadModelSelectionTableTestCase(unittest.TestCase):
self.subject = self.module.MstLoader
self.model_selection_table = 'mst_table'
self.model_arch_table = 'model_arch_library'
- self.model_arch_id_list = [1]
+ self.model_id_list = [1]
self.compile_params_list = [
"""
loss='categorical_crossentropy',
@@ -80,7 +80,7 @@ class LoadModelSelectionTableTestCase(unittest.TestCase):
generate_mst = self.subject(
self.model_selection_table,
self.model_arch_table,
- self.model_arch_id_list,
+ self.model_id_list,
self.compile_params_list,
self.fit_params_list
)
@@ -94,13 +94,13 @@ class LoadModelSelectionTableTestCase(unittest.TestCase):
generate_mst = self.subject(
self.model_selection_table,
self.model_arch_table,
- self.model_arch_id_list,
+ self.model_id_list,
self.compile_params_list,
self.fit_params_list
)
def test_duplicate_params(self):
- self.model_arch_id_list = [1, 1, 2]
+ self.model_id_list = [1, 1, 2]
self.compile_params_list = [
"""
loss='categorical_crossentropy',
@@ -126,7 +126,7 @@ class LoadModelSelectionTableTestCase(unittest.TestCase):
generate_mst = self.subject(
self.model_selection_table,
self.model_arch_table,
- self.model_arch_id_list,
+ self.model_id_list,
self.compile_params_list,
self.fit_params_list
)