You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@madlib.apache.org by fm...@apache.org on 2019/07/08 20:05:54 UTC
[madlib-site] branch asf-site updated: upload new 1.16 notebooks
and organize into directories
This is an automated email from the ASF dual-hosted git repository.
fmcquillan pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/madlib-site.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 4d44c6e upload new 1.16 notebooks and organize into directories
4d44c6e is described below
commit 4d44c6e5c9fd6479943fd740eba0cfc2cfb3156a
Author: Frank McQuillan <fm...@pivotal.io>
AuthorDate: Mon Jul 8 13:05:39 2019 -0700
upload new 1.16 notebooks and organize into directories
---
.../Encoding-categorical-variables-v2.ipynb | 0
.../Path-demo-4.ipynb | 0
.../Pivot-demo-v3.ipynb | 0
.../Sessionize-demo-2.ipynb | 0
.../Deep-learning/Load-model-architecture-v1.ipynb | 596 ++++
.../Deep-learning/MADlib-Keras-MLP-v1.ipynb | 3206 ++++++++++++++++++++
.../MADlib-Keras-cifar10-cnn-v1.ipynb | 1249 ++++++++
.../MADlib-Keras-transfer-learning-v1.ipynb | 1620 ++++++++++
.../Madlib Image Loader Demo.ipynb | 0
.../Deep-learning/Preprocessor-for-images-v1.ipynb | 1625 ++++++++++
.../{ => Deep-learning}/madlib_image_loader.py | 0
community-artifacts/{ => Graph}/APSP-v1.ipynb | 0
.../{ => Graph}/Breadth-first-search-v1.ipynb | 0
.../{ => Graph}/Graph-measures-v1.ipynb | 0
community-artifacts/{ => Graph}/HITS-v1.ipynb | 0
community-artifacts/{ => Graph}/PageRank-v2.ipynb | 0
community-artifacts/{ => Graph}/SSSP-v2.ipynb | 0
.../{ => Graph}/Weakly-connected-cpts-v2.ipynb | 0
.../Model-selection/Cross_validation_v1.ipynb | 1130 +++++++
.../Prediction-metrics-demo-1.ipynb | 0
.../Train-test-split-v1.ipynb | 0
.../{ => Sampling}/Balanced-sampling-v1.ipynb | 0
.../{ => Sampling}/Stratified-sampling-v2.ipynb | 0
.../Covariance-and-correlation-v1.ipynb | 0
.../{ => Statistics}/Summary-v2.ipynb | 0
.../Decision-trees-v2.ipynb | 0
.../{ => Supervised-learning}/Elastic-net-v3.ipynb | 0
.../KNN-v5.ipynb} | 256 +-
.../Linear-regression-v1.ipynb | 0
.../Logistic-regression-v1.ipynb | 0
.../{ => Supervised-learning}/MLP-mnist-v3.ipynb | 0
.../{ => Supervised-learning}/MLP-v4.ipynb | 0
.../Random-forest-v2.ipynb | 0
.../SVM-novelty-detection-v2.ipynb | 0
.../{ => Supervised-learning}/SVM-v1.ipynb | 0
.../{ => Unsupervised-learning}/Kmeans-v2.ipynb | 0
.../{ => Unsupervised-learning}/LDA-v1.ipynb | 0
.../PCA-project-v1.ipynb | 0
.../{ => Unsupervised-learning}/PCA-train-v1.ipynb | 0
.../Column-vector-operations-v1.ipynb | 0
.../Minibatch-preprocessor-v1.ipynb | 0
.../{ => Utilities}/Term-frequency-v1.ipynb | 0
42 files changed, 9650 insertions(+), 32 deletions(-)
diff --git a/community-artifacts/Encoding-categorical-variables-v2.ipynb b/community-artifacts/Data-types-and-transformations/Encoding-categorical-variables-v2.ipynb
similarity index 100%
rename from community-artifacts/Encoding-categorical-variables-v2.ipynb
rename to community-artifacts/Data-types-and-transformations/Encoding-categorical-variables-v2.ipynb
diff --git a/community-artifacts/Path-demo-4.ipynb b/community-artifacts/Data-types-and-transformations/Path-demo-4.ipynb
similarity index 100%
rename from community-artifacts/Path-demo-4.ipynb
rename to community-artifacts/Data-types-and-transformations/Path-demo-4.ipynb
diff --git a/community-artifacts/Pivot-demo-v3.ipynb b/community-artifacts/Data-types-and-transformations/Pivot-demo-v3.ipynb
similarity index 100%
rename from community-artifacts/Pivot-demo-v3.ipynb
rename to community-artifacts/Data-types-and-transformations/Pivot-demo-v3.ipynb
diff --git a/community-artifacts/Sessionize-demo-2.ipynb b/community-artifacts/Data-types-and-transformations/Sessionize-demo-2.ipynb
similarity index 100%
rename from community-artifacts/Sessionize-demo-2.ipynb
rename to community-artifacts/Data-types-and-transformations/Sessionize-demo-2.ipynb
diff --git a/community-artifacts/Deep-learning/Load-model-architecture-v1.ipynb b/community-artifacts/Deep-learning/Load-model-architecture-v1.ipynb
new file mode 100644
index 0000000..e7cd2f1
--- /dev/null
+++ b/community-artifacts/Deep-learning/Load-model-architecture-v1.ipynb
@@ -0,0 +1,596 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Load model architecture\n",
+ "This utility function loads model architectures and weights into a table for use by deep learning algorithms in Keras. \n",
+ "\n",
+ "The model architecture loader was added in MADlib 1.16.\n",
+ "\n",
+ "## Table of contents\n",
+ "\n",
+ "<a href=\"#define_model_arch\">1. Define model architecture</a>\n",
+ "\n",
+ "<a href=\"#load_model_arch\">2. Load model architecture</a>\n",
+ "\n",
+ "<a href=\"#load_model_wts\">3. Load model weights</a>\n",
+ "* <a href=\"#load_model_wts_madlib\">3a. Load weights from previous MADlib run</a>\n",
+ "* <a href=\"#load_model_wts_keras1\">3b. Load weights from Keras using a PL/Python function</a>\n",
+ "* <a href=\"#load_model_wts_keras2\">3c. Load weights from Keras using psycopg2</a>\n",
+ "\n",
+ "<a href=\"#delete_model\">4. Delete model</a>"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/config.py:13: ShimWarning: The `IPython.config` package has been deprecated since IPython 4.0. You should import from traitlets.config instead.\n",
+ " \"You should import from traitlets.config instead.\", ShimWarning)\n",
+ "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/utils/traitlets.py:5: UserWarning: IPython.utils.traitlets has moved to a top-level traitlets package.\n",
+ " warn(\"IPython.utils.traitlets has moved to a top-level traitlets package.\")\n"
+ ]
+ }
+ ],
+ "source": [
+ "%load_ext sql"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "u'Connected: gpadmin@madlib'"
+ ]
+ },
+ "execution_count": 2,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Greenplum Database 5.x on GCP (PM demo machine)\n",
+ "#%sql postgresql://gpadmin@35.184.232.200:5432/madlib\n",
+ " \n",
+ "# Greenplum Database 5.x on GCP for deep learning (PM demo machine)\n",
+ "%sql postgresql://gpadmin@35.239.240.26:5432/madlib\n",
+ " \n",
+ "# PostgreSQL local\n",
+ "#%sql postgresql://fmcquillan@localhost:5432/madlib"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>version</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>MADlib version: 1.16-dev, git revision: rel/v1.15.1-119-gea1e0ac, cmake configuration time: Sat Jun 8 00:55:28 UTC 2019, build type: release, build system: Linux-3.10.0-957.12.1.el7.x86_64, C compiler: gcc 4.8.5, C++ compiler: g++ 4.8.5</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'MADlib version: 1.16-dev, git revision: rel/v1.15.1-119-gea1e0ac, cmake configuration time: Sat Jun 8 00:55:28 UTC 2019, build type: release, build system: Linux-3.10.0-957.12.1.el7.x86_64, C compiler: gcc 4.8.5, C++ compiler: g++ 4.8.5',)]"
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%sql select madlib.version();\n",
+ "#%sql select version();"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"define_model_arch\"></a>\n",
+ "# 1. Define model architecture\n",
+ "\n",
+ "Import Keras libraries"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 28,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import keras\n",
+ "from keras.models import Sequential\n",
+ "from keras.layers import Dense"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Define model architecture"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 30,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "_________________________________________________________________\n",
+ "Layer (type) Output Shape Param # \n",
+ "=================================================================\n",
+ "dense_13 (Dense) (None, 10) 50 \n",
+ "_________________________________________________________________\n",
+ "dense_14 (Dense) (None, 10) 110 \n",
+ "_________________________________________________________________\n",
+ "dense_15 (Dense) (None, 3) 33 \n",
+ "=================================================================\n",
+ "Total params: 193\n",
+ "Trainable params: 193\n",
+ "Non-trainable params: 0\n",
+ "_________________________________________________________________\n"
+ ]
+ }
+ ],
+ "source": [
+ "model = Sequential()\n",
+ "model.add(Dense(10, activation='relu', input_shape=(4,)))\n",
+ "model.add(Dense(10, activation='relu'))\n",
+ "model.add(Dense(3, activation='softmax'))\n",
+ " \n",
+ "model.summary()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 31,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'{\"class_name\": \"Sequential\", \"keras_version\": \"2.1.6\", \"config\": [{\"class_name\": \"Dense\", \"config\": {\"kernel_initializer\": {\"class_name\": \"VarianceScaling\", \"config\": {\"distribution\": \"uniform\", \"scale\": 1.0, \"seed\": null, \"mode\": \"fan_avg\"}}, \"name\": \"dense_13\", \"kernel_constraint\": null, \"bias_regularizer\": null, \"bias_constraint\": null, \"dtype\": \"float32\", \"activation\": \"relu\", \"trainable\": true, \"kernel_regularizer\": [...]
+ ]
+ },
+ "execution_count": 31,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "model.to_json()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "'{\"class_name\": \"Sequential\", \"keras_version\": \"2.1.6\", \"config\": [{\"class_name\": \"Dense\", \"config\": {\"kernel_initializer\": {\"class_name\": \"VarianceScaling\", \"config\": {\"distribution\": \"uniform\", \"scale\": 1.0, \"seed\": null, \"mode\": \"fan_avg\"}}, \"name\": \"dense_1\", \"kernel_constraint\": null, \"bias_regularizer\": null, \"bias_constraint\": null, \"dtype\": \"float32\", \"activation\": \"linear\", \"trainable\": true, \"kernel_regularizer\": nu [...]
+ " "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"load_model_arch\"></a>\n",
+ "# 2. Load model architecture\n",
+ "\n",
+ "Load into model architecture table:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 46,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>model_id</th>\n",
+ " <th>model_arch</th>\n",
+ " <th>model_weights</th>\n",
+ " <th>name</th>\n",
+ " <th>description</th>\n",
+ " <th>__internal_madlib_id__</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>{u'class_name': u'Sequential', u'keras_version': u'2.1.6', u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer': {u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform', u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1', u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint': None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True, u'kernel_regularizer': None, u'bias_initia [...]
+ " <td>None</td>\n",
+ " <td>Sophie</td>\n",
+ " <td>A simple model</td>\n",
+ " <td>__madlib_temp_22335672_1560210445_1050509__</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, {u'class_name': u'Sequential', u'keras_version': u'2.1.6', u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer': {u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform', u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1', u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint': None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True, u'kernel_regularizer': None, u'bias_initializer': [...]
+ ]
+ },
+ "execution_count": 46,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS model_arch_library;\n",
+ "SELECT madlib.load_keras_model('model_arch_library', -- Output table,\n",
+ " \n",
+ "$$\n",
+ "{\"class_name\": \"Sequential\", \"keras_version\": \"2.1.6\", \"config\": [{\"class_name\": \"Dense\", \"config\": {\"kernel_initializer\": {\"class_name\": \"VarianceScaling\", \"config\": {\"distribution\": \"uniform\", \"scale\": 1.0, \"seed\": null, \"mode\": \"fan_avg\"}}, \"name\": \"dense_1\", \"kernel_constraint\": null, \"bias_regularizer\": null, \"bias_constraint\": null, \"dtype\": \"float32\", \"activation\": \"relu\", \"trainable\": true, \"kernel_regularizer\": null, [...]
+ "$$\n",
+ "::json, -- JSON blob\n",
+ " NULL, -- Weights\n",
+ " 'Sophie', -- Name\n",
+ " 'A simple model' -- Descr\n",
+ ");\n",
+ "\n",
+ "SELECT * FROM model_arch_library;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Load another model architecture:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 45,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n",
+ "2 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>model_id</th>\n",
+ " <th>model_arch</th>\n",
+ " <th>model_weights</th>\n",
+ " <th>name</th>\n",
+ " <th>description</th>\n",
+ " <th>__internal_madlib_id__</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>{u'class_name': u'Sequential', u'keras_version': u'2.1.6', u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer': {u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform', u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1', u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint': None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True, u'kernel_regularizer': None, u'bias_initia [...]
+ " <td>None</td>\n",
+ " <td>Sophie</td>\n",
+ " <td>A simple model</td>\n",
+ " <td>__madlib_temp_81479542_1560210311_41108954__</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>{u'class_name': u'Sequential', u'keras_version': u'2.1.6', u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer': {u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform', u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1', u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint': None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True, u'kernel_regularizer': None, u'bias_initia [...]
+ " <td>None</td>\n",
+ " <td>Maria</td>\n",
+ " <td>Also a simple model</td>\n",
+ " <td>__madlib_temp_7879812_1560210313_52185533__</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, {u'class_name': u'Sequential', u'keras_version': u'2.1.6', u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer': {u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform', u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1', u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint': None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True, u'kernel_regularizer': None, u'bias_initializer': [...]
+ " (2, {u'class_name': u'Sequential', u'keras_version': u'2.1.6', u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer': {u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform', u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1', u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint': None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True, u'kernel_regularizer': None, u'bias_initializer': [...]
+ ]
+ },
+ "execution_count": 45,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT madlib.load_keras_model('model_arch_library', -- Output table,\n",
+ " \n",
+ "$$\n",
+ "{\"class_name\": \"Sequential\", \"keras_version\": \"2.1.6\", \"config\": [{\"class_name\": \"Dense\", \"config\": {\"kernel_initializer\": {\"class_name\": \"VarianceScaling\", \"config\": {\"distribution\": \"uniform\", \"scale\": 1.0, \"seed\": null, \"mode\": \"fan_avg\"}}, \"name\": \"dense_1\", \"kernel_constraint\": null, \"bias_regularizer\": null, \"bias_constraint\": null, \"dtype\": \"float32\", \"activation\": \"relu\", \"trainable\": true, \"kernel_regularizer\": null, [...]
+ "$$\n",
+ "::json, -- JSON blob\n",
+ " NULL, -- Weights\n",
+ " 'Maria', -- Name\n",
+ " 'Also a simple model' -- Descr\n",
+ ");\n",
+ "\n",
+ "SELECT * FROM model_arch_library;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"load_model_wts\"></a>\n",
+ "# 3. Load model weights\n",
+ "\n",
+ "<a id=\"load_model_wts_madlib\"></a>\n",
+ "## 3a. Load weights from previous MADlib run\n",
+ "\n",
+ "Use UPDATE to load directly into the table. For example, if 'model_data' are the weights in the output table 'iris_model' from a previous run of 'madlib_keras_fit()' :"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%sql\n",
+ "UPDATE model_arch_library SET model_weights = model_data FROM iris_model WHERE model_id = 2;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"load_model_wts_keras1\"></a>\n",
+ "## 3b. Load weights from Keras using a PL/Python function \n",
+ "We need to flatten then serialize the weights to store as a PostgreSQL binary data type. Byte format is more efficient on space and memory compared to a numeric array. The model weights will be de-serialized when passed to Keras functions."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 47,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>count</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1L,)]"
+ ]
+ },
+ "execution_count": 47,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "CREATE OR REPLACE FUNCTION load_weights() RETURNS VOID AS\n",
+ "$$\n",
+ "from keras.layers import *\n",
+ "from keras import Sequential\n",
+ "import numpy as np\n",
+ "import plpy\n",
+ "\n",
+ "# create model\n",
+ "model = Sequential()\n",
+ "model.add(Dense(10, activation='relu', input_shape=(4,)))\n",
+ "model.add(Dense(10, activation='relu'))\n",
+ "model.add(Dense(3, activation='softmax'))\n",
+ "\n",
+ "# get weights, flatten and serialize\n",
+ "weights = model.get_weights()\n",
+ "weights_flat = [ w.flatten() for w in weights ]\n",
+ "weights1d = np.array([j for sub in weights_flat for j in sub])\n",
+ "weights_bytea = weights1d.tostring()\n",
+ "\n",
+ "# load query\n",
+ "load_query = plpy.prepare(\"\"\"SELECT madlib.load_keras_model(\n",
+ " 'model_arch_library',\n",
+ " $1, $2)\n",
+ " \"\"\", ['json','bytea'])\n",
+ "plpy.execute(load_query, [model.to_json(), weights_bytea])\n",
+ "$$ language plpythonu;\n",
+ "\n",
+ "-- Call load function\n",
+ "SELECT load_weights();\n",
+ "\n",
+ "-- Check weights loaded OK\n",
+ "SELECT COUNT(*) FROM model_arch_library WHERE model_weights IS NOT NULL;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"load_model_wts_keras2\"></a>\n",
+ "## 3c. Load weights from Keras using psycopg2\n",
+ "Psycopg is a PostgreSQL database adapter for the Python programming language. As above we need to flatten then serialize the weights to store as a PostgreSQL binary data type."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 48,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>count</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(2L,)]"
+ ]
+ },
+ "execution_count": 48,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "import psycopg2 as p2\n",
+ "conn = p2.connect('postgresql://gpadmin@35.239.240.26:5432/madlib')\n",
+ "cur = conn.cursor()\n",
+ "\n",
+ "from keras.layers import *\n",
+ "from keras import Sequential\n",
+ "import numpy as np\n",
+ "\n",
+ "# create model\n",
+ "model = Sequential()\n",
+ "model.add(Dense(10, activation='relu', input_shape=(4,)))\n",
+ "model.add(Dense(10, activation='relu'))\n",
+ "model.add(Dense(3, activation='softmax'))\n",
+ "\n",
+ "# get weights, flatten and serialize\n",
+ "weights = model.get_weights()\n",
+ "weights_flat = [ w.flatten() for w in weights ]\n",
+ "weights1d = np.array([j for sub in weights_flat for j in sub])\n",
+ "weights_bytea = psycopg2.Binary(weights1d.tostring())\n",
+ "\n",
+ "query = \"SELECT madlib.load_keras_model('model_arch_library', %s,%s)\"\n",
+ "cur.execute(query,[model.to_json(),weights_bytea])\n",
+ "conn.commit()\n",
+ "\n",
+ "# check weights loaded OK\n",
+ "%sql SELECT COUNT(*) FROM model_arch_library WHERE model_weights IS NOT NULL;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"delete_model\"></a>\n",
+ "# 4. Delete model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 43,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>count</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(3L,)]"
+ ]
+ },
+ "execution_count": 43,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT madlib.delete_keras_model('model_arch_library', -- Output table\n",
+ " 1 -- Model id\n",
+ " );\n",
+ "\n",
+ "SELECT COUNT(*) FROM model_arch_library;"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 2",
+ "language": "python",
+ "name": "python2"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.10"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
diff --git a/community-artifacts/Deep-learning/MADlib-Keras-MLP-v1.ipynb b/community-artifacts/Deep-learning/MADlib-Keras-MLP-v1.ipynb
new file mode 100644
index 0000000..aa61065
--- /dev/null
+++ b/community-artifacts/Deep-learning/MADlib-Keras-MLP-v1.ipynb
@@ -0,0 +1,3206 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Multilayer Perceptron Using Keras and MADlib\n",
+ "\n",
+ "E2E classification example using MADlib calling a Keras MLP.\n",
+ "\n",
+ "Deep learning works best on very large datasets, but that is not convenient for a quick introduction to the syntax. So in this workbook we use the well known iris data set from https://archive.ics.uci.edu/ml/datasets/iris to help get you started. It is similar to the example in user docs http://madlib.apache.org/docs/latest/index.html\n",
+ "\n",
+ "For more realistic examples with images please refer to the deep learning notebooks at\n",
+ "https://github.com/apache/madlib-site/tree/asf-site/community-artifacts\n",
+ "\n",
+ "## Table of contents\n",
+ "\n",
+ "<a href=\"#class\">Classification</a>\n",
+ "\n",
+ "* <a href=\"#create_input_data\">1. Create input data</a>\n",
+ "\n",
+ "* <a href=\"#pp\">2. Call preprocessor for deep learning</a>\n",
+ "\n",
+ "* <a href=\"#load\">3. Define and load model architecture</a>\n",
+ "\n",
+ "* <a href=\"#train\">4. Train</a>\n",
+ "\n",
+ "* <a href=\"#eval\">5. Evaluate</a>\n",
+ "\n",
+ "* <a href=\"#pred\">6. Predict</a>\n",
+ "\n",
+ "<a href=\"#class2\">Classification with Other Parameters</a>\n",
+ "\n",
+ "* <a href=\"#val_dataset\">1. Validation dataset</a>\n",
+ "\n",
+ "* <a href=\"#pred_prob\">2. Predict probabilities</a>\n",
+ "\n",
+ "* <a href=\"#warm_start\">3. Warm start</a>\n",
+ "\n",
+ "<a href=\"#transfer_learn\">Transfer learning</a>\n",
+ "\n",
+ "* <a href=\"#load2\">1. Define and load model architecture with some layers frozen</a>\n",
+ "\n",
+ "* <a href=\"#train2\">2. Train transfer model</a>"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/config.py:13: ShimWarning: The `IPython.config` package has been deprecated since IPython 4.0. You should import from traitlets.config instead.\n",
+ " \"You should import from traitlets.config instead.\", ShimWarning)\n",
+ "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/utils/traitlets.py:5: UserWarning: IPython.utils.traitlets has moved to a top-level traitlets package.\n",
+ " warn(\"IPython.utils.traitlets has moved to a top-level traitlets package.\")\n"
+ ]
+ }
+ ],
+ "source": [
+ "%load_ext sql"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "u'Connected: fmcquillan@madlib'"
+ ]
+ },
+ "execution_count": 2,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Greenplum Database 5.x on GCP (PM demo machine)\n",
+ "#%sql postgresql://gpadmin@35.184.232.200:5432/madlib\n",
+ " \n",
+ "# Greenplum Database 5.x on GCP for deep learning (PM demo machine)\n",
+ "#%sql postgresql://gpadmin@35.239.240.26:5432/madlib\n",
+ " \n",
+ "# PostgreSQL local\n",
+ "%sql postgresql://fmcquillan@localhost:5432/madlib"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 30,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>version</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>PostgreSQL 11.4 on x86_64-apple-darwin16.7.0, compiled by gcc (Homebrew gcc 5.2.0) 5.2.0, 64-bit</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'PostgreSQL 11.4 on x86_64-apple-darwin16.7.0, compiled by gcc (Homebrew gcc 5.2.0) 5.2.0, 64-bit',)]"
+ ]
+ },
+ "execution_count": 30,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "#%sql select madlib.version();\n",
+ "%sql select version();"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"class\"></a>\n",
+ "# Classification\n",
+ "\n",
+ "<a id=\"create_input_data\"></a>\n",
+ "# 1. Create input data\n",
+ "\n",
+ "Load iris data set."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "Done.\n",
+ "150 rows affected.\n",
+ "150 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>attributes</th>\n",
+ " <th>class_text</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>[Decimal('5.1'), Decimal('3.5'), Decimal('1.4'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>[Decimal('4.9'), Decimal('3.0'), Decimal('1.4'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>[Decimal('4.7'), Decimal('3.2'), Decimal('1.3'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>[Decimal('4.6'), Decimal('3.1'), Decimal('1.5'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>[Decimal('5.0'), Decimal('3.6'), Decimal('1.4'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>[Decimal('5.4'), Decimal('3.9'), Decimal('1.7'), Decimal('0.4')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>7</td>\n",
+ " <td>[Decimal('4.6'), Decimal('3.4'), Decimal('1.4'), Decimal('0.3')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>8</td>\n",
+ " <td>[Decimal('5.0'), Decimal('3.4'), Decimal('1.5'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>9</td>\n",
+ " <td>[Decimal('4.4'), Decimal('2.9'), Decimal('1.4'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>10</td>\n",
+ " <td>[Decimal('4.9'), Decimal('3.1'), Decimal('1.5'), Decimal('0.1')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>11</td>\n",
+ " <td>[Decimal('5.4'), Decimal('3.7'), Decimal('1.5'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>12</td>\n",
+ " <td>[Decimal('4.8'), Decimal('3.4'), Decimal('1.6'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>13</td>\n",
+ " <td>[Decimal('4.8'), Decimal('3.0'), Decimal('1.4'), Decimal('0.1')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>14</td>\n",
+ " <td>[Decimal('4.3'), Decimal('3.0'), Decimal('1.1'), Decimal('0.1')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>15</td>\n",
+ " <td>[Decimal('5.8'), Decimal('4.0'), Decimal('1.2'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>16</td>\n",
+ " <td>[Decimal('5.7'), Decimal('4.4'), Decimal('1.5'), Decimal('0.4')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>17</td>\n",
+ " <td>[Decimal('5.4'), Decimal('3.9'), Decimal('1.3'), Decimal('0.4')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>18</td>\n",
+ " <td>[Decimal('5.1'), Decimal('3.5'), Decimal('1.4'), Decimal('0.3')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>19</td>\n",
+ " <td>[Decimal('5.7'), Decimal('3.8'), Decimal('1.7'), Decimal('0.3')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>20</td>\n",
+ " <td>[Decimal('5.1'), Decimal('3.8'), Decimal('1.5'), Decimal('0.3')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>21</td>\n",
+ " <td>[Decimal('5.4'), Decimal('3.4'), Decimal('1.7'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>22</td>\n",
+ " <td>[Decimal('5.1'), Decimal('3.7'), Decimal('1.5'), Decimal('0.4')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>23</td>\n",
+ " <td>[Decimal('4.6'), Decimal('3.6'), Decimal('1.0'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>24</td>\n",
+ " <td>[Decimal('5.1'), Decimal('3.3'), Decimal('1.7'), Decimal('0.5')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>25</td>\n",
+ " <td>[Decimal('4.8'), Decimal('3.4'), Decimal('1.9'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>26</td>\n",
+ " <td>[Decimal('5.0'), Decimal('3.0'), Decimal('1.6'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>27</td>\n",
+ " <td>[Decimal('5.0'), Decimal('3.4'), Decimal('1.6'), Decimal('0.4')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>28</td>\n",
+ " <td>[Decimal('5.2'), Decimal('3.5'), Decimal('1.5'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>29</td>\n",
+ " <td>[Decimal('5.2'), Decimal('3.4'), Decimal('1.4'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>30</td>\n",
+ " <td>[Decimal('4.7'), Decimal('3.2'), Decimal('1.6'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>31</td>\n",
+ " <td>[Decimal('4.8'), Decimal('3.1'), Decimal('1.6'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>32</td>\n",
+ " <td>[Decimal('5.4'), Decimal('3.4'), Decimal('1.5'), Decimal('0.4')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>33</td>\n",
+ " <td>[Decimal('5.2'), Decimal('4.1'), Decimal('1.5'), Decimal('0.1')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>34</td>\n",
+ " <td>[Decimal('5.5'), Decimal('4.2'), Decimal('1.4'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>35</td>\n",
+ " <td>[Decimal('4.9'), Decimal('3.1'), Decimal('1.5'), Decimal('0.1')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>36</td>\n",
+ " <td>[Decimal('5.0'), Decimal('3.2'), Decimal('1.2'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>37</td>\n",
+ " <td>[Decimal('5.5'), Decimal('3.5'), Decimal('1.3'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>38</td>\n",
+ " <td>[Decimal('4.9'), Decimal('3.1'), Decimal('1.5'), Decimal('0.1')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>39</td>\n",
+ " <td>[Decimal('4.4'), Decimal('3.0'), Decimal('1.3'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>40</td>\n",
+ " <td>[Decimal('5.1'), Decimal('3.4'), Decimal('1.5'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>41</td>\n",
+ " <td>[Decimal('5.0'), Decimal('3.5'), Decimal('1.3'), Decimal('0.3')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>42</td>\n",
+ " <td>[Decimal('4.5'), Decimal('2.3'), Decimal('1.3'), Decimal('0.3')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>43</td>\n",
+ " <td>[Decimal('4.4'), Decimal('3.2'), Decimal('1.3'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>44</td>\n",
+ " <td>[Decimal('5.0'), Decimal('3.5'), Decimal('1.6'), Decimal('0.6')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>45</td>\n",
+ " <td>[Decimal('5.1'), Decimal('3.8'), Decimal('1.9'), Decimal('0.4')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>46</td>\n",
+ " <td>[Decimal('4.8'), Decimal('3.0'), Decimal('1.4'), Decimal('0.3')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>47</td>\n",
+ " <td>[Decimal('5.1'), Decimal('3.8'), Decimal('1.6'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>48</td>\n",
+ " <td>[Decimal('4.6'), Decimal('3.2'), Decimal('1.4'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>49</td>\n",
+ " <td>[Decimal('5.3'), Decimal('3.7'), Decimal('1.5'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>50</td>\n",
+ " <td>[Decimal('5.0'), Decimal('3.3'), Decimal('1.4'), Decimal('0.2')]</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>51</td>\n",
+ " <td>[Decimal('7.0'), Decimal('3.2'), Decimal('4.7'), Decimal('1.4')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>52</td>\n",
+ " <td>[Decimal('6.4'), Decimal('3.2'), Decimal('4.5'), Decimal('1.5')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>53</td>\n",
+ " <td>[Decimal('6.9'), Decimal('3.1'), Decimal('4.9'), Decimal('1.5')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>54</td>\n",
+ " <td>[Decimal('5.5'), Decimal('2.3'), Decimal('4.0'), Decimal('1.3')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>55</td>\n",
+ " <td>[Decimal('6.5'), Decimal('2.8'), Decimal('4.6'), Decimal('1.5')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>56</td>\n",
+ " <td>[Decimal('5.7'), Decimal('2.8'), Decimal('4.5'), Decimal('1.3')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>57</td>\n",
+ " <td>[Decimal('6.3'), Decimal('3.3'), Decimal('4.7'), Decimal('1.6')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>58</td>\n",
+ " <td>[Decimal('4.9'), Decimal('2.4'), Decimal('3.3'), Decimal('1.0')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>59</td>\n",
+ " <td>[Decimal('6.6'), Decimal('2.9'), Decimal('4.6'), Decimal('1.3')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>60</td>\n",
+ " <td>[Decimal('5.2'), Decimal('2.7'), Decimal('3.9'), Decimal('1.4')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>61</td>\n",
+ " <td>[Decimal('5.0'), Decimal('2.0'), Decimal('3.5'), Decimal('1.0')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>62</td>\n",
+ " <td>[Decimal('5.9'), Decimal('3.0'), Decimal('4.2'), Decimal('1.5')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>63</td>\n",
+ " <td>[Decimal('6.0'), Decimal('2.2'), Decimal('4.0'), Decimal('1.0')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>64</td>\n",
+ " <td>[Decimal('6.1'), Decimal('2.9'), Decimal('4.7'), Decimal('1.4')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>65</td>\n",
+ " <td>[Decimal('5.6'), Decimal('2.9'), Decimal('3.6'), Decimal('1.3')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>66</td>\n",
+ " <td>[Decimal('6.7'), Decimal('3.1'), Decimal('4.4'), Decimal('1.4')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>67</td>\n",
+ " <td>[Decimal('5.6'), Decimal('3.0'), Decimal('4.5'), Decimal('1.5')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>68</td>\n",
+ " <td>[Decimal('5.8'), Decimal('2.7'), Decimal('4.1'), Decimal('1.0')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>69</td>\n",
+ " <td>[Decimal('6.2'), Decimal('2.2'), Decimal('4.5'), Decimal('1.5')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>70</td>\n",
+ " <td>[Decimal('5.6'), Decimal('2.5'), Decimal('3.9'), Decimal('1.1')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>71</td>\n",
+ " <td>[Decimal('5.9'), Decimal('3.2'), Decimal('4.8'), Decimal('1.8')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>72</td>\n",
+ " <td>[Decimal('6.1'), Decimal('2.8'), Decimal('4.0'), Decimal('1.3')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>73</td>\n",
+ " <td>[Decimal('6.3'), Decimal('2.5'), Decimal('4.9'), Decimal('1.5')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>74</td>\n",
+ " <td>[Decimal('6.1'), Decimal('2.8'), Decimal('4.7'), Decimal('1.2')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>75</td>\n",
+ " <td>[Decimal('6.4'), Decimal('2.9'), Decimal('4.3'), Decimal('1.3')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>76</td>\n",
+ " <td>[Decimal('6.6'), Decimal('3.0'), Decimal('4.4'), Decimal('1.4')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>77</td>\n",
+ " <td>[Decimal('6.8'), Decimal('2.8'), Decimal('4.8'), Decimal('1.4')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>78</td>\n",
+ " <td>[Decimal('6.7'), Decimal('3.0'), Decimal('5.0'), Decimal('1.7')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>79</td>\n",
+ " <td>[Decimal('6.0'), Decimal('2.9'), Decimal('4.5'), Decimal('1.5')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>80</td>\n",
+ " <td>[Decimal('5.7'), Decimal('2.6'), Decimal('3.5'), Decimal('1.0')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>81</td>\n",
+ " <td>[Decimal('5.5'), Decimal('2.4'), Decimal('3.8'), Decimal('1.1')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>82</td>\n",
+ " <td>[Decimal('5.5'), Decimal('2.4'), Decimal('3.7'), Decimal('1.0')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>83</td>\n",
+ " <td>[Decimal('5.8'), Decimal('2.7'), Decimal('3.9'), Decimal('1.2')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>84</td>\n",
+ " <td>[Decimal('6.0'), Decimal('2.7'), Decimal('5.1'), Decimal('1.6')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>85</td>\n",
+ " <td>[Decimal('5.4'), Decimal('3.0'), Decimal('4.5'), Decimal('1.5')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>86</td>\n",
+ " <td>[Decimal('6.0'), Decimal('3.4'), Decimal('4.5'), Decimal('1.6')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>87</td>\n",
+ " <td>[Decimal('6.7'), Decimal('3.1'), Decimal('4.7'), Decimal('1.5')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>88</td>\n",
+ " <td>[Decimal('6.3'), Decimal('2.3'), Decimal('4.4'), Decimal('1.3')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>89</td>\n",
+ " <td>[Decimal('5.6'), Decimal('3.0'), Decimal('4.1'), Decimal('1.3')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>90</td>\n",
+ " <td>[Decimal('5.5'), Decimal('2.5'), Decimal('4.0'), Decimal('1.3')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>91</td>\n",
+ " <td>[Decimal('5.5'), Decimal('2.6'), Decimal('4.4'), Decimal('1.2')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>92</td>\n",
+ " <td>[Decimal('6.1'), Decimal('3.0'), Decimal('4.6'), Decimal('1.4')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>93</td>\n",
+ " <td>[Decimal('5.8'), Decimal('2.6'), Decimal('4.0'), Decimal('1.2')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>94</td>\n",
+ " <td>[Decimal('5.0'), Decimal('2.3'), Decimal('3.3'), Decimal('1.0')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>95</td>\n",
+ " <td>[Decimal('5.6'), Decimal('2.7'), Decimal('4.2'), Decimal('1.3')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>96</td>\n",
+ " <td>[Decimal('5.7'), Decimal('3.0'), Decimal('4.2'), Decimal('1.2')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>97</td>\n",
+ " <td>[Decimal('5.7'), Decimal('2.9'), Decimal('4.2'), Decimal('1.3')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>98</td>\n",
+ " <td>[Decimal('6.2'), Decimal('2.9'), Decimal('4.3'), Decimal('1.3')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>99</td>\n",
+ " <td>[Decimal('5.1'), Decimal('2.5'), Decimal('3.0'), Decimal('1.1')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>100</td>\n",
+ " <td>[Decimal('5.7'), Decimal('2.8'), Decimal('4.1'), Decimal('1.3')]</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>101</td>\n",
+ " <td>[Decimal('6.3'), Decimal('3.3'), Decimal('6.0'), Decimal('2.5')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>102</td>\n",
+ " <td>[Decimal('5.8'), Decimal('2.7'), Decimal('5.1'), Decimal('1.9')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>103</td>\n",
+ " <td>[Decimal('7.1'), Decimal('3.0'), Decimal('5.9'), Decimal('2.1')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>104</td>\n",
+ " <td>[Decimal('6.3'), Decimal('2.9'), Decimal('5.6'), Decimal('1.8')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>105</td>\n",
+ " <td>[Decimal('6.5'), Decimal('3.0'), Decimal('5.8'), Decimal('2.2')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>106</td>\n",
+ " <td>[Decimal('7.6'), Decimal('3.0'), Decimal('6.6'), Decimal('2.1')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>107</td>\n",
+ " <td>[Decimal('4.9'), Decimal('2.5'), Decimal('4.5'), Decimal('1.7')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>108</td>\n",
+ " <td>[Decimal('7.3'), Decimal('2.9'), Decimal('6.3'), Decimal('1.8')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>109</td>\n",
+ " <td>[Decimal('6.7'), Decimal('2.5'), Decimal('5.8'), Decimal('1.8')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>110</td>\n",
+ " <td>[Decimal('7.2'), Decimal('3.6'), Decimal('6.1'), Decimal('2.5')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>111</td>\n",
+ " <td>[Decimal('6.5'), Decimal('3.2'), Decimal('5.1'), Decimal('2.0')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>112</td>\n",
+ " <td>[Decimal('6.4'), Decimal('2.7'), Decimal('5.3'), Decimal('1.9')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>113</td>\n",
+ " <td>[Decimal('6.8'), Decimal('3.0'), Decimal('5.5'), Decimal('2.1')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>114</td>\n",
+ " <td>[Decimal('5.7'), Decimal('2.5'), Decimal('5.0'), Decimal('2.0')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>115</td>\n",
+ " <td>[Decimal('5.8'), Decimal('2.8'), Decimal('5.1'), Decimal('2.4')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>116</td>\n",
+ " <td>[Decimal('6.4'), Decimal('3.2'), Decimal('5.3'), Decimal('2.3')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>117</td>\n",
+ " <td>[Decimal('6.5'), Decimal('3.0'), Decimal('5.5'), Decimal('1.8')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>118</td>\n",
+ " <td>[Decimal('7.7'), Decimal('3.8'), Decimal('6.7'), Decimal('2.2')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>119</td>\n",
+ " <td>[Decimal('7.7'), Decimal('2.6'), Decimal('6.9'), Decimal('2.3')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>120</td>\n",
+ " <td>[Decimal('6.0'), Decimal('2.2'), Decimal('5.0'), Decimal('1.5')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>121</td>\n",
+ " <td>[Decimal('6.9'), Decimal('3.2'), Decimal('5.7'), Decimal('2.3')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>122</td>\n",
+ " <td>[Decimal('5.6'), Decimal('2.8'), Decimal('4.9'), Decimal('2.0')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>123</td>\n",
+ " <td>[Decimal('7.7'), Decimal('2.8'), Decimal('6.7'), Decimal('2.0')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>124</td>\n",
+ " <td>[Decimal('6.3'), Decimal('2.7'), Decimal('4.9'), Decimal('1.8')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>125</td>\n",
+ " <td>[Decimal('6.7'), Decimal('3.3'), Decimal('5.7'), Decimal('2.1')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>126</td>\n",
+ " <td>[Decimal('7.2'), Decimal('3.2'), Decimal('6.0'), Decimal('1.8')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>127</td>\n",
+ " <td>[Decimal('6.2'), Decimal('2.8'), Decimal('4.8'), Decimal('1.8')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>128</td>\n",
+ " <td>[Decimal('6.1'), Decimal('3.0'), Decimal('4.9'), Decimal('1.8')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>129</td>\n",
+ " <td>[Decimal('6.4'), Decimal('2.8'), Decimal('5.6'), Decimal('2.1')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>130</td>\n",
+ " <td>[Decimal('7.2'), Decimal('3.0'), Decimal('5.8'), Decimal('1.6')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>131</td>\n",
+ " <td>[Decimal('7.4'), Decimal('2.8'), Decimal('6.1'), Decimal('1.9')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>132</td>\n",
+ " <td>[Decimal('7.9'), Decimal('3.8'), Decimal('6.4'), Decimal('2.0')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>133</td>\n",
+ " <td>[Decimal('6.4'), Decimal('2.8'), Decimal('5.6'), Decimal('2.2')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>134</td>\n",
+ " <td>[Decimal('6.3'), Decimal('2.8'), Decimal('5.1'), Decimal('1.5')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>135</td>\n",
+ " <td>[Decimal('6.1'), Decimal('2.6'), Decimal('5.6'), Decimal('1.4')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>136</td>\n",
+ " <td>[Decimal('7.7'), Decimal('3.0'), Decimal('6.1'), Decimal('2.3')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>137</td>\n",
+ " <td>[Decimal('6.3'), Decimal('3.4'), Decimal('5.6'), Decimal('2.4')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>138</td>\n",
+ " <td>[Decimal('6.4'), Decimal('3.1'), Decimal('5.5'), Decimal('1.8')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>139</td>\n",
+ " <td>[Decimal('6.0'), Decimal('3.0'), Decimal('4.8'), Decimal('1.8')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>140</td>\n",
+ " <td>[Decimal('6.9'), Decimal('3.1'), Decimal('5.4'), Decimal('2.1')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>141</td>\n",
+ " <td>[Decimal('6.7'), Decimal('3.1'), Decimal('5.6'), Decimal('2.4')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>142</td>\n",
+ " <td>[Decimal('6.9'), Decimal('3.1'), Decimal('5.1'), Decimal('2.3')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>143</td>\n",
+ " <td>[Decimal('5.8'), Decimal('2.7'), Decimal('5.1'), Decimal('1.9')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>144</td>\n",
+ " <td>[Decimal('6.8'), Decimal('3.2'), Decimal('5.9'), Decimal('2.3')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>145</td>\n",
+ " <td>[Decimal('6.7'), Decimal('3.3'), Decimal('5.7'), Decimal('2.5')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>146</td>\n",
+ " <td>[Decimal('6.7'), Decimal('3.0'), Decimal('5.2'), Decimal('2.3')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>147</td>\n",
+ " <td>[Decimal('6.3'), Decimal('2.5'), Decimal('5.0'), Decimal('1.9')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>148</td>\n",
+ " <td>[Decimal('6.5'), Decimal('3.0'), Decimal('5.2'), Decimal('2.0')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>149</td>\n",
+ " <td>[Decimal('6.2'), Decimal('3.4'), Decimal('5.4'), Decimal('2.3')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>150</td>\n",
+ " <td>[Decimal('5.9'), Decimal('3.0'), Decimal('5.1'), Decimal('1.8')]</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, [Decimal('5.1'), Decimal('3.5'), Decimal('1.4'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (2, [Decimal('4.9'), Decimal('3.0'), Decimal('1.4'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (3, [Decimal('4.7'), Decimal('3.2'), Decimal('1.3'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (4, [Decimal('4.6'), Decimal('3.1'), Decimal('1.5'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (5, [Decimal('5.0'), Decimal('3.6'), Decimal('1.4'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (6, [Decimal('5.4'), Decimal('3.9'), Decimal('1.7'), Decimal('0.4')], u'Iris-setosa'),\n",
+ " (7, [Decimal('4.6'), Decimal('3.4'), Decimal('1.4'), Decimal('0.3')], u'Iris-setosa'),\n",
+ " (8, [Decimal('5.0'), Decimal('3.4'), Decimal('1.5'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (9, [Decimal('4.4'), Decimal('2.9'), Decimal('1.4'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (10, [Decimal('4.9'), Decimal('3.1'), Decimal('1.5'), Decimal('0.1')], u'Iris-setosa'),\n",
+ " (11, [Decimal('5.4'), Decimal('3.7'), Decimal('1.5'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (12, [Decimal('4.8'), Decimal('3.4'), Decimal('1.6'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (13, [Decimal('4.8'), Decimal('3.0'), Decimal('1.4'), Decimal('0.1')], u'Iris-setosa'),\n",
+ " (14, [Decimal('4.3'), Decimal('3.0'), Decimal('1.1'), Decimal('0.1')], u'Iris-setosa'),\n",
+ " (15, [Decimal('5.8'), Decimal('4.0'), Decimal('1.2'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (16, [Decimal('5.7'), Decimal('4.4'), Decimal('1.5'), Decimal('0.4')], u'Iris-setosa'),\n",
+ " (17, [Decimal('5.4'), Decimal('3.9'), Decimal('1.3'), Decimal('0.4')], u'Iris-setosa'),\n",
+ " (18, [Decimal('5.1'), Decimal('3.5'), Decimal('1.4'), Decimal('0.3')], u'Iris-setosa'),\n",
+ " (19, [Decimal('5.7'), Decimal('3.8'), Decimal('1.7'), Decimal('0.3')], u'Iris-setosa'),\n",
+ " (20, [Decimal('5.1'), Decimal('3.8'), Decimal('1.5'), Decimal('0.3')], u'Iris-setosa'),\n",
+ " (21, [Decimal('5.4'), Decimal('3.4'), Decimal('1.7'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (22, [Decimal('5.1'), Decimal('3.7'), Decimal('1.5'), Decimal('0.4')], u'Iris-setosa'),\n",
+ " (23, [Decimal('4.6'), Decimal('3.6'), Decimal('1.0'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (24, [Decimal('5.1'), Decimal('3.3'), Decimal('1.7'), Decimal('0.5')], u'Iris-setosa'),\n",
+ " (25, [Decimal('4.8'), Decimal('3.4'), Decimal('1.9'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (26, [Decimal('5.0'), Decimal('3.0'), Decimal('1.6'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (27, [Decimal('5.0'), Decimal('3.4'), Decimal('1.6'), Decimal('0.4')], u'Iris-setosa'),\n",
+ " (28, [Decimal('5.2'), Decimal('3.5'), Decimal('1.5'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (29, [Decimal('5.2'), Decimal('3.4'), Decimal('1.4'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (30, [Decimal('4.7'), Decimal('3.2'), Decimal('1.6'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (31, [Decimal('4.8'), Decimal('3.1'), Decimal('1.6'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (32, [Decimal('5.4'), Decimal('3.4'), Decimal('1.5'), Decimal('0.4')], u'Iris-setosa'),\n",
+ " (33, [Decimal('5.2'), Decimal('4.1'), Decimal('1.5'), Decimal('0.1')], u'Iris-setosa'),\n",
+ " (34, [Decimal('5.5'), Decimal('4.2'), Decimal('1.4'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (35, [Decimal('4.9'), Decimal('3.1'), Decimal('1.5'), Decimal('0.1')], u'Iris-setosa'),\n",
+ " (36, [Decimal('5.0'), Decimal('3.2'), Decimal('1.2'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (37, [Decimal('5.5'), Decimal('3.5'), Decimal('1.3'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (38, [Decimal('4.9'), Decimal('3.1'), Decimal('1.5'), Decimal('0.1')], u'Iris-setosa'),\n",
+ " (39, [Decimal('4.4'), Decimal('3.0'), Decimal('1.3'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (40, [Decimal('5.1'), Decimal('3.4'), Decimal('1.5'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (41, [Decimal('5.0'), Decimal('3.5'), Decimal('1.3'), Decimal('0.3')], u'Iris-setosa'),\n",
+ " (42, [Decimal('4.5'), Decimal('2.3'), Decimal('1.3'), Decimal('0.3')], u'Iris-setosa'),\n",
+ " (43, [Decimal('4.4'), Decimal('3.2'), Decimal('1.3'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (44, [Decimal('5.0'), Decimal('3.5'), Decimal('1.6'), Decimal('0.6')], u'Iris-setosa'),\n",
+ " (45, [Decimal('5.1'), Decimal('3.8'), Decimal('1.9'), Decimal('0.4')], u'Iris-setosa'),\n",
+ " (46, [Decimal('4.8'), Decimal('3.0'), Decimal('1.4'), Decimal('0.3')], u'Iris-setosa'),\n",
+ " (47, [Decimal('5.1'), Decimal('3.8'), Decimal('1.6'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (48, [Decimal('4.6'), Decimal('3.2'), Decimal('1.4'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (49, [Decimal('5.3'), Decimal('3.7'), Decimal('1.5'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (50, [Decimal('5.0'), Decimal('3.3'), Decimal('1.4'), Decimal('0.2')], u'Iris-setosa'),\n",
+ " (51, [Decimal('7.0'), Decimal('3.2'), Decimal('4.7'), Decimal('1.4')], u'Iris-versicolor'),\n",
+ " (52, [Decimal('6.4'), Decimal('3.2'), Decimal('4.5'), Decimal('1.5')], u'Iris-versicolor'),\n",
+ " (53, [Decimal('6.9'), Decimal('3.1'), Decimal('4.9'), Decimal('1.5')], u'Iris-versicolor'),\n",
+ " (54, [Decimal('5.5'), Decimal('2.3'), Decimal('4.0'), Decimal('1.3')], u'Iris-versicolor'),\n",
+ " (55, [Decimal('6.5'), Decimal('2.8'), Decimal('4.6'), Decimal('1.5')], u'Iris-versicolor'),\n",
+ " (56, [Decimal('5.7'), Decimal('2.8'), Decimal('4.5'), Decimal('1.3')], u'Iris-versicolor'),\n",
+ " (57, [Decimal('6.3'), Decimal('3.3'), Decimal('4.7'), Decimal('1.6')], u'Iris-versicolor'),\n",
+ " (58, [Decimal('4.9'), Decimal('2.4'), Decimal('3.3'), Decimal('1.0')], u'Iris-versicolor'),\n",
+ " (59, [Decimal('6.6'), Decimal('2.9'), Decimal('4.6'), Decimal('1.3')], u'Iris-versicolor'),\n",
+ " (60, [Decimal('5.2'), Decimal('2.7'), Decimal('3.9'), Decimal('1.4')], u'Iris-versicolor'),\n",
+ " (61, [Decimal('5.0'), Decimal('2.0'), Decimal('3.5'), Decimal('1.0')], u'Iris-versicolor'),\n",
+ " (62, [Decimal('5.9'), Decimal('3.0'), Decimal('4.2'), Decimal('1.5')], u'Iris-versicolor'),\n",
+ " (63, [Decimal('6.0'), Decimal('2.2'), Decimal('4.0'), Decimal('1.0')], u'Iris-versicolor'),\n",
+ " (64, [Decimal('6.1'), Decimal('2.9'), Decimal('4.7'), Decimal('1.4')], u'Iris-versicolor'),\n",
+ " (65, [Decimal('5.6'), Decimal('2.9'), Decimal('3.6'), Decimal('1.3')], u'Iris-versicolor'),\n",
+ " (66, [Decimal('6.7'), Decimal('3.1'), Decimal('4.4'), Decimal('1.4')], u'Iris-versicolor'),\n",
+ " (67, [Decimal('5.6'), Decimal('3.0'), Decimal('4.5'), Decimal('1.5')], u'Iris-versicolor'),\n",
+ " (68, [Decimal('5.8'), Decimal('2.7'), Decimal('4.1'), Decimal('1.0')], u'Iris-versicolor'),\n",
+ " (69, [Decimal('6.2'), Decimal('2.2'), Decimal('4.5'), Decimal('1.5')], u'Iris-versicolor'),\n",
+ " (70, [Decimal('5.6'), Decimal('2.5'), Decimal('3.9'), Decimal('1.1')], u'Iris-versicolor'),\n",
+ " (71, [Decimal('5.9'), Decimal('3.2'), Decimal('4.8'), Decimal('1.8')], u'Iris-versicolor'),\n",
+ " (72, [Decimal('6.1'), Decimal('2.8'), Decimal('4.0'), Decimal('1.3')], u'Iris-versicolor'),\n",
+ " (73, [Decimal('6.3'), Decimal('2.5'), Decimal('4.9'), Decimal('1.5')], u'Iris-versicolor'),\n",
+ " (74, [Decimal('6.1'), Decimal('2.8'), Decimal('4.7'), Decimal('1.2')], u'Iris-versicolor'),\n",
+ " (75, [Decimal('6.4'), Decimal('2.9'), Decimal('4.3'), Decimal('1.3')], u'Iris-versicolor'),\n",
+ " (76, [Decimal('6.6'), Decimal('3.0'), Decimal('4.4'), Decimal('1.4')], u'Iris-versicolor'),\n",
+ " (77, [Decimal('6.8'), Decimal('2.8'), Decimal('4.8'), Decimal('1.4')], u'Iris-versicolor'),\n",
+ " (78, [Decimal('6.7'), Decimal('3.0'), Decimal('5.0'), Decimal('1.7')], u'Iris-versicolor'),\n",
+ " (79, [Decimal('6.0'), Decimal('2.9'), Decimal('4.5'), Decimal('1.5')], u'Iris-versicolor'),\n",
+ " (80, [Decimal('5.7'), Decimal('2.6'), Decimal('3.5'), Decimal('1.0')], u'Iris-versicolor'),\n",
+ " (81, [Decimal('5.5'), Decimal('2.4'), Decimal('3.8'), Decimal('1.1')], u'Iris-versicolor'),\n",
+ " (82, [Decimal('5.5'), Decimal('2.4'), Decimal('3.7'), Decimal('1.0')], u'Iris-versicolor'),\n",
+ " (83, [Decimal('5.8'), Decimal('2.7'), Decimal('3.9'), Decimal('1.2')], u'Iris-versicolor'),\n",
+ " (84, [Decimal('6.0'), Decimal('2.7'), Decimal('5.1'), Decimal('1.6')], u'Iris-versicolor'),\n",
+ " (85, [Decimal('5.4'), Decimal('3.0'), Decimal('4.5'), Decimal('1.5')], u'Iris-versicolor'),\n",
+ " (86, [Decimal('6.0'), Decimal('3.4'), Decimal('4.5'), Decimal('1.6')], u'Iris-versicolor'),\n",
+ " (87, [Decimal('6.7'), Decimal('3.1'), Decimal('4.7'), Decimal('1.5')], u'Iris-versicolor'),\n",
+ " (88, [Decimal('6.3'), Decimal('2.3'), Decimal('4.4'), Decimal('1.3')], u'Iris-versicolor'),\n",
+ " (89, [Decimal('5.6'), Decimal('3.0'), Decimal('4.1'), Decimal('1.3')], u'Iris-versicolor'),\n",
+ " (90, [Decimal('5.5'), Decimal('2.5'), Decimal('4.0'), Decimal('1.3')], u'Iris-versicolor'),\n",
+ " (91, [Decimal('5.5'), Decimal('2.6'), Decimal('4.4'), Decimal('1.2')], u'Iris-versicolor'),\n",
+ " (92, [Decimal('6.1'), Decimal('3.0'), Decimal('4.6'), Decimal('1.4')], u'Iris-versicolor'),\n",
+ " (93, [Decimal('5.8'), Decimal('2.6'), Decimal('4.0'), Decimal('1.2')], u'Iris-versicolor'),\n",
+ " (94, [Decimal('5.0'), Decimal('2.3'), Decimal('3.3'), Decimal('1.0')], u'Iris-versicolor'),\n",
+ " (95, [Decimal('5.6'), Decimal('2.7'), Decimal('4.2'), Decimal('1.3')], u'Iris-versicolor'),\n",
+ " (96, [Decimal('5.7'), Decimal('3.0'), Decimal('4.2'), Decimal('1.2')], u'Iris-versicolor'),\n",
+ " (97, [Decimal('5.7'), Decimal('2.9'), Decimal('4.2'), Decimal('1.3')], u'Iris-versicolor'),\n",
+ " (98, [Decimal('6.2'), Decimal('2.9'), Decimal('4.3'), Decimal('1.3')], u'Iris-versicolor'),\n",
+ " (99, [Decimal('5.1'), Decimal('2.5'), Decimal('3.0'), Decimal('1.1')], u'Iris-versicolor'),\n",
+ " (100, [Decimal('5.7'), Decimal('2.8'), Decimal('4.1'), Decimal('1.3')], u'Iris-versicolor'),\n",
+ " (101, [Decimal('6.3'), Decimal('3.3'), Decimal('6.0'), Decimal('2.5')], u'Iris-virginica'),\n",
+ " (102, [Decimal('5.8'), Decimal('2.7'), Decimal('5.1'), Decimal('1.9')], u'Iris-virginica'),\n",
+ " (103, [Decimal('7.1'), Decimal('3.0'), Decimal('5.9'), Decimal('2.1')], u'Iris-virginica'),\n",
+ " (104, [Decimal('6.3'), Decimal('2.9'), Decimal('5.6'), Decimal('1.8')], u'Iris-virginica'),\n",
+ " (105, [Decimal('6.5'), Decimal('3.0'), Decimal('5.8'), Decimal('2.2')], u'Iris-virginica'),\n",
+ " (106, [Decimal('7.6'), Decimal('3.0'), Decimal('6.6'), Decimal('2.1')], u'Iris-virginica'),\n",
+ " (107, [Decimal('4.9'), Decimal('2.5'), Decimal('4.5'), Decimal('1.7')], u'Iris-virginica'),\n",
+ " (108, [Decimal('7.3'), Decimal('2.9'), Decimal('6.3'), Decimal('1.8')], u'Iris-virginica'),\n",
+ " (109, [Decimal('6.7'), Decimal('2.5'), Decimal('5.8'), Decimal('1.8')], u'Iris-virginica'),\n",
+ " (110, [Decimal('7.2'), Decimal('3.6'), Decimal('6.1'), Decimal('2.5')], u'Iris-virginica'),\n",
+ " (111, [Decimal('6.5'), Decimal('3.2'), Decimal('5.1'), Decimal('2.0')], u'Iris-virginica'),\n",
+ " (112, [Decimal('6.4'), Decimal('2.7'), Decimal('5.3'), Decimal('1.9')], u'Iris-virginica'),\n",
+ " (113, [Decimal('6.8'), Decimal('3.0'), Decimal('5.5'), Decimal('2.1')], u'Iris-virginica'),\n",
+ " (114, [Decimal('5.7'), Decimal('2.5'), Decimal('5.0'), Decimal('2.0')], u'Iris-virginica'),\n",
+ " (115, [Decimal('5.8'), Decimal('2.8'), Decimal('5.1'), Decimal('2.4')], u'Iris-virginica'),\n",
+ " (116, [Decimal('6.4'), Decimal('3.2'), Decimal('5.3'), Decimal('2.3')], u'Iris-virginica'),\n",
+ " (117, [Decimal('6.5'), Decimal('3.0'), Decimal('5.5'), Decimal('1.8')], u'Iris-virginica'),\n",
+ " (118, [Decimal('7.7'), Decimal('3.8'), Decimal('6.7'), Decimal('2.2')], u'Iris-virginica'),\n",
+ " (119, [Decimal('7.7'), Decimal('2.6'), Decimal('6.9'), Decimal('2.3')], u'Iris-virginica'),\n",
+ " (120, [Decimal('6.0'), Decimal('2.2'), Decimal('5.0'), Decimal('1.5')], u'Iris-virginica'),\n",
+ " (121, [Decimal('6.9'), Decimal('3.2'), Decimal('5.7'), Decimal('2.3')], u'Iris-virginica'),\n",
+ " (122, [Decimal('5.6'), Decimal('2.8'), Decimal('4.9'), Decimal('2.0')], u'Iris-virginica'),\n",
+ " (123, [Decimal('7.7'), Decimal('2.8'), Decimal('6.7'), Decimal('2.0')], u'Iris-virginica'),\n",
+ " (124, [Decimal('6.3'), Decimal('2.7'), Decimal('4.9'), Decimal('1.8')], u'Iris-virginica'),\n",
+ " (125, [Decimal('6.7'), Decimal('3.3'), Decimal('5.7'), Decimal('2.1')], u'Iris-virginica'),\n",
+ " (126, [Decimal('7.2'), Decimal('3.2'), Decimal('6.0'), Decimal('1.8')], u'Iris-virginica'),\n",
+ " (127, [Decimal('6.2'), Decimal('2.8'), Decimal('4.8'), Decimal('1.8')], u'Iris-virginica'),\n",
+ " (128, [Decimal('6.1'), Decimal('3.0'), Decimal('4.9'), Decimal('1.8')], u'Iris-virginica'),\n",
+ " (129, [Decimal('6.4'), Decimal('2.8'), Decimal('5.6'), Decimal('2.1')], u'Iris-virginica'),\n",
+ " (130, [Decimal('7.2'), Decimal('3.0'), Decimal('5.8'), Decimal('1.6')], u'Iris-virginica'),\n",
+ " (131, [Decimal('7.4'), Decimal('2.8'), Decimal('6.1'), Decimal('1.9')], u'Iris-virginica'),\n",
+ " (132, [Decimal('7.9'), Decimal('3.8'), Decimal('6.4'), Decimal('2.0')], u'Iris-virginica'),\n",
+ " (133, [Decimal('6.4'), Decimal('2.8'), Decimal('5.6'), Decimal('2.2')], u'Iris-virginica'),\n",
+ " (134, [Decimal('6.3'), Decimal('2.8'), Decimal('5.1'), Decimal('1.5')], u'Iris-virginica'),\n",
+ " (135, [Decimal('6.1'), Decimal('2.6'), Decimal('5.6'), Decimal('1.4')], u'Iris-virginica'),\n",
+ " (136, [Decimal('7.7'), Decimal('3.0'), Decimal('6.1'), Decimal('2.3')], u'Iris-virginica'),\n",
+ " (137, [Decimal('6.3'), Decimal('3.4'), Decimal('5.6'), Decimal('2.4')], u'Iris-virginica'),\n",
+ " (138, [Decimal('6.4'), Decimal('3.1'), Decimal('5.5'), Decimal('1.8')], u'Iris-virginica'),\n",
+ " (139, [Decimal('6.0'), Decimal('3.0'), Decimal('4.8'), Decimal('1.8')], u'Iris-virginica'),\n",
+ " (140, [Decimal('6.9'), Decimal('3.1'), Decimal('5.4'), Decimal('2.1')], u'Iris-virginica'),\n",
+ " (141, [Decimal('6.7'), Decimal('3.1'), Decimal('5.6'), Decimal('2.4')], u'Iris-virginica'),\n",
+ " (142, [Decimal('6.9'), Decimal('3.1'), Decimal('5.1'), Decimal('2.3')], u'Iris-virginica'),\n",
+ " (143, [Decimal('5.8'), Decimal('2.7'), Decimal('5.1'), Decimal('1.9')], u'Iris-virginica'),\n",
+ " (144, [Decimal('6.8'), Decimal('3.2'), Decimal('5.9'), Decimal('2.3')], u'Iris-virginica'),\n",
+ " (145, [Decimal('6.7'), Decimal('3.3'), Decimal('5.7'), Decimal('2.5')], u'Iris-virginica'),\n",
+ " (146, [Decimal('6.7'), Decimal('3.0'), Decimal('5.2'), Decimal('2.3')], u'Iris-virginica'),\n",
+ " (147, [Decimal('6.3'), Decimal('2.5'), Decimal('5.0'), Decimal('1.9')], u'Iris-virginica'),\n",
+ " (148, [Decimal('6.5'), Decimal('3.0'), Decimal('5.2'), Decimal('2.0')], u'Iris-virginica'),\n",
+ " (149, [Decimal('6.2'), Decimal('3.4'), Decimal('5.4'), Decimal('2.3')], u'Iris-virginica'),\n",
+ " (150, [Decimal('5.9'), Decimal('3.0'), Decimal('5.1'), Decimal('1.8')], u'Iris-virginica')]"
+ ]
+ },
+ "execution_count": 4,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql \n",
+ "DROP TABLE IF EXISTS iris_data;\n",
+ "\n",
+ "CREATE TABLE iris_data(\n",
+ " id serial,\n",
+ " attributes numeric[],\n",
+ " class_text varchar\n",
+ ");\n",
+ "\n",
+ "INSERT INTO iris_data(id, attributes, class_text) VALUES\n",
+ "(1,ARRAY[5.1,3.5,1.4,0.2],'Iris-setosa'),\n",
+ "(2,ARRAY[4.9,3.0,1.4,0.2],'Iris-setosa'),\n",
+ "(3,ARRAY[4.7,3.2,1.3,0.2],'Iris-setosa'),\n",
+ "(4,ARRAY[4.6,3.1,1.5,0.2],'Iris-setosa'),\n",
+ "(5,ARRAY[5.0,3.6,1.4,0.2],'Iris-setosa'),\n",
+ "(6,ARRAY[5.4,3.9,1.7,0.4],'Iris-setosa'),\n",
+ "(7,ARRAY[4.6,3.4,1.4,0.3],'Iris-setosa'),\n",
+ "(8,ARRAY[5.0,3.4,1.5,0.2],'Iris-setosa'),\n",
+ "(9,ARRAY[4.4,2.9,1.4,0.2],'Iris-setosa'),\n",
+ "(10,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa'),\n",
+ "(11,ARRAY[5.4,3.7,1.5,0.2],'Iris-setosa'),\n",
+ "(12,ARRAY[4.8,3.4,1.6,0.2],'Iris-setosa'),\n",
+ "(13,ARRAY[4.8,3.0,1.4,0.1],'Iris-setosa'),\n",
+ "(14,ARRAY[4.3,3.0,1.1,0.1],'Iris-setosa'),\n",
+ "(15,ARRAY[5.8,4.0,1.2,0.2],'Iris-setosa'),\n",
+ "(16,ARRAY[5.7,4.4,1.5,0.4],'Iris-setosa'),\n",
+ "(17,ARRAY[5.4,3.9,1.3,0.4],'Iris-setosa'),\n",
+ "(18,ARRAY[5.1,3.5,1.4,0.3],'Iris-setosa'),\n",
+ "(19,ARRAY[5.7,3.8,1.7,0.3],'Iris-setosa'),\n",
+ "(20,ARRAY[5.1,3.8,1.5,0.3],'Iris-setosa'),\n",
+ "(21,ARRAY[5.4,3.4,1.7,0.2],'Iris-setosa'),\n",
+ "(22,ARRAY[5.1,3.7,1.5,0.4],'Iris-setosa'),\n",
+ "(23,ARRAY[4.6,3.6,1.0,0.2],'Iris-setosa'),\n",
+ "(24,ARRAY[5.1,3.3,1.7,0.5],'Iris-setosa'),\n",
+ "(25,ARRAY[4.8,3.4,1.9,0.2],'Iris-setosa'),\n",
+ "(26,ARRAY[5.0,3.0,1.6,0.2],'Iris-setosa'),\n",
+ "(27,ARRAY[5.0,3.4,1.6,0.4],'Iris-setosa'),\n",
+ "(28,ARRAY[5.2,3.5,1.5,0.2],'Iris-setosa'),\n",
+ "(29,ARRAY[5.2,3.4,1.4,0.2],'Iris-setosa'),\n",
+ "(30,ARRAY[4.7,3.2,1.6,0.2],'Iris-setosa'),\n",
+ "(31,ARRAY[4.8,3.1,1.6,0.2],'Iris-setosa'),\n",
+ "(32,ARRAY[5.4,3.4,1.5,0.4],'Iris-setosa'),\n",
+ "(33,ARRAY[5.2,4.1,1.5,0.1],'Iris-setosa'),\n",
+ "(34,ARRAY[5.5,4.2,1.4,0.2],'Iris-setosa'),\n",
+ "(35,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa'),\n",
+ "(36,ARRAY[5.0,3.2,1.2,0.2],'Iris-setosa'),\n",
+ "(37,ARRAY[5.5,3.5,1.3,0.2],'Iris-setosa'),\n",
+ "(38,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa'),\n",
+ "(39,ARRAY[4.4,3.0,1.3,0.2],'Iris-setosa'),\n",
+ "(40,ARRAY[5.1,3.4,1.5,0.2],'Iris-setosa'),\n",
+ "(41,ARRAY[5.0,3.5,1.3,0.3],'Iris-setosa'),\n",
+ "(42,ARRAY[4.5,2.3,1.3,0.3],'Iris-setosa'),\n",
+ "(43,ARRAY[4.4,3.2,1.3,0.2],'Iris-setosa'),\n",
+ "(44,ARRAY[5.0,3.5,1.6,0.6],'Iris-setosa'),\n",
+ "(45,ARRAY[5.1,3.8,1.9,0.4],'Iris-setosa'),\n",
+ "(46,ARRAY[4.8,3.0,1.4,0.3],'Iris-setosa'),\n",
+ "(47,ARRAY[5.1,3.8,1.6,0.2],'Iris-setosa'),\n",
+ "(48,ARRAY[4.6,3.2,1.4,0.2],'Iris-setosa'),\n",
+ "(49,ARRAY[5.3,3.7,1.5,0.2],'Iris-setosa'),\n",
+ "(50,ARRAY[5.0,3.3,1.4,0.2],'Iris-setosa'),\n",
+ "(51,ARRAY[7.0,3.2,4.7,1.4],'Iris-versicolor'),\n",
+ "(52,ARRAY[6.4,3.2,4.5,1.5],'Iris-versicolor'),\n",
+ "(53,ARRAY[6.9,3.1,4.9,1.5],'Iris-versicolor'),\n",
+ "(54,ARRAY[5.5,2.3,4.0,1.3],'Iris-versicolor'),\n",
+ "(55,ARRAY[6.5,2.8,4.6,1.5],'Iris-versicolor'),\n",
+ "(56,ARRAY[5.7,2.8,4.5,1.3],'Iris-versicolor'),\n",
+ "(57,ARRAY[6.3,3.3,4.7,1.6],'Iris-versicolor'),\n",
+ "(58,ARRAY[4.9,2.4,3.3,1.0],'Iris-versicolor'),\n",
+ "(59,ARRAY[6.6,2.9,4.6,1.3],'Iris-versicolor'),\n",
+ "(60,ARRAY[5.2,2.7,3.9,1.4],'Iris-versicolor'),\n",
+ "(61,ARRAY[5.0,2.0,3.5,1.0],'Iris-versicolor'),\n",
+ "(62,ARRAY[5.9,3.0,4.2,1.5],'Iris-versicolor'),\n",
+ "(63,ARRAY[6.0,2.2,4.0,1.0],'Iris-versicolor'),\n",
+ "(64,ARRAY[6.1,2.9,4.7,1.4],'Iris-versicolor'),\n",
+ "(65,ARRAY[5.6,2.9,3.6,1.3],'Iris-versicolor'),\n",
+ "(66,ARRAY[6.7,3.1,4.4,1.4],'Iris-versicolor'),\n",
+ "(67,ARRAY[5.6,3.0,4.5,1.5],'Iris-versicolor'),\n",
+ "(68,ARRAY[5.8,2.7,4.1,1.0],'Iris-versicolor'),\n",
+ "(69,ARRAY[6.2,2.2,4.5,1.5],'Iris-versicolor'),\n",
+ "(70,ARRAY[5.6,2.5,3.9,1.1],'Iris-versicolor'),\n",
+ "(71,ARRAY[5.9,3.2,4.8,1.8],'Iris-versicolor'),\n",
+ "(72,ARRAY[6.1,2.8,4.0,1.3],'Iris-versicolor'),\n",
+ "(73,ARRAY[6.3,2.5,4.9,1.5],'Iris-versicolor'),\n",
+ "(74,ARRAY[6.1,2.8,4.7,1.2],'Iris-versicolor'),\n",
+ "(75,ARRAY[6.4,2.9,4.3,1.3],'Iris-versicolor'),\n",
+ "(76,ARRAY[6.6,3.0,4.4,1.4],'Iris-versicolor'),\n",
+ "(77,ARRAY[6.8,2.8,4.8,1.4],'Iris-versicolor'),\n",
+ "(78,ARRAY[6.7,3.0,5.0,1.7],'Iris-versicolor'),\n",
+ "(79,ARRAY[6.0,2.9,4.5,1.5],'Iris-versicolor'),\n",
+ "(80,ARRAY[5.7,2.6,3.5,1.0],'Iris-versicolor'),\n",
+ "(81,ARRAY[5.5,2.4,3.8,1.1],'Iris-versicolor'),\n",
+ "(82,ARRAY[5.5,2.4,3.7,1.0],'Iris-versicolor'),\n",
+ "(83,ARRAY[5.8,2.7,3.9,1.2],'Iris-versicolor'),\n",
+ "(84,ARRAY[6.0,2.7,5.1,1.6],'Iris-versicolor'),\n",
+ "(85,ARRAY[5.4,3.0,4.5,1.5],'Iris-versicolor'),\n",
+ "(86,ARRAY[6.0,3.4,4.5,1.6],'Iris-versicolor'),\n",
+ "(87,ARRAY[6.7,3.1,4.7,1.5],'Iris-versicolor'),\n",
+ "(88,ARRAY[6.3,2.3,4.4,1.3],'Iris-versicolor'),\n",
+ "(89,ARRAY[5.6,3.0,4.1,1.3],'Iris-versicolor'),\n",
+ "(90,ARRAY[5.5,2.5,4.0,1.3],'Iris-versicolor'),\n",
+ "(91,ARRAY[5.5,2.6,4.4,1.2],'Iris-versicolor'),\n",
+ "(92,ARRAY[6.1,3.0,4.6,1.4],'Iris-versicolor'),\n",
+ "(93,ARRAY[5.8,2.6,4.0,1.2],'Iris-versicolor'),\n",
+ "(94,ARRAY[5.0,2.3,3.3,1.0],'Iris-versicolor'),\n",
+ "(95,ARRAY[5.6,2.7,4.2,1.3],'Iris-versicolor'),\n",
+ "(96,ARRAY[5.7,3.0,4.2,1.2],'Iris-versicolor'),\n",
+ "(97,ARRAY[5.7,2.9,4.2,1.3],'Iris-versicolor'),\n",
+ "(98,ARRAY[6.2,2.9,4.3,1.3],'Iris-versicolor'),\n",
+ "(99,ARRAY[5.1,2.5,3.0,1.1],'Iris-versicolor'),\n",
+ "(100,ARRAY[5.7,2.8,4.1,1.3],'Iris-versicolor'),\n",
+ "(101,ARRAY[6.3,3.3,6.0,2.5],'Iris-virginica'),\n",
+ "(102,ARRAY[5.8,2.7,5.1,1.9],'Iris-virginica'),\n",
+ "(103,ARRAY[7.1,3.0,5.9,2.1],'Iris-virginica'),\n",
+ "(104,ARRAY[6.3,2.9,5.6,1.8],'Iris-virginica'),\n",
+ "(105,ARRAY[6.5,3.0,5.8,2.2],'Iris-virginica'),\n",
+ "(106,ARRAY[7.6,3.0,6.6,2.1],'Iris-virginica'),\n",
+ "(107,ARRAY[4.9,2.5,4.5,1.7],'Iris-virginica'),\n",
+ "(108,ARRAY[7.3,2.9,6.3,1.8],'Iris-virginica'),\n",
+ "(109,ARRAY[6.7,2.5,5.8,1.8],'Iris-virginica'),\n",
+ "(110,ARRAY[7.2,3.6,6.1,2.5],'Iris-virginica'),\n",
+ "(111,ARRAY[6.5,3.2,5.1,2.0],'Iris-virginica'),\n",
+ "(112,ARRAY[6.4,2.7,5.3,1.9],'Iris-virginica'),\n",
+ "(113,ARRAY[6.8,3.0,5.5,2.1],'Iris-virginica'),\n",
+ "(114,ARRAY[5.7,2.5,5.0,2.0],'Iris-virginica'),\n",
+ "(115,ARRAY[5.8,2.8,5.1,2.4],'Iris-virginica'),\n",
+ "(116,ARRAY[6.4,3.2,5.3,2.3],'Iris-virginica'),\n",
+ "(117,ARRAY[6.5,3.0,5.5,1.8],'Iris-virginica'),\n",
+ "(118,ARRAY[7.7,3.8,6.7,2.2],'Iris-virginica'),\n",
+ "(119,ARRAY[7.7,2.6,6.9,2.3],'Iris-virginica'),\n",
+ "(120,ARRAY[6.0,2.2,5.0,1.5],'Iris-virginica'),\n",
+ "(121,ARRAY[6.9,3.2,5.7,2.3],'Iris-virginica'),\n",
+ "(122,ARRAY[5.6,2.8,4.9,2.0],'Iris-virginica'),\n",
+ "(123,ARRAY[7.7,2.8,6.7,2.0],'Iris-virginica'),\n",
+ "(124,ARRAY[6.3,2.7,4.9,1.8],'Iris-virginica'),\n",
+ "(125,ARRAY[6.7,3.3,5.7,2.1],'Iris-virginica'),\n",
+ "(126,ARRAY[7.2,3.2,6.0,1.8],'Iris-virginica'),\n",
+ "(127,ARRAY[6.2,2.8,4.8,1.8],'Iris-virginica'),\n",
+ "(128,ARRAY[6.1,3.0,4.9,1.8],'Iris-virginica'),\n",
+ "(129,ARRAY[6.4,2.8,5.6,2.1],'Iris-virginica'),\n",
+ "(130,ARRAY[7.2,3.0,5.8,1.6],'Iris-virginica'),\n",
+ "(131,ARRAY[7.4,2.8,6.1,1.9],'Iris-virginica'),\n",
+ "(132,ARRAY[7.9,3.8,6.4,2.0],'Iris-virginica'),\n",
+ "(133,ARRAY[6.4,2.8,5.6,2.2],'Iris-virginica'),\n",
+ "(134,ARRAY[6.3,2.8,5.1,1.5],'Iris-virginica'),\n",
+ "(135,ARRAY[6.1,2.6,5.6,1.4],'Iris-virginica'),\n",
+ "(136,ARRAY[7.7,3.0,6.1,2.3],'Iris-virginica'),\n",
+ "(137,ARRAY[6.3,3.4,5.6,2.4],'Iris-virginica'),\n",
+ "(138,ARRAY[6.4,3.1,5.5,1.8],'Iris-virginica'),\n",
+ "(139,ARRAY[6.0,3.0,4.8,1.8],'Iris-virginica'),\n",
+ "(140,ARRAY[6.9,3.1,5.4,2.1],'Iris-virginica'),\n",
+ "(141,ARRAY[6.7,3.1,5.6,2.4],'Iris-virginica'),\n",
+ "(142,ARRAY[6.9,3.1,5.1,2.3],'Iris-virginica'),\n",
+ "(143,ARRAY[5.8,2.7,5.1,1.9],'Iris-virginica'),\n",
+ "(144,ARRAY[6.8,3.2,5.9,2.3],'Iris-virginica'),\n",
+ "(145,ARRAY[6.7,3.3,5.7,2.5],'Iris-virginica'),\n",
+ "(146,ARRAY[6.7,3.0,5.2,2.3],'Iris-virginica'),\n",
+ "(147,ARRAY[6.3,2.5,5.0,1.9],'Iris-virginica'),\n",
+ "(148,ARRAY[6.5,3.0,5.2,2.0],'Iris-virginica'),\n",
+ "(149,ARRAY[6.2,3.4,5.4,2.3],'Iris-virginica'),\n",
+ "(150,ARRAY[5.9,3.0,5.1,1.8],'Iris-virginica');\n",
+ "\n",
+ "SELECT * FROM iris_data ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Create a test/validation dataset from the training data"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>count</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>120</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(120L,)]"
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS iris_train, iris_test;\n",
+ "\n",
+ "-- Set seed so results are reproducible\n",
+ "SELECT setseed(0);\n",
+ "\n",
+ "SELECT madlib.train_test_split('iris_data', -- Source table\n",
+ " 'iris', -- Output table root name\n",
+ " 0.8, -- Train proportion\n",
+ " NULL, -- Test proportion (0.2)\n",
+ " NULL, -- Strata definition\n",
+ " NULL, -- Output all columns\n",
+ " NULL, -- Sample without replacement\n",
+ " TRUE -- Separate output tables\n",
+ " );\n",
+ "\n",
+ "SELECT COUNT(*) FROM iris_train;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"pp\"></a>\n",
+ "# 2. Call preprocessor for deep learning\n",
+ "Training dataset (uses training preprocessor):"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>independent_var</th>\n",
+ " <th>dependent_var</th>\n",
+ " <th>buffer_id</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[5.3, 3.7, 1.5, 0.2], [6.3, 3.4, 5.6, 2.4], [7.2, 3.2, 6.0, 1.8], [5.7, 4.4, 1.5, 0.4], [5.4, 3.7, 1.5, 0.2], [5.0, 3.0, 1.6, 0.2], [4.7, 3.2, 1.3, 0.2], [5.0, 2.3, 3.3, 1.0], [5.7, 2.8, 4.5, 1.3], [4.8, 3.0, 1.4, 0.1], [5.7, 3.8, 1.7, 0.3], [5.5, 4.2, 1.4, 0.2], [7.7, 3.8, 6.7, 2.2], [6.0, 2.2, 5.0, 1.5], [6.5, 3.0, 5.2, 2.0], [4.6, 3.6, 1.0, 0.2], [6.9, 3.1, 4.9, 1.5], [5.1, 3.8, 1.9, 0.4], [6.7, 2.5, 5.8, 1.8], [6.0, 2.7, 5.1, 1.6], [6.3, 3.3, 4.7, 1.6], [6.7, 3.1 [...]
+ " <td>[[1, 0, 0], [0, 0, 1], [0, 0, 1], [1, 0, 0], [1, 0, 0], [1, 0, 0], [1, 0, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [1, 0, 0], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 0, 1], [1, 0, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 0, 1], [1, 0, 0], [0, 0, 1], [1, 0, 0], [1, 0, 0], [0, 0, 1] [...]
+ " <td>0</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[([[5.3, 3.7, 1.5, 0.2], [6.3, 3.4, 5.6, 2.4], [7.2, 3.2, 6.0, 1.8], [5.7, 4.4, 1.5, 0.4], [5.4, 3.7, 1.5, 0.2], [5.0, 3.0, 1.6, 0.2], [4.7, 3.2, 1.3, 0.2], [5.0, 2.3, 3.3, 1.0], [5.7, 2.8, 4.5, 1.3], [4.8, 3.0, 1.4, 0.1], [5.7, 3.8, 1.7, 0.3], [5.5, 4.2, 1.4, 0.2], [7.7, 3.8, 6.7, 2.2], [6.0, 2.2, 5.0, 1.5], [6.5, 3.0, 5.2, 2.0], [4.6, 3.6, 1.0, 0.2], [6.9, 3.1, 4.9, 1.5], [5.1, 3.8, 1.9, 0.4], [6.7, 2.5, 5.8, 1.8], [6.0, 2.7, 5.1, 1.6], [6.3, 3.3, 4.7, 1.6], [6.7, 3.1, 4.7, 1.5 [...]
+ ]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS iris_train_packed, iris_train_packed_summary;\n",
+ "\n",
+ "SELECT madlib.training_preprocessor_dl('iris_train', -- Source table\n",
+ " 'iris_train_packed', -- Output table\n",
+ " 'class_text', -- Dependent variable\n",
+ " 'attributes' -- Independent variable\n",
+ " ); \n",
+ "\n",
+ "SELECT * FROM iris_train_packed ORDER BY buffer_id;"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>source_table</th>\n",
+ " <th>output_table</th>\n",
+ " <th>dependent_varname</th>\n",
+ " <th>independent_varname</th>\n",
+ " <th>dependent_vartype</th>\n",
+ " <th>class_values</th>\n",
+ " <th>buffer_size</th>\n",
+ " <th>normalizing_const</th>\n",
+ " <th>num_classes</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>iris_train</td>\n",
+ " <td>iris_train_packed</td>\n",
+ " <td>class_text</td>\n",
+ " <td>attributes</td>\n",
+ " <td>character varying</td>\n",
+ " <td>[u'Iris-setosa', u'Iris-versicolor', u'Iris-virginica']</td>\n",
+ " <td>120</td>\n",
+ " <td>1.0</td>\n",
+ " <td>3</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'iris_train', u'iris_train_packed', u'class_text', u'attributes', u'character varying', [u'Iris-setosa', u'Iris-versicolor', u'Iris-virginica'], 120, 1.0, 3)]"
+ ]
+ },
+ "execution_count": 7,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM iris_train_packed_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Validation dataset (uses validation preprocessor):"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>independent_var</th>\n",
+ " <th>dependent_var</th>\n",
+ " <th>buffer_id</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[5.1, 3.4, 1.5, 0.2], [5.7, 2.8, 4.1, 1.3], [6.6, 3.0, 4.4, 1.4], [5.4, 3.9, 1.3, 0.4], [5.6, 2.7, 4.2, 1.3], [6.9, 3.1, 5.4, 2.1], [6.5, 3.0, 5.5, 1.8], [6.1, 3.0, 4.6, 1.4], [4.4, 3.2, 1.3, 0.2], [4.9, 2.5, 4.5, 1.7], [6.5, 2.8, 4.6, 1.5], [4.6, 3.1, 1.5, 0.2], [6.3, 2.3, 4.4, 1.3], [5.0, 3.3, 1.4, 0.2], [6.3, 2.9, 5.6, 1.8], [6.5, 3.2, 5.1, 2.0], [6.3, 2.8, 5.1, 1.5], [5.8, 2.7, 5.1, 1.9], [5.8, 2.7, 3.9, 1.2], [6.4, 3.2, 4.5, 1.5], [6.0, 3.4, 4.5, 1.6], [5.1, 3.3 [...]
+ " <td>[[1, 0, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [1, 0, 0]]</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[([[5.1, 3.4, 1.5, 0.2], [5.7, 2.8, 4.1, 1.3], [6.6, 3.0, 4.4, 1.4], [5.4, 3.9, 1.3, 0.4], [5.6, 2.7, 4.2, 1.3], [6.9, 3.1, 5.4, 2.1], [6.5, 3.0, 5.5, 1.8], [6.1, 3.0, 4.6, 1.4], [4.4, 3.2, 1.3, 0.2], [4.9, 2.5, 4.5, 1.7], [6.5, 2.8, 4.6, 1.5], [4.6, 3.1, 1.5, 0.2], [6.3, 2.3, 4.4, 1.3], [5.0, 3.3, 1.4, 0.2], [6.3, 2.9, 5.6, 1.8], [6.5, 3.2, 5.1, 2.0], [6.3, 2.8, 5.1, 1.5], [5.8, 2.7, 5.1, 1.9], [5.8, 2.7, 3.9, 1.2], [6.4, 3.2, 4.5, 1.5], [6.0, 3.4, 4.5, 1.6], [5.1, 3.3, 1.7, 0.5 [...]
+ ]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS iris_test_packed, iris_test_packed_summary;\n",
+ "\n",
+ "SELECT madlib.validation_preprocessor_dl('iris_test', -- Source table\n",
+ " 'iris_test_packed', -- Output table\n",
+ " 'class_text', -- Dependent variable\n",
+ " 'attributes', -- Independent variable\n",
+ " 'iris_train_packed' -- From training preprocessor step\n",
+ " ); \n",
+ "\n",
+ "SELECT * FROM iris_test_packed ORDER BY buffer_id;"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>source_table</th>\n",
+ " <th>output_table</th>\n",
+ " <th>dependent_varname</th>\n",
+ " <th>independent_varname</th>\n",
+ " <th>dependent_vartype</th>\n",
+ " <th>class_values</th>\n",
+ " <th>buffer_size</th>\n",
+ " <th>normalizing_const</th>\n",
+ " <th>num_classes</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>iris_test</td>\n",
+ " <td>iris_test_packed</td>\n",
+ " <td>class_text</td>\n",
+ " <td>attributes</td>\n",
+ " <td>character varying</td>\n",
+ " <td>[u'Iris-setosa', u'Iris-versicolor', u'Iris-virginica']</td>\n",
+ " <td>30</td>\n",
+ " <td>1.0</td>\n",
+ " <td>3</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'iris_test', u'iris_test_packed', u'class_text', u'attributes', u'character varying', [u'Iris-setosa', u'Iris-versicolor', u'Iris-virginica'], 30, 1.0, 3)]"
+ ]
+ },
+ "execution_count": 9,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM iris_test_packed_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"load\"></a>\n",
+ "# 3. Define and load model architecture\n",
+ "Import Keras libraries"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Using TensorFlow backend.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Couldn't import dot_parser, loading of dot files will not be possible.\n"
+ ]
+ }
+ ],
+ "source": [
+ "import keras\n",
+ "from keras.models import Sequential\n",
+ "from keras.layers import Dense"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Define model architecture"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "_________________________________________________________________\n",
+ "Layer (type) Output Shape Param # \n",
+ "=================================================================\n",
+ "dense_1 (Dense) (None, 10) 50 \n",
+ "_________________________________________________________________\n",
+ "dense_2 (Dense) (None, 10) 110 \n",
+ "_________________________________________________________________\n",
+ "dense_3 (Dense) (None, 3) 33 \n",
+ "=================================================================\n",
+ "Total params: 193\n",
+ "Trainable params: 193\n",
+ "Non-trainable params: 0\n",
+ "_________________________________________________________________\n"
+ ]
+ }
+ ],
+ "source": [
+ "model_simple = Sequential()\n",
+ "model_simple.add(Dense(10, activation='relu', input_shape=(4,)))\n",
+ "model_simple.add(Dense(10, activation='relu'))\n",
+ "model_simple.add(Dense(3, activation='softmax'))\n",
+ " \n",
+ "model_simple.summary()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'{\"class_name\": \"Sequential\", \"keras_version\": \"2.1.6\", \"config\": [{\"class_name\": \"Dense\", \"config\": {\"kernel_initializer\": {\"class_name\": \"VarianceScaling\", \"config\": {\"distribution\": \"uniform\", \"scale\": 1.0, \"seed\": null, \"mode\": \"fan_avg\"}}, \"name\": \"dense_1\", \"kernel_constraint\": null, \"bias_regularizer\": null, \"bias_constraint\": null, \"dtype\": \"float32\", \"activation\": \"relu\", \"trainable\": true, \"kernel_regularizer\": n [...]
+ ]
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "model_simple.to_json()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Load into model architecture table"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>model_id</th>\n",
+ " <th>model_arch</th>\n",
+ " <th>model_weights</th>\n",
+ " <th>name</th>\n",
+ " <th>description</th>\n",
+ " <th>__internal_madlib_id__</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>{u'class_name': u'Sequential', u'keras_version': u'2.1.6', u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer': {u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform', u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1', u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint': None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True, u'kernel_regularizer': None, u'bias_initia [...]
+ " <td>None</td>\n",
+ " <td>Sophie</td>\n",
+ " <td>A simple model</td>\n",
+ " <td>__madlib_temp_62550369_1562173248_86696923__</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, {u'class_name': u'Sequential', u'keras_version': u'2.1.6', u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer': {u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform', u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1', u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint': None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True, u'kernel_regularizer': None, u'bias_initializer': [...]
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS model_arch_library;\n",
+ "SELECT madlib.load_keras_model('model_arch_library', -- Output table,\n",
+ " \n",
+ "$$\n",
+ "{\"class_name\": \"Sequential\", \"keras_version\": \"2.1.6\", \"config\": [{\"class_name\": \"Dense\", \"config\": {\"kernel_initializer\": {\"class_name\": \"VarianceScaling\", \"config\": {\"distribution\": \"uniform\", \"scale\": 1.0, \"seed\": null, \"mode\": \"fan_avg\"}}, \"name\": \"dense_1\", \"kernel_constraint\": null, \"bias_regularizer\": null, \"bias_constraint\": null, \"dtype\": \"float32\", \"activation\": \"relu\", \"trainable\": true, \"kernel_regularizer\": null, [...]
+ "$$\n",
+ "::json, -- JSON blob\n",
+ " NULL, -- Weights\n",
+ " 'Sophie', -- Name\n",
+ " 'A simple model' -- Descr\n",
+ ");\n",
+ "\n",
+ "SELECT * FROM model_arch_library;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"train\"></a>\n",
+ "# 4. Train\n",
+ "Train the model:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>madlib_keras_fit</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td></td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[('',)]"
+ ]
+ },
+ "execution_count": 13,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS iris_model, iris_model_summary;\n",
+ "\n",
+ "SELECT madlib.madlib_keras_fit('iris_train_packed', -- source table\n",
+ " 'iris_model', -- model output table\n",
+ " 'model_arch_library', -- model arch table\n",
+ " 1, -- model arch id\n",
+ " $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$, -- compile_params\n",
+ " $$ batch_size=5, epochs=3 $$, -- fit_params\n",
+ " 10 -- num_iterations\n",
+ " );"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "View the model summary:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>source_table</th>\n",
+ " <th>model</th>\n",
+ " <th>dependent_varname</th>\n",
+ " <th>independent_varname</th>\n",
+ " <th>model_arch_table</th>\n",
+ " <th>model_arch_id</th>\n",
+ " <th>compile_params</th>\n",
+ " <th>fit_params</th>\n",
+ " <th>num_iterations</th>\n",
+ " <th>validation_table</th>\n",
+ " <th>metrics_compute_frequency</th>\n",
+ " <th>name</th>\n",
+ " <th>description</th>\n",
+ " <th>model_type</th>\n",
+ " <th>model_size</th>\n",
+ " <th>start_training_time</th>\n",
+ " <th>end_training_time</th>\n",
+ " <th>metrics_elapsed_time</th>\n",
+ " <th>madlib_version</th>\n",
+ " <th>num_classes</th>\n",
+ " <th>class_values</th>\n",
+ " <th>dependent_vartype</th>\n",
+ " <th>normalizing_const</th>\n",
+ " <th>metrics_type</th>\n",
+ " <th>training_metrics_final</th>\n",
+ " <th>training_loss_final</th>\n",
+ " <th>training_metrics</th>\n",
+ " <th>training_loss</th>\n",
+ " <th>validation_metrics_final</th>\n",
+ " <th>validation_loss_final</th>\n",
+ " <th>validation_metrics</th>\n",
+ " <th>validation_loss</th>\n",
+ " <th>metrics_iters</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>iris_train_packed</td>\n",
+ " <td>iris_model</td>\n",
+ " <td>class_text</td>\n",
+ " <td>attributes</td>\n",
+ " <td>model_arch_library</td>\n",
+ " <td>1</td>\n",
+ " <td> loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] </td>\n",
+ " <td> batch_size=5, epochs=3 </td>\n",
+ " <td>10</td>\n",
+ " <td>None</td>\n",
+ " <td>10</td>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " <td>madlib_keras</td>\n",
+ " <td>0.7900390625</td>\n",
+ " <td>2019-07-03 10:00:52.477704</td>\n",
+ " <td>2019-07-03 10:00:58.552077</td>\n",
+ " <td>[6.07431507110596]</td>\n",
+ " <td>1.16</td>\n",
+ " <td>3</td>\n",
+ " <td>[u'Iris-setosa', u'Iris-versicolor', u'Iris-virginica']</td>\n",
+ " <td>character varying</td>\n",
+ " <td>1.0</td>\n",
+ " <td>[u'accuracy']</td>\n",
+ " <td>0.883333325386</td>\n",
+ " <td>0.347580313683</td>\n",
+ " <td>[0.883333325386047]</td>\n",
+ " <td>[0.347580313682556]</td>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " <td>[10]</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'iris_train_packed', u'iris_model', u'class_text', u'attributes', u'model_arch_library', 1, u\" loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] \", u' batch_size=5, epochs=3 ', 10, None, 10, None, None, u'madlib_keras', 0.7900390625, datetime.datetime(2019, 7, 3, 10, 0, 52, 477704), datetime.datetime(2019, 7, 3, 10, 0, 58, 552077), [6.07431507110596], u'1.16', 3, [u'Iris-setosa', u'Iris-versicolor', u'Iris-virginica'], u'character varying', 1.0, [u'accu [...]
+ ]
+ },
+ "execution_count": 14,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM iris_model_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"eval\"></a>\n",
+ "# 5. Evaluate\n",
+ "\n",
+ "Now run evaluate using model we built above:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>loss</th>\n",
+ " <th>metric</th>\n",
+ " <th>metrics_type</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>0.43478807807</td>\n",
+ " <td>0.899999976158</td>\n",
+ " <td>[u'accuracy']</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(0.434788078069687, 0.899999976158142, [u'accuracy'])]"
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS iris_validate;\n",
+ "\n",
+ "SELECT madlib.madlib_keras_evaluate('iris_model', -- model\n",
+ " 'iris_test_packed', -- test table\n",
+ " 'iris_validate' -- output table\n",
+ " );\n",
+ "\n",
+ "SELECT * FROM iris_validate;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"pred\"></a>\n",
+ "# 6. Predict\n",
+ "\n",
+ "Now predict using model we built. We will use the validation data set for prediction as well, which is not usual but serves to show the syntax. The prediction is in the estimated_class_text column:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "30 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>estimated_class_text</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>17</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>24</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>30</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>39</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>40</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>43</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>50</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>52</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>54</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>55</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>59</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>61</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>76</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>81</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>83</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>86</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>88</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>92</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>93</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>95</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>100</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>104</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>107</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>111</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>117</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>134</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>140</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>143</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>150</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(4, u'Iris-setosa'),\n",
+ " (17, u'Iris-setosa'),\n",
+ " (24, u'Iris-setosa'),\n",
+ " (30, u'Iris-setosa'),\n",
+ " (39, u'Iris-setosa'),\n",
+ " (40, u'Iris-setosa'),\n",
+ " (43, u'Iris-setosa'),\n",
+ " (50, u'Iris-setosa'),\n",
+ " (52, u'Iris-versicolor'),\n",
+ " (54, u'Iris-virginica'),\n",
+ " (55, u'Iris-virginica'),\n",
+ " (59, u'Iris-versicolor'),\n",
+ " (61, u'Iris-versicolor'),\n",
+ " (76, u'Iris-versicolor'),\n",
+ " (81, u'Iris-versicolor'),\n",
+ " (83, u'Iris-versicolor'),\n",
+ " (86, u'Iris-versicolor'),\n",
+ " (88, u'Iris-virginica'),\n",
+ " (92, u'Iris-versicolor'),\n",
+ " (93, u'Iris-versicolor'),\n",
+ " (95, u'Iris-versicolor'),\n",
+ " (100, u'Iris-versicolor'),\n",
+ " (104, u'Iris-virginica'),\n",
+ " (107, u'Iris-virginica'),\n",
+ " (111, u'Iris-virginica'),\n",
+ " (117, u'Iris-virginica'),\n",
+ " (134, u'Iris-virginica'),\n",
+ " (140, u'Iris-virginica'),\n",
+ " (143, u'Iris-virginica'),\n",
+ " (150, u'Iris-virginica')]"
+ ]
+ },
+ "execution_count": 16,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS iris_predict;\n",
+ "\n",
+ "SELECT madlib.madlib_keras_predict('iris_model', -- model\n",
+ " 'iris_test', -- test_table\n",
+ " 'id', -- id column\n",
+ " 'attributes', -- independent var\n",
+ " 'iris_predict' -- output table\n",
+ " );\n",
+ "\n",
+ "SELECT * FROM iris_predict ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Count missclassifications"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>count</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(3L,)]"
+ ]
+ },
+ "execution_count": 17,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT COUNT(*) FROM iris_predict JOIN iris_test USING (id) \n",
+ "WHERE iris_predict.estimated_class_text != iris_test.class_text;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Percent missclassifications"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>test_accuracy_percent</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>90.00</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(Decimal('90.00'),)]"
+ ]
+ },
+ "execution_count": 18,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT round(count(*)*100/(150*0.2),2) as test_accuracy_percent from\n",
+ " (select iris_test.class_text as actual, iris_predict.estimated_class_text as estimated\n",
+ " from iris_predict inner join iris_test\n",
+ " on iris_test.id=iris_predict.id) q\n",
+ "WHERE q.actual=q.estimated;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"class2\"></a>\n",
+ "# Classification with Other Parameters\n",
+ "\n",
+ "<a id=\"val_dataset\"></a>\n",
+ "# 1. Validation dataset\n",
+ "Now use a validation dataset and compute metrics every 5th iteration using the 'metrics_compute_frequency' parameter. This can help reduce run time if you do not need metrics computed at every iteration."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>madlib_keras_fit</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td></td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[('',)]"
+ ]
+ },
+ "execution_count": 19,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS iris_model, iris_model_summary;\n",
+ "\n",
+ "SELECT madlib.madlib_keras_fit('iris_train_packed', -- source table\n",
+ " 'iris_model', -- model output table\n",
+ " 'model_arch_library', -- model arch table\n",
+ " 1, -- model arch id\n",
+ " $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$, -- compile_params\n",
+ " $$ batch_size=5, epochs=3 $$, -- fit_params\n",
+ " 10, -- num_iterations\n",
+ " 0, -- GPUs per host\n",
+ " 'iris_test_packed', -- validation dataset\n",
+ " 3, -- metrics compute frequency\n",
+ " FALSE, -- warm start\n",
+ " 'Sophie L.', -- name\n",
+ " 'Simple MLP for iris dataset' -- description\n",
+ " );"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "View the model summary:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>source_table</th>\n",
+ " <th>model</th>\n",
+ " <th>dependent_varname</th>\n",
+ " <th>independent_varname</th>\n",
+ " <th>model_arch_table</th>\n",
+ " <th>model_arch_id</th>\n",
+ " <th>compile_params</th>\n",
+ " <th>fit_params</th>\n",
+ " <th>num_iterations</th>\n",
+ " <th>validation_table</th>\n",
+ " <th>metrics_compute_frequency</th>\n",
+ " <th>name</th>\n",
+ " <th>description</th>\n",
+ " <th>model_type</th>\n",
+ " <th>model_size</th>\n",
+ " <th>start_training_time</th>\n",
+ " <th>end_training_time</th>\n",
+ " <th>metrics_elapsed_time</th>\n",
+ " <th>madlib_version</th>\n",
+ " <th>num_classes</th>\n",
+ " <th>class_values</th>\n",
+ " <th>dependent_vartype</th>\n",
+ " <th>normalizing_const</th>\n",
+ " <th>metrics_type</th>\n",
+ " <th>training_metrics_final</th>\n",
+ " <th>training_loss_final</th>\n",
+ " <th>training_metrics</th>\n",
+ " <th>training_loss</th>\n",
+ " <th>validation_metrics_final</th>\n",
+ " <th>validation_loss_final</th>\n",
+ " <th>validation_metrics</th>\n",
+ " <th>validation_loss</th>\n",
+ " <th>metrics_iters</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>iris_train_packed</td>\n",
+ " <td>iris_model</td>\n",
+ " <td>class_text</td>\n",
+ " <td>attributes</td>\n",
+ " <td>model_arch_library</td>\n",
+ " <td>1</td>\n",
+ " <td> loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] </td>\n",
+ " <td> batch_size=5, epochs=3 </td>\n",
+ " <td>10</td>\n",
+ " <td>iris_test_packed</td>\n",
+ " <td>3</td>\n",
+ " <td>Sophie L.</td>\n",
+ " <td>Simple MLP for iris dataset</td>\n",
+ " <td>madlib_keras</td>\n",
+ " <td>0.7900390625</td>\n",
+ " <td>2019-07-03 10:01:36.883439</td>\n",
+ " <td>2019-07-03 10:01:43.631505</td>\n",
+ " <td>[2.01867508888245, 3.89846897125244, 5.95652198791504, 6.74802613258362]</td>\n",
+ " <td>1.16</td>\n",
+ " <td>3</td>\n",
+ " <td>[u'Iris-setosa', u'Iris-versicolor', u'Iris-virginica']</td>\n",
+ " <td>character varying</td>\n",
+ " <td>1.0</td>\n",
+ " <td>[u'accuracy']</td>\n",
+ " <td>0.941666662693</td>\n",
+ " <td>0.371823817492</td>\n",
+ " <td>[0.741666674613953, 0.800000011920929, 0.916666686534882, 0.941666662693024]</td>\n",
+ " <td>[0.829422652721405, 0.578483581542969, 0.405027717351913, 0.371823817491531]</td>\n",
+ " <td>0.866666674614</td>\n",
+ " <td>0.492831081152</td>\n",
+ " <td>[0.566666662693024, 0.566666662693024, 0.833333313465118, 0.866666674613953]</td>\n",
+ " <td>[0.945741951465607, 0.735191941261292, 0.535544157028198, 0.492831081151962]</td>\n",
+ " <td>[3, 6, 9, 10]</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'iris_train_packed', u'iris_model', u'class_text', u'attributes', u'model_arch_library', 1, u\" loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] \", u' batch_size=5, epochs=3 ', 10, u'iris_test_packed', 3, u'Sophie L.', u'Simple MLP for iris dataset', u'madlib_keras', 0.7900390625, datetime.datetime(2019, 7, 3, 10, 1, 36, 883439), datetime.datetime(2019, 7, 3, 10, 1, 43, 631505), [2.01867508888245, 3.89846897125244, 5.95652198791504, 6.74802613258362], u [...]
+ ]
+ },
+ "execution_count": 20,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM iris_model_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"pred_prob\"></a>\n",
+ "# 2. Predict probabilities\n",
+ "Predict with probabilities for each class:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "30 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>prob_Iris-setosa</th>\n",
+ " <th>prob_Iris-versicolor</th>\n",
+ " <th>prob_Iris-virginica</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>0.89618546</td>\n",
+ " <td>0.10063652</td>\n",
+ " <td>0.0031779788</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>17</td>\n",
+ " <td>0.9432447</td>\n",
+ " <td>0.055964436</td>\n",
+ " <td>0.00079082645</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>24</td>\n",
+ " <td>0.87476385</td>\n",
+ " <td>0.121528275</td>\n",
+ " <td>0.0037078795</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>30</td>\n",
+ " <td>0.8998875</td>\n",
+ " <td>0.09730224</td>\n",
+ " <td>0.0028102635</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>39</td>\n",
+ " <td>0.89937997</td>\n",
+ " <td>0.09733549</td>\n",
+ " <td>0.003284483</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>40</td>\n",
+ " <td>0.9124368</td>\n",
+ " <td>0.08554972</td>\n",
+ " <td>0.0020134845</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>43</td>\n",
+ " <td>0.91849124</td>\n",
+ " <td>0.079369105</td>\n",
+ " <td>0.0021397525</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>50</td>\n",
+ " <td>0.909191</td>\n",
+ " <td>0.08853077</td>\n",
+ " <td>0.002278217</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>52</td>\n",
+ " <td>0.087838314</td>\n",
+ " <td>0.5821711</td>\n",
+ " <td>0.3299906</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>54</td>\n",
+ " <td>0.040643755</td>\n",
+ " <td>0.41653973</td>\n",
+ " <td>0.5428166</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>55</td>\n",
+ " <td>0.028924335</td>\n",
+ " <td>0.42645925</td>\n",
+ " <td>0.54461634</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>59</td>\n",
+ " <td>0.043529026</td>\n",
+ " <td>0.48816994</td>\n",
+ " <td>0.468301</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>61</td>\n",
+ " <td>0.06540182</td>\n",
+ " <td>0.44180837</td>\n",
+ " <td>0.4927898</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>76</td>\n",
+ " <td>0.059951168</td>\n",
+ " <td>0.5297594</td>\n",
+ " <td>0.41028935</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>81</td>\n",
+ " <td>0.0791236</td>\n",
+ " <td>0.49948806</td>\n",
+ " <td>0.42138833</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>83</td>\n",
+ " <td>0.103119485</td>\n",
+ " <td>0.54959977</td>\n",
+ " <td>0.34728083</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>86</td>\n",
+ " <td>0.15161447</td>\n",
+ " <td>0.6227044</td>\n",
+ " <td>0.22568108</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>88</td>\n",
+ " <td>0.013850493</td>\n",
+ " <td>0.3046302</td>\n",
+ " <td>0.6815193</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>92</td>\n",
+ " <td>0.070333414</td>\n",
+ " <td>0.545334</td>\n",
+ " <td>0.3843325</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>93</td>\n",
+ " <td>0.07561784</td>\n",
+ " <td>0.5140931</td>\n",
+ " <td>0.41028905</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>95</td>\n",
+ " <td>0.08128024</td>\n",
+ " <td>0.52868783</td>\n",
+ " <td>0.39003193</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>100</td>\n",
+ " <td>0.10329097</td>\n",
+ " <td>0.5575698</td>\n",
+ " <td>0.33913925</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>104</td>\n",
+ " <td>0.010344159</td>\n",
+ " <td>0.31838065</td>\n",
+ " <td>0.6712752</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>107</td>\n",
+ " <td>0.039944757</td>\n",
+ " <td>0.4255802</td>\n",
+ " <td>0.5344751</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>111</td>\n",
+ " <td>0.026599577</td>\n",
+ " <td>0.44653893</td>\n",
+ " <td>0.52686155</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>117</td>\n",
+ " <td>0.012989115</td>\n",
+ " <td>0.35122624</td>\n",
+ " <td>0.6357847</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>134</td>\n",
+ " <td>0.019927882</td>\n",
+ " <td>0.38532582</td>\n",
+ " <td>0.5947463</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>140</td>\n",
+ " <td>0.009378748</td>\n",
+ " <td>0.31938007</td>\n",
+ " <td>0.6712412</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>143</td>\n",
+ " <td>0.013671607</td>\n",
+ " <td>0.3261078</td>\n",
+ " <td>0.66022056</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>150</td>\n",
+ " <td>0.031985275</td>\n",
+ " <td>0.45162654</td>\n",
+ " <td>0.51638824</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(4, 0.89618546, 0.10063652, 0.0031779788),\n",
+ " (17, 0.9432447, 0.055964436, 0.00079082645),\n",
+ " (24, 0.87476385, 0.121528275, 0.0037078795),\n",
+ " (30, 0.8998875, 0.09730224, 0.0028102635),\n",
+ " (39, 0.89937997, 0.09733549, 0.003284483),\n",
+ " (40, 0.9124368, 0.08554972, 0.0020134845),\n",
+ " (43, 0.91849124, 0.079369105, 0.0021397525),\n",
+ " (50, 0.909191, 0.08853077, 0.002278217),\n",
+ " (52, 0.087838314, 0.5821711, 0.3299906),\n",
+ " (54, 0.040643755, 0.41653973, 0.5428166),\n",
+ " (55, 0.028924335, 0.42645925, 0.54461634),\n",
+ " (59, 0.043529026, 0.48816994, 0.468301),\n",
+ " (61, 0.06540182, 0.44180837, 0.4927898),\n",
+ " (76, 0.059951168, 0.5297594, 0.41028935),\n",
+ " (81, 0.0791236, 0.49948806, 0.42138833),\n",
+ " (83, 0.103119485, 0.54959977, 0.34728083),\n",
+ " (86, 0.15161447, 0.6227044, 0.22568108),\n",
+ " (88, 0.013850493, 0.3046302, 0.6815193),\n",
+ " (92, 0.070333414, 0.545334, 0.3843325),\n",
+ " (93, 0.07561784, 0.5140931, 0.41028905),\n",
+ " (95, 0.08128024, 0.52868783, 0.39003193),\n",
+ " (100, 0.10329097, 0.5575698, 0.33913925),\n",
+ " (104, 0.010344159, 0.31838065, 0.6712752),\n",
+ " (107, 0.039944757, 0.4255802, 0.5344751),\n",
+ " (111, 0.026599577, 0.44653893, 0.52686155),\n",
+ " (117, 0.012989115, 0.35122624, 0.6357847),\n",
+ " (134, 0.019927882, 0.38532582, 0.5947463),\n",
+ " (140, 0.009378748, 0.31938007, 0.6712412),\n",
+ " (143, 0.013671607, 0.3261078, 0.66022056),\n",
+ " (150, 0.031985275, 0.45162654, 0.51638824)]"
+ ]
+ },
+ "execution_count": 21,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS iris_predict;\n",
+ "\n",
+ "SELECT madlib.madlib_keras_predict('iris_model', -- model\n",
+ " 'iris_test', -- test_table\n",
+ " 'id', -- id column\n",
+ " 'attributes', -- independent var\n",
+ " 'iris_predict', -- output table\n",
+ " 'prob' -- response type\n",
+ " );\n",
+ "\n",
+ "SELECT * FROM iris_predict ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"warm_start\"></a>\n",
+ "# 3. Warm start\n",
+ "Next, use the warm_start parameter to continue learning, using the coefficients from the run above. Note that we don't drop the model table or model summary table:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>madlib_keras_fit</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td></td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[('',)]"
+ ]
+ },
+ "execution_count": 22,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT madlib.madlib_keras_fit('iris_train_packed', -- source table\n",
+ " 'iris_model', -- model output table\n",
+ " 'model_arch_library', -- model arch table\n",
+ " 1, -- model arch id\n",
+ " $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$, -- compile_params\n",
+ " $$ batch_size=5, epochs=3 $$, -- fit_params\n",
+ " 5, -- num_iterations\n",
+ " 0, -- GPUs per host\n",
+ " 'iris_test_packed', -- validation dataset\n",
+ " 1, -- metrics compute frequency\n",
+ " TRUE, -- warm start\n",
+ " 'Sophie L.', -- name \n",
+ " 'Simple MLP for iris dataset' -- description\n",
+ " );"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "In the summary table note that the loss and accuracy values pick up from where the previous run left off:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 23,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>source_table</th>\n",
+ " <th>model</th>\n",
+ " <th>dependent_varname</th>\n",
+ " <th>independent_varname</th>\n",
+ " <th>model_arch_table</th>\n",
+ " <th>model_arch_id</th>\n",
+ " <th>compile_params</th>\n",
+ " <th>fit_params</th>\n",
+ " <th>num_iterations</th>\n",
+ " <th>validation_table</th>\n",
+ " <th>metrics_compute_frequency</th>\n",
+ " <th>name</th>\n",
+ " <th>description</th>\n",
+ " <th>model_type</th>\n",
+ " <th>model_size</th>\n",
+ " <th>start_training_time</th>\n",
+ " <th>end_training_time</th>\n",
+ " <th>metrics_elapsed_time</th>\n",
+ " <th>madlib_version</th>\n",
+ " <th>num_classes</th>\n",
+ " <th>class_values</th>\n",
+ " <th>dependent_vartype</th>\n",
+ " <th>normalizing_const</th>\n",
+ " <th>metrics_type</th>\n",
+ " <th>training_metrics_final</th>\n",
+ " <th>training_loss_final</th>\n",
+ " <th>training_metrics</th>\n",
+ " <th>training_loss</th>\n",
+ " <th>validation_metrics_final</th>\n",
+ " <th>validation_loss_final</th>\n",
+ " <th>validation_metrics</th>\n",
+ " <th>validation_loss</th>\n",
+ " <th>metrics_iters</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>iris_train_packed</td>\n",
+ " <td>iris_model</td>\n",
+ " <td>class_text</td>\n",
+ " <td>attributes</td>\n",
+ " <td>model_arch_library</td>\n",
+ " <td>1</td>\n",
+ " <td> loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] </td>\n",
+ " <td> batch_size=5, epochs=3 </td>\n",
+ " <td>5</td>\n",
+ " <td>iris_test_packed</td>\n",
+ " <td>1</td>\n",
+ " <td>Sophie L.</td>\n",
+ " <td>Simple MLP for iris dataset</td>\n",
+ " <td>madlib_keras</td>\n",
+ " <td>0.7900390625</td>\n",
+ " <td>2019-07-03 10:01:52.495132</td>\n",
+ " <td>2019-07-03 10:01:56.620862</td>\n",
+ " <td>[0.777317047119141, 1.62117600440979, 2.46934199333191, 3.33750104904175, 4.12567687034607]</td>\n",
+ " <td>1.16</td>\n",
+ " <td>3</td>\n",
+ " <td>[u'Iris-setosa', u'Iris-versicolor', u'Iris-virginica']</td>\n",
+ " <td>character varying</td>\n",
+ " <td>1.0</td>\n",
+ " <td>[u'accuracy']</td>\n",
+ " <td>0.941666662693</td>\n",
+ " <td>0.256453901529</td>\n",
+ " <td>[0.941666662693024, 0.941666662693024, 0.941666662693024, 0.941666662693024, 0.941666662693024]</td>\n",
+ " <td>[0.341408282518387, 0.313415616750717, 0.293370455503464, 0.273623049259186, 0.256453901529312]</td>\n",
+ " <td>0.899999976158</td>\n",
+ " <td>0.36899459362</td>\n",
+ " <td>[0.833333313465118, 0.866666674613953, 0.899999976158142, 0.899999976158142, 0.899999976158142]</td>\n",
+ " <td>[0.468038767576218, 0.429012358188629, 0.406685948371887, 0.386049389839172, 0.3689945936203]</td>\n",
+ " <td>[1, 2, 3, 4, 5]</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'iris_train_packed', u'iris_model', u'class_text', u'attributes', u'model_arch_library', 1, u\" loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] \", u' batch_size=5, epochs=3 ', 5, u'iris_test_packed', 1, u'Sophie L.', u'Simple MLP for iris dataset', u'madlib_keras', 0.7900390625, datetime.datetime(2019, 7, 3, 10, 1, 52, 495132), datetime.datetime(2019, 7, 3, 10, 1, 56, 620862), [0.777317047119141, 1.62117600440979, 2.46934199333191, 3.33750104904175, 4. [...]
+ ]
+ },
+ "execution_count": 23,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM iris_model_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"transfer_learn\"></a>\n",
+ "# Transfer learning\n",
+ "\n",
+ "<a id=\"load2\"></a>\n",
+ "# 1. Define and load model architecture with some layers frozen\n",
+ "Here we want to start with initial weights from a pre-trained model rather than training from scratch. We also want to use a model architecture with the earlier feature layer(s) frozen to save on training time. The example below is somewhat contrived but gives you the idea of the steps.\n",
+ "\n",
+ "First define a model architecture with the 1st hidden layer frozen:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 24,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "_________________________________________________________________\n",
+ "Layer (type) Output Shape Param # \n",
+ "=================================================================\n",
+ "dense_4 (Dense) (None, 10) 50 \n",
+ "_________________________________________________________________\n",
+ "dense_5 (Dense) (None, 10) 110 \n",
+ "_________________________________________________________________\n",
+ "dense_6 (Dense) (None, 3) 33 \n",
+ "=================================================================\n",
+ "Total params: 193\n",
+ "Trainable params: 143\n",
+ "Non-trainable params: 50\n",
+ "_________________________________________________________________\n"
+ ]
+ }
+ ],
+ "source": [
+ "model_transfer = Sequential()\n",
+ "model_transfer.add(Dense(10, activation='relu', input_shape=(4,), trainable=False))\n",
+ "model_transfer.add(Dense(10, activation='relu'))\n",
+ "model_transfer.add(Dense(3, activation='softmax'))\n",
+ " \n",
+ "model_transfer.summary()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'{\"class_name\": \"Sequential\", \"keras_version\": \"2.1.6\", \"config\": [{\"class_name\": \"Dense\", \"config\": {\"kernel_initializer\": {\"class_name\": \"VarianceScaling\", \"config\": {\"distribution\": \"uniform\", \"scale\": 1.0, \"seed\": null, \"mode\": \"fan_avg\"}}, \"name\": \"dense_4\", \"kernel_constraint\": null, \"bias_regularizer\": null, \"bias_constraint\": null, \"dtype\": \"float32\", \"activation\": \"relu\", \"trainable\": false, \"kernel_regularizer\": [...]
+ ]
+ },
+ "execution_count": 25,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "model_transfer.to_json()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Load transfer model into model architecture table"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n",
+ "2 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>model_id</th>\n",
+ " <th>model_arch</th>\n",
+ " <th>model_weights</th>\n",
+ " <th>name</th>\n",
+ " <th>description</th>\n",
+ " <th>__internal_madlib_id__</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>{u'class_name': u'Sequential', u'keras_version': u'2.1.6', u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer': {u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform', u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1', u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint': None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True, u'kernel_regularizer': None, u'bias_initia [...]
+ " <td>None</td>\n",
+ " <td>Sophie</td>\n",
+ " <td>A simple model</td>\n",
+ " <td>__madlib_temp_62550369_1562173248_86696923__</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>{u'class_name': u'Sequential', u'keras_version': u'2.1.6', u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer': {u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform', u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_2', u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint': None, u'dtype': u'float32', u'activation': u'relu', u'trainable': False, u'kernel_regularizer': None, u'bias_initi [...]
+ " <td>None</td>\n",
+ " <td>Maria</td>\n",
+ " <td>A transfer model</td>\n",
+ " <td>__madlib_temp_71661491_1562173329_6849629__</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, {u'class_name': u'Sequential', u'keras_version': u'2.1.6', u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer': {u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform', u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1', u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint': None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True, u'kernel_regularizer': None, u'bias_initializer': [...]
+ " (2, {u'class_name': u'Sequential', u'keras_version': u'2.1.6', u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer': {u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform', u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_2', u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint': None, u'dtype': u'float32', u'activation': u'relu', u'trainable': False, u'kernel_regularizer': None, u'bias_initializer' [...]
+ ]
+ },
+ "execution_count": 26,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT madlib.load_keras_model('model_arch_library', -- Output table,\n",
+ " \n",
+ "$$\n",
+ "{\"class_name\": \"Sequential\", \"keras_version\": \"2.1.6\", \"config\": [{\"class_name\": \"Dense\", \"config\": {\"kernel_initializer\": {\"class_name\": \"VarianceScaling\", \"config\": {\"distribution\": \"uniform\", \"scale\": 1.0, \"seed\": null, \"mode\": \"fan_avg\"}}, \"name\": \"dense_2\", \"kernel_constraint\": null, \"bias_regularizer\": null, \"bias_constraint\": null, \"dtype\": \"float32\", \"activation\": \"relu\", \"trainable\": false, \"kernel_regularizer\": null [...]
+ "$$\n",
+ "::json, -- JSON blob\n",
+ " NULL, -- Weights\n",
+ " 'Maria', -- Name\n",
+ " 'A transfer model' -- Descr\n",
+ ");\n",
+ "\n",
+ "SELECT * FROM model_arch_library;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"train2\"></a>\n",
+ "# 2. Train transfer model\n",
+ "\n",
+ "Fetch the weights from a previous MADlib run. (Normally these would be downloaded from a source that trained the same model architecture on a related dataset.)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "[]"
+ ]
+ },
+ "execution_count": 27,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "UPDATE model_arch_library SET model_weights = model_data FROM iris_model WHERE model_id = 2;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Now train the model using the transfer model and the pre-trained weights:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 28,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>madlib_keras_fit</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td></td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[('',)]"
+ ]
+ },
+ "execution_count": 28,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS iris_model, iris_model_summary;\n",
+ "\n",
+ "SELECT madlib.madlib_keras_fit('iris_train_packed', -- source table\n",
+ " 'iris_model', -- model output table\n",
+ " 'model_arch_library', -- model arch table\n",
+ " 2, -- model arch id\n",
+ " $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$, -- compile_params\n",
+ " $$ batch_size=5, epochs=3 $$, -- fit_params\n",
+ " 10 -- num_iterations\n",
+ " );"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 29,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>source_table</th>\n",
+ " <th>model</th>\n",
+ " <th>dependent_varname</th>\n",
+ " <th>independent_varname</th>\n",
+ " <th>model_arch_table</th>\n",
+ " <th>model_arch_id</th>\n",
+ " <th>compile_params</th>\n",
+ " <th>fit_params</th>\n",
+ " <th>num_iterations</th>\n",
+ " <th>validation_table</th>\n",
+ " <th>metrics_compute_frequency</th>\n",
+ " <th>name</th>\n",
+ " <th>description</th>\n",
+ " <th>model_type</th>\n",
+ " <th>model_size</th>\n",
+ " <th>start_training_time</th>\n",
+ " <th>end_training_time</th>\n",
+ " <th>metrics_elapsed_time</th>\n",
+ " <th>madlib_version</th>\n",
+ " <th>num_classes</th>\n",
+ " <th>class_values</th>\n",
+ " <th>dependent_vartype</th>\n",
+ " <th>normalizing_const</th>\n",
+ " <th>metrics_type</th>\n",
+ " <th>training_metrics_final</th>\n",
+ " <th>training_loss_final</th>\n",
+ " <th>training_metrics</th>\n",
+ " <th>training_loss</th>\n",
+ " <th>validation_metrics_final</th>\n",
+ " <th>validation_loss_final</th>\n",
+ " <th>validation_metrics</th>\n",
+ " <th>validation_loss</th>\n",
+ " <th>metrics_iters</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>iris_train_packed</td>\n",
+ " <td>iris_model</td>\n",
+ " <td>class_text</td>\n",
+ " <td>attributes</td>\n",
+ " <td>model_arch_library</td>\n",
+ " <td>2</td>\n",
+ " <td> loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] </td>\n",
+ " <td> batch_size=5, epochs=3 </td>\n",
+ " <td>10</td>\n",
+ " <td>None</td>\n",
+ " <td>10</td>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " <td>madlib_keras</td>\n",
+ " <td>0.7900390625</td>\n",
+ " <td>2019-07-03 10:02:15.366826</td>\n",
+ " <td>2019-07-03 10:02:20.437775</td>\n",
+ " <td>[5.07090902328491]</td>\n",
+ " <td>1.16</td>\n",
+ " <td>3</td>\n",
+ " <td>[u'Iris-setosa', u'Iris-versicolor', u'Iris-virginica']</td>\n",
+ " <td>character varying</td>\n",
+ " <td>1.0</td>\n",
+ " <td>[u'accuracy']</td>\n",
+ " <td>0.933333337307</td>\n",
+ " <td>0.188783079386</td>\n",
+ " <td>[0.933333337306976]</td>\n",
+ " <td>[0.188783079385757]</td>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " <td>[10]</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'iris_train_packed', u'iris_model', u'class_text', u'attributes', u'model_arch_library', 2, u\" loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] \", u' batch_size=5, epochs=3 ', 10, None, 10, None, None, u'madlib_keras', 0.7900390625, datetime.datetime(2019, 7, 3, 10, 2, 15, 366826), datetime.datetime(2019, 7, 3, 10, 2, 20, 437775), [5.07090902328491], u'1.16', 3, [u'Iris-setosa', u'Iris-versicolor', u'Iris-virginica'], u'character varying', 1.0, [u'accu [...]
+ ]
+ },
+ "execution_count": 29,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM iris_model_summary;"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 2",
+ "language": "python",
+ "name": "python2"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.10"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
diff --git a/community-artifacts/Deep-learning/MADlib-Keras-cifar10-cnn-v1.ipynb b/community-artifacts/Deep-learning/MADlib-Keras-cifar10-cnn-v1.ipynb
new file mode 100644
index 0000000..6ba5acc
--- /dev/null
+++ b/community-artifacts/Deep-learning/MADlib-Keras-cifar10-cnn-v1.ipynb
@@ -0,0 +1,1249 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# CNN Using Keras and MADlib\n",
+ "\n",
+ "E2E classification example using MADlib calling a Keras CNN. Based on model architecture in https://keras.io/examples/cifar10_cnn/\n",
+ "\n",
+ "## Table of contents\n",
+ "<a href=\"#import_libraries\">1. Import libraries</a>\n",
+ "\n",
+ "<a href=\"#load_and_prepare_data\">2. Load dataset into table</a>\n",
+ "\n",
+ "<a href=\"#image_preproc\">3. Call image preprocessor</a>\n",
+ "\n",
+ "<a href=\"#define_and_load_model\">4. Define and load model architecture</a>\n",
+ "\n",
+ "<a href=\"#train\">5. Train</a>\n",
+ "\n",
+ "<a href=\"#plot\">6. Plots by iteration and time</a>"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/config.py:13: ShimWarning: The `IPython.config` package has been deprecated since IPython 4.0. You should import from traitlets.config instead.\n",
+ " \"You should import from traitlets.config instead.\", ShimWarning)\n",
+ "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/utils/traitlets.py:5: UserWarning: IPython.utils.traitlets has moved to a top-level traitlets package.\n",
+ " warn(\"IPython.utils.traitlets has moved to a top-level traitlets package.\")\n"
+ ]
+ }
+ ],
+ "source": [
+ "%load_ext sql"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "u'Connected: gpadmin@cifar10'"
+ ]
+ },
+ "execution_count": 2,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Greenplum Database 5.x on GCP for deep learning (PM demo machine)\n",
+ "%sql postgresql://gpadmin@35.239.240.26:5432/madlib\n",
+ " \n",
+ "# Greenplum Database 5.x on GCP (dev machine)\n",
+ "#%sql postgresql://gpadmin@35.247.88.252:5432/cifar10\n",
+ " \n",
+ "# PostgreSQL local\n",
+ "#%sql postgresql://fmcquillan@localhost:5432/madlib"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>version</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>PostgreSQL 8.3.23 (Greenplum Database 5.7.0 build commit:f7c6eb5cc61b25a7ff9c5a657d6f903befbae013) on x86_64-pc-linux-gnu, compiled by GCC gcc (GCC) 6.2.0, 64-bit compiled on Mar 30 2018 14:19:06</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'PostgreSQL 8.3.23 (Greenplum Database 5.7.0 build commit:f7c6eb5cc61b25a7ff9c5a657d6f903befbae013) on x86_64-pc-linux-gnu, compiled by GCC gcc (GCC) 6.2.0, 64-bit compiled on Mar 30 2018 14:19:06',)]"
+ ]
+ },
+ "execution_count": 7,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%sql select madlib.version();\n",
+ "#%sql select version();"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"import_libraries\"></a>\n",
+ "# 1. Import libraries\n",
+ "From https://keras.io/examples/mnist_transfer_cnn/ import libraries and define some params"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Using TensorFlow backend.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Couldn't import dot_parser, loading of dot files will not be possible.\n"
+ ]
+ }
+ ],
+ "source": [
+ "from __future__ import print_function\n",
+ "import keras\n",
+ "from keras.datasets import cifar10\n",
+ "from keras.preprocessing.image import ImageDataGenerator\n",
+ "from keras.models import Sequential\n",
+ "from keras.layers import Dense, Dropout, Activation, Flatten\n",
+ "from keras.layers import Conv2D, MaxPooling2D\n",
+ "import os\n",
+ "\n",
+ "batch_size = 32\n",
+ "num_classes = 10\n",
+ "epochs = 100"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Others needed in this workbook"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "import numpy as np\n",
+ "import sys\n",
+ "import os\n",
+ "from matplotlib import pyplot as plt"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"load_and_prepare_data\"></a>\n",
+ "# 2. Set up image loader and load dataset into table\n",
+ "\n",
+ "First set up image loader."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "madlib_site_dir = '/Users/fmcquillan/Documents/Product/MADlib/Demos/data'\n",
+ "sys.path.append(madlib_site_dir)\n",
+ "\n",
+ "# Import image loader module\n",
+ "from madlib_image_loader import ImageLoader, DbCredentials\n",
+ "\n",
+ "# Specify database credentials, for connecting to db\n",
+ "db_creds = DbCredentials(user='gpadmin',\n",
+ " host='35.239.240.26',\n",
+ " port='5432',\n",
+ " password='')\n",
+ "\n",
+ "# Specify database credentials, for connecting to db\n",
+ "#db_creds = DbCredentials(user='gpadmin', \n",
+ "# db_name='cifar10',\n",
+ "# host='35.247.88.252',\n",
+ "# port='5432',\n",
+ "# password='')\n",
+ "\n",
+ "# Initialize ImageLoader (increase num_workers to run faster)\n",
+ "iloader = ImageLoader(num_workers=5, db_creds=db_creds)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "First load CIFAR-10 data from Keras consisting of 50,000 32x32 color training images, labeled over 10 categories, and 10,000 test images."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "MainProcess: Connected to cifar10 db.\n",
+ "Executing: CREATE TABLE cifar_10_train_data (id SERIAL, x REAL[], y TEXT)\n",
+ "CREATE TABLE\n",
+ "Created table cifar_10_train_data in cifar10 db\n",
+ "Spawning 5 workers...\n",
+ "Initializing PoolWorker-1 [pid 38560]\n",
+ "PoolWorker-1: Created temporary directory PoolWorker-1\n",
+ "Initializing PoolWorker-2 [pid 38561]\n",
+ "PoolWorker-2: Created temporary directory PoolWorker-2\n",
+ "Initializing PoolWorker-3 [pid 38562]\n",
+ "PoolWorker-3: Created temporary directory PoolWorker-3\n",
+ "Initializing PoolWorker-4 [pid 38563]\n",
+ "PoolWorker-4: Created temporary directory PoolWorker-4\n",
+ "Initializing PoolWorker-5 [pid 38564]\n",
+ "PoolWorker-5: Created temporary directory PoolWorker-5\n",
+ "PoolWorker-1: Connected to cifar10 db.\n",
+ "PoolWorker-2: Connected to cifar10 db.\n",
+ "PoolWorker-3: Connected to cifar10 db.\n",
+ "PoolWorker-4: Connected to cifar10 db.\n",
+ "PoolWorker-5: Connected to cifar10 db.\n",
+ "PoolWorker-1: Wrote 1000 images to /tmp/madlib_ASclSSNtOB/cifar_10_train_data0000.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to /tmp/madlib_va8z5K5H7g/cifar_10_train_data0000.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to /tmp/madlib_JcEAyIUK2N/cifar_10_train_data0000.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to /tmp/madlib_x6Z6qduBSV/cifar_10_train_data0000.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to /tmp/madlib_MwcNPSdCVQ/cifar_10_train_data0000.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to /tmp/madlib_ASclSSNtOB/cifar_10_train_data0001.tmp\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-3: Wrote 1000 images to /tmp/madlib_JcEAyIUK2N/cifar_10_train_data0001.tmp\n",
+ "PoolWorker-1: Wrote 1000 images to /tmp/madlib_ASclSSNtOB/cifar_10_train_data0002.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to /tmp/madlib_va8z5K5H7g/cifar_10_train_data0001.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to /tmp/madlib_ASclSSNtOB/cifar_10_train_data0003.tmp\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-3: Wrote 1000 images to /tmp/madlib_JcEAyIUK2N/cifar_10_train_data0002.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to /tmp/madlib_x6Z6qduBSV/cifar_10_train_data0001.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-2: Wrote 1000 images to /tmp/madlib_va8z5K5H7g/cifar_10_train_data0002.tmp\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to /tmp/madlib_ASclSSNtOB/cifar_10_train_data0004.tmp\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-3: Wrote 1000 images to /tmp/madlib_JcEAyIUK2N/cifar_10_train_data0003.tmp\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-2: Wrote 1000 images to /tmp/madlib_va8z5K5H7g/cifar_10_train_data0003.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to /tmp/madlib_x6Z6qduBSV/cifar_10_train_data0002.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to /tmp/madlib_MwcNPSdCVQ/cifar_10_train_data0001.tmp\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to /tmp/madlib_ASclSSNtOB/cifar_10_train_data0005.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to /tmp/madlib_JcEAyIUK2N/cifar_10_train_data0004.tmp\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-2: Wrote 1000 images to /tmp/madlib_va8z5K5H7g/cifar_10_train_data0004.tmp\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-4: Wrote 1000 images to /tmp/madlib_x6Z6qduBSV/cifar_10_train_data0003.tmp\n",
+ "PoolWorker-1: Wrote 1000 images to /tmp/madlib_ASclSSNtOB/cifar_10_train_data0006.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to /tmp/madlib_JcEAyIUK2N/cifar_10_train_data0005.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to /tmp/madlib_MwcNPSdCVQ/cifar_10_train_data0002.tmp\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-2: Wrote 1000 images to /tmp/madlib_va8z5K5H7g/cifar_10_train_data0005.tmp\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-4: Wrote 1000 images to /tmp/madlib_x6Z6qduBSV/cifar_10_train_data0004.tmp\n",
+ "PoolWorker-1: Wrote 1000 images to /tmp/madlib_ASclSSNtOB/cifar_10_train_data0007.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to /tmp/madlib_JcEAyIUK2N/cifar_10_train_data0006.tmp\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-2: Wrote 1000 images to /tmp/madlib_va8z5K5H7g/cifar_10_train_data0006.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to /tmp/madlib_MwcNPSdCVQ/cifar_10_train_data0003.tmp\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-4: Wrote 1000 images to /tmp/madlib_x6Z6qduBSV/cifar_10_train_data0005.tmp\n",
+ "PoolWorker-1: Wrote 1000 images to /tmp/madlib_ASclSSNtOB/cifar_10_train_data0008.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to /tmp/madlib_JcEAyIUK2N/cifar_10_train_data0007.tmp\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-2: Wrote 1000 images to /tmp/madlib_va8z5K5H7g/cifar_10_train_data0007.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-5: Wrote 1000 images to /tmp/madlib_MwcNPSdCVQ/cifar_10_train_data0004.tmp\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-4: Wrote 1000 images to /tmp/madlib_x6Z6qduBSV/cifar_10_train_data0006.tmp\n",
+ "PoolWorker-1: Wrote 1000 images to /tmp/madlib_ASclSSNtOB/cifar_10_train_data0009.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to /tmp/madlib_JcEAyIUK2N/cifar_10_train_data0008.tmp\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-2: Wrote 1000 images to /tmp/madlib_va8z5K5H7g/cifar_10_train_data0008.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to /tmp/madlib_MwcNPSdCVQ/cifar_10_train_data0005.tmp\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-4: Wrote 1000 images to /tmp/madlib_x6Z6qduBSV/cifar_10_train_data0007.tmp\n",
+ "PoolWorker-1: Wrote 1000 images to /tmp/madlib_ASclSSNtOB/cifar_10_train_data0010.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to /tmp/madlib_JcEAyIUK2N/cifar_10_train_data0009.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to /tmp/madlib_MwcNPSdCVQ/cifar_10_train_data0006.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to /tmp/madlib_ASclSSNtOB/cifar_10_train_data0011.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to /tmp/madlib_x6Z6qduBSV/cifar_10_train_data0008.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to /tmp/madlib_JcEAyIUK2N/cifar_10_train_data0010.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to /tmp/madlib_MwcNPSdCVQ/cifar_10_train_data0007.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-3: Wrote 1000 images to /tmp/madlib_JcEAyIUK2N/cifar_10_train_data0011.tmp\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-5: Removed temporary directory PoolWorker-5\n",
+ "PoolWorker-2: Removed temporary directory PoolWorker-2\n",
+ "PoolWorker-4: Removed temporary directory PoolWorker-4\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "PoolWorker-1: Removed temporary directory PoolWorker-1\n",
+ "PoolWorker-3: Removed temporary directory PoolWorker-3\n",
+ "Done! Loaded 50000 images in 48.6765198708s\n",
+ "MainProcess: Connected to cifar10 db.\n",
+ "Executing: CREATE TABLE cifar_10_test_data (id SERIAL, x REAL[], y TEXT)\n",
+ "CREATE TABLE\n",
+ "Created table cifar_10_test_data in cifar10 db\n",
+ "Spawning 5 workers...\n",
+ "Initializing PoolWorker-6 [pid 38571]\n",
+ "PoolWorker-6: Created temporary directory PoolWorker-6\n",
+ "Initializing PoolWorker-7 [pid 38572]\n",
+ "PoolWorker-7: Created temporary directory PoolWorker-7\n",
+ "Initializing PoolWorker-8 [pid 38573]\n",
+ "PoolWorker-8: Created temporary directory PoolWorker-8\n",
+ "Initializing PoolWorker-9 [pid 38574]\n",
+ "PoolWorker-9: Created temporary directory PoolWorker-9\n",
+ "Initializing PoolWorker-10 [pid 38575]\n",
+ "PoolWorker-10: Created temporary directory PoolWorker-10\n",
+ "PoolWorker-6: Connected to cifar10 db.\n",
+ "PoolWorker-7: Connected to cifar10 db.\n",
+ "PoolWorker-8: Connected to cifar10 db.\n",
+ "PoolWorker-9: Connected to cifar10 db.\n",
+ "PoolWorker-10: Connected to cifar10 db.\n",
+ "PoolWorker-6: Wrote 1000 images to /tmp/madlib_aDPItemTNl/cifar_10_test_data0000.tmp\n",
+ "PoolWorker-7: Wrote 1000 images to /tmp/madlib_pvthRdGxuQ/cifar_10_test_data0000.tmp\n",
+ "PoolWorker-8: Wrote 1000 images to /tmp/madlib_wDeaZQyk8B/cifar_10_test_data0000.tmp\n",
+ "PoolWorker-9: Wrote 1000 images to /tmp/madlib_C43d97TTQR/cifar_10_test_data0000.tmp\n",
+ "PoolWorker-10: Wrote 1000 images to /tmp/madlib_z9IdD2HoMV/cifar_10_test_data0000.tmp\n",
+ "PoolWorker-7: Loaded 1000 images into cifar_10_test_data\n",
+ "PoolWorker-9: Loaded 1000 images into cifar_10_test_data\n",
+ "PoolWorker-6: Loaded 1000 images into cifar_10_test_data\n",
+ "PoolWorker-7: Wrote 1000 images to /tmp/madlib_pvthRdGxuQ/cifar_10_test_data0001.tmp\n",
+ "PoolWorker-9: Wrote 1000 images to /tmp/madlib_C43d97TTQR/cifar_10_test_data0001.tmp\n",
+ "PoolWorker-10: Loaded 1000 images into cifar_10_test_data\n",
+ "PoolWorker-6: Wrote 1000 images to /tmp/madlib_aDPItemTNl/cifar_10_test_data0001.tmp\n",
+ "PoolWorker-8: Loaded 1000 images into cifar_10_test_data\n",
+ "PoolWorker-7: Loaded 1000 images into cifar_10_test_data\n",
+ "PoolWorker-9: Loaded 1000 images into cifar_10_test_data\n",
+ "PoolWorker-10: Wrote 1000 images to /tmp/madlib_z9IdD2HoMV/cifar_10_test_data0001.tmp\n",
+ "PoolWorker-6: Loaded 1000 images into cifar_10_test_data\n",
+ "PoolWorker-8: Wrote 1000 images to /tmp/madlib_wDeaZQyk8B/cifar_10_test_data0001.tmp\n",
+ "PoolWorker-10: Loaded 1000 images into cifar_10_test_data\n",
+ "PoolWorker-8: Loaded 1000 images into cifar_10_test_data\n",
+ "PoolWorker-9: Removed temporary directory PoolWorker-9\n",
+ "PoolWorker-6: Removed temporary directory PoolWorker-6\n",
+ "PoolWorker-10: Removed temporary directory PoolWorker-10\n",
+ "PoolWorker-8: Removed temporary directory PoolWorker-8\n",
+ "PoolWorker-7: Removed temporary directory PoolWorker-7\n",
+ "Done! Loaded 10000 images in 11.5516760349s\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Load dataset into np array\n",
+ "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
+ "\n",
+ "#%sql DROP TABLE IF EXISTS cifar_10_train_data, cifar_10_test_data;\n",
+ "\n",
+ "# Save images to temporary directories and load into database\n",
+ "iloader.load_np_array_to_table(x_train, y_train, 'cifar_10_train_data', append=False, img_names=None)\n",
+ "iloader.load_np_array_to_table(x_test, y_test, 'cifar_10_test_data', append=False, img_names=None)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>count</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>50000</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(50000L,)]"
+ ]
+ },
+ "execution_count": 14,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%sql select count(*) from cifar_10_train_data;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"image_preproc\"></a>\n",
+ "# 3. Call image preprocessor\n",
+ "\n",
+ "Transforms from one image per row to multiple images per row for batch optimization. Also normalizes and one-hot encodes.\n",
+ "\n",
+ "Training data"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>source_table</th>\n",
+ " <th>output_table</th>\n",
+ " <th>dependent_varname</th>\n",
+ " <th>independent_varname</th>\n",
+ " <th>dependent_vartype</th>\n",
+ " <th>class_values</th>\n",
+ " <th>buffer_size</th>\n",
+ " <th>normalizing_const</th>\n",
+ " <th>num_classes</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>cifar_10_train_data</td>\n",
+ " <td>cifar_10_train_data_packed</td>\n",
+ " <td>y</td>\n",
+ " <td>x</td>\n",
+ " <td>text</td>\n",
+ " <td>[u'0', u'1', u'2', u'3', u'4', u'5', u'6', u'7', u'8', u'9']</td>\n",
+ " <td>24415</td>\n",
+ " <td>255.0</td>\n",
+ " <td>10</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'cifar_10_train_data', u'cifar_10_train_data_packed', u'y', u'x', u'text', [u'0', u'1', u'2', u'3', u'4', u'5', u'6', u'7', u'8', u'9'], 24415, 255.0, 10)]"
+ ]
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS cifar_10_train_data_packed, cifar_10_train_data_packed_summary;\n",
+ "\n",
+ "SELECT madlib.training_preprocessor_dl('cifar_10_train_data', -- Source table\n",
+ " 'cifar_10_train_data_packed', -- Output table\n",
+ " 'y', -- Dependent variable\n",
+ " 'x', -- Independent variable\n",
+ " 1000, -- Buffer size\n",
+ " 255 -- Normalizing constant\n",
+ " );\n",
+ "\n",
+ "SELECT * FROM cifar_10_train_data_packed_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Test data"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>source_table</th>\n",
+ " <th>output_table</th>\n",
+ " <th>dependent_varname</th>\n",
+ " <th>independent_varname</th>\n",
+ " <th>dependent_vartype</th>\n",
+ " <th>class_values</th>\n",
+ " <th>buffer_size</th>\n",
+ " <th>normalizing_const</th>\n",
+ " <th>num_classes</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>cifar_10_test_data</td>\n",
+ " <td>cifar_10_test_data_packed</td>\n",
+ " <td>y</td>\n",
+ " <td>x</td>\n",
+ " <td>text</td>\n",
+ " <td>[u'0', u'1', u'2', u'3', u'4', u'5', u'6', u'7', u'8', u'9']</td>\n",
+ " <td>5000</td>\n",
+ " <td>255.0</td>\n",
+ " <td>10</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'cifar_10_test_data', u'cifar_10_test_data_packed', u'y', u'x', u'text', [u'0', u'1', u'2', u'3', u'4', u'5', u'6', u'7', u'8', u'9'], 5000, 255.0, 10)]"
+ ]
+ },
+ "execution_count": 11,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS cifar_10_test_data_packed, cifar_10_test_data_packed_summary;\n",
+ "\n",
+ "SELECT madlib.validation_preprocessor_dl('cifar_10_test_data', -- Source table\n",
+ " 'cifar_10_test_data_packed', -- Output table\n",
+ " 'y', -- Dependent variable\n",
+ " 'x', -- Independent variable\n",
+ " 'cifar_10_train_data_packed', -- Training preproc table\n",
+ " 1000 -- Buffer size\n",
+ " );\n",
+ "\n",
+ "SELECT * FROM cifar_10_test_data_packed_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"define_and_load_model\"></a>\n",
+ "# 4. Define and load model architecture\n",
+ "\n",
+ "Model architecture from https://keras.io/examples/cifar10_cnn/"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "_________________________________________________________________\n",
+ "Layer (type) Output Shape Param # \n",
+ "=================================================================\n",
+ "conv2d_1 (Conv2D) (None, 32, 32, 32) 896 \n",
+ "_________________________________________________________________\n",
+ "activation_1 (Activation) (None, 32, 32, 32) 0 \n",
+ "_________________________________________________________________\n",
+ "conv2d_2 (Conv2D) (None, 30, 30, 32) 9248 \n",
+ "_________________________________________________________________\n",
+ "activation_2 (Activation) (None, 30, 30, 32) 0 \n",
+ "_________________________________________________________________\n",
+ "max_pooling2d_1 (MaxPooling2 (None, 15, 15, 32) 0 \n",
+ "_________________________________________________________________\n",
+ "dropout_1 (Dropout) (None, 15, 15, 32) 0 \n",
+ "_________________________________________________________________\n",
+ "conv2d_3 (Conv2D) (None, 15, 15, 64) 18496 \n",
+ "_________________________________________________________________\n",
+ "activation_3 (Activation) (None, 15, 15, 64) 0 \n",
+ "_________________________________________________________________\n",
+ "conv2d_4 (Conv2D) (None, 13, 13, 64) 36928 \n",
+ "_________________________________________________________________\n",
+ "activation_4 (Activation) (None, 13, 13, 64) 0 \n",
+ "_________________________________________________________________\n",
+ "max_pooling2d_2 (MaxPooling2 (None, 6, 6, 64) 0 \n",
+ "_________________________________________________________________\n",
+ "dropout_2 (Dropout) (None, 6, 6, 64) 0 \n",
+ "_________________________________________________________________\n",
+ "flatten_1 (Flatten) (None, 2304) 0 \n",
+ "_________________________________________________________________\n",
+ "dense_1 (Dense) (None, 512) 1180160 \n",
+ "_________________________________________________________________\n",
+ "activation_5 (Activation) (None, 512) 0 \n",
+ "_________________________________________________________________\n",
+ "dropout_3 (Dropout) (None, 512) 0 \n",
+ "_________________________________________________________________\n",
+ "dense_2 (Dense) (None, 10) 5130 \n",
+ "_________________________________________________________________\n",
+ "activation_6 (Activation) (None, 10) 0 \n",
+ "=================================================================\n",
+ "Total params: 1,250,858\n",
+ "Trainable params: 1,250,858\n",
+ "Non-trainable params: 0\n",
+ "_________________________________________________________________\n"
+ ]
+ }
+ ],
+ "source": [
+ "model = Sequential()\n",
+ "model.add(Conv2D(32, (3, 3), padding='same',\n",
+ " input_shape=x_train.shape[1:]))\n",
+ "model.add(Activation('relu'))\n",
+ "model.add(Conv2D(32, (3, 3)))\n",
+ "model.add(Activation('relu'))\n",
+ "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
+ "model.add(Dropout(0.25))\n",
+ "\n",
+ "model.add(Conv2D(64, (3, 3), padding='same'))\n",
+ "model.add(Activation('relu'))\n",
+ "model.add(Conv2D(64, (3, 3)))\n",
+ "model.add(Activation('relu'))\n",
+ "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
+ "model.add(Dropout(0.25))\n",
+ "\n",
+ "model.add(Flatten())\n",
+ "model.add(Dense(512))\n",
+ "model.add(Activation('relu'))\n",
+ "model.add(Dropout(0.5))\n",
+ "model.add(Dense(num_classes))\n",
+ "model.add(Activation('softmax'))\n",
+ "\n",
+ "model.summary()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Load into model architecture table using psycopg2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>model_id</th>\n",
+ " <th>name</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>CNN from Keras docs for CIFAR-10</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, u'CNN from Keras docs for CIFAR-10')]"
+ ]
+ },
+ "execution_count": 13,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "import psycopg2 as p2\n",
+ "conn = p2.connect('postgresql://gpadmin@35.239.240.26:5432/madlib')\n",
+ "cur = conn.cursor()\n",
+ "\n",
+ "%sql DROP TABLE IF EXISTS model_arch_library;\n",
+ "query = \"SELECT madlib.load_keras_model('model_arch_library', %s, NULL, %s)\"\n",
+ "cur.execute(query,[model.to_json(), \"CNN from Keras docs for CIFAR-10\"])\n",
+ "conn.commit()\n",
+ "\n",
+ "# check model loaded OK\n",
+ "%sql SELECT model_id, name FROM model_arch_library;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"train\"></a>\n",
+ "# 5. Train"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>madlib_keras_fit</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td></td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[('',)]"
+ ]
+ },
+ "execution_count": 22,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS cifar_10_model, cifar_10_model_summary;\n",
+ "\n",
+ "SELECT madlib.madlib_keras_fit('cifar_10_train_data_packed', -- source table\n",
+ " 'cifar_10_model', -- model output table\n",
+ " 'model_arch_library', -- model arch table\n",
+ " 1, -- model arch id\n",
+ " $$ loss='categorical_crossentropy', optimizer='rmsprop(lr=0.0001, decay=1e-6)', metrics=['accuracy']$$, -- compile_params\n",
+ " $$ batch_size=32, epochs=3 $$, -- fit_params\n",
+ " 20, -- num_iterations\n",
+ " 0, -- GPUs per host\n",
+ " 'cifar_10_test_data_packed', -- validation dataset\n",
+ " 2 -- metrics compute frequency\n",
+ " );"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "View the model summary:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 110,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>source_table</th>\n",
+ " <th>model</th>\n",
+ " <th>dependent_varname</th>\n",
+ " <th>independent_varname</th>\n",
+ " <th>model_arch_table</th>\n",
+ " <th>model_arch_id</th>\n",
+ " <th>compile_params</th>\n",
+ " <th>fit_params</th>\n",
+ " <th>num_iterations</th>\n",
+ " <th>validation_table</th>\n",
+ " <th>metrics_compute_frequency</th>\n",
+ " <th>name</th>\n",
+ " <th>description</th>\n",
+ " <th>model_type</th>\n",
+ " <th>model_size</th>\n",
+ " <th>start_training_time</th>\n",
+ " <th>end_training_time</th>\n",
+ " <th>metrics_elapsed_time</th>\n",
+ " <th>madlib_version</th>\n",
+ " <th>num_classes</th>\n",
+ " <th>class_values</th>\n",
+ " <th>dependent_vartype</th>\n",
+ " <th>normalizing_const</th>\n",
+ " <th>metrics_type</th>\n",
+ " <th>training_metrics_final</th>\n",
+ " <th>training_loss_final</th>\n",
+ " <th>training_metrics</th>\n",
+ " <th>training_loss</th>\n",
+ " <th>validation_metrics_final</th>\n",
+ " <th>validation_loss_final</th>\n",
+ " <th>validation_metrics</th>\n",
+ " <th>validation_loss</th>\n",
+ " <th>metrics_iters</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>cifar_10_train_data_packed</td>\n",
+ " <td>cifar_10_model</td>\n",
+ " <td>y</td>\n",
+ " <td>x</td>\n",
+ " <td>model_arch_library</td>\n",
+ " <td>1</td>\n",
+ " <td> loss='categorical_crossentropy', optimizer='rmsprop(lr=0.0001, decay=1e-6)', metrics=['accuracy']</td>\n",
+ " <td> batch_size=32, epochs=3 </td>\n",
+ " <td>20</td>\n",
+ " <td>cifar_10_test_data_packed</td>\n",
+ " <td>2</td>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " <td>madlib_keras</td>\n",
+ " <td>4886.20019531</td>\n",
+ " <td>2019-06-25 05:40:29.287703</td>\n",
+ " <td>2019-06-25 07:59:52.961506</td>\n",
+ " <td>[798.95044708252, 1616.68976902962, 2447.13853096962, 3273.68762302399, 4116.44566893578, 4962.07483291626, 5805.66080999374, 6665.33687210083, 7526.0603749752, 8363.67366909981]</td>\n",
+ " <td>1.16-dev</td>\n",
+ " <td>10</td>\n",
+ " <td>[u'0', u'1', u'2', u'3', u'4', u'5', u'6', u'7', u'8', u'9']</td>\n",
+ " <td>text</td>\n",
+ " <td>255.0</td>\n",
+ " <td>[u'accuracy']</td>\n",
+ " <td>0.836480021477</td>\n",
+ " <td>0.500134825706</td>\n",
+ " <td>[0.579240024089813, 0.672980010509491, 0.723999977111816, 0.75764000415802, 0.783959984779358, 0.79475998878479, 0.811240017414093, 0.822780013084412, 0.829559981822968, 0.836480021476746]</td>\n",
+ " <td>[1.19081699848175, 0.940543830394745, 0.800645172595978, 0.700933694839478, 0.636690974235535, 0.599389910697937, 0.556614756584167, 0.53840559720993, 0.517430067062378, 0.500134825706482]</td>\n",
+ " <td>0.778900027275</td>\n",
+ " <td>0.661625564098</td>\n",
+ " <td>[0.57150000333786, 0.653800010681152, 0.692200005054474, 0.721300005912781, 0.740000009536743, 0.751299977302551, 0.756099998950958, 0.769999980926514, 0.77240002155304, 0.778900027275085]</td>\n",
+ " <td>[1.20945084095001, 0.987037718296051, 0.871006071567535, 0.800125658512115, 0.751632690429688, 0.72808450460434, 0.704570233821869, 0.684175074100494, 0.675221920013428, 0.661625564098358]</td>\n",
+ " <td>[2, 4, 6, 8, 10, 12, 14, 16, 18, 20]</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'cifar_10_train_data_packed', u'cifar_10_model', u'y', u'x', u'model_arch_library', 1, u\" loss='categorical_crossentropy', optimizer='rmsprop(lr=0.0001, decay=1e-6)', metrics=['accuracy']\", u' batch_size=32, epochs=3 ', 20, u'cifar_10_test_data_packed', 2, None, None, u'madlib_keras', 4886.20019531, datetime.datetime(2019, 6, 25, 5, 40, 29, 287703), datetime.datetime(2019, 6, 25, 7, 59, 52, 961506), [798.95044708252, 1616.68976902962, 2447.13853096962, 3273.68762302399, 4116 [...]
+ ]
+ },
+ "execution_count": 110,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM cifar_10_model_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Evaluate using test data (same values as last iteration from the fit output summary above)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>loss</th>\n",
+ " <th>metric</th>\n",
+ " <th>metrics_type</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>0.661625564098</td>\n",
+ " <td>0.778900027275</td>\n",
+ " <td>[u'accuracy']</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(0.661625564098358, 0.778900027275085, [u'accuracy'])]"
+ ]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS cifar10_validate;\n",
+ "\n",
+ "SELECT madlib.madlib_keras_evaluate('cifar_10_model', -- model\n",
+ " 'cifar_10_test_data_packed', -- test table\n",
+ " 'cifar10_validate' -- output table\n",
+ " );\n",
+ "\n",
+ "SELECT * FROM cifar10_validate;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"plot\"></a>\n",
+ "# 6. Plots by iteration and by time\n",
+ "Accuracy by iteration"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 100,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "<matplotlib.legend.Legend at 0x1195ef9d0>"
+ ]
+ },
+ "execution_count": 100,
+ "metadata": {},
+ "output_type": "execute_result"
+ },
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzs3Xd4FFX3wPHvSSD0Jr0TqVKkikZBg6gUXxEVESuK/rCLIio2xIIdrKivr6KiCHZEBBExUZCIgHSQ3pEWaoAQkj2/P+4kLCEN2M2mnM/z7LO7M3dmzk42c/bOnXtHVBVjjDEmK2GhDsAYY0zeZ8nCGGNMtixZGGOMyZYlC2OMMdmyZGGMMSZbliyMMcZky5KFMQWAiFwkIusCtK4PROSxQKzrFGJYLiIdQxmDOZYli0JMRK4TkTkikiAi/4rIZBHp4M0bKiKf+ZVVETng [...]
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "# get accuracy and iteration number\n",
+ "iters_proxy = %sql SELECT metrics_iters FROM cifar_10_model_summary;\n",
+ "train_accuracy_proxy = %sql SELECT training_metrics FROM cifar_10_model_summary;\n",
+ "test_accuracy_proxy = %sql SELECT validation_metrics FROM cifar_10_model_summary;\n",
+ "\n",
+ "# get number of points\n",
+ "num_points_proxy = %sql SELECT array_length(metrics_iters,1) FROM cifar_10_model_summary;\n",
+ "num_points = num_points_proxy[0]\n",
+ "\n",
+ "# reshape to np arrays\n",
+ "iters = np.array(iters_proxy).reshape(num_points)\n",
+ "train_accuracy = np.array(train_accuracy_proxy).reshape(num_points)\n",
+ "test_accuracy = np.array(test_accuracy_proxy).reshape(num_points)\n",
+ "\n",
+ "#plot\n",
+ "plt.title('CIFAR-10 accuracy by iteration')\n",
+ "plt.xlabel('Iteration number')\n",
+ "plt.ylabel('Accuracy')\n",
+ "plt.grid(True)\n",
+ "plt.plot(iters, train_accuracy, 'g.-', label='Train')\n",
+ "plt.plot(iters, test_accuracy, 'r.-', label='Test')\n",
+ "plt.legend()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Loss by iteration"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 101,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "<matplotlib.legend.Legend at 0x119279910>"
+ ]
+ },
+ "execution_count": 101,
+ "metadata": {},
+ "output_type": "execute_result"
+ },
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzs3XmcTfX/wPHXe3aMdWjIjOy7yNqEDCqiaJEsUSStX60qUUkSpW/1LeUbSvxoSqlUKmUJX6MQMZQlyR7GNpRlzPv3x7kzc41ZmTN3lvfz8TiPe+85n3POe+7cue/5fD7nfD6iqhhjjDEAfr4OwBhjTP5hScEYY0wKSwrGGGNSWFIwxhiTwpKCMcaYFJYUjDHGpLCkYIo8ERkkIovy6Fw1RSRXrgMXkadFZGJuHOsCYpgnIn19GYPJXZYUTKZEpI+IrBSRYyKyR0S+FpE2 [...]
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "# get loss\n",
+ "train_loss_proxy = %sql SELECT training_loss FROM cifar_10_model_summary;\n",
+ "test_loss_proxy = %sql SELECT validation_loss FROM cifar_10_model_summary;\n",
+ "\n",
+ "# reshape to np arrays\n",
+ "train_loss = np.array(train_loss_proxy).reshape(num_points)\n",
+ "test_loss = np.array(test_loss_proxy).reshape(num_points)\n",
+ "\n",
+ "#plot\n",
+ "plt.title('CIFAR-10 loss by iteration')\n",
+ "plt.xlabel('Iteration number')\n",
+ "plt.ylabel('Loss')\n",
+ "plt.grid(True)\n",
+ "plt.plot(iters, train_loss, 'g.-', label='Train')\n",
+ "plt.plot(iters, test_loss, 'r.-', label='Test')\n",
+ "plt.legend()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Accuracy by time"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 108,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "<matplotlib.legend.Legend at 0x119664410>"
+ ]
+ },
+ "execution_count": 108,
+ "metadata": {},
+ "output_type": "execute_result"
+ },
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xd4FFX3wPHvSSD0DqLShdCUJghEUKNYaBYUFRBULPx8bSgWxMqrrwj6+trAilgARQFFUEAUiYAgIFJUpEvvLRBa2vn9cSdhE1KWsJtNOZ/n2Sc7M3dmzu7Cnr1z79wrqooxxhiTlbBQB2CMMSbvs2RhjDEmW5YsjDHGZMuShTHGmGxZsjDGGJMtSxbGGGOyZcnCmEJIRC4XkQ1BPP4qEbkoWMc3uc+ShfGbiPQSkd9EJE5EtovINBFp720bLCJjfMqqiBz2ysaJyIF0 [...]
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "# get time\n",
+ "time_proxy = %sql SELECT metrics_elapsed_time FROM cifar_10_model_summary;\n",
+ "\n",
+ "# reshape to np arrays\n",
+ "time = np.array(time_proxy).reshape(num_points)/60.0\n",
+ "\n",
+ "#plot\n",
+ "plt.title('CIFAR-10 accuracy by time')\n",
+ "plt.xlabel('Time (min)')\n",
+ "plt.ylabel('Accuracy')\n",
+ "plt.grid(True)\n",
+ "plt.plot(time, train_accuracy, 'g.-', label='Train')\n",
+ "plt.plot(time, test_accuracy, 'r.-', label='Test')\n",
+ "plt.legend()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Time to achieve a given accuracy"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 109,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "<matplotlib.legend.Legend at 0x119628690>"
+ ]
+ },
+ "execution_count": 109,
+ "metadata": {},
+ "output_type": "execute_result"
+ },
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEWCAYAAABIVsEJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xd4VFX6wPHvm0AIJRCkC2iwAFKkBEuQXYPIWhbFgoJIUVyxrWXRFcR1cXFta+/tJ2tgEcKCBVkUBQ2KREqA0KX33hIChJDk/f1xLzFAICGZmTuTeT/Pk2dm7r1z73syMG/OOfecI6qKMcYYc7oivA7AGGNMaLIEYowxplQsgRhjjCkVSyDGGGNKxRKIMcaYUrEEYowxplQsgRhzHBH5PxEZGoDrXCki6/x9HWP8xRKIOSkR6S0ic0UkS0S2isjXItLJ3fe0iPyn0LEq [...]
+ "text/plain": [
+ "<Figure size 432x288 with 1 Axes>"
+ ]
+ },
+ "metadata": {
+ "needs_background": "light"
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "#plot\n",
+ "plt.title('CIFAR-10 time by accuracy')\n",
+ "plt.xlabel('Accuracy')\n",
+ "plt.ylabel('Time (min)')\n",
+ "plt.grid(True)\n",
+ "plt.plot(train_accuracy, time, 'g.-', label='Train')\n",
+ "plt.plot(test_accuracy, time, 'r.-', label='Test')\n",
+ "plt.legend()"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 2",
+ "language": "python",
+ "name": "python2"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.12"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
diff --git a/community-artifacts/Deep-learning/MADlib-Keras-transfer-learning-v1.ipynb b/community-artifacts/Deep-learning/MADlib-Keras-transfer-learning-v1.ipynb
new file mode 100644
index 0000000..faaad68
--- /dev/null
+++ b/community-artifacts/Deep-learning/MADlib-Keras-transfer-learning-v1.ipynb
@@ -0,0 +1,1620 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Transfer Learning Using Keras and MADlib\n",
+ "\n",
+ "This is a transfer learning example based on https://keras.io/examples/mnist_transfer_cnn/ \n",
+ "\n",
+ "## Table of contents\n",
+ "<a href=\"#import_libraries\">1. Import libraries</a>\n",
+ "\n",
+ "<a href=\"#load_and_prepare_data\">2. Load and prepare data</a>\n",
+ "\n",
+ "<a href=\"#image_preproc\">3. Call image preprocessor</a>\n",
+ "\n",
+ "<a href=\"#define_and_load_model\">4. Define and load model architecture</a>\n",
+ "\n",
+ "<a href=\"#train\">5. Train</a>\n",
+ "\n",
+ "<a href=\"#transfer_learning\">6. Transfer learning</a>"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/config.py:13: ShimWarning: The `IPython.config` package has been deprecated since IPython 4.0. You should import from traitlets.config instead.\n",
+ " \"You should import from traitlets.config instead.\", ShimWarning)\n",
+ "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/utils/traitlets.py:5: UserWarning: IPython.utils.traitlets has moved to a top-level traitlets package.\n",
+ " warn(\"IPython.utils.traitlets has moved to a top-level traitlets package.\")\n"
+ ]
+ }
+ ],
+ "source": [
+ "%load_ext sql"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "u'Connected: gpadmin@madlib'"
+ ]
+ },
+ "execution_count": 2,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Greenplum Database 5.x on GCP (PM demo machine)\n",
+ "#%sql postgresql://gpadmin@35.184.232.200:5432/madlib\n",
+ " \n",
+ "# Greenplum Database 5.x on GCP for deep learning (PM demo machine)\n",
+ "%sql postgresql://gpadmin@35.239.240.26:5432/madlib\n",
+ " \n",
+ "# PostgreSQL local\n",
+ "#%sql postgresql://fmcquillan@localhost:5432/madlib"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>version</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>MADlib version: 1.16-dev, git revision: rel/v1.15.1-129-g954609a, cmake configuration time: Fri Jun 21 18:18:35 UTC 2019, build type: release, build system: Linux-3.10.0-957.12.1.el7.x86_64, C compiler: gcc 4.8.5, C++ compiler: g++ 4.8.5</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'MADlib version: 1.16-dev, git revision: rel/v1.15.1-129-g954609a, cmake configuration time: Fri Jun 21 18:18:35 UTC 2019, build type: release, build system: Linux-3.10.0-957.12.1.el7.x86_64, C compiler: gcc 4.8.5, C++ compiler: g++ 4.8.5',)]"
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%sql select madlib.version();\n",
+ "#%sql select version();"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"import_libraries\"></a>\n",
+ "# 1. Import libraries\n",
+ "From https://keras.io/examples/mnist_transfer_cnn/ import libraries and define some params"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Using TensorFlow backend.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Couldn't import dot_parser, loading of dot files will not be possible.\n"
+ ]
+ }
+ ],
+ "source": [
+ "from __future__ import print_function\n",
+ "\n",
+ "import datetime\n",
+ "import keras\n",
+ "from keras.datasets import mnist\n",
+ "from keras.models import Sequential\n",
+ "from keras.layers import Dense, Dropout, Activation, Flatten\n",
+ "from keras.layers import Conv2D, MaxPooling2D\n",
+ "from keras import backend as K\n",
+ "\n",
+ "now = datetime.datetime.now\n",
+ "\n",
+ "batch_size = 128\n",
+ "num_classes = 5\n",
+ "epochs = 5\n",
+ "\n",
+ "# input image dimensions\n",
+ "img_rows, img_cols = 28, 28\n",
+ "# number of convolutional filters to use\n",
+ "filters = 32\n",
+ "# size of pooling area for max pooling\n",
+ "pool_size = 2\n",
+ "# convolution kernel size\n",
+ "kernel_size = 3\n",
+ "\n",
+ "if K.image_data_format() == 'channels_first':\n",
+ " input_shape = (1, img_rows, img_cols)\n",
+ "else:\n",
+ " input_shape = (img_rows, img_cols, 1)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Others needed in this workbook"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "import numpy as np"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"load_and_prepare_data\"></a>\n",
+ "# 2. Load and prepare data\n",
+ "\n",
+ "First load MNIST data from Keras, consisting of 60,000 28x28 grayscale images of the 10 digits, along with a test set of 10,000 images."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(4861, 28, 28)\n",
+ "(4861, 28, 28, 1)\n"
+ ]
+ }
+ ],
+ "source": [
+ "# the data, split between train and test sets\n",
+ "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
+ "\n",
+ "# create two datasets one with digits below 5 and one with 5 and above\n",
+ "x_train_lt5 = x_train[y_train < 5]\n",
+ "y_train_lt5 = y_train[y_train < 5]\n",
+ "x_test_lt5 = x_test[y_test < 5]\n",
+ "y_test_lt5 = y_test[y_test < 5]\n",
+ "\n",
+ "x_train_gte5 = x_train[y_train >= 5]\n",
+ "y_train_gte5 = y_train[y_train >= 5] - 5\n",
+ "x_test_gte5 = x_test[y_test >= 5]\n",
+ "y_test_gte5 = y_test[y_test >= 5] - 5\n",
+ "\n",
+ "# reshape to match model architecture\n",
+ "print(x_test_gte5.shape)\n",
+ "x_train_lt5=x_train_lt5.reshape(len(x_train_lt5), *input_shape)\n",
+ "x_test_lt5 = x_test_lt5.reshape(len(x_test_lt5), *input_shape)\n",
+ "x_train_gte5=x_train_gte5.reshape(len(x_train_gte5), *input_shape)\n",
+ "x_test_gte5 = x_test_gte5.reshape(len(x_test_gte5), *input_shape)\n",
+ "print(x_test_gte5.shape)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Load datasets into tables using image loader scripts"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# MADlib tools directory\n",
+ "import sys\n",
+ "import os\n",
+ "madlib_site_dir = '/Users/fmcquillan/Documents/Product/MADlib/Demos/data'\n",
+ "sys.path.append(madlib_site_dir)\n",
+ "\n",
+ "# Import image loader module\n",
+ "from madlib_image_loader import ImageLoader, DbCredentials"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Specify database credentials, for connecting to db\n",
+ "db_creds = DbCredentials(user='gpadmin',\n",
+ " host='35.239.240.26',\n",
+ " port='5432',\n",
+ " password='')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Initialize ImageLoader (increase num_workers to run faster)\n",
+ "iloader = ImageLoader(num_workers=5, db_creds=db_creds)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "Executing: CREATE TABLE train_lt5 (id SERIAL, x REAL[], y TEXT)\n",
+ "CREATE TABLE\n",
+ "Created table train_lt5 in madlib db\n",
+ "Spawning 5 workers...\n",
+ "Initializing PoolWorker-31 [pid 36504]\n",
+ "PoolWorker-31: Created temporary directory PoolWorker-31\n",
+ "Initializing PoolWorker-32 [pid 36505]\n",
+ "PoolWorker-32: Created temporary directory PoolWorker-32\n",
+ "Initializing PoolWorker-33 [pid 36506]\n",
+ "PoolWorker-33: Created temporary directory PoolWorker-33\n",
+ "Initializing PoolWorker-34 [pid 36507]\n",
+ "PoolWorker-34: Created temporary directory PoolWorker-34\n",
+ "Initializing PoolWorker-35 [pid 36508]\n",
+ "PoolWorker-35: Created temporary directory PoolWorker-35\n",
+ "PoolWorker-31: Connected to madlib db.\n",
+ "PoolWorker-32: Connected to madlib db.\n",
+ "PoolWorker-33: Connected to madlib db.\n",
+ "PoolWorker-34: Connected to madlib db.\n",
+ "PoolWorker-35: Connected to madlib db.\n",
+ "PoolWorker-32: Wrote 1000 images to /tmp/madlib_BaxJbXtAOV/train_lt50000.tmp\n",
+ "PoolWorker-31: Wrote 1000 images to /tmp/madlib_gijRoaCfJc/train_lt50000.tmp\n",
+ "PoolWorker-33: Wrote 1000 images to /tmp/madlib_OpZNic57mi/train_lt50000.tmp\n",
+ "PoolWorker-35: Wrote 1000 images to /tmp/madlib_l9HyKE46Lp/train_lt50000.tmp\n",
+ "PoolWorker-34: Wrote 1000 images to /tmp/madlib_92vUGUI7Kg/train_lt50000.tmp\n",
+ "PoolWorker-31: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-32: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-34: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-33: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-31: Wrote 1000 images to /tmp/madlib_gijRoaCfJc/train_lt50001.tmp\n",
+ "PoolWorker-35: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-32: Wrote 1000 images to /tmp/madlib_BaxJbXtAOV/train_lt50001.tmp\n",
+ "PoolWorker-34: Wrote 1000 images to /tmp/madlib_92vUGUI7Kg/train_lt50001.tmp\n",
+ "PoolWorker-33: Wrote 1000 images to /tmp/madlib_OpZNic57mi/train_lt50001.tmp\n",
+ "PoolWorker-35: Wrote 1000 images to /tmp/madlib_l9HyKE46Lp/train_lt50001.tmp\n",
+ "PoolWorker-31: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-32: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-34: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-31: Wrote 1000 images to /tmp/madlib_gijRoaCfJc/train_lt50002.tmp\n",
+ "PoolWorker-32: Wrote 1000 images to /tmp/madlib_BaxJbXtAOV/train_lt50002.tmp\n",
+ "PoolWorker-33: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-34: Wrote 1000 images to /tmp/madlib_92vUGUI7Kg/train_lt50002.tmp\n",
+ "PoolWorker-35: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-33: Wrote 1000 images to /tmp/madlib_OpZNic57mi/train_lt50002.tmp\n",
+ "PoolWorker-31: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-35: Wrote 1000 images to /tmp/madlib_l9HyKE46Lp/train_lt50002.tmp\n",
+ "PoolWorker-34: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-32: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-31: Wrote 1000 images to /tmp/madlib_gijRoaCfJc/train_lt50003.tmp\n",
+ "PoolWorker-34: Wrote 1000 images to /tmp/madlib_92vUGUI7Kg/train_lt50003.tmp\n",
+ "PoolWorker-32: Wrote 1000 images to /tmp/madlib_BaxJbXtAOV/train_lt50003.tmp\n",
+ "PoolWorker-33: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-31: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-33: Wrote 1000 images to /tmp/madlib_OpZNic57mi/train_lt50003.tmp\n",
+ "PoolWorker-35: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-31: Wrote 1000 images to /tmp/madlib_gijRoaCfJc/train_lt50004.tmp\n",
+ "PoolWorker-34: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-35: Wrote 1000 images to /tmp/madlib_l9HyKE46Lp/train_lt50003.tmp\n",
+ "PoolWorker-32: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-34: Wrote 1000 images to /tmp/madlib_92vUGUI7Kg/train_lt50004.tmp\n",
+ "PoolWorker-31: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-33: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-32: Wrote 1000 images to /tmp/madlib_BaxJbXtAOV/train_lt50004.tmp\n",
+ "PoolWorker-31: Wrote 1000 images to /tmp/madlib_gijRoaCfJc/train_lt50005.tmp\n",
+ "PoolWorker-33: Wrote 1000 images to /tmp/madlib_OpZNic57mi/train_lt50004.tmp\n",
+ "PoolWorker-34: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-35: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-32: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-31: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-35: Wrote 1000 images to /tmp/madlib_l9HyKE46Lp/train_lt50004.tmp\n",
+ "PoolWorker-34: Wrote 1000 images to /tmp/madlib_92vUGUI7Kg/train_lt50005.tmp\n",
+ "PoolWorker-31: Wrote 596 images to /tmp/madlib_gijRoaCfJc/train_lt50006.tmp\n",
+ "PoolWorker-33: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-32: Wrote 1000 images to /tmp/madlib_BaxJbXtAOV/train_lt50005.tmp\n",
+ "PoolWorker-33: Wrote 1000 images to /tmp/madlib_OpZNic57mi/train_lt50005.tmp\n",
+ "PoolWorker-31: Loaded 596 images into train_lt5\n",
+ "PoolWorker-34: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-32: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-35: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-33: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-35: Wrote 1000 images to /tmp/madlib_l9HyKE46Lp/train_lt50005.tmp\n",
+ "PoolWorker-35: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-35: Removed temporary directory PoolWorker-35\n",
+ "PoolWorker-34: Removed temporary directory PoolWorker-34\n",
+ "PoolWorker-33: Removed temporary directory PoolWorker-33\n",
+ "PoolWorker-31: Removed temporary directory PoolWorker-31\n",
+ "PoolWorker-32: Removed temporary directory PoolWorker-32\n",
+ "Done! Loaded 30596 images in 13.3909709454s\n",
+ "MainProcess: Connected to madlib db.\n",
+ "Executing: CREATE TABLE test_lt5 (id SERIAL, x REAL[], y TEXT)\n",
+ "CREATE TABLE\n",
+ "Created table test_lt5 in madlib db\n",
+ "Spawning 5 workers...\n",
+ "Initializing PoolWorker-36 [pid 36510]\n",
+ "PoolWorker-36: Created temporary directory PoolWorker-36\n",
+ "Initializing PoolWorker-37 [pid 36511]\n",
+ "PoolWorker-37: Created temporary directory PoolWorker-37\n",
+ "Initializing PoolWorker-38 [pid 36512]\n",
+ "PoolWorker-38: Created temporary directory PoolWorker-38\n",
+ "Initializing PoolWorker-39 [pid 36513]\n",
+ "PoolWorker-39: Created temporary directory PoolWorker-39\n",
+ "Initializing PoolWorker-40 [pid 36514]\n",
+ "PoolWorker-40: Created temporary directory PoolWorker-40\n",
+ "PoolWorker-36: Connected to madlib db.\n",
+ "PoolWorker-37: Connected to madlib db.\n",
+ "PoolWorker-38: Connected to madlib db.\n",
+ "PoolWorker-39: Connected to madlib db.\n",
+ "PoolWorker-40: Connected to madlib db.\n",
+ "PoolWorker-37: Wrote 1000 images to /tmp/madlib_TPjU7lz34X/test_lt50000.tmp\n",
+ "PoolWorker-38: Wrote 1000 images to /tmp/madlib_PNUF8ABUjF/test_lt50000.tmp\n",
+ "PoolWorker-39: Wrote 1000 images to /tmp/madlib_4X2ZzvvwFm/test_lt50000.tmp\n",
+ "PoolWorker-40: Wrote 1000 images to /tmp/madlib_neDGAgy83W/test_lt50000.tmp\n",
+ "PoolWorker-36: Wrote 1000 images to /tmp/madlib_L6JjsmxTxZ/test_lt50000.tmp\n",
+ "PoolWorker-36: Loaded 1000 images into test_lt5\n",
+ "PoolWorker-37: Loaded 1000 images into test_lt5\n",
+ "PoolWorker-36: Wrote 139 images to /tmp/madlib_L6JjsmxTxZ/test_lt50001.tmp\n",
+ "PoolWorker-38: Loaded 1000 images into test_lt5\n",
+ "PoolWorker-40: Loaded 1000 images into test_lt5\n",
+ "PoolWorker-36: Loaded 139 images into test_lt5\n",
+ "PoolWorker-39: Loaded 1000 images into test_lt5\n",
+ "PoolWorker-38: Removed temporary directory PoolWorker-38\n",
+ "PoolWorker-37: Removed temporary directory PoolWorker-37\n",
+ "PoolWorker-36: Removed temporary directory PoolWorker-36\n",
+ "PoolWorker-40: Removed temporary directory PoolWorker-40\n",
+ "PoolWorker-39: Removed temporary directory PoolWorker-39\n",
+ "Done! Loaded 5139 images in 3.34779882431s\n",
+ "MainProcess: Connected to madlib db.\n",
+ "Executing: CREATE TABLE train_gte5 (id SERIAL, x REAL[], y TEXT)\n",
+ "CREATE TABLE\n",
+ "Created table train_gte5 in madlib db\n",
+ "Spawning 5 workers...\n",
+ "Initializing PoolWorker-41 [pid 36515]\n",
+ "PoolWorker-41: Created temporary directory PoolWorker-41\n",
+ "Initializing PoolWorker-42 [pid 36516]\n",
+ "PoolWorker-42: Created temporary directory PoolWorker-42\n",
+ "Initializing PoolWorker-43 [pid 36517]\n",
+ "PoolWorker-43: Created temporary directory PoolWorker-43\n",
+ "Initializing PoolWorker-44 [pid 36518]\n",
+ "PoolWorker-44: Created temporary directory PoolWorker-44\n",
+ "Initializing PoolWorker-45 [pid 36519]\n",
+ "PoolWorker-45: Created temporary directory PoolWorker-45\n",
+ "PoolWorker-41: Connected to madlib db.\n",
+ "PoolWorker-42: Connected to madlib db.\n",
+ "PoolWorker-43: Connected to madlib db.\n",
+ "PoolWorker-44: Connected to madlib db.\n",
+ "PoolWorker-45: Connected to madlib db.\n",
+ "PoolWorker-41: Wrote 1000 images to /tmp/madlib_4culpKfGHi/train_gte50000.tmp\n",
+ "PoolWorker-42: Wrote 1000 images to /tmp/madlib_V6Nqwp374Q/train_gte50000.tmp\n",
+ "PoolWorker-43: Wrote 1000 images to /tmp/madlib_fHsDrEhTBh/train_gte50000.tmp\n",
+ "PoolWorker-44: Wrote 1000 images to /tmp/madlib_CU2CNf1Yxz/train_gte50000.tmp\n",
+ "PoolWorker-45: Wrote 1000 images to /tmp/madlib_1ZkWBU7pyz/train_gte50000.tmp\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "PoolWorker-43: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-44: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-41: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-43: Wrote 1000 images to /tmp/madlib_fHsDrEhTBh/train_gte50001.tmp\n",
+ "PoolWorker-44: Wrote 1000 images to /tmp/madlib_CU2CNf1Yxz/train_gte50001.tmp\n",
+ "PoolWorker-41: Wrote 1000 images to /tmp/madlib_4culpKfGHi/train_gte50001.tmp\n",
+ "PoolWorker-43: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-42: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-45: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-43: Wrote 1000 images to /tmp/madlib_fHsDrEhTBh/train_gte50002.tmp\n",
+ "PoolWorker-41: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-42: Wrote 1000 images to /tmp/madlib_V6Nqwp374Q/train_gte50001.tmp\n",
+ "PoolWorker-45: Wrote 1000 images to /tmp/madlib_1ZkWBU7pyz/train_gte50001.tmp\n",
+ "PoolWorker-41: Wrote 1000 images to /tmp/madlib_4culpKfGHi/train_gte50002.tmp\n",
+ "PoolWorker-43: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-44: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-43: Wrote 1000 images to /tmp/madlib_fHsDrEhTBh/train_gte50003.tmp\n",
+ "PoolWorker-44: Wrote 1000 images to /tmp/madlib_CU2CNf1Yxz/train_gte50002.tmp\n",
+ "PoolWorker-41: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-42: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-43: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-41: Wrote 1000 images to /tmp/madlib_4culpKfGHi/train_gte50003.tmp\n",
+ "PoolWorker-45: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-42: Wrote 1000 images to /tmp/madlib_V6Nqwp374Q/train_gte50002.tmp\n",
+ "PoolWorker-43: Wrote 1000 images to /tmp/madlib_fHsDrEhTBh/train_gte50004.tmp\n",
+ "PoolWorker-44: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-45: Wrote 1000 images to /tmp/madlib_1ZkWBU7pyz/train_gte50002.tmp\n",
+ "PoolWorker-41: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-43: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-44: Wrote 1000 images to /tmp/madlib_CU2CNf1Yxz/train_gte50003.tmp\n",
+ "PoolWorker-42: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-41: Wrote 1000 images to /tmp/madlib_4culpKfGHi/train_gte50004.tmp\n",
+ "PoolWorker-43: Wrote 1000 images to /tmp/madlib_fHsDrEhTBh/train_gte50005.tmp\n",
+ "PoolWorker-42: Wrote 1000 images to /tmp/madlib_V6Nqwp374Q/train_gte50003.tmp\n",
+ "PoolWorker-43: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-41: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-45: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-44: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-42: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-43: Wrote 1000 images to /tmp/madlib_fHsDrEhTBh/train_gte50006.tmp\n",
+ "PoolWorker-41: Wrote 1000 images to /tmp/madlib_4culpKfGHi/train_gte50005.tmp\n",
+ "PoolWorker-45: Wrote 1000 images to /tmp/madlib_1ZkWBU7pyz/train_gte50003.tmp\n",
+ "PoolWorker-44: Wrote 1000 images to /tmp/madlib_CU2CNf1Yxz/train_gte50004.tmp\n",
+ "PoolWorker-42: Wrote 1000 images to /tmp/madlib_V6Nqwp374Q/train_gte50004.tmp\n",
+ "PoolWorker-43: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-41: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-43: Wrote 1000 images to /tmp/madlib_fHsDrEhTBh/train_gte50007.tmp\n",
+ "PoolWorker-42: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-44: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-45: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-42: Wrote 404 images to /tmp/madlib_V6Nqwp374Q/train_gte50005.tmp\n",
+ "PoolWorker-44: Wrote 1000 images to /tmp/madlib_CU2CNf1Yxz/train_gte50005.tmp\n",
+ "PoolWorker-42: Loaded 404 images into train_gte5\n",
+ "PoolWorker-43: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-44: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-45: Removed temporary directory PoolWorker-45\n",
+ "PoolWorker-44: Removed temporary directory PoolWorker-44\n",
+ "PoolWorker-42: Removed temporary directory PoolWorker-42\n",
+ "PoolWorker-41: Removed temporary directory PoolWorker-41\n",
+ "PoolWorker-43: Removed temporary directory PoolWorker-43\n",
+ "Done! Loaded 29404 images in 12.6191689968s\n",
+ "MainProcess: Connected to madlib db.\n",
+ "Executing: CREATE TABLE test_gte5 (id SERIAL, x REAL[], y TEXT)\n",
+ "CREATE TABLE\n",
+ "Created table test_gte5 in madlib db\n",
+ "Spawning 5 workers...\n",
+ "Initializing PoolWorker-46 [pid 36521]\n",
+ "PoolWorker-46: Created temporary directory PoolWorker-46\n",
+ "Initializing PoolWorker-47 [pid 36522]\n",
+ "PoolWorker-47: Created temporary directory PoolWorker-47\n",
+ "Initializing PoolWorker-48 [pid 36523]\n",
+ "PoolWorker-50: Created temporary directory PoolWorker-50\n",
+ "PoolWorker-48: Created temporary directory PoolWorker-48\n",
+ "Initializing PoolWorker-49 [pid 36524]\n",
+ "PoolWorker-49: Created temporary directory PoolWorker-49\n",
+ "Initializing PoolWorker-50 [pid 36525]\n",
+ "PoolWorker-46: Connected to madlib db.\n",
+ "PoolWorker-47: Connected to madlib db.\n",
+ "PoolWorker-48: Connected to madlib db.\n",
+ "PoolWorker-49: Connected to madlib db.\n",
+ "PoolWorker-50: Connected to madlib db.\n",
+ "PoolWorker-46: Wrote 1000 images to /tmp/madlib_DTfIW4fes1/test_gte50000.tmp\n",
+ "PoolWorker-50: Wrote 861 images to /tmp/madlib_dkZDQsdCfI/test_gte50000.tmp\n",
+ "PoolWorker-48: Wrote 1000 images to /tmp/madlib_eLa5ZPTJoU/test_gte50000.tmp\n",
+ "PoolWorker-47: Wrote 1000 images to /tmp/madlib_agBlKzPnjs/test_gte50000.tmp\n",
+ "PoolWorker-49: Wrote 1000 images to /tmp/madlib_89sgwkcIn8/test_gte50000.tmp\n",
+ "PoolWorker-50: Loaded 861 images into test_gte5\n",
+ "PoolWorker-48: Loaded 1000 images into test_gte5\n",
+ "PoolWorker-47: Loaded 1000 images into test_gte5\n",
+ "PoolWorker-49: Loaded 1000 images into test_gte5\n",
+ "PoolWorker-46: Loaded 1000 images into test_gte5\n",
+ "PoolWorker-50: Removed temporary directory PoolWorker-50\n",
+ "PoolWorker-49: Removed temporary directory PoolWorker-49\n",
+ "PoolWorker-46: Removed temporary directory PoolWorker-46\n",
+ "PoolWorker-47: Removed temporary directory PoolWorker-47\n",
+ "PoolWorker-48: Removed temporary directory PoolWorker-48\n",
+ "Done! Loaded 4861 images in 3.05431604385s\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Drop tables\n",
+ "%sql DROP TABLE IF EXISTS train_lt5, test_lt5, train_gte5, test_gte5\n",
+ "\n",
+ "# Save images to temporary directories and load into database\n",
+ "iloader.load_np_array_to_table(x_train_lt5, y_train_lt5, 'train_lt5', append=False, img_names=None)\n",
+ "iloader.load_np_array_to_table(x_test_lt5, y_test_lt5, 'test_lt5', append=False, img_names=None)\n",
+ "iloader.load_np_array_to_table(x_train_gte5, y_train_gte5, 'train_gte5', append=False, img_names=None)\n",
+ "iloader.load_np_array_to_table(x_test_gte5, y_test_gte5, 'test_gte5', append=False, img_names=None)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"image_preproc\"></a>\n",
+ "# 3. Call image preprocessor\n",
+ "\n",
+ "Transforms from one image per row to multiple images per row for batch optimization. Also normalizes and one-hot encodes.\n",
+ "\n",
+ "Training dataset < 5"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>source_table</th>\n",
+ " <th>output_table</th>\n",
+ " <th>dependent_varname</th>\n",
+ " <th>independent_varname</th>\n",
+ " <th>dependent_vartype</th>\n",
+ " <th>class_values</th>\n",
+ " <th>buffer_size</th>\n",
+ " <th>normalizing_const</th>\n",
+ " <th>num_classes</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>train_lt5</td>\n",
+ " <td>train_lt5_packed</td>\n",
+ " <td>y</td>\n",
+ " <td>x</td>\n",
+ " <td>text</td>\n",
+ " <td>[u'0', u'1', u'2', u'3', u'4']</td>\n",
+ " <td>15298</td>\n",
+ " <td>255.0</td>\n",
+ " <td>5</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'train_lt5', u'train_lt5_packed', u'y', u'x', u'text', [u'0', u'1', u'2', u'3', u'4'], 15298, 255.0, 5)]"
+ ]
+ },
+ "execution_count": 11,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS train_lt5_packed, train_lt5_packed_summary;\n",
+ "\n",
+ "SELECT madlib.training_preprocessor_dl('train_lt5', -- Source table\n",
+ " 'train_lt5_packed', -- Output table\n",
+ " 'y', -- Dependent variable\n",
+ " 'x', -- Independent variable\n",
+ " NULL, -- Buffer size\n",
+ " 255 -- Normalizing constant\n",
+ " );\n",
+ "\n",
+ "SELECT * FROM train_lt5_packed_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Test dataset < 5"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>source_table</th>\n",
+ " <th>output_table</th>\n",
+ " <th>dependent_varname</th>\n",
+ " <th>independent_varname</th>\n",
+ " <th>dependent_vartype</th>\n",
+ " <th>class_values</th>\n",
+ " <th>buffer_size</th>\n",
+ " <th>normalizing_const</th>\n",
+ " <th>num_classes</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>test_lt5</td>\n",
+ " <td>test_lt5_packed</td>\n",
+ " <td>y</td>\n",
+ " <td>x</td>\n",
+ " <td>text</td>\n",
+ " <td>[u'0', u'1', u'2', u'3', u'4']</td>\n",
+ " <td>2570</td>\n",
+ " <td>255.0</td>\n",
+ " <td>5</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'test_lt5', u'test_lt5_packed', u'y', u'x', u'text', [u'0', u'1', u'2', u'3', u'4'], 2570, 255.0, 5)]"
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS test_lt5_packed, test_lt5_packed_summary;\n",
+ "\n",
+ "SELECT madlib.validation_preprocessor_dl('test_lt5', -- Source table\n",
+ " 'test_lt5_packed', -- Output table\n",
+ " 'y', -- Dependent variable\n",
+ " 'x', -- Independent variable\n",
+ " 'train_lt5_packed' -- Training preproc table\n",
+ " );\n",
+ "\n",
+ "SELECT * FROM test_lt5_packed_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Training dataset >= 5"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>source_table</th>\n",
+ " <th>output_table</th>\n",
+ " <th>dependent_varname</th>\n",
+ " <th>independent_varname</th>\n",
+ " <th>dependent_vartype</th>\n",
+ " <th>class_values</th>\n",
+ " <th>buffer_size</th>\n",
+ " <th>normalizing_const</th>\n",
+ " <th>num_classes</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>train_gte5</td>\n",
+ " <td>train_gte5_packed</td>\n",
+ " <td>y</td>\n",
+ " <td>x</td>\n",
+ " <td>text</td>\n",
+ " <td>[u'0', u'1', u'2', u'3', u'4']</td>\n",
+ " <td>14702</td>\n",
+ " <td>255.0</td>\n",
+ " <td>5</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'train_gte5', u'train_gte5_packed', u'y', u'x', u'text', [u'0', u'1', u'2', u'3', u'4'], 14702, 255.0, 5)]"
+ ]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS train_gte5_packed, train_gte5_packed_summary;\n",
+ "\n",
+ "SELECT madlib.training_preprocessor_dl('train_gte5', -- Source table\n",
+ " 'train_gte5_packed', -- Output table\n",
+ " 'y', -- Dependent variable\n",
+ " 'x', -- Independent variable\n",
+ " NULL, -- Buffer size\n",
+ " 255 -- Normalizing constant\n",
+ " );\n",
+ "\n",
+ "SELECT * FROM train_gte5_packed_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Test dataset >= 5"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>source_table</th>\n",
+ " <th>output_table</th>\n",
+ " <th>dependent_varname</th>\n",
+ " <th>independent_varname</th>\n",
+ " <th>dependent_vartype</th>\n",
+ " <th>class_values</th>\n",
+ " <th>buffer_size</th>\n",
+ " <th>normalizing_const</th>\n",
+ " <th>num_classes</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>test_gte5</td>\n",
+ " <td>test_gte5_packed</td>\n",
+ " <td>y</td>\n",
+ " <td>x</td>\n",
+ " <td>text</td>\n",
+ " <td>[u'0', u'1', u'2', u'3', u'4']</td>\n",
+ " <td>2431</td>\n",
+ " <td>255.0</td>\n",
+ " <td>5</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'test_gte5', u'test_gte5_packed', u'y', u'x', u'text', [u'0', u'1', u'2', u'3', u'4'], 2431, 255.0, 5)]"
+ ]
+ },
+ "execution_count": 17,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS test_gte5_packed, test_gte5_packed_summary;\n",
+ "\n",
+ "SELECT madlib.validation_preprocessor_dl('test_gte5', -- Source table\n",
+ " 'test_gte5_packed', -- Output table\n",
+ " 'y', -- Dependent variable\n",
+ " 'x', -- Independent variable\n",
+ " 'train_gte5_packed' -- Training preproc table\n",
+ " );\n",
+ "\n",
+ "SELECT * FROM test_gte5_packed_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"define_and_load_model\"></a>\n",
+ "# 4. Define and load model architecture\n",
+ "\n",
+ "Model with feature and classification layers trainable"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "_________________________________________________________________\n",
+ "Layer (type) Output Shape Param # \n",
+ "=================================================================\n",
+ "conv2d_1 (Conv2D) (None, 26, 26, 32) 320 \n",
+ "_________________________________________________________________\n",
+ "activation_1 (Activation) (None, 26, 26, 32) 0 \n",
+ "_________________________________________________________________\n",
+ "conv2d_2 (Conv2D) (None, 24, 24, 32) 9248 \n",
+ "_________________________________________________________________\n",
+ "activation_2 (Activation) (None, 24, 24, 32) 0 \n",
+ "_________________________________________________________________\n",
+ "max_pooling2d_1 (MaxPooling2 (None, 12, 12, 32) 0 \n",
+ "_________________________________________________________________\n",
+ "dropout_1 (Dropout) (None, 12, 12, 32) 0 \n",
+ "_________________________________________________________________\n",
+ "flatten_1 (Flatten) (None, 4608) 0 \n",
+ "_________________________________________________________________\n",
+ "dense_1 (Dense) (None, 128) 589952 \n",
+ "_________________________________________________________________\n",
+ "activation_3 (Activation) (None, 128) 0 \n",
+ "_________________________________________________________________\n",
+ "dropout_2 (Dropout) (None, 128) 0 \n",
+ "_________________________________________________________________\n",
+ "dense_2 (Dense) (None, 5) 645 \n",
+ "_________________________________________________________________\n",
+ "activation_4 (Activation) (None, 5) 0 \n",
+ "=================================================================\n",
+ "Total params: 600,165\n",
+ "Trainable params: 600,165\n",
+ "Non-trainable params: 0\n",
+ "_________________________________________________________________\n"
+ ]
+ }
+ ],
+ "source": [
+ "# define two groups of layers: feature (convolutions) and classification (dense)\n",
+ "feature_layers = [\n",
+ " Conv2D(filters, kernel_size,\n",
+ " padding='valid',\n",
+ " input_shape=input_shape),\n",
+ " Activation('relu'),\n",
+ " Conv2D(filters, kernel_size),\n",
+ " Activation('relu'),\n",
+ " MaxPooling2D(pool_size=pool_size),\n",
+ " Dropout(0.25),\n",
+ " Flatten(),\n",
+ "]\n",
+ "\n",
+ "classification_layers = [\n",
+ " Dense(128),\n",
+ " Activation('relu'),\n",
+ " Dropout(0.5),\n",
+ " Dense(num_classes),\n",
+ " Activation('softmax')\n",
+ "]\n",
+ "\n",
+ "# create complete model\n",
+ "model = Sequential(feature_layers + classification_layers)\n",
+ "\n",
+ "model.summary()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Load into model architecture table using psycopg2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>model_id</th>\n",
+ " <th>name</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>feature + classification layers trainable</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, u'feature + classification layers trainable')]"
+ ]
+ },
+ "execution_count": 20,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "import psycopg2 as p2\n",
+ "conn = p2.connect('postgresql://gpadmin@35.239.240.26:5432/madlib')\n",
+ "cur = conn.cursor()\n",
+ "\n",
+ "%sql DROP TABLE IF EXISTS model_arch_library;\n",
+ "query = \"SELECT madlib.load_keras_model('model_arch_library', %s, NULL, %s)\"\n",
+ "cur.execute(query,[model.to_json(), \"feature + classification layers trainable\"])\n",
+ "conn.commit()\n",
+ "\n",
+ "# check model loaded OK\n",
+ "%sql SELECT model_id, name FROM model_arch_library;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Model with feature layers frozen"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "_________________________________________________________________\n",
+ "Layer (type) Output Shape Param # \n",
+ "=================================================================\n",
+ "conv2d_1 (Conv2D) (None, 26, 26, 32) 320 \n",
+ "_________________________________________________________________\n",
+ "activation_1 (Activation) (None, 26, 26, 32) 0 \n",
+ "_________________________________________________________________\n",
+ "conv2d_2 (Conv2D) (None, 24, 24, 32) 9248 \n",
+ "_________________________________________________________________\n",
+ "activation_2 (Activation) (None, 24, 24, 32) 0 \n",
+ "_________________________________________________________________\n",
+ "max_pooling2d_1 (MaxPooling2 (None, 12, 12, 32) 0 \n",
+ "_________________________________________________________________\n",
+ "dropout_1 (Dropout) (None, 12, 12, 32) 0 \n",
+ "_________________________________________________________________\n",
+ "flatten_1 (Flatten) (None, 4608) 0 \n",
+ "_________________________________________________________________\n",
+ "dense_1 (Dense) (None, 128) 589952 \n",
+ "_________________________________________________________________\n",
+ "activation_3 (Activation) (None, 128) 0 \n",
+ "_________________________________________________________________\n",
+ "dropout_2 (Dropout) (None, 128) 0 \n",
+ "_________________________________________________________________\n",
+ "dense_2 (Dense) (None, 5) 645 \n",
+ "_________________________________________________________________\n",
+ "activation_4 (Activation) (None, 5) 0 \n",
+ "=================================================================\n",
+ "Total params: 600,165\n",
+ "Trainable params: 590,597\n",
+ "Non-trainable params: 9,568\n",
+ "_________________________________________________________________\n"
+ ]
+ }
+ ],
+ "source": [
+ "# freeze feature layers\n",
+ "for l in feature_layers:\n",
+ " l.trainable = False\n",
+ "\n",
+ "model.summary()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Load into transfer model architecture table using psycopg2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "2 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>model_id</th>\n",
+ " <th>name</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>feature + classification layers trainable</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>only classification layers trainable</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, u'feature + classification layers trainable'),\n",
+ " (2, u'only classification layers trainable')]"
+ ]
+ },
+ "execution_count": 22,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "cur.execute(query,[model.to_json(), \"only classification layers trainable\"])\n",
+ "conn.commit()\n",
+ "\n",
+ "# check model loaded OK\n",
+ "%sql SELECT model_id, name FROM model_arch_library ORDER BY model_id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"train\"></a>\n",
+ "# 5. Train\n",
+ "Train the model for 5-digit classification [0..4] "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>madlib_keras_fit</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td></td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[('',)]"
+ ]
+ },
+ "execution_count": 22,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS mnist_model, mnist_model_summary;\n",
+ "\n",
+ "SELECT madlib.madlib_keras_fit('train_lt5_packed', -- source table\n",
+ " 'mnist_model', -- model output table\n",
+ " 'model_arch_library', -- model arch table\n",
+ " 1, -- model arch id\n",
+ " $$ loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy']$$, -- compile_params\n",
+ " $$ batch_size=128, epochs=1 $$, -- fit_params\n",
+ " 5 -- num_iterations\n",
+ " );"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "View the model summary:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 23,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>source_table</th>\n",
+ " <th>model</th>\n",
+ " <th>dependent_varname</th>\n",
+ " <th>independent_varname</th>\n",
+ " <th>model_arch_table</th>\n",
+ " <th>model_arch_id</th>\n",
+ " <th>compile_params</th>\n",
+ " <th>fit_params</th>\n",
+ " <th>num_iterations</th>\n",
+ " <th>validation_table</th>\n",
+ " <th>metrics_compute_frequency</th>\n",
+ " <th>name</th>\n",
+ " <th>description</th>\n",
+ " <th>model_type</th>\n",
+ " <th>model_size</th>\n",
+ " <th>start_training_time</th>\n",
+ " <th>end_training_time</th>\n",
+ " <th>metrics_elapsed_time</th>\n",
+ " <th>madlib_version</th>\n",
+ " <th>num_classes</th>\n",
+ " <th>class_values</th>\n",
+ " <th>dependent_vartype</th>\n",
+ " <th>normalizing_const</th>\n",
+ " <th>metrics_type</th>\n",
+ " <th>training_metrics_final</th>\n",
+ " <th>training_loss_final</th>\n",
+ " <th>training_metrics</th>\n",
+ " <th>training_loss</th>\n",
+ " <th>validation_metrics_final</th>\n",
+ " <th>validation_loss_final</th>\n",
+ " <th>validation_metrics</th>\n",
+ " <th>validation_loss</th>\n",
+ " <th>metrics_iters</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>train_lt5_packed</td>\n",
+ " <td>mnist_model</td>\n",
+ " <td>y</td>\n",
+ " <td>x</td>\n",
+ " <td>model_arch_library</td>\n",
+ " <td>1</td>\n",
+ " <td> loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy']</td>\n",
+ " <td> batch_size=128, epochs=1 </td>\n",
+ " <td>5</td>\n",
+ " <td>None</td>\n",
+ " <td>5</td>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " <td>madlib_keras</td>\n",
+ " <td>2344.43066406</td>\n",
+ " <td>2019-06-24 19:08:31.328530</td>\n",
+ " <td>2019-06-24 19:13:50.944601</td>\n",
+ " <td>[319.616029977798]</td>\n",
+ " <td>1.16-dev</td>\n",
+ " <td>5</td>\n",
+ " <td>[u'0', u'1', u'2', u'3', u'4']</td>\n",
+ " <td>text</td>\n",
+ " <td>255.0</td>\n",
+ " <td>[u'accuracy']</td>\n",
+ " <td>0.996045231819</td>\n",
+ " <td>0.0139331035316</td>\n",
+ " <td>[0.996045231819153]</td>\n",
+ " <td>[0.013933103531599]</td>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " <td>[5]</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'train_lt5_packed', u'mnist_model', u'y', u'x', u'model_arch_library', 1, u\" loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy']\", u' batch_size=128, epochs=1 ', 5, None, 5, None, None, u'madlib_keras', 2344.43066406, datetime.datetime(2019, 6, 24, 19, 8, 31, 328530), datetime.datetime(2019, 6, 24, 19, 13, 50, 944601), [319.616029977798], u'1.16-dev', 5, [u'0', u'1', u'2', u'3', u'4'], u'text', 255.0, [u'accuracy'], 0.996045231819, 0.0139331035316, [0 [...]
+ ]
+ },
+ "execution_count": 23,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM mnist_model_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Evaluate using test data"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 24,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>loss</th>\n",
+ " <th>metric</th>\n",
+ " <th>metrics_type</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>0.00919340737164</td>\n",
+ " <td>0.997081160545</td>\n",
+ " <td>[u'accuracy']</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(0.00919340737164021, 0.997081160545349, [u'accuracy'])]"
+ ]
+ },
+ "execution_count": 24,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS mnist_validate;\n",
+ "\n",
+ "SELECT madlib.madlib_keras_evaluate('mnist_model', -- model\n",
+ " 'test_lt5_packed', -- test table\n",
+ " 'mnist_validate' -- output table\n",
+ " );\n",
+ "\n",
+ "SELECT * FROM mnist_validate;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"transfer_learning\"></a>\n",
+ "# 6. Transfer learning"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Use UPDATE to load trained weights from previous run into the model library table:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "[]"
+ ]
+ },
+ "execution_count": 25,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "UPDATE model_arch_library SET model_weights = model_data FROM mnist_model WHERE model_id = 2;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Transfer: train dense layers for new classification task [5..9]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>madlib_keras_fit</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td></td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[('',)]"
+ ]
+ },
+ "execution_count": 26,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS mnist_transfer_model, mnist_transfer_model_summary;\n",
+ "\n",
+ "SELECT madlib.madlib_keras_fit('train_gte5_packed', -- source table\n",
+ " 'mnist_transfer_model',-- model output table\n",
+ " 'model_arch_library', -- model arch table\n",
+ " 2, -- model arch id\n",
+ " $$ loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy']$$, -- compile_params\n",
+ " $$ batch_size=128, epochs=1 $$, -- fit_params\n",
+ " 5 -- num_iterations\n",
+ " );"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "View the model summary"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>source_table</th>\n",
+ " <th>model</th>\n",
+ " <th>dependent_varname</th>\n",
+ " <th>independent_varname</th>\n",
+ " <th>model_arch_table</th>\n",
+ " <th>model_arch_id</th>\n",
+ " <th>compile_params</th>\n",
+ " <th>fit_params</th>\n",
+ " <th>num_iterations</th>\n",
+ " <th>validation_table</th>\n",
+ " <th>metrics_compute_frequency</th>\n",
+ " <th>name</th>\n",
+ " <th>description</th>\n",
+ " <th>model_type</th>\n",
+ " <th>model_size</th>\n",
+ " <th>start_training_time</th>\n",
+ " <th>end_training_time</th>\n",
+ " <th>metrics_elapsed_time</th>\n",
+ " <th>madlib_version</th>\n",
+ " <th>num_classes</th>\n",
+ " <th>class_values</th>\n",
+ " <th>dependent_vartype</th>\n",
+ " <th>normalizing_const</th>\n",
+ " <th>metrics_type</th>\n",
+ " <th>training_metrics_final</th>\n",
+ " <th>training_loss_final</th>\n",
+ " <th>training_metrics</th>\n",
+ " <th>training_loss</th>\n",
+ " <th>validation_metrics_final</th>\n",
+ " <th>validation_loss_final</th>\n",
+ " <th>validation_metrics</th>\n",
+ " <th>validation_loss</th>\n",
+ " <th>metrics_iters</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>train_gte5_packed</td>\n",
+ " <td>mnist_transfer_model</td>\n",
+ " <td>y</td>\n",
+ " <td>x</td>\n",
+ " <td>model_arch_library</td>\n",
+ " <td>2</td>\n",
+ " <td> loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy']</td>\n",
+ " <td> batch_size=128, epochs=1 </td>\n",
+ " <td>5</td>\n",
+ " <td>None</td>\n",
+ " <td>5</td>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " <td>madlib_keras</td>\n",
+ " <td>2344.43066406</td>\n",
+ " <td>2019-06-24 19:16:55.336042</td>\n",
+ " <td>2019-06-24 19:19:53.589704</td>\n",
+ " <td>[178.253571987152]</td>\n",
+ " <td>1.16-dev</td>\n",
+ " <td>5</td>\n",
+ " <td>[u'0', u'1', u'2', u'3', u'4']</td>\n",
+ " <td>text</td>\n",
+ " <td>255.0</td>\n",
+ " <td>[u'accuracy']</td>\n",
+ " <td>0.991429746151</td>\n",
+ " <td>0.0280887652189</td>\n",
+ " <td>[0.99142974615097]</td>\n",
+ " <td>[0.028088765218854]</td>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " <td>[5]</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'train_gte5_packed', u'mnist_transfer_model', u'y', u'x', u'model_arch_library', 2, u\" loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy']\", u' batch_size=128, epochs=1 ', 5, None, 5, None, None, u'madlib_keras', 2344.43066406, datetime.datetime(2019, 6, 24, 19, 16, 55, 336042), datetime.datetime(2019, 6, 24, 19, 19, 53, 589704), [178.253571987152], u'1.16-dev', 5, [u'0', u'1', u'2', u'3', u'4'], u'text', 255.0, [u'accuracy'], 0.991429746151, 0.028088 [...]
+ ]
+ },
+ "execution_count": 27,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM mnist_transfer_model_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Evaluate using test data"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 30,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>loss</th>\n",
+ " <th>metric</th>\n",
+ " <th>metrics_type</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>0.0312170274556</td>\n",
+ " <td>0.989714026451</td>\n",
+ " <td>[u'accuracy']</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(0.0312170274555683, 0.989714026451111, [u'accuracy'])]"
+ ]
+ },
+ "execution_count": 30,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS mnist_transfer_validate;\n",
+ "\n",
+ "SELECT madlib.madlib_keras_evaluate('mnist_transfer_model', -- model\n",
+ " 'test_gte5_packed', -- test table\n",
+ " 'mnist_transfer_validate' -- output table\n",
+ " );\n",
+ "\n",
+ "SELECT * FROM mnist_transfer_validate;"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 2",
+ "language": "python",
+ "name": "python2"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.10"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
diff --git a/community-artifacts/Madlib Image Loader Demo.ipynb b/community-artifacts/Deep-learning/Madlib Image Loader Demo.ipynb
similarity index 100%
rename from community-artifacts/Madlib Image Loader Demo.ipynb
rename to community-artifacts/Deep-learning/Madlib Image Loader Demo.ipynb
diff --git a/community-artifacts/Deep-learning/Preprocessor-for-images-v1.ipynb b/community-artifacts/Deep-learning/Preprocessor-for-images-v1.ipynb
new file mode 100644
index 0000000..2ab4b91
--- /dev/null
+++ b/community-artifacts/Deep-learning/Preprocessor-for-images-v1.ipynb
@@ -0,0 +1,1625 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Preprocessor for image data\n",
+ "This is a mini-batch preprocessor utility for image data:\n",
+ "* training_preprocessor_dl() for training datasets\n",
+ "* validation_preprocessor_dl() for validation datasets\n",
+ "\n",
+ "Note that there is a separate mini-batch preprocessor utility for general use cases\n",
+ "http://madlib.apache.org/docs/latest/group__grp__minibatch__preprocessing.html\n",
+ "\n",
+ "The preprocessor for image data was added in MADlib 1.16.\n",
+ "\n",
+ "## Table of contents\n",
+ "\n",
+ "<a href=\"#load_data\">1. Load data</a>\n",
+ "\n",
+ "<a href=\"#pp_train\">2. Run preprocessor for training image data</a>\n",
+ "\n",
+ "<a href=\"#pp_val\">3. Run preprocessor for validation image data</a>\n",
+ "\n",
+ "<a href=\"#load_data2\">4. Load data, another format</a>\n",
+ "\n",
+ "<a href=\"#pp_train2\">5. Run preprocessor for training image data</a>\n",
+ "\n",
+ "<a href=\"#pp_val2\">6. Run preprocessor for validation image data</a>\n",
+ "\n",
+ "<a href=\"#change_buffer\">7. Change buffer size</a>\n",
+ "\n",
+ "<a href=\"#set_num_classes\">8. Setting number of classes</a>"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/config.py:13: ShimWarning: The `IPython.config` package has been deprecated since IPython 4.0. You should import from traitlets.config instead.\n",
+ " \"You should import from traitlets.config instead.\", ShimWarning)\n",
+ "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/utils/traitlets.py:5: UserWarning: IPython.utils.traitlets has moved to a top-level traitlets package.\n",
+ " warn(\"IPython.utils.traitlets has moved to a top-level traitlets package.\")\n"
+ ]
+ }
+ ],
+ "source": [
+ "%load_ext sql"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "u'Connected: gpadmin@madlib'"
+ ]
+ },
+ "execution_count": 2,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Greenplum Database 5.x on GCP (PM demo machine)\n",
+ "%sql postgresql://gpadmin@35.239.240.26:5432/madlib\n",
+ " \n",
+ "# PostgreSQL local\n",
+ "#%sql postgresql://fmcquillan@localhost:5432/madlib"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>version</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>MADlib version: 1.16-dev, git revision: rel/v1.15.1-98-g544a8e5, cmake configuration time: Mon May 20 16:40:50 UTC 2019, build type: release, build system: Linux-3.10.0-957.12.1.el7.x86_64, C compiler: gcc 4.8.5, C++ compiler: g++ 4.8.5</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'MADlib version: 1.16-dev, git revision: rel/v1.15.1-98-g544a8e5, cmake configuration time: Mon May 20 16:40:50 UTC 2019, build type: release, build system: Linux-3.10.0-957.12.1.el7.x86_64, C compiler: gcc 4.8.5, C++ compiler: g++ 4.8.5',)]"
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%sql select madlib.version();\n",
+ "#%sql select version();"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"load_data\"></a>\n",
+ "# 1. Load data\n",
+ "\n",
+ "Create an artificial 2x2 resolution color image data set with 3 possible classifications. The RGB values are per-pixel arrays:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "52 rows affected.\n",
+ "52 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>rgb</th>\n",
+ " <th>species</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[76, 125, 240], [191, 13, 20]], [[153, 77, 7], [41, 143, 172]]]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[15, 126, 174], [246, 129, 81]], [[143, 220, 157], [96, 207, 223]]]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[84, 24, 1], [201, 28, 77]], [[70, 12, 11], [83, 33, 165]]]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[40, 206, 47], [211, 138, 62]], [[82, 56, 52], [210, 137, 195]]]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[38, 35, 125], [5, 18, 209]], [[29, 19, 153], [57, 95, 223]]]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[107, 50, 102], [15, 210, 142]], [[222, 1, 97], [103, 63, 179]]]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[115, 133, 40], [55, 51, 78]], [[89, 176, 83], [108, 129, 112]]]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[27, 169, 222], [249, 239, 73]], [[43, 85, 88], [253, 227, 54]]]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[68, 157, 61], [246, 60, 176]], [[123, 100, 230], [175, 178, 64]]]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[4, 172, 224], [116, 42, 251]], [[30, 8, 244], [12, 81, 31]]]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[169, 28, 68], [223, 26, 136]], [[124, 87, 126], [184, 7, 250]]]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[237, 168, 205], [45, 7, 210]], [[217, 231, 70], [3, 226, 100]]]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[214, 112, 91], [246, 209, 4]], [[18, 21, 227], [44, 157, 95]]]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[27, 22, 138], [21, 50, 119]], [[189, 255, 164], [196, 209, 125]]]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[23, 128, 141], [123, 138, 99]], [[236, 230, 88], [189, 234, 106]]]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[205, 151, 111], [44, 26, 139]], [[66, 163, 159], [116, 26, 92]]]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[191, 32, 68], [60, 203, 92]], [[188, 88, 215], [70, 186, 195]]]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[19, 128, 22], [125, 82, 227]], [[20, 193, 14], [45, 76, 80]]]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[235, 196, 235], [71, 55, 170]], [[103, 123, 230], [50, 215, 161]]]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[174, 231, 68], [112, 19, 87]], [[240, 41, 212], [66, 12, 232]]]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[26, 21, 79], [106, 230, 59]], [[46, 209, 130], [101, 123, 233]]]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[96, 27, 183], [1, 164, 100]], [[232, 232, 213], [251, 62, 197]]]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[18, 7, 47], [250, 10, 73]], [[15, 89, 180], [244, 148, 226]]]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[22, 71, 64], [255, 39, 160]], [[26, 222, 161], [190, 66, 137]]]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[23, 132, 228], [220, 168, 247]], [[226, 215, 241], [236, 32, 255]]]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[212, 244, 217], [182, 185, 239]], [[253, 249, 238], [36, 153, 7]]]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[202, 170, 34], [234, 24, 7]], [[99, 34, 11], [185, 160, 246]]]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[231, 138, 156], [250, 92, 165]], [[215, 8, 125], [201, 61, 208]]]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[6, 175, 244], [189, 184, 190]], [[103, 218, 167], [127, 225, 10]]]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[236, 195, 65], [226, 86, 41]], [[108, 242, 35], [200, 150, 250]]]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[19, 196, 13], [228, 219, 19]], [[147, 207, 208], [75, 141, 54]]]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[52, 181, 6], [63, 87, 243]], [[2, 152, 212], [88, 193, 64]]]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[227, 8, 224], [222, 216, 243]], [[161, 229, 215], [125, 248, 106]]]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[200, 181, 217], [254, 218, 13]], [[179, 224, 76], [10, 210, 78]]]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[167, 166, 99], [231, 239, 70]], [[239, 207, 36], [200, 194, 197]]]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[152, 66, 164], [2, 142, 108]], [[182, 102, 106], [144, 116, 29]]]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[191, 39, 66], [13, 202, 233]], [[179, 44, 209], [162, 114, 192]]]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[150, 136, 50], [91, 52, 202]], [[157, 217, 204], [43, 68, 130]]]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[174, 18, 5], [204, 130, 196]], [[243, 197, 210], [189, 174, 133]]]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[127, 38, 91], [63, 151, 242]], [[198, 201, 77], [250, 147, 234]]]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[95, 21, 24], [226, 167, 198]], [[244, 172, 146], [119, 113, 133]]]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[66, 66, 234], [199, 43, 105]], [[237, 134, 168], [132, 120, 110]]]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[197, 104, 225], [175, 59, 64]], [[197, 83, 34], [108, 25, 22]]]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[171, 141, 136], [48, 201, 203]], [[113, 179, 145], [156, 27, 127]]]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[195, 3, 155], [49, 80, 96]], [[153, 49, 15], [212, 113, 212]]]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[147, 63, 64], [169, 87, 235]], [[54, 223, 26], [254, 170, 139]]]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[59, 40, 204], [186, 74, 143]], [[189, 229, 192], [14, 69, 89]]]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[83, 45, 175], [39, 84, 66]], [[102, 149, 235], [189, 127, 32]]]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[153, 31, 71], [37, 207, 130]], [[76, 155, 61], [151, 42, 250]]]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[234, 7, 193], [67, 70, 20]], [[112, 245, 59], [196, 55, 161]]]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[34, 95, 216], [67, 252, 113]], [[97, 67, 150], [49, 197, 226]]]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[2, 121, 246], [252, 245, 224]], [[3, 182, 35], [73, 202, 147]]]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[([[[76, 125, 240], [191, 13, 20]], [[153, 77, 7], [41, 143, 172]]], u'bird'),\n",
+ " ([[[15, 126, 174], [246, 129, 81]], [[143, 220, 157], [96, 207, 223]]], u'bird'),\n",
+ " ([[[84, 24, 1], [201, 28, 77]], [[70, 12, 11], [83, 33, 165]]], u'dog'),\n",
+ " ([[[40, 206, 47], [211, 138, 62]], [[82, 56, 52], [210, 137, 195]]], u'bird'),\n",
+ " ([[[38, 35, 125], [5, 18, 209]], [[29, 19, 153], [57, 95, 223]]], u'cat'),\n",
+ " ([[[107, 50, 102], [15, 210, 142]], [[222, 1, 97], [103, 63, 179]]], u'dog'),\n",
+ " ([[[115, 133, 40], [55, 51, 78]], [[89, 176, 83], [108, 129, 112]]], u'dog'),\n",
+ " ([[[27, 169, 222], [249, 239, 73]], [[43, 85, 88], [253, 227, 54]]], u'bird'),\n",
+ " ([[[68, 157, 61], [246, 60, 176]], [[123, 100, 230], [175, 178, 64]]], u'dog'),\n",
+ " ([[[4, 172, 224], [116, 42, 251]], [[30, 8, 244], [12, 81, 31]]], u'dog'),\n",
+ " ([[[169, 28, 68], [223, 26, 136]], [[124, 87, 126], [184, 7, 250]]], u'cat'),\n",
+ " ([[[237, 168, 205], [45, 7, 210]], [[217, 231, 70], [3, 226, 100]]], u'cat'),\n",
+ " ([[[214, 112, 91], [246, 209, 4]], [[18, 21, 227], [44, 157, 95]]], u'dog'),\n",
+ " ([[[27, 22, 138], [21, 50, 119]], [[189, 255, 164], [196, 209, 125]]], u'bird'),\n",
+ " ([[[23, 128, 141], [123, 138, 99]], [[236, 230, 88], [189, 234, 106]]], u'bird'),\n",
+ " ([[[205, 151, 111], [44, 26, 139]], [[66, 163, 159], [116, 26, 92]]], u'dog'),\n",
+ " ([[[191, 32, 68], [60, 203, 92]], [[188, 88, 215], [70, 186, 195]]], u'cat'),\n",
+ " ([[[19, 128, 22], [125, 82, 227]], [[20, 193, 14], [45, 76, 80]]], u'bird'),\n",
+ " ([[[235, 196, 235], [71, 55, 170]], [[103, 123, 230], [50, 215, 161]]], u'dog'),\n",
+ " ([[[174, 231, 68], [112, 19, 87]], [[240, 41, 212], [66, 12, 232]]], u'cat'),\n",
+ " ([[[26, 21, 79], [106, 230, 59]], [[46, 209, 130], [101, 123, 233]]], u'bird'),\n",
+ " ([[[96, 27, 183], [1, 164, 100]], [[232, 232, 213], [251, 62, 197]]], u'cat'),\n",
+ " ([[[18, 7, 47], [250, 10, 73]], [[15, 89, 180], [244, 148, 226]]], u'bird'),\n",
+ " ([[[22, 71, 64], [255, 39, 160]], [[26, 222, 161], [190, 66, 137]]], u'dog'),\n",
+ " ([[[23, 132, 228], [220, 168, 247]], [[226, 215, 241], [236, 32, 255]]], u'cat'),\n",
+ " ([[[212, 244, 217], [182, 185, 239]], [[253, 249, 238], [36, 153, 7]]], u'cat'),\n",
+ " ([[[202, 170, 34], [234, 24, 7]], [[99, 34, 11], [185, 160, 246]]], u'dog'),\n",
+ " ([[[231, 138, 156], [250, 92, 165]], [[215, 8, 125], [201, 61, 208]]], u'cat'),\n",
+ " ([[[6, 175, 244], [189, 184, 190]], [[103, 218, 167], [127, 225, 10]]], u'dog'),\n",
+ " ([[[236, 195, 65], [226, 86, 41]], [[108, 242, 35], [200, 150, 250]]], u'bird'),\n",
+ " ([[[19, 196, 13], [228, 219, 19]], [[147, 207, 208], [75, 141, 54]]], u'cat'),\n",
+ " ([[[52, 181, 6], [63, 87, 243]], [[2, 152, 212], [88, 193, 64]]], u'cat'),\n",
+ " ([[[227, 8, 224], [222, 216, 243]], [[161, 229, 215], [125, 248, 106]]], u'cat'),\n",
+ " ([[[200, 181, 217], [254, 218, 13]], [[179, 224, 76], [10, 210, 78]]], u'dog'),\n",
+ " ([[[167, 166, 99], [231, 239, 70]], [[239, 207, 36], [200, 194, 197]]], u'bird'),\n",
+ " ([[[152, 66, 164], [2, 142, 108]], [[182, 102, 106], [144, 116, 29]]], u'dog'),\n",
+ " ([[[191, 39, 66], [13, 202, 233]], [[179, 44, 209], [162, 114, 192]]], u'dog'),\n",
+ " ([[[150, 136, 50], [91, 52, 202]], [[157, 217, 204], [43, 68, 130]]], u'dog'),\n",
+ " ([[[174, 18, 5], [204, 130, 196]], [[243, 197, 210], [189, 174, 133]]], u'bird'),\n",
+ " ([[[127, 38, 91], [63, 151, 242]], [[198, 201, 77], [250, 147, 234]]], u'bird'),\n",
+ " ([[[95, 21, 24], [226, 167, 198]], [[244, 172, 146], [119, 113, 133]]], u'cat'),\n",
+ " ([[[66, 66, 234], [199, 43, 105]], [[237, 134, 168], [132, 120, 110]]], u'cat'),\n",
+ " ([[[197, 104, 225], [175, 59, 64]], [[197, 83, 34], [108, 25, 22]]], u'cat'),\n",
+ " ([[[171, 141, 136], [48, 201, 203]], [[113, 179, 145], [156, 27, 127]]], u'cat'),\n",
+ " ([[[195, 3, 155], [49, 80, 96]], [[153, 49, 15], [212, 113, 212]]], u'cat'),\n",
+ " ([[[147, 63, 64], [169, 87, 235]], [[54, 223, 26], [254, 170, 139]]], u'bird'),\n",
+ " ([[[59, 40, 204], [186, 74, 143]], [[189, 229, 192], [14, 69, 89]]], u'cat'),\n",
+ " ([[[83, 45, 175], [39, 84, 66]], [[102, 149, 235], [189, 127, 32]]], u'dog'),\n",
+ " ([[[153, 31, 71], [37, 207, 130]], [[76, 155, 61], [151, 42, 250]]], u'dog'),\n",
+ " ([[[234, 7, 193], [67, 70, 20]], [[112, 245, 59], [196, 55, 161]]], u'dog'),\n",
+ " ([[[34, 95, 216], [67, 252, 113]], [[97, 67, 150], [49, 197, 226]]], u'bird'),\n",
+ " ([[[2, 121, 246], [252, 245, 224]], [[3, 182, 35], [73, 202, 147]]], u'cat')]"
+ ]
+ },
+ "execution_count": 4,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS image_data;\n",
+ "\n",
+ "CREATE TABLE image_data AS (\n",
+ " SELECT ARRAY[\n",
+ " ARRAY[\n",
+ " ARRAY[(random() * 256)::integer, -- pixel (1,1)\n",
+ " (random() * 256)::integer,\n",
+ " (random() * 256)::integer],\n",
+ " ARRAY[(random() * 256)::integer, -- pixel (2,1)\n",
+ " (random() * 256)::integer,\n",
+ " (random() * 256)::integer]\n",
+ " ],\n",
+ " ARRAY[\n",
+ " ARRAY[(random() * 256)::integer, -- pixel (1,2)\n",
+ " (random() * 256)::integer,\n",
+ " (random() * 256)::integer],\n",
+ " ARRAY[(random() * 256)::integer, -- pixel (2,1)\n",
+ " (random() * 256)::integer,\n",
+ " (random() * 256)::integer]\n",
+ " ]\n",
+ " ] as rgb, ('{cat,dog,bird}'::text[])[ceil(random()*3)] as species\n",
+ " FROM generate_series(1, 52)\n",
+ ");\n",
+ "\n",
+ "SELECT * FROM image_data;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"pp_train\"></a>\n",
+ "# 2. Run preprocessor for training image data\n",
+ "\n",
+ "Run the preprocessor to generate the packed output table:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "2 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>independent_var</th>\n",
+ " <th>dependent_var</th>\n",
+ " <th>buffer_id</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.0862745, 0.278431, 0.25098], [1.0, 0.152941, 0.627451]], [[0.101961, 0.870588, 0.631373], [0.745098, 0.258824, 0.537255]]], [[[0.0588235, 0.494118, 0.682353], [0.964706, 0.505882, 0.317647]], [[0.560784, 0.862745, 0.615686], [0.376471, 0.811765, 0.87451]]], [[[0.156863, 0.807843, 0.184314], [0.827451, 0.541176, 0.243137]], [[0.321569, 0.219608, 0.203922], [0.823529, 0.537255, 0.764706]]], [[[0.419608, 0.196078, 0.4], [0.0588235, 0.823529, 0.556863]], [[0.870588, [...]
+ " <td>[[0, 0, 1], [1, 0, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [1, 0, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 0, 1], [1, 0, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0]]</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.45098, 0.521569, 0.156863], [0.215686, 0.2, 0.305882]], [[0.34902, 0.690196, 0.32549], [0.423529, 0.505882, 0.439216]]], [[[0.0901961, 0.501961, 0.552941], [0.482353, 0.541176, 0.388235]], [[0.92549, 0.901961, 0.345098], [0.741176, 0.917647, 0.415686]]], [[[0.921569, 0.768628, 0.921569], [0.278431, 0.215686, 0.666667]], [[0.403922, 0.482353, 0.901961], [0.196078, 0.843137, 0.631373]]], [[[0.772549, 0.407843, 0.882353], [0.686275, 0.231373, 0.25098]], [[0.772549, [...]
+ " <td>[[0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0]]</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[([[[[0.0862745, 0.278431, 0.25098], [1.0, 0.152941, 0.627451]], [[0.101961, 0.870588, 0.631373], [0.745098, 0.258824, 0.537255]]], [[[0.0588235, 0.494118, 0.682353], [0.964706, 0.505882, 0.317647]], [[0.560784, 0.862745, 0.615686], [0.376471, 0.811765, 0.87451]]], [[[0.156863, 0.807843, 0.184314], [0.827451, 0.541176, 0.243137]], [[0.321569, 0.219608, 0.203922], [0.823529, 0.537255, 0.764706]]], [[[0.419608, 0.196078, 0.4], [0.0588235, 0.823529, 0.556863]], [[0.870588, 0.0039215 [...]
+ " ([[[[0.45098, 0.521569, 0.156863], [0.215686, 0.2, 0.305882]], [[0.34902, 0.690196, 0.32549], [0.423529, 0.505882, 0.439216]]], [[[0.0901961, 0.501961, 0.552941], [0.482353, 0.541176, 0.388235]], [[0.92549, 0.901961, 0.345098], [0.741176, 0.917647, 0.415686]]], [[[0.921569, 0.768628, 0.921569], [0.278431, 0.215686, 0.666667]], [[0.403922, 0.482353, 0.901961], [0.196078, 0.843137, 0.631373]]], [[[0.772549, 0.407843, 0.882353], [0.686275, 0.231373, 0.25098]], [[0.772549, 0.32549, [...]
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS image_data_packed, image_data_packed_summary;\n",
+ "\n",
+ "SELECT madlib.training_preprocessor_dl('image_data', -- Source table\n",
+ " 'image_data_packed', -- Output table\n",
+ " 'species', -- Dependent variable\n",
+ " 'rgb', -- Independent variable\n",
+ " NULL, -- Buffer size\n",
+ " 255 -- Normalizing constant\n",
+ " );\n",
+ "\n",
+ "SELECT * FROM image_data_packed ORDER BY buffer_id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "For small datasets like in this example, buffer size is mainly determined by the number of segments in the database. For a Greenplum database with 2 segments, there will be 2 rows with a buffer size of 26. For PostgresSQL, there would be only one row with a buffer size of 52 since it is a single node database. For larger data sets, other factors go into computing buffers size besides number of segments. \n",
+ "\n",
+ "Review the output summary table:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>source_table</th>\n",
+ " <th>output_table</th>\n",
+ " <th>dependent_varname</th>\n",
+ " <th>independent_varname</th>\n",
+ " <th>dependent_vartype</th>\n",
+ " <th>class_values</th>\n",
+ " <th>buffer_size</th>\n",
+ " <th>normalizing_const</th>\n",
+ " <th>num_classes</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>image_data</td>\n",
+ " <td>image_data_packed</td>\n",
+ " <td>species</td>\n",
+ " <td>rgb</td>\n",
+ " <td>text</td>\n",
+ " <td>[u'bird', u'cat', u'dog']</td>\n",
+ " <td>26</td>\n",
+ " <td>255.0</td>\n",
+ " <td>3</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'image_data', u'image_data_packed', u'species', u'rgb', u'text', [u'bird', u'cat', u'dog'], 26, Decimal('255.0'), 3)]"
+ ]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM image_data_packed_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"pp_val\"></a>\n",
+ "# 3. Run preprocessor for validation image data\n",
+ "\n",
+ "Run the preprocessor for the validation dataset. In this example, we use the same images for validation to demonstrate, but normally validation data is different than training data:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "26 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>independent_var</th>\n",
+ " <th>dependent_var</th>\n",
+ " <th>buffer_id</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.290196, 0.105882, 0.152941], [0.47451, 0.670588, 0.356863]], [[0.101961, 0.458824, 0.188235], [0.584314, 0.921569, 0.854902]]], [[[0.941177, 0.92549, 0.34902], [0.137255, 0.360784, 0.411765]], [[0.0627451, 0.917647, 0.898039], [0.203922, 0.313726, 0.247059]]]]</td>\n",
+ " <td>[[1, 0, 0], [0, 0, 1]]</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.815686, 0.294118, 0.568627], [0.603922, 0.662745, 0.870588]], [[0.913726, 0.352941, 0.0745098], [0.0117647, 0.596078, 0.6]]], [[[0.0509804, 0.262745, 0.933333], [0.894118, 0.603922, 0.0901961]], [[0.643137, 0.12549, 0.623529], [0.0392157, 0.713726, 0.819608]]]]</td>\n",
+ " <td>[[0, 1, 0], [0, 0, 1]]</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.14902, 0.52549, 0.6], [0.784314, 0.619608, 0.823529]], [[0.0823529, 0.0862745, 0.454902], [0.835294, 0.231373, 0.996078]]], [[[0.713726, 0.803922, 0.0156863], [0.678431, 0.415686, 0.470588]], [[0.156863, 0.85098, 0.941177], [0.27451, 0.141176, 0.72549]]]]</td>\n",
+ " <td>[[1, 0, 0], [0, 0, 1]]</td>\n",
+ " <td>2</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.584314, 0.890196, 0.94902], [0.419608, 0.709804, 0.760784]], [[0.713726, 0.270588, 0.360784], [0.372549, 0.141176, 0.270588]]], [[[0.819608, 0.823529, 0.0980392], [0.262745, 0.713726, 0.552941]], [[0.203922, 0.890196, 0.945098], [0.156863, 0.235294, 0.466667]]]]</td>\n",
+ " <td>[[1, 0, 0], [0, 1, 0]]</td>\n",
+ " <td>3</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.215686, 0.282353, 0.321569], [0.815686, 0.588235, 0.466667]], [[0.337255, 0.184314, 0.247059], [0.956863, 0.00392157, 0.329412]]], [[[0.129412, 0.388235, 0.270588], [0.980392, 0.623529, 0.984314]], [[0.780392, 0.639216, 0.658824], [0.192157, 0.105882, 0.815686]]]]</td>\n",
+ " <td>[[0, 1, 0], [0, 1, 0]]</td>\n",
+ " <td>4</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.458824, 0.160784, 0.270588], [0.45098, 0.94902, 0.854902]], [[0.337255, 0.894118, 0.27451], [0.0431373, 0.65098, 0.988235]]], [[[0.0431373, 0.0862745, 0.180392], [0.772549, 0.615686, 1.0]], [[0.588235, 0.713726, 0.254902], [0.298039, 0.262745, 0.458824]]]]</td>\n",
+ " <td>[[0, 1, 0], [0, 1, 0]]</td>\n",
+ " <td>5</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.00784314, 0.356863, 0.454902], [0.282353, 0.0784314, 0.670588]], [[0.564706, 0.4, 0.478431], [0.14902, 0.866667, 0.815686]]], [[[0.207843, 0.615686, 0.419608], [0.670588, 0.760784, 0.54902]], [[0.054902, 0.0313726, 0.52549], [0.678431, 0.0117647, 0.298039]]]]</td>\n",
+ " <td>[[0, 1, 0], [0, 1, 0]]</td>\n",
+ " <td>6</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.666667, 0.490196, 0.419608], [0.478431, 0.529412, 0.462745]], [[0.564706, 0.709804, 0.231373], [0.176471, 0.701961, 0.819608]]], [[[0.113725, 0.764706, 0.337255], [0.439216, 0.803922, 0.796079]], [[0.6, 0.0745098, 0.243137], [0.54902, 0.929412, 0.580392]]]]</td>\n",
+ " <td>[[1, 0, 0], [0, 0, 1]]</td>\n",
+ " <td>7</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.196078, 0.619608, 0.0862745], [0.180392, 0.933333, 0.0941177]], [[0.537255, 0.384314, 0.376471], [0.619608, 0.0509804, 0.941177]]], [[[0.960784, 0.113725, 0.14902], [0.415686, 0.301961, 0.356863]], [[0.027451, 0.721569, 0.0235294], [0.788235, 0.266667, 0.0784314]]]]</td>\n",
+ " <td>[[0, 1, 0], [1, 0, 0]]</td>\n",
+ " <td>8</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.529412, 0.0862745, 0.882353], [0.341176, 0.415686, 0.996078]], [[0.101961, 0.752941, 0.431373], [0.909804, 0.545098, 0.027451]]], [[[0.792157, 0.760784, 0.827451], [0.0862745, 0.0705882, 0.490196]], [[0.576471, 0.490196, 0.972549], [0.101961, 0.952941, 0.533333]]]]</td>\n",
+ " <td>[[1, 0, 0], [1, 0, 0]]</td>\n",
+ " <td>9</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.180392, 0.709804, 0.509804], [1.0, 0.592157, 0.466667]], [[0.113725, 0.741176, 0.882353], [0.415686, 0.0941177, 0.905882]]], [[[0.784314, 0.576471, 0.905882], [0.360784, 0.0117647, 0.0980392]], [[0.980392, 0.0980392, 0.282353], [0.913726, 0.196078, 0.819608]]]]</td>\n",
+ " <td>[[0, 1, 0], [0, 1, 0]]</td>\n",
+ " <td>10</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.572549, 0.431373, 0.345098], [0.509804, 0.447059, 0.87451]], [[0.592157, 0.32549, 0.211765], [0.00784314, 0.313726, 0.313726]]], [[[0.117647, 0.694118, 0.4], [0.196078, 0.505882, 0.188235]], [[0.956863, 0.329412, 0.27451], [0.0235294, 0.823529, 0.854902]]]]</td>\n",
+ " <td>[[1, 0, 0], [0, 0, 1]]</td>\n",
+ " <td>11</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.792157, 0.956863, 0.458824], [0.317647, 0.764706, 0.639216]], [[0.0235294, 0.270588, 0.635294], [0.615686, 0.737255, 0.74902]]], [[[0.745098, 0.219608, 0.301961], [0.776471, 0.196078, 0.0823529]], [[0.34902, 0.0980392, 0.443137], [0.360784, 0.196078, 0.419608]]]]</td>\n",
+ " <td>[[0, 0, 1], [0, 0, 1]]</td>\n",
+ " <td>12</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.478431, 0.329412, 0.654902], [0.290196, 0.623529, 0.223529]], [[0.721569, 0.964706, 0.729412], [0.164706, 0.835294, 0.321569]]], [[[0.615686, 0.156863, 0.447059], [0.521569, 0.290196, 0.564706]], [[0.207843, 0.690196, 0.760784], [0.717647, 0.878431, 0.713726]]]]</td>\n",
+ " <td>[[0, 0, 1], [0, 1, 0]]</td>\n",
+ " <td>13</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.0431373, 0.32549, 0.803922], [0.356863, 0.0823529, 0.54902]], [[0.572549, 0.384314, 0.321569], [0.768628, 0.466667, 0.670588]]], [[[0.14902, 0.737255, 0.866667], [0.0, 0.243137, 0.65098]], [[0.956863, 0.705882, 0.972549], [0.721569, 0.341176, 0.996078]]]]</td>\n",
+ " <td>[[1, 0, 0], [1, 0, 0]]</td>\n",
+ " <td>14</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.976471, 0.607843, 0.72549], [0.721569, 0.960784, 0.341176]], [[0.878431, 0.4, 0.858824], [0.164706, 0.964706, 0.0666667]]], [[[0.909804, 0.027451, 0.0588235], [0.32549, 0.486275, 0.537255]], [[0.658824, 0.137255, 0.827451], [0.27451, 0.360784, 0.545098]]]]</td>\n",
+ " <td>[[1, 0, 0], [0, 1, 0]]</td>\n",
+ " <td>15</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.0862745, 0.709804, 0.0705882], [0.403922, 0.196078, 0.109804]], [[0.733333, 1.0, 0.466667], [0.815686, 0.541176, 0.0352941]]], [[[0.721569, 0.780392, 0.729412], [0.431373, 0.823529, 0.882353]], [[0.164706, 0.686275, 0.882353], [0.407843, 0.333333, 0.835294]]]]</td>\n",
+ " <td>[[0, 1, 0], [0, 1, 0]]</td>\n",
+ " <td>16</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.862745, 0.803922, 0.662745], [0.529412, 0.666667, 0.568627]], [[0.552941, 0.72549, 0.894118], [0.0352941, 0.254902, 0.54902]]], [[[0.301961, 0.552941, 0.447059], [0.294118, 0.541176, 0.419608]], [[0.898039, 0.266667, 0.137255], [0.854902, 0.603922, 0.0117647]]]]</td>\n",
+ " <td>[[0, 1, 0], [0, 1, 0]]</td>\n",
+ " <td>17</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.0784314, 0.823529, 0.533333], [0.623529, 0.0588235, 0.619608]], [[0.329412, 0.12549, 0.0196078], [0.52549, 0.235294, 0.752941]]], [[[0.462745, 0.180392, 0.211765], [0.52549, 0.0313726, 0.933333]], [[0.305882, 0.760784, 0.360784], [0.12549, 0.639216, 0.52549]]]]</td>\n",
+ " <td>[[0, 0, 1], [1, 0, 0]]</td>\n",
+ " <td>18</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.701961, 0.564706, 0.0588235], [0.737255, 0.760784, 0.921569]], [[0.537255, 0.415686, 0.447059], [0.2, 0.984314, 1.0]]], [[[0.517647, 0.933333, 0.141176], [0.352941, 0.0352941, 0.447059]], [[0.905882, 0.486275, 0.737255], [0.443137, 0.905882, 0.631373]]]]</td>\n",
+ " <td>[[1, 0, 0], [1, 0, 0]]</td>\n",
+ " <td>19</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.878431, 0.0352941, 0.176471], [0.419608, 0.207843, 0.258824]], [[0.243137, 0.741176, 0.882353], [0.298039, 0.356863, 0.207843]]], [[[0.0392157, 0.482353, 0.309804], [0.0509804, 0.737255, 0.768628]], [[0.231373, 0.94902, 0.290196], [0.262745, 0.878431, 0.596078]]]]</td>\n",
+ " <td>[[0, 0, 1], [0, 1, 0]]</td>\n",
+ " <td>20</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.376471, 0.733333, 0.662745], [0.121569, 0.25098, 0.360784]], [[0.686275, 0.309804, 0.0941177], [0.443137, 0.231373, 0.631373]]], [[[0.239216, 0.721569, 0.658824], [0.764706, 0.529412, 0.172549]], [[0.694118, 0.670588, 0.52549], [0.729412, 0.113725, 0.427451]]]]</td>\n",
+ " <td>[[1, 0, 0], [0, 1, 0]]</td>\n",
+ " <td>21</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.85098, 0.866667, 0.113725], [0.478431, 0.572549, 0.152941]], [[0.964706, 0.878431, 0.203922], [0.694118, 0.647059, 0.431373]]], [[[0.67451, 0.831373, 0.839216], [0.67451, 0.752941, 0.713726]], [[0.705882, 0.933333, 0.129412], [0.917647, 0.184314, 0.372549]]]]</td>\n",
+ " <td>[[0, 0, 1], [0, 0, 1]]</td>\n",
+ " <td>22</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.0627451, 0.670588, 0.00392157], [0.270588, 0.0941177, 0.380392]], [[0.0, 0.752941, 0.505882], [0.25098, 0.109804, 0.188235]]], [[[0.937255, 0.694118, 0.513726], [0.529412, 0.713726, 0.752941]], [[0.247059, 0.368627, 0.513726], [0.776471, 0.541176, 0.2]]]]</td>\n",
+ " <td>[[0, 0, 1], [0, 0, 1]]</td>\n",
+ " <td>23</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.203922, 0.627451, 0.792157], [0.835294, 0.482353, 0.462745]], [[0.662745, 0.321569, 0.133333], [0.411765, 0.027451, 0.839216]]], [[[0.0627451, 0.929412, 0.552941], [0.490196, 0.137255, 0.4]], [[0.352941, 0.25098, 0.882353], [0.92549, 0.403922, 0.839216]]]]</td>\n",
+ " <td>[[0, 0, 1], [1, 0, 0]]</td>\n",
+ " <td>24</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[[[0.603922, 0.529412, 0.443137], [0.0352941, 0.164706, 0.376471]], [[0.729412, 0.678431, 0.905882], [0.439216, 0.427451, 0.14902]]], [[[0.160784, 0.752941, 0.52549], [0.533333, 0.403922, 0.588235]], [[0.2, 0.407843, 0.858824], [0.290196, 0.788235, 0.858824]]]]</td>\n",
+ " <td>[[1, 0, 0], [0, 1, 0]]</td>\n",
+ " <td>25</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[([[[[0.290196, 0.105882, 0.152941], [0.47451, 0.670588, 0.356863]], [[0.101961, 0.458824, 0.188235], [0.584314, 0.921569, 0.854902]]], [[[0.941177, 0.92549, 0.34902], [0.137255, 0.360784, 0.411765]], [[0.0627451, 0.917647, 0.898039], [0.203922, 0.313726, 0.247059]]]], [[1, 0, 0], [0, 0, 1]], 0),\n",
+ " ([[[[0.815686, 0.294118, 0.568627], [0.603922, 0.662745, 0.870588]], [[0.913726, 0.352941, 0.0745098], [0.0117647, 0.596078, 0.6]]], [[[0.0509804, 0.262745, 0.933333], [0.894118, 0.603922, 0.0901961]], [[0.643137, 0.12549, 0.623529], [0.0392157, 0.713726, 0.819608]]]], [[0, 1, 0], [0, 0, 1]], 1),\n",
+ " ([[[[0.14902, 0.52549, 0.6], [0.784314, 0.619608, 0.823529]], [[0.0823529, 0.0862745, 0.454902], [0.835294, 0.231373, 0.996078]]], [[[0.713726, 0.803922, 0.0156863], [0.678431, 0.415686, 0.470588]], [[0.156863, 0.85098, 0.941177], [0.27451, 0.141176, 0.72549]]]], [[1, 0, 0], [0, 0, 1]], 2),\n",
+ " ([[[[0.584314, 0.890196, 0.94902], [0.419608, 0.709804, 0.760784]], [[0.713726, 0.270588, 0.360784], [0.372549, 0.141176, 0.270588]]], [[[0.819608, 0.823529, 0.0980392], [0.262745, 0.713726, 0.552941]], [[0.203922, 0.890196, 0.945098], [0.156863, 0.235294, 0.466667]]]], [[1, 0, 0], [0, 1, 0]], 3),\n",
+ " ([[[[0.215686, 0.282353, 0.321569], [0.815686, 0.588235, 0.466667]], [[0.337255, 0.184314, 0.247059], [0.956863, 0.00392157, 0.329412]]], [[[0.129412, 0.388235, 0.270588], [0.980392, 0.623529, 0.984314]], [[0.780392, 0.639216, 0.658824], [0.192157, 0.105882, 0.815686]]]], [[0, 1, 0], [0, 1, 0]], 4),\n",
+ " ([[[[0.458824, 0.160784, 0.270588], [0.45098, 0.94902, 0.854902]], [[0.337255, 0.894118, 0.27451], [0.0431373, 0.65098, 0.988235]]], [[[0.0431373, 0.0862745, 0.180392], [0.772549, 0.615686, 1.0]], [[0.588235, 0.713726, 0.254902], [0.298039, 0.262745, 0.458824]]]], [[0, 1, 0], [0, 1, 0]], 5),\n",
+ " ([[[[0.00784314, 0.356863, 0.454902], [0.282353, 0.0784314, 0.670588]], [[0.564706, 0.4, 0.478431], [0.14902, 0.866667, 0.815686]]], [[[0.207843, 0.615686, 0.419608], [0.670588, 0.760784, 0.54902]], [[0.054902, 0.0313726, 0.52549], [0.678431, 0.0117647, 0.298039]]]], [[0, 1, 0], [0, 1, 0]], 6),\n",
+ " ([[[[0.666667, 0.490196, 0.419608], [0.478431, 0.529412, 0.462745]], [[0.564706, 0.709804, 0.231373], [0.176471, 0.701961, 0.819608]]], [[[0.113725, 0.764706, 0.337255], [0.439216, 0.803922, 0.796079]], [[0.6, 0.0745098, 0.243137], [0.54902, 0.929412, 0.580392]]]], [[1, 0, 0], [0, 0, 1]], 7),\n",
+ " ([[[[0.196078, 0.619608, 0.0862745], [0.180392, 0.933333, 0.0941177]], [[0.537255, 0.384314, 0.376471], [0.619608, 0.0509804, 0.941177]]], [[[0.960784, 0.113725, 0.14902], [0.415686, 0.301961, 0.356863]], [[0.027451, 0.721569, 0.0235294], [0.788235, 0.266667, 0.0784314]]]], [[0, 1, 0], [1, 0, 0]], 8),\n",
+ " ([[[[0.529412, 0.0862745, 0.882353], [0.341176, 0.415686, 0.996078]], [[0.101961, 0.752941, 0.431373], [0.909804, 0.545098, 0.027451]]], [[[0.792157, 0.760784, 0.827451], [0.0862745, 0.0705882, 0.490196]], [[0.576471, 0.490196, 0.972549], [0.101961, 0.952941, 0.533333]]]], [[1, 0, 0], [1, 0, 0]], 9),\n",
+ " ([[[[0.180392, 0.709804, 0.509804], [1.0, 0.592157, 0.466667]], [[0.113725, 0.741176, 0.882353], [0.415686, 0.0941177, 0.905882]]], [[[0.784314, 0.576471, 0.905882], [0.360784, 0.0117647, 0.0980392]], [[0.980392, 0.0980392, 0.282353], [0.913726, 0.196078, 0.819608]]]], [[0, 1, 0], [0, 1, 0]], 10),\n",
+ " ([[[[0.572549, 0.431373, 0.345098], [0.509804, 0.447059, 0.87451]], [[0.592157, 0.32549, 0.211765], [0.00784314, 0.313726, 0.313726]]], [[[0.117647, 0.694118, 0.4], [0.196078, 0.505882, 0.188235]], [[0.956863, 0.329412, 0.27451], [0.0235294, 0.823529, 0.854902]]]], [[1, 0, 0], [0, 0, 1]], 11),\n",
+ " ([[[[0.792157, 0.956863, 0.458824], [0.317647, 0.764706, 0.639216]], [[0.0235294, 0.270588, 0.635294], [0.615686, 0.737255, 0.74902]]], [[[0.745098, 0.219608, 0.301961], [0.776471, 0.196078, 0.0823529]], [[0.34902, 0.0980392, 0.443137], [0.360784, 0.196078, 0.419608]]]], [[0, 0, 1], [0, 0, 1]], 12),\n",
+ " ([[[[0.478431, 0.329412, 0.654902], [0.290196, 0.623529, 0.223529]], [[0.721569, 0.964706, 0.729412], [0.164706, 0.835294, 0.321569]]], [[[0.615686, 0.156863, 0.447059], [0.521569, 0.290196, 0.564706]], [[0.207843, 0.690196, 0.760784], [0.717647, 0.878431, 0.713726]]]], [[0, 0, 1], [0, 1, 0]], 13),\n",
+ " ([[[[0.0431373, 0.32549, 0.803922], [0.356863, 0.0823529, 0.54902]], [[0.572549, 0.384314, 0.321569], [0.768628, 0.466667, 0.670588]]], [[[0.14902, 0.737255, 0.866667], [0.0, 0.243137, 0.65098]], [[0.956863, 0.705882, 0.972549], [0.721569, 0.341176, 0.996078]]]], [[1, 0, 0], [1, 0, 0]], 14),\n",
+ " ([[[[0.976471, 0.607843, 0.72549], [0.721569, 0.960784, 0.341176]], [[0.878431, 0.4, 0.858824], [0.164706, 0.964706, 0.0666667]]], [[[0.909804, 0.027451, 0.0588235], [0.32549, 0.486275, 0.537255]], [[0.658824, 0.137255, 0.827451], [0.27451, 0.360784, 0.545098]]]], [[1, 0, 0], [0, 1, 0]], 15),\n",
+ " ([[[[0.0862745, 0.709804, 0.0705882], [0.403922, 0.196078, 0.109804]], [[0.733333, 1.0, 0.466667], [0.815686, 0.541176, 0.0352941]]], [[[0.721569, 0.780392, 0.729412], [0.431373, 0.823529, 0.882353]], [[0.164706, 0.686275, 0.882353], [0.407843, 0.333333, 0.835294]]]], [[0, 1, 0], [0, 1, 0]], 16),\n",
+ " ([[[[0.862745, 0.803922, 0.662745], [0.529412, 0.666667, 0.568627]], [[0.552941, 0.72549, 0.894118], [0.0352941, 0.254902, 0.54902]]], [[[0.301961, 0.552941, 0.447059], [0.294118, 0.541176, 0.419608]], [[0.898039, 0.266667, 0.137255], [0.854902, 0.603922, 0.0117647]]]], [[0, 1, 0], [0, 1, 0]], 17),\n",
+ " ([[[[0.0784314, 0.823529, 0.533333], [0.623529, 0.0588235, 0.619608]], [[0.329412, 0.12549, 0.0196078], [0.52549, 0.235294, 0.752941]]], [[[0.462745, 0.180392, 0.211765], [0.52549, 0.0313726, 0.933333]], [[0.305882, 0.760784, 0.360784], [0.12549, 0.639216, 0.52549]]]], [[0, 0, 1], [1, 0, 0]], 18),\n",
+ " ([[[[0.701961, 0.564706, 0.0588235], [0.737255, 0.760784, 0.921569]], [[0.537255, 0.415686, 0.447059], [0.2, 0.984314, 1.0]]], [[[0.517647, 0.933333, 0.141176], [0.352941, 0.0352941, 0.447059]], [[0.905882, 0.486275, 0.737255], [0.443137, 0.905882, 0.631373]]]], [[1, 0, 0], [1, 0, 0]], 19),\n",
+ " ([[[[0.878431, 0.0352941, 0.176471], [0.419608, 0.207843, 0.258824]], [[0.243137, 0.741176, 0.882353], [0.298039, 0.356863, 0.207843]]], [[[0.0392157, 0.482353, 0.309804], [0.0509804, 0.737255, 0.768628]], [[0.231373, 0.94902, 0.290196], [0.262745, 0.878431, 0.596078]]]], [[0, 0, 1], [0, 1, 0]], 20),\n",
+ " ([[[[0.376471, 0.733333, 0.662745], [0.121569, 0.25098, 0.360784]], [[0.686275, 0.309804, 0.0941177], [0.443137, 0.231373, 0.631373]]], [[[0.239216, 0.721569, 0.658824], [0.764706, 0.529412, 0.172549]], [[0.694118, 0.670588, 0.52549], [0.729412, 0.113725, 0.427451]]]], [[1, 0, 0], [0, 1, 0]], 21),\n",
+ " ([[[[0.85098, 0.866667, 0.113725], [0.478431, 0.572549, 0.152941]], [[0.964706, 0.878431, 0.203922], [0.694118, 0.647059, 0.431373]]], [[[0.67451, 0.831373, 0.839216], [0.67451, 0.752941, 0.713726]], [[0.705882, 0.933333, 0.129412], [0.917647, 0.184314, 0.372549]]]], [[0, 0, 1], [0, 0, 1]], 22),\n",
+ " ([[[[0.0627451, 0.670588, 0.00392157], [0.270588, 0.0941177, 0.380392]], [[0.0, 0.752941, 0.505882], [0.25098, 0.109804, 0.188235]]], [[[0.937255, 0.694118, 0.513726], [0.529412, 0.713726, 0.752941]], [[0.247059, 0.368627, 0.513726], [0.776471, 0.541176, 0.2]]]], [[0, 0, 1], [0, 0, 1]], 23),\n",
+ " ([[[[0.203922, 0.627451, 0.792157], [0.835294, 0.482353, 0.462745]], [[0.662745, 0.321569, 0.133333], [0.411765, 0.027451, 0.839216]]], [[[0.0627451, 0.929412, 0.552941], [0.490196, 0.137255, 0.4]], [[0.352941, 0.25098, 0.882353], [0.92549, 0.403922, 0.839216]]]], [[0, 0, 1], [1, 0, 0]], 24),\n",
+ " ([[[[0.603922, 0.529412, 0.443137], [0.0352941, 0.164706, 0.376471]], [[0.729412, 0.678431, 0.905882], [0.439216, 0.427451, 0.14902]]], [[[0.160784, 0.752941, 0.52549], [0.533333, 0.403922, 0.588235]], [[0.2, 0.407843, 0.858824], [0.290196, 0.788235, 0.858824]]]], [[1, 0, 0], [0, 1, 0]], 25)]"
+ ]
+ },
+ "execution_count": 7,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS val_image_data_packed, val_image_data_packed_summary;\n",
+ "SELECT madlib.validation_preprocessor_dl(\n",
+ " 'image_data', -- Source table\n",
+ " 'val_image_data_packed', -- Output table\n",
+ " 'species', -- Dependent variable\n",
+ " 'rgb', -- Independent variable\n",
+ " 'image_data_packed', -- From training preprocessor step\n",
+ " 2 -- Buffer size\n",
+ " );\n",
+ "SELECT * FROM val_image_data_packed ORDER BY buffer_id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Review the output summary table:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>source_table</th>\n",
+ " <th>output_table</th>\n",
+ " <th>dependent_varname</th>\n",
+ " <th>independent_varname</th>\n",
+ " <th>dependent_vartype</th>\n",
+ " <th>class_values</th>\n",
+ " <th>buffer_size</th>\n",
+ " <th>normalizing_const</th>\n",
+ " <th>num_classes</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>image_data</td>\n",
+ " <td>val_image_data_packed</td>\n",
+ " <td>species</td>\n",
+ " <td>rgb</td>\n",
+ " <td>text</td>\n",
+ " <td>[u'bird', u'cat', u'dog']</td>\n",
+ " <td>2</td>\n",
+ " <td>255.0</td>\n",
+ " <td>3</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'image_data', u'val_image_data_packed', u'species', u'rgb', u'text', [u'bird', u'cat', u'dog'], 2, Decimal('255.0'), 3)]"
+ ]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM val_image_data_packed_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"load_data2\"></a>\n",
+ "# 4. Load data, another format\n",
+ "Create an artificial 2x2 resolution color image data set with 3 possible classifications. The RGB values are unrolled in to a flat array:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "52 rows affected.\n",
+ "52 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>rgb</th>\n",
+ " <th>species</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[87, 118, 166, 176, 152, 5, 135, 219, 1, 249, 60, 67]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[109, 9, 113, 57, 10, 234, 47, 6, 223, 16, 9, 148]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[82, 94, 197, 145, 99, 28, 7, 8, 203, 159, 13, 83]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[14, 75, 182, 81, 218, 36, 90, 74, 93, 100, 52, 140]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[19, 156, 115, 167, 206, 198, 5, 147, 86, 104, 175, 93]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[122, 252, 125, 205, 118, 140, 24, 44, 221, 242, 80, 55]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[173, 155, 112, 57, 6, 131, 212, 121, 42, 162, 63, 47]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[149, 150, 227, 242, 6, 93, 238, 132, 42, 100, 15, 66]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[236, 52, 225, 36, 112, 141, 191, 224, 198, 197, 98, 154]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[140, 60, 125, 187, 113, 18, 81, 84, 5, 88, 178, 243]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[220, 87, 235, 30, 232, 216, 82, 200, 251, 194, 85, 186]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[27, 127, 5, 181, 189, 145, 241, 58, 76, 97, 76, 157]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[80, 245, 103, 67, 209, 67, 154, 188, 97, 130, 148, 179]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[143, 117, 160, 74, 23, 187, 200, 28, 111, 133, 173, 96]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[249, 193, 10, 150, 119, 91, 139, 222, 158, 92, 33, 56]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[130, 186, 172, 53, 4, 59, 170, 164, 133, 193, 94, 77]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[206, 210, 138, 46, 145, 131, 239, 156, 24, 102, 246, 163]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[148, 255, 101, 204, 23, 231, 134, 195, 27, 138, 254, 197]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[131, 134, 140, 208, 100, 90, 162, 238, 136, 52, 112, 119]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[137, 221, 198, 44, 34, 90, 42, 135, 38, 65, 109, 171]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[136, 54, 1, 78, 99, 132, 212, 239, 84, 56, 73, 246]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[209, 42, 150, 72, 249, 30, 37, 191, 74, 71, 24, 116]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[62, 181, 59, 233, 185, 195, 31, 187, 17, 130, 63, 229]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[147, 28, 187, 137, 66, 140, 179, 215, 211, 172, 246, 249]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[64, 63, 131, 180, 13, 193, 105, 72, 170, 35, 11, 201]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[28, 75, 28, 0, 189, 175, 29, 120, 56, 94, 3, 235]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[215, 151, 44, 207, 2, 107, 15, 133, 31, 28, 71, 137]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[197, 68, 195, 107, 92, 71, 80, 55, 239, 70, 26, 198]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[6, 230, 54, 75, 186, 42, 36, 112, 227, 19, 109, 220]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[145, 117, 8, 147, 175, 205, 215, 113, 57, 51, 184, 136]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[167, 207, 132, 109, 6, 138, 83, 60, 213, 13, 102, 249]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[73, 12, 234, 37, 13, 123, 154, 21, 14, 72, 226, 229]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[26, 24, 114, 163, 130, 25, 114, 6, 134, 119, 144, 217]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[101, 230, 25, 94, 99, 99, 106, 77, 136, 119, 199, 34]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[213, 106, 109, 186, 36, 136, 210, 151, 43, 84, 176, 156]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[54, 20, 235, 15, 199, 80, 245, 224, 174, 87, 67, 24]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[203, 143, 107, 237, 26, 65, 87, 136, 251, 123, 16, 205]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[58, 34, 193, 214, 124, 248, 234, 103, 7, 177, 183, 251]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[101, 83, 212, 125, 247, 159, 12, 98, 139, 38, 163, 226]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[157, 93, 189, 107, 110, 248, 140, 48, 206, 8, 39, 184]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[46, 105, 38, 41, 250, 139, 124, 206, 8, 115, 109, 19]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[248, 57, 119, 218, 231, 21, 55, 164, 127, 166, 156, 11]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[106, 20, 253, 34, 131, 43, 139, 170, 84, 133, 53, 208]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[60, 67, 191, 79, 24, 184, 136, 143, 146, 111, 164, 201]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[35, 110, 175, 47, 68, 25, 67, 65, 59, 198, 107, 198]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[191, 74, 164, 144, 157, 224, 211, 92, 48, 234, 20, 184]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[166, 39, 29, 110, 59, 65, 221, 234, 111, 33, 3, 178]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[62, 120, 205, 4, 231, 140, 78, 139, 28, 235, 108, 238]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[155, 217, 91, 83, 82, 0, 122, 111, 110, 181, 176, 75]</td>\n",
+ " <td>dog</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[31, 108, 162, 209, 205, 224, 73, 154, 228, 48, 38, 50]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[65, 29, 39, 48, 100, 194, 8, 190, 21, 90, 190, 144]</td>\n",
+ " <td>bird</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[45, 69, 122, 120, 228, 153, 228, 134, 106, 177, 103, 179]</td>\n",
+ " <td>cat</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[([87, 118, 166, 176, 152, 5, 135, 219, 1, 249, 60, 67], u'dog'),\n",
+ " ([109, 9, 113, 57, 10, 234, 47, 6, 223, 16, 9, 148], u'cat'),\n",
+ " ([82, 94, 197, 145, 99, 28, 7, 8, 203, 159, 13, 83], u'dog'),\n",
+ " ([14, 75, 182, 81, 218, 36, 90, 74, 93, 100, 52, 140], u'dog'),\n",
+ " ([19, 156, 115, 167, 206, 198, 5, 147, 86, 104, 175, 93], u'dog'),\n",
+ " ([122, 252, 125, 205, 118, 140, 24, 44, 221, 242, 80, 55], u'cat'),\n",
+ " ([173, 155, 112, 57, 6, 131, 212, 121, 42, 162, 63, 47], u'cat'),\n",
+ " ([149, 150, 227, 242, 6, 93, 238, 132, 42, 100, 15, 66], u'dog'),\n",
+ " ([236, 52, 225, 36, 112, 141, 191, 224, 198, 197, 98, 154], u'cat'),\n",
+ " ([140, 60, 125, 187, 113, 18, 81, 84, 5, 88, 178, 243], u'bird'),\n",
+ " ([220, 87, 235, 30, 232, 216, 82, 200, 251, 194, 85, 186], u'dog'),\n",
+ " ([27, 127, 5, 181, 189, 145, 241, 58, 76, 97, 76, 157], u'bird'),\n",
+ " ([80, 245, 103, 67, 209, 67, 154, 188, 97, 130, 148, 179], u'cat'),\n",
+ " ([143, 117, 160, 74, 23, 187, 200, 28, 111, 133, 173, 96], u'bird'),\n",
+ " ([249, 193, 10, 150, 119, 91, 139, 222, 158, 92, 33, 56], u'cat'),\n",
+ " ([130, 186, 172, 53, 4, 59, 170, 164, 133, 193, 94, 77], u'bird'),\n",
+ " ([206, 210, 138, 46, 145, 131, 239, 156, 24, 102, 246, 163], u'cat'),\n",
+ " ([148, 255, 101, 204, 23, 231, 134, 195, 27, 138, 254, 197], u'cat'),\n",
+ " ([131, 134, 140, 208, 100, 90, 162, 238, 136, 52, 112, 119], u'bird'),\n",
+ " ([137, 221, 198, 44, 34, 90, 42, 135, 38, 65, 109, 171], u'cat'),\n",
+ " ([136, 54, 1, 78, 99, 132, 212, 239, 84, 56, 73, 246], u'cat'),\n",
+ " ([209, 42, 150, 72, 249, 30, 37, 191, 74, 71, 24, 116], u'bird'),\n",
+ " ([62, 181, 59, 233, 185, 195, 31, 187, 17, 130, 63, 229], u'dog'),\n",
+ " ([147, 28, 187, 137, 66, 140, 179, 215, 211, 172, 246, 249], u'dog'),\n",
+ " ([64, 63, 131, 180, 13, 193, 105, 72, 170, 35, 11, 201], u'bird'),\n",
+ " ([28, 75, 28, 0, 189, 175, 29, 120, 56, 94, 3, 235], u'cat'),\n",
+ " ([215, 151, 44, 207, 2, 107, 15, 133, 31, 28, 71, 137], u'dog'),\n",
+ " ([197, 68, 195, 107, 92, 71, 80, 55, 239, 70, 26, 198], u'cat'),\n",
+ " ([6, 230, 54, 75, 186, 42, 36, 112, 227, 19, 109, 220], u'cat'),\n",
+ " ([145, 117, 8, 147, 175, 205, 215, 113, 57, 51, 184, 136], u'dog'),\n",
+ " ([167, 207, 132, 109, 6, 138, 83, 60, 213, 13, 102, 249], u'dog'),\n",
+ " ([73, 12, 234, 37, 13, 123, 154, 21, 14, 72, 226, 229], u'bird'),\n",
+ " ([26, 24, 114, 163, 130, 25, 114, 6, 134, 119, 144, 217], u'bird'),\n",
+ " ([101, 230, 25, 94, 99, 99, 106, 77, 136, 119, 199, 34], u'dog'),\n",
+ " ([213, 106, 109, 186, 36, 136, 210, 151, 43, 84, 176, 156], u'dog'),\n",
+ " ([54, 20, 235, 15, 199, 80, 245, 224, 174, 87, 67, 24], u'dog'),\n",
+ " ([203, 143, 107, 237, 26, 65, 87, 136, 251, 123, 16, 205], u'cat'),\n",
+ " ([58, 34, 193, 214, 124, 248, 234, 103, 7, 177, 183, 251], u'dog'),\n",
+ " ([101, 83, 212, 125, 247, 159, 12, 98, 139, 38, 163, 226], u'bird'),\n",
+ " ([157, 93, 189, 107, 110, 248, 140, 48, 206, 8, 39, 184], u'dog'),\n",
+ " ([46, 105, 38, 41, 250, 139, 124, 206, 8, 115, 109, 19], u'bird'),\n",
+ " ([248, 57, 119, 218, 231, 21, 55, 164, 127, 166, 156, 11], u'bird'),\n",
+ " ([106, 20, 253, 34, 131, 43, 139, 170, 84, 133, 53, 208], u'cat'),\n",
+ " ([60, 67, 191, 79, 24, 184, 136, 143, 146, 111, 164, 201], u'cat'),\n",
+ " ([35, 110, 175, 47, 68, 25, 67, 65, 59, 198, 107, 198], u'dog'),\n",
+ " ([191, 74, 164, 144, 157, 224, 211, 92, 48, 234, 20, 184], u'dog'),\n",
+ " ([166, 39, 29, 110, 59, 65, 221, 234, 111, 33, 3, 178], u'dog'),\n",
+ " ([62, 120, 205, 4, 231, 140, 78, 139, 28, 235, 108, 238], u'cat'),\n",
+ " ([155, 217, 91, 83, 82, 0, 122, 111, 110, 181, 176, 75], u'dog'),\n",
+ " ([31, 108, 162, 209, 205, 224, 73, 154, 228, 48, 38, 50], u'bird'),\n",
+ " ([65, 29, 39, 48, 100, 194, 8, 190, 21, 90, 190, 144], u'bird'),\n",
+ " ([45, 69, 122, 120, 228, 153, 228, 134, 106, 177, 103, 179], u'cat')]"
+ ]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS image_data;\n",
+ "\n",
+ "CREATE TABLE image_data AS (\n",
+ "SELECT ARRAY[\n",
+ " (random() * 256)::integer, -- R values\n",
+ " (random() * 256)::integer,\n",
+ " (random() * 256)::integer,\n",
+ " (random() * 256)::integer,\n",
+ " (random() * 256)::integer, -- G values\n",
+ " (random() * 256)::integer,\n",
+ " (random() * 256)::integer,\n",
+ " (random() * 256)::integer,\n",
+ " (random() * 256)::integer, -- B values\n",
+ " (random() * 256)::integer,\n",
+ " (random() * 256)::integer,\n",
+ " (random() * 256)::integer\n",
+ " ] as rgb, ('{cat,dog,bird}'::text[])[ceil(random()*3)] as species\n",
+ "FROM generate_series(1, 52)\n",
+ ");\n",
+ "\n",
+ "SELECT * FROM image_data;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"pp_train2\"></a>\n",
+ "# 5. Run preprocessor for training image data\n",
+ "\n",
+ "Run the preprocessor to generate the packed output table:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "2 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>independent_var</th>\n",
+ " <th>dependent_var</th>\n",
+ " <th>buffer_id</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[0.105882, 0.498039, 0.0196078, 0.709804, 0.741176, 0.568627, 0.945098, 0.227451, 0.298039, 0.380392, 0.298039, 0.615686], [0.537255, 0.866667, 0.776471, 0.172549, 0.133333, 0.352941, 0.164706, 0.529412, 0.14902, 0.254902, 0.427451, 0.670588], [0.235294, 0.262745, 0.74902, 0.309804, 0.0941177, 0.721569, 0.533333, 0.560784, 0.572549, 0.435294, 0.643137, 0.788235], [0.560784, 0.458824, 0.627451, 0.290196, 0.0901961, 0.733333, 0.784314, 0.109804, 0.435294, 0.521569, 0.6 [...]
+ " <td>[[1, 0, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [1, 0, 0], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 1, 0]]</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[0.313726, 0.960784, 0.403922, 0.262745, 0.819608, 0.262745, 0.603922, 0.737255, 0.380392, 0.509804, 0.580392, 0.701961], [0.654902, 0.811765, 0.517647, 0.427451, 0.0235294, 0.541176, 0.32549, 0.235294, 0.835294, 0.0509804, 0.4, 0.976471], [0.180392, 0.411765, 0.14902, 0.160784, 0.980392, 0.545098, 0.486275, 0.807843, 0.0313726, 0.45098, 0.427451, 0.0745098], [0.0235294, 0.901961, 0.211765, 0.294118, 0.729412, 0.164706, 0.141176, 0.439216, 0.890196, 0.0745098, 0.4274 [...]
+ " <td>[[0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 0, 1], [0, 0, 1], [1, 0, 0]]</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[([[0.105882, 0.498039, 0.0196078, 0.709804, 0.741176, 0.568627, 0.945098, 0.227451, 0.298039, 0.380392, 0.298039, 0.615686], [0.537255, 0.866667, 0.776471, 0.172549, 0.133333, 0.352941, 0.164706, 0.529412, 0.14902, 0.254902, 0.427451, 0.670588], [0.235294, 0.262745, 0.74902, 0.309804, 0.0941177, 0.721569, 0.533333, 0.560784, 0.572549, 0.435294, 0.643137, 0.788235], [0.560784, 0.458824, 0.627451, 0.290196, 0.0901961, 0.733333, 0.784314, 0.109804, 0.435294, 0.521569, 0.678431, 0.3 [...]
+ " ([[0.313726, 0.960784, 0.403922, 0.262745, 0.819608, 0.262745, 0.603922, 0.737255, 0.380392, 0.509804, 0.580392, 0.701961], [0.654902, 0.811765, 0.517647, 0.427451, 0.0235294, 0.541176, 0.32549, 0.235294, 0.835294, 0.0509804, 0.4, 0.976471], [0.180392, 0.411765, 0.14902, 0.160784, 0.980392, 0.545098, 0.486275, 0.807843, 0.0313726, 0.45098, 0.427451, 0.0745098], [0.0235294, 0.901961, 0.211765, 0.294118, 0.729412, 0.164706, 0.141176, 0.439216, 0.890196, 0.0745098, 0.427451, 0.8627 [...]
+ ]
+ },
+ "execution_count": 7,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS image_data_packed, image_data_packed_summary;\n",
+ "\n",
+ "SELECT madlib.training_preprocessor_dl('image_data', -- Source table\n",
+ " 'image_data_packed', -- Output table\n",
+ " 'species', -- Dependent variable\n",
+ " 'rgb', -- Independent variable\n",
+ " NULL, -- Buffer size\n",
+ " 255 -- Normalizing constant\n",
+ " );\n",
+ "\n",
+ "SELECT * FROM image_data_packed ORDER BY buffer_id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"pp_val2\"></a>\n",
+ "# 6. Run preprocessor for validation image data\n",
+ "\n",
+ "Run the preprocessor for the validation dataset. In this example, we use the same images for validation to demonstrate, but normally validation data is different than training data:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "2 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>independent_var</th>\n",
+ " <th>dependent_var</th>\n",
+ " <th>buffer_id</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[0.576471, 0.701961, 0.580392, 0.627451, 0.964706, 0.509804, 0.517647, 0.564706, 0.6, 0.152941, 0.690196, 0.215686], [0.258824, 0.478431, 0.772549, 0.105882, 0.152941, 0.345098, 0.803922, 0.729412, 0.972549, 0.764706, 0.235294, 0.482353], [0.72549, 0.682353, 0.109804, 0.105882, 0.796079, 0.368627, 0.584314, 0.564706, 0.47451, 0.733333, 0.909804, 0.27451], [0.152941, 0.870588, 0.623529, 0.917647, 0.384314, 0.345098, 0.596078, 0.494118, 0.45098, 0.388235, 0.862745, 0.0 [...]
+ " <td>[[0, 1, 0], [0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [1, 0, 0], [1, 0, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 0, 1]]</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[0.231373, 0.423529, 0.152941, 0.521569, 0.533333, 0.305882, 0.996078, 0.2, 0.662745, 0.0941177, 0.654902, 0.85098], [0.4, 0.0352941, 0.639216, 0.968628, 0.141176, 0.87451, 0.388235, 0.294118, 0.388235, 0.921569, 0.6, 0.384314], [0.831373, 0.635294, 0.0117647, 0.0470588, 0.831373, 0.411765, 0.0784314, 0.466667, 0.372549, 0.223529, 0.337255, 0.764706], [0.878431, 0.0392157, 0.694118, 0.356863, 0.364706, 0.52549, 0.996078, 0.372549, 0.568627, 0.823529, 0.784314, 0.6509 [...]
+ " <td>[[1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 0, 1], [1, 0, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0]]</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[([[0.576471, 0.701961, 0.580392, 0.627451, 0.964706, 0.509804, 0.517647, 0.564706, 0.6, 0.152941, 0.690196, 0.215686], [0.258824, 0.478431, 0.772549, 0.105882, 0.152941, 0.345098, 0.803922, 0.729412, 0.972549, 0.764706, 0.235294, 0.482353], [0.72549, 0.682353, 0.109804, 0.105882, 0.796079, 0.368627, 0.584314, 0.564706, 0.47451, 0.733333, 0.909804, 0.27451], [0.152941, 0.870588, 0.623529, 0.917647, 0.384314, 0.345098, 0.596078, 0.494118, 0.45098, 0.388235, 0.862745, 0.0313726], [ [...]
+ " ([[0.231373, 0.423529, 0.152941, 0.521569, 0.533333, 0.305882, 0.996078, 0.2, 0.662745, 0.0941177, 0.654902, 0.85098], [0.4, 0.0352941, 0.639216, 0.968628, 0.141176, 0.87451, 0.388235, 0.294118, 0.388235, 0.921569, 0.6, 0.384314], [0.831373, 0.635294, 0.0117647, 0.0470588, 0.831373, 0.411765, 0.0784314, 0.466667, 0.372549, 0.223529, 0.337255, 0.764706], [0.878431, 0.0392157, 0.694118, 0.356863, 0.364706, 0.52549, 0.996078, 0.372549, 0.568627, 0.823529, 0.784314, 0.65098], [0.333 [...]
+ ]
+ },
+ "execution_count": 11,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS val_image_data_packed, val_image_data_packed_summary;\n",
+ "\n",
+ "SELECT madlib.validation_preprocessor_dl(\n",
+ " 'image_data', -- Source table\n",
+ " 'val_image_data_packed', -- Output table\n",
+ " 'species', -- Dependent variable\n",
+ " 'rgb', -- Independent variable\n",
+ " 'image_data_packed', -- From training preprocessor step\n",
+ " NULL -- Buffer size\n",
+ " );\n",
+ "\n",
+ "SELECT * FROM val_image_data_packed ORDER BY buffer_id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"change_buffer\"></a>\n",
+ "# 7. Change buffer size \n",
+ "\n",
+ "Generally the default buffer size will work well, but if you have occasion to change it:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "6 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>independent_var</th>\n",
+ " <th>dependent_var</th>\n",
+ " <th>buffer_id</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[0.72549, 0.682353, 0.109804, 0.105882, 0.796079, 0.368627, 0.584314, 0.564706, 0.47451, 0.733333, 0.909804, 0.27451], [0.521569, 0.733333, 0.968628, 0.776471, 0.945098, 0.443137, 0.760784, 0.129412, 0.235294, 0.847059, 0.0392157, 0.635294], [0.380392, 0.866667, 0.529412, 0.760784, 0.541176, 0.647059, 0.407843, 0.54902, 0.0352941, 0.894118, 0.619608, 0.533333], [0.145098, 0.203922, 0.878431, 0.258824, 0.858824, 0.882353, 0.490196, 0.796079, 0.478431, 0.854902, 0.2156 [...]
+ " <td>[[0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0]]</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[0.192157, 0.0392157, 0.556863, 0.74902, 0.211765, 0.74902, 0.541176, 0.588235, 0.67451, 0.776471, 0.917647, 0.137255], [0.231373, 0.423529, 0.152941, 0.521569, 0.533333, 0.305882, 0.996078, 0.2, 0.662745, 0.0941177, 0.654902, 0.85098], [0.0, 0.192157, 0.478431, 0.905882, 0.901961, 0.980392, 0.180392, 0.533333, 0.192157, 0.631373, 0.564706, 0.976471], [0.0666667, 0.780392, 0.67451, 0.0901961, 0.894118, 0.839216, 0.431373, 0.254902, 0.454902, 0.960784, 0.784314, 0.929 [...]
+ " <td>[[0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0]]</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[0.282353, 0.407843, 0.635294, 0.52549, 0.556863, 0.0117647, 0.384314, 0.0862745, 0.772549, 0.92549, 0.729412, 0.176471], [0.145098, 0.678431, 0.517647, 0.0235294, 0.470588, 0.0392157, 0.756863, 0.435294, 0.815686, 0.698039, 0.882353, 0.572549], [0.576471, 0.701961, 0.580392, 0.627451, 0.964706, 0.509804, 0.517647, 0.564706, 0.6, 0.152941, 0.690196, 0.215686], [0.152941, 0.870588, 0.623529, 0.917647, 0.384314, 0.345098, 0.596078, 0.494118, 0.45098, 0.388235, 0.862745 [...]
+ " <td>[[0, 0, 1], [1, 0, 0], [0, 1, 0], [1, 0, 0], [1, 0, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 0, 1]]</td>\n",
+ " <td>2</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[0.168627, 0.858824, 0.027451, 0.972549, 0.458824, 0.556863, 0.407843, 0.494118, 0.721569, 0.784314, 0.219608, 0.4], [0.113725, 0.419608, 0.862745, 0.74902, 0.560784, 0.443137, 0.509804, 0.788235, 0.478431, 0.831373, 0.478431, 0.109804], [0.960784, 0.545098, 0.396078, 0.521569, 0.254902, 0.458824, 0.298039, 0.933333, 0.54902, 0.192157, 0.768628, 0.980392], [0.403922, 0.564706, 0.384314, 0.690196, 0.658824, 0.341176, 0.521569, 0.717647, 0.207843, 0.623529, 0.380392, 0 [...]
+ " <td>[[1, 0, 0], [0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0]]</td>\n",
+ " <td>3</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[0.658824, 0.827451, 0.835294, 0.462745, 0.764706, 0.752941, 0.811765, 0.901961, 0.113725, 0.215686, 0.964706, 0.0235294], [0.00392157, 0.901961, 0.160784, 0.654902, 0.184314, 0.313726, 0.521569, 0.807843, 0.227451, 0.905882, 0.152941, 0.823529], [0.113725, 0.647059, 0.00392157, 0.396078, 0.486275, 0.0705882, 0.494118, 0.309804, 0.384314, 0.666667, 0.278431, 0.905882], [0.478431, 0.988235, 0.188235, 0.796079, 0.0901961, 0.913726, 0.4, 0.298039, 0.545098, 0.12549, 0.0 [...]
+ " <td>[[0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1]]</td>\n",
+ " <td>4</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[0.466667, 0.603922, 0.972549, 0.235294, 0.866667, 0.737255, 0.580392, 0.870588, 0.113725, 0.168627, 0.156863, 0.882353], [0.164706, 0.337255, 0.360784, 0.619608, 0.529412, 0.533333, 0.470588, 0.556863, 0.498039, 0.929412, 0.109804, 0.905882], [0.192157, 0.796079, 0.376471, 0.92549, 0.235294, 0.329412, 0.470588, 0.627451, 0.85098, 0.72549, 0.0823529, 0.14902], [0.509804, 0.815686, 0.270588, 0.768628, 0.843137, 0.623529, 0.00784314, 0.376471, 0.74902, 0.290196, 0.1019 [...]
+ " <td>[[1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0]]</td>\n",
+ " <td>5</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[([[0.72549, 0.682353, 0.109804, 0.105882, 0.796079, 0.368627, 0.584314, 0.564706, 0.47451, 0.733333, 0.909804, 0.27451], [0.521569, 0.733333, 0.968628, 0.776471, 0.945098, 0.443137, 0.760784, 0.129412, 0.235294, 0.847059, 0.0392157, 0.635294], [0.380392, 0.866667, 0.529412, 0.760784, 0.541176, 0.647059, 0.407843, 0.54902, 0.0352941, 0.894118, 0.619608, 0.533333], [0.145098, 0.203922, 0.878431, 0.258824, 0.858824, 0.882353, 0.490196, 0.796079, 0.478431, 0.854902, 0.215686, 0.2862 [...]
+ " ([[0.192157, 0.0392157, 0.556863, 0.74902, 0.211765, 0.74902, 0.541176, 0.588235, 0.67451, 0.776471, 0.917647, 0.137255], [0.231373, 0.423529, 0.152941, 0.521569, 0.533333, 0.305882, 0.996078, 0.2, 0.662745, 0.0941177, 0.654902, 0.85098], [0.0, 0.192157, 0.478431, 0.905882, 0.901961, 0.980392, 0.180392, 0.533333, 0.192157, 0.631373, 0.564706, 0.976471], [0.0666667, 0.780392, 0.67451, 0.0901961, 0.894118, 0.839216, 0.431373, 0.254902, 0.454902, 0.960784, 0.784314, 0.929412], [0.2 [...]
+ " ([[0.282353, 0.407843, 0.635294, 0.52549, 0.556863, 0.0117647, 0.384314, 0.0862745, 0.772549, 0.92549, 0.729412, 0.176471], [0.145098, 0.678431, 0.517647, 0.0235294, 0.470588, 0.0392157, 0.756863, 0.435294, 0.815686, 0.698039, 0.882353, 0.572549], [0.576471, 0.701961, 0.580392, 0.627451, 0.964706, 0.509804, 0.517647, 0.564706, 0.6, 0.152941, 0.690196, 0.215686], [0.152941, 0.870588, 0.623529, 0.917647, 0.384314, 0.345098, 0.596078, 0.494118, 0.45098, 0.388235, 0.862745, 0.031372 [...]
+ " ([[0.168627, 0.858824, 0.027451, 0.972549, 0.458824, 0.556863, 0.407843, 0.494118, 0.721569, 0.784314, 0.219608, 0.4], [0.113725, 0.419608, 0.862745, 0.74902, 0.560784, 0.443137, 0.509804, 0.788235, 0.478431, 0.831373, 0.478431, 0.109804], [0.960784, 0.545098, 0.396078, 0.521569, 0.254902, 0.458824, 0.298039, 0.933333, 0.54902, 0.192157, 0.768628, 0.980392], [0.403922, 0.564706, 0.384314, 0.690196, 0.658824, 0.341176, 0.521569, 0.717647, 0.207843, 0.623529, 0.380392, 0.380392], [...]
+ " ([[0.658824, 0.827451, 0.835294, 0.462745, 0.764706, 0.752941, 0.811765, 0.901961, 0.113725, 0.215686, 0.964706, 0.0235294], [0.00392157, 0.901961, 0.160784, 0.654902, 0.184314, 0.313726, 0.521569, 0.807843, 0.227451, 0.905882, 0.152941, 0.823529], [0.113725, 0.647059, 0.00392157, 0.396078, 0.486275, 0.0705882, 0.494118, 0.309804, 0.384314, 0.666667, 0.278431, 0.905882], [0.478431, 0.988235, 0.188235, 0.796079, 0.0901961, 0.913726, 0.4, 0.298039, 0.545098, 0.12549, 0.0823529, 0. [...]
+ " ([[0.466667, 0.603922, 0.972549, 0.235294, 0.866667, 0.737255, 0.580392, 0.870588, 0.113725, 0.168627, 0.156863, 0.882353], [0.164706, 0.337255, 0.360784, 0.619608, 0.529412, 0.533333, 0.470588, 0.556863, 0.498039, 0.929412, 0.109804, 0.905882], [0.192157, 0.796079, 0.376471, 0.92549, 0.235294, 0.329412, 0.470588, 0.627451, 0.85098, 0.72549, 0.0823529, 0.14902], [0.509804, 0.815686, 0.270588, 0.768628, 0.843137, 0.623529, 0.00784314, 0.376471, 0.74902, 0.290196, 0.101961, 0.9098 [...]
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS image_data_packed, image_data_packed_summary;\n",
+ "\n",
+ "SELECT madlib.training_preprocessor_dl('image_data', -- Source table\n",
+ " 'image_data_packed', -- Output table\n",
+ " 'species', -- Dependent variable\n",
+ " 'rgb', -- Independent variable\n",
+ " 10, -- Buffer size\n",
+ " 255 -- Normalizing constant\n",
+ " );\n",
+ "\n",
+ "SELECT * FROM image_data_packed ORDER BY buffer_id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Review the output summary data:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>source_table</th>\n",
+ " <th>output_table</th>\n",
+ " <th>dependent_varname</th>\n",
+ " <th>independent_varname</th>\n",
+ " <th>dependent_vartype</th>\n",
+ " <th>class_values</th>\n",
+ " <th>buffer_size</th>\n",
+ " <th>normalizing_const</th>\n",
+ " <th>num_classes</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>image_data</td>\n",
+ " <td>image_data_packed</td>\n",
+ " <td>species</td>\n",
+ " <td>rgb</td>\n",
+ " <td>text</td>\n",
+ " <td>[u'bird', u'cat', u'dog']</td>\n",
+ " <td>10</td>\n",
+ " <td>255.0</td>\n",
+ " <td>3</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'image_data', u'image_data_packed', u'species', u'rgb', u'text', [u'bird', u'cat', u'dog'], 10, Decimal('255.0'), 3)]"
+ ]
+ },
+ "execution_count": 13,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM image_data_packed_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"set_num_classes\"></a>\n",
+ "# 8. Setting number of classes\n",
+ "\n",
+ "If want the 1-hot encoded vector to have more classes than present in the data, use the num_classes param "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "2 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>independent_var</th>\n",
+ " <th>dependent_var</th>\n",
+ " <th>buffer_id</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[0.258824, 0.478431, 0.772549, 0.105882, 0.152941, 0.345098, 0.803922, 0.729412, 0.972549, 0.764706, 0.235294, 0.482353], [0.482353, 0.211765, 0.886275, 0.32549, 0.745098, 0.72549, 0.172549, 0.717647, 0.647059, 0.4, 0.694118, 0.466667], [0.72549, 0.603922, 0.207843, 0.631373, 0.733333, 0.792157, 0.913726, 0.443137, 0.384314, 0.14902, 0.407843, 0.772549], [0.580392, 0.282353, 0.713726, 0.596078, 0.239216, 0.968628, 0.388235, 0.109804, 0.360784, 0.576471, 0.745098, 0.6 [...]
+ " <td>[[0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [1, 0, 0, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [1, 0, 0, 0, 0], [0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [1, 0, 0, 0, 0]]</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[[0.509804, 0.815686, 0.270588, 0.768628, 0.843137, 0.623529, 0.00784314, 0.376471, 0.74902, 0.290196, 0.101961, 0.909804], [0.831373, 0.635294, 0.0117647, 0.0470588, 0.831373, 0.411765, 0.0784314, 0.466667, 0.372549, 0.223529, 0.337255, 0.764706], [0.192157, 0.0392157, 0.556863, 0.74902, 0.211765, 0.74902, 0.541176, 0.588235, 0.67451, 0.776471, 0.917647, 0.137255], [0.164706, 0.337255, 0.360784, 0.619608, 0.529412, 0.533333, 0.470588, 0.556863, 0.498039, 0.929412, 0. [...]
+ " <td>[[0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [0, 0, 1, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [1, 0, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [0, 0, 1, 0, 0], [1, 0, 0, 0, 0], [0, 0, 1, 0, 0], [1, 0, 0, 0, 0], [1, 0, 0, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [0, 0, 1, 0, 0], [0, 0, 1, 0, 0]]</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[([[0.258824, 0.478431, 0.772549, 0.105882, 0.152941, 0.345098, 0.803922, 0.729412, 0.972549, 0.764706, 0.235294, 0.482353], [0.482353, 0.211765, 0.886275, 0.32549, 0.745098, 0.72549, 0.172549, 0.717647, 0.647059, 0.4, 0.694118, 0.466667], [0.72549, 0.603922, 0.207843, 0.631373, 0.733333, 0.792157, 0.913726, 0.443137, 0.384314, 0.14902, 0.407843, 0.772549], [0.580392, 0.282353, 0.713726, 0.596078, 0.239216, 0.968628, 0.388235, 0.109804, 0.360784, 0.576471, 0.745098, 0.615686], [0 [...]
+ " ([[0.509804, 0.815686, 0.270588, 0.768628, 0.843137, 0.623529, 0.00784314, 0.376471, 0.74902, 0.290196, 0.101961, 0.909804], [0.831373, 0.635294, 0.0117647, 0.0470588, 0.831373, 0.411765, 0.0784314, 0.466667, 0.372549, 0.223529, 0.337255, 0.764706], [0.192157, 0.0392157, 0.556863, 0.74902, 0.211765, 0.74902, 0.541176, 0.588235, 0.67451, 0.776471, 0.917647, 0.137255], [0.164706, 0.337255, 0.360784, 0.619608, 0.529412, 0.533333, 0.470588, 0.556863, 0.498039, 0.929412, 0.109804, 0. [...]
+ ]
+ },
+ "execution_count": 14,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS image_data_packed, image_data_packed_summary;\n",
+ "\n",
+ "SELECT madlib.training_preprocessor_dl('image_data', -- Source table\n",
+ " 'image_data_packed', -- Output table\n",
+ " 'species', -- Dependent variable\n",
+ " 'rgb', -- Independent variable\n",
+ " NULL, -- Buffer size\n",
+ " 255, -- Normalizing constant\n",
+ " 5 -- Number of desired class values\n",
+ " );\n",
+ "\n",
+ "SELECT * FROM image_data_packed ORDER BY buffer_id;"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>source_table</th>\n",
+ " <th>output_table</th>\n",
+ " <th>dependent_varname</th>\n",
+ " <th>independent_varname</th>\n",
+ " <th>dependent_vartype</th>\n",
+ " <th>class_values</th>\n",
+ " <th>buffer_size</th>\n",
+ " <th>normalizing_const</th>\n",
+ " <th>num_classes</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>image_data</td>\n",
+ " <td>image_data_packed</td>\n",
+ " <td>species</td>\n",
+ " <td>rgb</td>\n",
+ " <td>text</td>\n",
+ " <td>[u'bird', u'cat', u'dog', None, None]</td>\n",
+ " <td>26</td>\n",
+ " <td>255.0</td>\n",
+ " <td>5</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'image_data', u'image_data_packed', u'species', u'rgb', u'text', [u'bird', u'cat', u'dog', None, None], 26, Decimal('255.0'), 5)]"
+ ]
+ },
+ "execution_count": 16,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM image_data_packed_summary;"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 2",
+ "language": "python",
+ "name": "python2"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.12"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
diff --git a/community-artifacts/madlib_image_loader.py b/community-artifacts/Deep-learning/madlib_image_loader.py
similarity index 100%
rename from community-artifacts/madlib_image_loader.py
rename to community-artifacts/Deep-learning/madlib_image_loader.py
diff --git a/community-artifacts/APSP-v1.ipynb b/community-artifacts/Graph/APSP-v1.ipynb
similarity index 100%
rename from community-artifacts/APSP-v1.ipynb
rename to community-artifacts/Graph/APSP-v1.ipynb
diff --git a/community-artifacts/Breadth-first-search-v1.ipynb b/community-artifacts/Graph/Breadth-first-search-v1.ipynb
similarity index 100%
rename from community-artifacts/Breadth-first-search-v1.ipynb
rename to community-artifacts/Graph/Breadth-first-search-v1.ipynb
diff --git a/community-artifacts/Graph-measures-v1.ipynb b/community-artifacts/Graph/Graph-measures-v1.ipynb
similarity index 100%
rename from community-artifacts/Graph-measures-v1.ipynb
rename to community-artifacts/Graph/Graph-measures-v1.ipynb
diff --git a/community-artifacts/HITS-v1.ipynb b/community-artifacts/Graph/HITS-v1.ipynb
similarity index 100%
rename from community-artifacts/HITS-v1.ipynb
rename to community-artifacts/Graph/HITS-v1.ipynb
diff --git a/community-artifacts/PageRank-v2.ipynb b/community-artifacts/Graph/PageRank-v2.ipynb
similarity index 100%
rename from community-artifacts/PageRank-v2.ipynb
rename to community-artifacts/Graph/PageRank-v2.ipynb
diff --git a/community-artifacts/SSSP-v2.ipynb b/community-artifacts/Graph/SSSP-v2.ipynb
similarity index 100%
rename from community-artifacts/SSSP-v2.ipynb
rename to community-artifacts/Graph/SSSP-v2.ipynb
diff --git a/community-artifacts/Weakly-connected-cpts-v2.ipynb b/community-artifacts/Graph/Weakly-connected-cpts-v2.ipynb
similarity index 100%
rename from community-artifacts/Weakly-connected-cpts-v2.ipynb
rename to community-artifacts/Graph/Weakly-connected-cpts-v2.ipynb
diff --git a/community-artifacts/Model-selection/Cross_validation_v1.ipynb b/community-artifacts/Model-selection/Cross_validation_v1.ipynb
new file mode 100644
index 0000000..743ef06
--- /dev/null
+++ b/community-artifacts/Model-selection/Cross_validation_v1.ipynb
@@ -0,0 +1,1130 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Cross validation using general function\n",
+ "\n",
+ "Examples for \n",
+ "http://madlib.apache.org/docs/latest/group__grp__validation.html"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/config.py:13: ShimWarning: The `IPython.config` package has been deprecated since IPython 4.0. You should import from traitlets.config instead.\n",
+ " \"You should import from traitlets.config instead.\", ShimWarning)\n",
+ "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/utils/traitlets.py:5: UserWarning: IPython.utils.traitlets has moved to a top-level traitlets package.\n",
+ " warn(\"IPython.utils.traitlets has moved to a top-level traitlets package.\")\n"
+ ]
+ }
+ ],
+ "source": [
+ "%load_ext sql"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "u'Connected: fmcquillan@madlib'"
+ ]
+ },
+ "execution_count": 2,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Greenplum Database 5.4.0 on GCP (demo machine)\n",
+ "#%sql postgresql://gpadmin@35.184.232.200:5432/madlib\n",
+ " \n",
+ "# PostgreSQL local\n",
+ "%sql postgresql://fmcquillan@localhost:5432/madlib"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>version</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>MADlib version: 1.15.1, git revision: rc/1.15.1-rc1, cmake configuration time: Wed Oct 10 04:29:25 UTC 2018, build type: Release, build system: Darwin-17.7.0, C compiler: Clang, C++ compiler: Clang</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'MADlib version: 1.15.1, git revision: rc/1.15.1-rc1, cmake configuration time: Wed Oct 10 04:29:25 UTC 2018, build type: Release, build system: Darwin-17.7.0, C compiler: Clang, C++ compiler: Clang',)]"
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%sql select madlib.version();\n",
+ "#%sql select version();"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 1. Create data set\n",
+ "House prices and characteristics."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "Done.\n",
+ "36 rows affected.\n",
+ "36 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>tax</th>\n",
+ " <th>bedroom</th>\n",
+ " <th>bath</th>\n",
+ " <th>size</th>\n",
+ " <th>lot</th>\n",
+ " <th>zipcode</th>\n",
+ " <th>price</th>\n",
+ " <th>high_priced</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>590</td>\n",
+ " <td>2</td>\n",
+ " <td>1.0</td>\n",
+ " <td>770</td>\n",
+ " <td>22100</td>\n",
+ " <td>94301</td>\n",
+ " <td>50000</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>1050</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>1410</td>\n",
+ " <td>12000</td>\n",
+ " <td>94301</td>\n",
+ " <td>85000</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>20</td>\n",
+ " <td>3</td>\n",
+ " <td>1.0</td>\n",
+ " <td>1060</td>\n",
+ " <td>3500</td>\n",
+ " <td>94301</td>\n",
+ " <td>22500</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>870</td>\n",
+ " <td>2</td>\n",
+ " <td>2.0</td>\n",
+ " <td>1300</td>\n",
+ " <td>17500</td>\n",
+ " <td>94301</td>\n",
+ " <td>90000</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>1320</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>1500</td>\n",
+ " <td>30000</td>\n",
+ " <td>94301</td>\n",
+ " <td>133000</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>1350</td>\n",
+ " <td>2</td>\n",
+ " <td>1.0</td>\n",
+ " <td>820</td>\n",
+ " <td>25700</td>\n",
+ " <td>94301</td>\n",
+ " <td>90500</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>7</td>\n",
+ " <td>2790</td>\n",
+ " <td>3</td>\n",
+ " <td>2.5</td>\n",
+ " <td>2130</td>\n",
+ " <td>25000</td>\n",
+ " <td>94301</td>\n",
+ " <td>260000</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>8</td>\n",
+ " <td>680</td>\n",
+ " <td>2</td>\n",
+ " <td>1.0</td>\n",
+ " <td>1170</td>\n",
+ " <td>22000</td>\n",
+ " <td>94301</td>\n",
+ " <td>142500</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>9</td>\n",
+ " <td>1840</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>1500</td>\n",
+ " <td>19000</td>\n",
+ " <td>94301</td>\n",
+ " <td>160000</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>10</td>\n",
+ " <td>3680</td>\n",
+ " <td>4</td>\n",
+ " <td>2.0</td>\n",
+ " <td>2790</td>\n",
+ " <td>20000</td>\n",
+ " <td>94301</td>\n",
+ " <td>240000</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>11</td>\n",
+ " <td>1660</td>\n",
+ " <td>3</td>\n",
+ " <td>1.0</td>\n",
+ " <td>1030</td>\n",
+ " <td>17500</td>\n",
+ " <td>94301</td>\n",
+ " <td>87000</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>12</td>\n",
+ " <td>1620</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>1250</td>\n",
+ " <td>20000</td>\n",
+ " <td>94301</td>\n",
+ " <td>118600</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>13</td>\n",
+ " <td>3100</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>1760</td>\n",
+ " <td>38000</td>\n",
+ " <td>94301</td>\n",
+ " <td>140000</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>14</td>\n",
+ " <td>2070</td>\n",
+ " <td>2</td>\n",
+ " <td>3.0</td>\n",
+ " <td>1550</td>\n",
+ " <td>14000</td>\n",
+ " <td>94301</td>\n",
+ " <td>148000</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>15</td>\n",
+ " <td>650</td>\n",
+ " <td>3</td>\n",
+ " <td>1.5</td>\n",
+ " <td>1450</td>\n",
+ " <td>12000</td>\n",
+ " <td>94301</td>\n",
+ " <td>65000</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>16</td>\n",
+ " <td>770</td>\n",
+ " <td>2</td>\n",
+ " <td>2.0</td>\n",
+ " <td>1300</td>\n",
+ " <td>17500</td>\n",
+ " <td>76010</td>\n",
+ " <td>91000</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>17</td>\n",
+ " <td>1220</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>1500</td>\n",
+ " <td>30000</td>\n",
+ " <td>76010</td>\n",
+ " <td>132300</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>18</td>\n",
+ " <td>1150</td>\n",
+ " <td>2</td>\n",
+ " <td>1.0</td>\n",
+ " <td>820</td>\n",
+ " <td>25700</td>\n",
+ " <td>76010</td>\n",
+ " <td>91100</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>19</td>\n",
+ " <td>2690</td>\n",
+ " <td>3</td>\n",
+ " <td>2.5</td>\n",
+ " <td>2130</td>\n",
+ " <td>25000</td>\n",
+ " <td>76010</td>\n",
+ " <td>260011</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>20</td>\n",
+ " <td>780</td>\n",
+ " <td>2</td>\n",
+ " <td>1.0</td>\n",
+ " <td>1170</td>\n",
+ " <td>22000</td>\n",
+ " <td>76010</td>\n",
+ " <td>141800</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>21</td>\n",
+ " <td>1910</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>1500</td>\n",
+ " <td>19000</td>\n",
+ " <td>76010</td>\n",
+ " <td>160900</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>22</td>\n",
+ " <td>3600</td>\n",
+ " <td>4</td>\n",
+ " <td>2.0</td>\n",
+ " <td>2790</td>\n",
+ " <td>20000</td>\n",
+ " <td>76010</td>\n",
+ " <td>239000</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>23</td>\n",
+ " <td>1600</td>\n",
+ " <td>3</td>\n",
+ " <td>1.0</td>\n",
+ " <td>1030</td>\n",
+ " <td>17500</td>\n",
+ " <td>76010</td>\n",
+ " <td>81010</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>24</td>\n",
+ " <td>1590</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>1250</td>\n",
+ " <td>20000</td>\n",
+ " <td>76010</td>\n",
+ " <td>117910</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>25</td>\n",
+ " <td>3200</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>1760</td>\n",
+ " <td>38000</td>\n",
+ " <td>76010</td>\n",
+ " <td>141100</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>26</td>\n",
+ " <td>2270</td>\n",
+ " <td>2</td>\n",
+ " <td>3.0</td>\n",
+ " <td>1550</td>\n",
+ " <td>14000</td>\n",
+ " <td>76010</td>\n",
+ " <td>148011</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>27</td>\n",
+ " <td>750</td>\n",
+ " <td>3</td>\n",
+ " <td>1.5</td>\n",
+ " <td>1450</td>\n",
+ " <td>12000</td>\n",
+ " <td>76010</td>\n",
+ " <td>66000</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>28</td>\n",
+ " <td>2690</td>\n",
+ " <td>3</td>\n",
+ " <td>2.5</td>\n",
+ " <td>2130</td>\n",
+ " <td>25000</td>\n",
+ " <td>76010</td>\n",
+ " <td>260011</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>29</td>\n",
+ " <td>780</td>\n",
+ " <td>2</td>\n",
+ " <td>1.0</td>\n",
+ " <td>1170</td>\n",
+ " <td>22000</td>\n",
+ " <td>76010</td>\n",
+ " <td>141800</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>30</td>\n",
+ " <td>1910</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>1500</td>\n",
+ " <td>19000</td>\n",
+ " <td>76010</td>\n",
+ " <td>160900</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>31</td>\n",
+ " <td>3600</td>\n",
+ " <td>4</td>\n",
+ " <td>2.0</td>\n",
+ " <td>2790</td>\n",
+ " <td>20000</td>\n",
+ " <td>76010</td>\n",
+ " <td>239000</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>32</td>\n",
+ " <td>1600</td>\n",
+ " <td>3</td>\n",
+ " <td>1.0</td>\n",
+ " <td>1030</td>\n",
+ " <td>17500</td>\n",
+ " <td>76010</td>\n",
+ " <td>81010</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>33</td>\n",
+ " <td>1590</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>1250</td>\n",
+ " <td>20000</td>\n",
+ " <td>76010</td>\n",
+ " <td>117910</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>34</td>\n",
+ " <td>3200</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>1760</td>\n",
+ " <td>38000</td>\n",
+ " <td>76010</td>\n",
+ " <td>141100</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>35</td>\n",
+ " <td>2270</td>\n",
+ " <td>2</td>\n",
+ " <td>3.0</td>\n",
+ " <td>1550</td>\n",
+ " <td>14000</td>\n",
+ " <td>76010</td>\n",
+ " <td>148011</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>36</td>\n",
+ " <td>750</td>\n",
+ " <td>3</td>\n",
+ " <td>1.5</td>\n",
+ " <td>1450</td>\n",
+ " <td>12000</td>\n",
+ " <td>76010</td>\n",
+ " <td>66000</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, 590, 2, 1.0, 770, 22100, 94301, 50000, False),\n",
+ " (2, 1050, 3, 2.0, 1410, 12000, 94301, 85000, False),\n",
+ " (3, 20, 3, 1.0, 1060, 3500, 94301, 22500, False),\n",
+ " (4, 870, 2, 2.0, 1300, 17500, 94301, 90000, False),\n",
+ " (5, 1320, 3, 2.0, 1500, 30000, 94301, 133000, True),\n",
+ " (6, 1350, 2, 1.0, 820, 25700, 94301, 90500, False),\n",
+ " (7, 2790, 3, 2.5, 2130, 25000, 94301, 260000, True),\n",
+ " (8, 680, 2, 1.0, 1170, 22000, 94301, 142500, True),\n",
+ " (9, 1840, 3, 2.0, 1500, 19000, 94301, 160000, True),\n",
+ " (10, 3680, 4, 2.0, 2790, 20000, 94301, 240000, True),\n",
+ " (11, 1660, 3, 1.0, 1030, 17500, 94301, 87000, False),\n",
+ " (12, 1620, 3, 2.0, 1250, 20000, 94301, 118600, True),\n",
+ " (13, 3100, 3, 2.0, 1760, 38000, 94301, 140000, True),\n",
+ " (14, 2070, 2, 3.0, 1550, 14000, 94301, 148000, True),\n",
+ " (15, 650, 3, 1.5, 1450, 12000, 94301, 65000, False),\n",
+ " (16, 770, 2, 2.0, 1300, 17500, 76010, 91000, False),\n",
+ " (17, 1220, 3, 2.0, 1500, 30000, 76010, 132300, True),\n",
+ " (18, 1150, 2, 1.0, 820, 25700, 76010, 91100, False),\n",
+ " (19, 2690, 3, 2.5, 2130, 25000, 76010, 260011, True),\n",
+ " (20, 780, 2, 1.0, 1170, 22000, 76010, 141800, True),\n",
+ " (21, 1910, 3, 2.0, 1500, 19000, 76010, 160900, True),\n",
+ " (22, 3600, 4, 2.0, 2790, 20000, 76010, 239000, True),\n",
+ " (23, 1600, 3, 1.0, 1030, 17500, 76010, 81010, False),\n",
+ " (24, 1590, 3, 2.0, 1250, 20000, 76010, 117910, False),\n",
+ " (25, 3200, 3, 2.0, 1760, 38000, 76010, 141100, True),\n",
+ " (26, 2270, 2, 3.0, 1550, 14000, 76010, 148011, True),\n",
+ " (27, 750, 3, 1.5, 1450, 12000, 76010, 66000, False),\n",
+ " (28, 2690, 3, 2.5, 2130, 25000, 76010, 260011, True),\n",
+ " (29, 780, 2, 1.0, 1170, 22000, 76010, 141800, True),\n",
+ " (30, 1910, 3, 2.0, 1500, 19000, 76010, 160900, True),\n",
+ " (31, 3600, 4, 2.0, 2790, 20000, 76010, 239000, True),\n",
+ " (32, 1600, 3, 1.0, 1030, 17500, 76010, 81010, False),\n",
+ " (33, 1590, 3, 2.0, 1250, 20000, 76010, 117910, False),\n",
+ " (34, 3200, 3, 2.0, 1760, 38000, 76010, 141100, True),\n",
+ " (35, 2270, 2, 3.0, 1550, 14000, 76010, 148011, True),\n",
+ " (36, 750, 3, 1.5, 1450, 12000, 76010, 66000, False)]"
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS houses;\n",
+ "\n",
+ "CREATE TABLE houses ( id INT,\n",
+ " tax INT,\n",
+ " bedroom INT,\n",
+ " bath FLOAT,\n",
+ " size INT,\n",
+ " lot INT,\n",
+ " zipcode INT,\n",
+ " price INT,\n",
+ " high_priced BOOLEAN\n",
+ " );\n",
+ "\n",
+ "INSERT INTO houses (id, tax, bedroom, bath, price, size, lot, zipcode, high_priced) VALUES\n",
+ "(1 , 590 , 2 , 1 , 50000 , 770 , 22100 , 94301, 'f'::boolean),\n",
+ "(2 , 1050 , 3 , 2 , 85000 , 1410 , 12000 , 94301, 'f'::boolean),\n",
+ "(3 , 20 , 3 , 1 , 22500 , 1060 , 3500 , 94301, 'f'::boolean),\n",
+ "(4 , 870 , 2 , 2 , 90000 , 1300 , 17500 , 94301, 'f'::boolean),\n",
+ "(5 , 1320 , 3 , 2 , 133000 , 1500 , 30000 , 94301, 't'::boolean),\n",
+ "(6 , 1350 , 2 , 1 , 90500 , 820 , 25700 , 94301, 'f'::boolean),\n",
+ "(7 , 2790 , 3 , 2.5 , 260000 , 2130 , 25000 , 94301, 't'::boolean),\n",
+ "(8 , 680 , 2 , 1 , 142500 , 1170 , 22000 , 94301, 't'::boolean),\n",
+ "(9 , 1840 , 3 , 2 , 160000 , 1500 , 19000 , 94301, 't'::boolean),\n",
+ "(10 , 3680 , 4 , 2 , 240000 , 2790 , 20000 , 94301, 't'::boolean),\n",
+ "(11 , 1660 , 3 , 1 , 87000 , 1030 , 17500 , 94301, 'f'::boolean),\n",
+ "(12 , 1620 , 3 , 2 , 118600 , 1250 , 20000 , 94301, 't'::boolean),\n",
+ "(13 , 3100 , 3 , 2 , 140000 , 1760 , 38000 , 94301, 't'::boolean),\n",
+ "(14 , 2070 , 2 , 3 , 148000 , 1550 , 14000 , 94301, 't'::boolean),\n",
+ "(15 , 650 , 3 , 1.5 , 65000 , 1450 , 12000 , 94301, 'f'::boolean),\n",
+ "(16 , 770 , 2 , 2 , 91000 , 1300 , 17500 , 76010, 'f'::boolean),\n",
+ "(17 , 1220 , 3 , 2 , 132300 , 1500 , 30000 , 76010, 't'::boolean),\n",
+ "(18 , 1150 , 2 , 1 , 91100 , 820 , 25700 , 76010, 'f'::boolean),\n",
+ "(19 , 2690 , 3 , 2.5 , 260011 , 2130 , 25000 , 76010, 't'::boolean),\n",
+ "(20 , 780 , 2 , 1 , 141800 , 1170 , 22000 , 76010, 't'::boolean),\n",
+ "(21 , 1910 , 3 , 2 , 160900 , 1500 , 19000 , 76010, 't'::boolean),\n",
+ "(22 , 3600 , 4 , 2 , 239000 , 2790 , 20000 , 76010, 't'::boolean),\n",
+ "(23 , 1600 , 3 , 1 , 81010 , 1030 , 17500 , 76010, 'f'::boolean),\n",
+ "(24 , 1590 , 3 , 2 , 117910 , 1250 , 20000 , 76010, 'f'::boolean),\n",
+ "(25 , 3200 , 3 , 2 , 141100 , 1760 , 38000 , 76010, 't'::boolean),\n",
+ "(26 , 2270 , 2 , 3 , 148011 , 1550 , 14000 , 76010, 't'::boolean),\n",
+ "(27 , 750 , 3 , 1.5 , 66000 , 1450 , 12000 , 76010, 'f'::boolean),\n",
+ "(28 , 2690 , 3 , 2.5 , 260011 , 2130 , 25000 , 76010, 't'::boolean),\n",
+ "(29 , 780 , 2 , 1 , 141800 , 1170 , 22000 , 76010, 't'::boolean),\n",
+ "(30 , 1910 , 3 , 2 , 160900 , 1500 , 19000 , 76010, 't'::boolean),\n",
+ "(31 , 3600 , 4 , 2 , 239000 , 2790 , 20000 , 76010, 't'::boolean),\n",
+ "(32 , 1600 , 3 , 1 , 81010 , 1030 , 17500 , 76010, 'f'::boolean),\n",
+ "(33 , 1590 , 3 , 2 , 117910 , 1250 , 20000 , 76010, 'f'::boolean),\n",
+ "(34 , 3200 , 3 , 2 , 141100 , 1760 , 38000 , 76010, 't'::boolean),\n",
+ "(35 , 2270 , 2 , 3 , 148011 , 1550 , 14000 , 76010, 't'::boolean),\n",
+ "(36 , 750 , 3 , 1.5 , 66000 , 1450 , 12000 , 76010, 'f'::boolean);\n",
+ "\n",
+ "SELECT * FROM houses ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 2. Elastic net\n",
+ "\n",
+ "Note that elastic net also has a built in cross validation function for selecting elastic net control parameter alpha and regularization value lambda\n",
+ "http://madlib.apache.org/docs/latest/group__grp__elasticnet.html\n",
+ "\n",
+ "But here we use the general function to explore lambda values:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>cross_validation_general</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td></td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[('',)]"
+ ]
+ },
+ "execution_count": 13,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS houses_cv_results;\n",
+ "\n",
+ "SELECT madlib.cross_validation_general(\n",
+ " -- modelling_func\n",
+ " 'madlib.elastic_net_train',\n",
+ " \n",
+ " -- modelling_params\n",
+ " '{%data%, %model%, price, \"array[tax, bath, size]\", gaussian, 0.5, lambda, TRUE, NULL, fista,\n",
+ " \"{eta = 2, max_stepsize = 2, use_active_set = t}\",\n",
+ " NULL, 10000, 1e-6}'::varchar[],\n",
+ " \n",
+ " -- modelling_params_type\n",
+ " '{varchar, varchar, varchar, varchar, varchar, double precision,\n",
+ " double precision, boolean, varchar, varchar, varchar, varchar,\n",
+ " integer, double precision}'::varchar[],\n",
+ " \n",
+ " -- param_explored\n",
+ " 'lambda',\n",
+ " \n",
+ " -- explore_values\n",
+ " '{0.1, 0.2}'::varchar[],\n",
+ " \n",
+ " -- predict_func\n",
+ " 'madlib.elastic_net_predict',\n",
+ " \n",
+ " -- predict_params\n",
+ " '{%model%, %data%, %id%, %prediction%}'::varchar[],\n",
+ " \n",
+ " -- predict_params_type\n",
+ " '{text, text, text, text}'::varchar[],\n",
+ " \n",
+ " -- metric_func\n",
+ " 'madlib.mse_error',\n",
+ " \n",
+ " -- metric_params\n",
+ " '{%prediction%, %data%, %id%, price, %error%}'::varchar[],\n",
+ " \n",
+ " -- metric_params_type\n",
+ " '{varchar, varchar, varchar, varchar, varchar}'::varchar[],\n",
+ " \n",
+ " -- data_tbl\n",
+ " 'houses',\n",
+ " \n",
+ " -- data_id\n",
+ " 'id',\n",
+ " \n",
+ " -- id_is_random\n",
+ " FALSE,\n",
+ " \n",
+ " -- validation_result\n",
+ " 'houses_cv_results',\n",
+ " \n",
+ " -- data_cols\n",
+ " NULL,\n",
+ " \n",
+ " -- fold_num\n",
+ " 3\n",
+ ");"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "2 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>lambda</th>\n",
+ " <th>mean_squared_error_avg</th>\n",
+ " <th>mean_squared_error_stddev</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>0.1</td>\n",
+ " <td>1194685622.16</td>\n",
+ " <td>366687470.78</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>0.2</td>\n",
+ " <td>1181768409.98</td>\n",
+ " <td>352203200.758</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(0.1, 1194685622.1604, 366687470.779826),\n",
+ " (0.2, 1181768409.98238, 352203200.758414)]"
+ ]
+ },
+ "execution_count": 14,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM houses_cv_results;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 3. Logistic regression \n",
+ "\n",
+ "Here we use the general function to explore maximum number of iterations:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>cross_validation_general</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td></td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[('',)]"
+ ]
+ },
+ "execution_count": 20,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS houses_logregr_cv;\n",
+ "\n",
+ "SELECT madlib.cross_validation_general(\n",
+ " -- modelling_func\n",
+ " 'madlib.logregr_train',\n",
+ " \n",
+ " -- modelling_params\n",
+ " '{%data%, %model%, high_priced, \"ARRAY[1, bedroom, bath, size]\", NULL, max_iter}'::varchar[],\n",
+ " \n",
+ " -- modelling_params_type\n",
+ " '{varchar, varchar, varchar, varchar, varchar, integer}'::varchar[],\n",
+ " \n",
+ " -- param_explored\n",
+ " 'max_iter',\n",
+ " \n",
+ " -- explore_values\n",
+ " '{2, 10, 40, 100}'::varchar[],\n",
+ " \n",
+ " -- predict_func\n",
+ " 'madlib.cv_logregr_predict',\n",
+ " \n",
+ " -- predict_params\n",
+ " '{%model%, %data%, \"ARRAY[1, bedroom, bath, size]\", id, %prediction%}'::varchar[],\n",
+ " \n",
+ " -- predict_params_type\n",
+ " '{varchar, varchar,varchar,varchar,varchar}'::varchar[],\n",
+ " \n",
+ " -- metric_func\n",
+ " 'madlib.misclassification_avg',\n",
+ " \n",
+ " -- metric_params\n",
+ " '{%prediction%, %data%, id, high_priced, %error%}'::varchar[],\n",
+ " \n",
+ " -- metric_params_type\n",
+ " '{varchar, varchar, varchar, varchar, varchar}'::varchar[],\n",
+ " \n",
+ " -- data_tbl\n",
+ " 'houses',\n",
+ " \n",
+ " -- data_id\n",
+ " 'id',\n",
+ " \n",
+ " -- id_is_random\n",
+ " FALSE,\n",
+ " \n",
+ " -- validation_result\n",
+ " 'houses_logregr_cv',\n",
+ " \n",
+ " -- data_cols\n",
+ " NULL,\n",
+ " \n",
+ " -- fold_num\n",
+ " 5\n",
+ ");"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "4 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>max_iter</th>\n",
+ " <th>error_rate_avg</th>\n",
+ " <th>error_rate_stddev</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>0.19642857142857142857</td>\n",
+ " <td>0.0818317088384971429780598253843971801653</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>10</td>\n",
+ " <td>0.22142857142857142857</td>\n",
+ " <td>0.0731925054711399884549944979733273803475</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>40</td>\n",
+ " <td>0.22142857142857142857</td>\n",
+ " <td>0.0731925054711399884549944979733273803475</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>100</td>\n",
+ " <td>0.22142857142857142857</td>\n",
+ " <td>0.0731925054711399884549944979733273803475</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(2, Decimal('0.19642857142857142857'), Decimal('0.0818317088384971429780598253843971801653')),\n",
+ " (10, Decimal('0.22142857142857142857'), Decimal('0.0731925054711399884549944979733273803475')),\n",
+ " (40, Decimal('0.22142857142857142857'), Decimal('0.0731925054711399884549944979733273803475')),\n",
+ " (100, Decimal('0.22142857142857142857'), Decimal('0.0731925054711399884549944979733273803475'))]"
+ ]
+ },
+ "execution_count": 21,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM houses_logregr_cv;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 4. Decision tree\n",
+ "\n",
+ "Here we use the general function to explore tree depth. First we need to create a wrapper function for predict that does a column rename:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 23,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "[]"
+ ]
+ },
+ "execution_count": 23,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "CREATE OR REPLACE FUNCTION tree_predict_rename_col(model_table VARCHAR, data_table VARCHAR, output_table VARCHAR,\n",
+ " orig_column VARCHAR, new_column VARCHAR)\n",
+ "RETURNS VOID AS $$\n",
+ "BEGIN\n",
+ " EXECUTE format('SELECT madlib.tree_predict(''%s'', ''%s'', ''%s'')', model_table, data_table, output_table);\n",
+ " EXECUTE 'ALTER TABLE ' || output_table || ' RENAME ' || orig_column || ' TO ' || new_column;\n",
+ "END\n",
+ "$$ LANGUAGE plpgsql VOLATILE;"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 24,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "4 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>max_depth</th>\n",
+ " <th>error_rate_avg</th>\n",
+ " <th>error_rate_stddev</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>0.16785714285714285714</td>\n",
+ " <td>0.1208494593977759334440468761527971235643</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>0.16785714285714285714</td>\n",
+ " <td>0.1208494593977759334440468761527971235643</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>0.17142857142857142857</td>\n",
+ " <td>0.1564921592871903181329101774752513216155</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>0.14285714285714285714</td>\n",
+ " <td>0.1428571428571428571449999999999999999999</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, Decimal('0.16785714285714285714'), Decimal('0.1208494593977759334440468761527971235643')),\n",
+ " (2, Decimal('0.16785714285714285714'), Decimal('0.1208494593977759334440468761527971235643')),\n",
+ " (3, Decimal('0.17142857142857142857'), Decimal('0.1564921592871903181329101774752513216155')),\n",
+ " (4, Decimal('0.14285714285714285714'), Decimal('0.1428571428571428571449999999999999999999'))]"
+ ]
+ },
+ "execution_count": 24,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS houses_dt_cv;\n",
+ "\n",
+ "SELECT madlib.cross_validation_general(\n",
+ " -- modelling_func\n",
+ " 'madlib.tree_train',\n",
+ " \n",
+ " -- modelling_params\n",
+ " '{%data%, %model%, id, high_priced, \"bedroom, bath, size, zipcode\", NULL, NULL, NULL, NULL, max_depth, 1, 1, 10}'::varchar[],\n",
+ " \n",
+ " -- modelling_params_type\n",
+ " '{varchar, varchar, varchar, varchar, varchar, varchar, varchar, varchar, varchar, integer, integer, integer, integer}',\n",
+ " \n",
+ " -- param_explored\n",
+ " 'max_depth',\n",
+ " \n",
+ " -- explore_values\n",
+ " '{1, 2, 3, 4}'::varchar[],\n",
+ " \n",
+ " -- predict_func\n",
+ " 'tree_predict_rename_col',\n",
+ " \n",
+ " -- predict_params\n",
+ " '{%model%, %data%, %prediction%, estimated_high_priced, prediction}'::varchar[],\n",
+ " \n",
+ " -- predict_params_type\n",
+ " '{varchar,varchar,varchar,varchar,varchar}'::varchar[],\n",
+ " \n",
+ " -- metric_func\n",
+ " 'madlib.misclassification_avg',\n",
+ " \n",
+ " -- metric_params\n",
+ " '{%prediction%, %data%, id, high_priced, %error%}'::varchar[],\n",
+ " \n",
+ " -- metric_params_type\n",
+ " '{varchar, varchar, varchar, varchar, varchar}'::varchar[],\n",
+ " \n",
+ " -- data_tbl\n",
+ " 'houses',\n",
+ " \n",
+ " -- data_id\n",
+ " 'id',\n",
+ " \n",
+ " -- id_is_random\n",
+ " FALSE,\n",
+ " \n",
+ " -- validation_result\n",
+ " 'houses_dt_cv',\n",
+ " \n",
+ " -- data_cols\n",
+ " NULL,\n",
+ " \n",
+ " -- fold_num\n",
+ " 5\n",
+ ");\n",
+ "SELECT * FROM houses_dt_cv;"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 2",
+ "language": "python",
+ "name": "python2"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.12"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
diff --git a/community-artifacts/Prediction-metrics-demo-1.ipynb b/community-artifacts/Model-selection/Prediction-metrics-demo-1.ipynb
similarity index 100%
rename from community-artifacts/Prediction-metrics-demo-1.ipynb
rename to community-artifacts/Model-selection/Prediction-metrics-demo-1.ipynb
diff --git a/community-artifacts/Train-test-split-v1.ipynb b/community-artifacts/Model-selection/Train-test-split-v1.ipynb
similarity index 100%
rename from community-artifacts/Train-test-split-v1.ipynb
rename to community-artifacts/Model-selection/Train-test-split-v1.ipynb
diff --git a/community-artifacts/Balanced-sampling-v1.ipynb b/community-artifacts/Sampling/Balanced-sampling-v1.ipynb
similarity index 100%
rename from community-artifacts/Balanced-sampling-v1.ipynb
rename to community-artifacts/Sampling/Balanced-sampling-v1.ipynb
diff --git a/community-artifacts/Stratified-sampling-v2.ipynb b/community-artifacts/Sampling/Stratified-sampling-v2.ipynb
similarity index 100%
rename from community-artifacts/Stratified-sampling-v2.ipynb
rename to community-artifacts/Sampling/Stratified-sampling-v2.ipynb
diff --git a/community-artifacts/Covariance-and-correlation-v1.ipynb b/community-artifacts/Statistics/Covariance-and-correlation-v1.ipynb
similarity index 100%
rename from community-artifacts/Covariance-and-correlation-v1.ipynb
rename to community-artifacts/Statistics/Covariance-and-correlation-v1.ipynb
diff --git a/community-artifacts/Summary-v2.ipynb b/community-artifacts/Statistics/Summary-v2.ipynb
similarity index 100%
rename from community-artifacts/Summary-v2.ipynb
rename to community-artifacts/Statistics/Summary-v2.ipynb
diff --git a/community-artifacts/Decision-trees-v2.ipynb b/community-artifacts/Supervised-learning/Decision-trees-v2.ipynb
similarity index 100%
rename from community-artifacts/Decision-trees-v2.ipynb
rename to community-artifacts/Supervised-learning/Decision-trees-v2.ipynb
diff --git a/community-artifacts/Elastic-net-v3.ipynb b/community-artifacts/Supervised-learning/Elastic-net-v3.ipynb
similarity index 100%
rename from community-artifacts/Elastic-net-v3.ipynb
rename to community-artifacts/Supervised-learning/Elastic-net-v3.ipynb
diff --git a/community-artifacts/KNN-v4.ipynb b/community-artifacts/Supervised-learning/KNN-v5.ipynb
similarity index 75%
rename from community-artifacts/KNN-v4.ipynb
rename to community-artifacts/Supervised-learning/KNN-v5.ipynb
index a4b3304..2f3d51a 100644
--- a/community-artifacts/KNN-v4.ipynb
+++ b/community-artifacts/Supervised-learning/KNN-v5.ipynb
@@ -5,7 +5,7 @@
"metadata": {},
"source": [
"# k-Nearest Neighbors\n",
- "Finds k nearest data points to a given data point and outputs majority vote value of output classes in case of classification, and average value of target values in case of regression. KNN was first added in MADlib 1.10 with updates in 1.13 and 1.14."
+ "Finds k nearest data points to a given data point and outputs majority vote value of output classes in case of classification, and average value of target values in case of regression. KNN was first added in MADlib 1.10 with multiple updates in subsequent releases."
]
},
{
@@ -17,7 +17,7 @@
"name": "stderr",
"output_type": "stream",
"text": [
- "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/config.py:13: ShimWarning: The `IPython.config` package has been deprecated. You should import from traitlets.config instead.\n",
+ "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/config.py:13: ShimWarning: The `IPython.config` package has been deprecated since IPython 4.0. You should import from traitlets.config instead.\n",
" \"You should import from traitlets.config instead.\", ShimWarning)\n",
"/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/utils/traitlets.py:5: UserWarning: IPython.utils.traitlets has moved to a top-level traitlets package.\n",
" warn(\"IPython.utils.traitlets has moved to a top-level traitlets package.\")\n"
@@ -30,7 +30,7 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 3,
"metadata": {},
"outputs": [
{
@@ -39,25 +39,22 @@
"u'Connected: gpadmin@madlib'"
]
},
- "execution_count": 2,
+ "execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
- "# Greenplum Database 5.4.0 on GCP (demo machine)\n",
- "%sql postgresql://gpadmin@35.184.253.255:5432/madlib\n",
+ "# Greenplum Database 5.x on GCP (demo machine)\n",
+ "%sql postgresql://gpadmin@35.184.232.200:5432/madlib\n",
" \n",
"# PostgreSQL local\n",
- "#%sql postgresql://fmcquillan@localhost:5432/madlib\n",
- "\n",
- "# Greenplum Database 4.3.10.0\n",
- "#%sql postgresql://gpdbchina@10.194.10.68:61000/madlib"
+ "#%sql postgresql://fmcquillan@localhost:5432/madlib"
]
},
{
"cell_type": "code",
- "execution_count": 67,
+ "execution_count": 4,
"metadata": {},
"outputs": [
{
@@ -75,15 +72,15 @@
" <th>version</th>\n",
" </tr>\n",
" <tr>\n",
- " <td>MADlib version: 1.14-dev, git revision: rc/1.13-rc1-12-gb8a306e, cmake configuration time: Mon Feb 12 19:57:54 UTC 2018, build type: release, build system: Linux-2.6.32-696.20.1.el6.x86_64, C compiler: gcc 4.4.7, C++ compiler: g++ 4.4.7</td>\n",
+ " <td>MADlib version: 1.16-dev, git revision: rc/1.15.1-rc1-29-g0ba6155, cmake configuration time: Wed Feb 20 17:40:16 UTC 2019, build type: release, build system: Linux-2.6.32-754.6.3.el6.x86_64, C compiler: gcc 4.4.7, C++ compiler: g++ 4.4.7</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
- "[(u'MADlib version: 1.14-dev, git revision: rc/1.13-rc1-12-gb8a306e, cmake configuration time: Mon Feb 12 19:57:54 UTC 2018, build type: release, build system: Linux-2.6.32-696.20.1.el6.x86_64, C compiler: gcc 4.4.7, C++ compiler: g++ 4.4.7',)]"
+ "[(u'MADlib version: 1.16-dev, git revision: rc/1.15.1-rc1-29-g0ba6155, cmake configuration time: Wed Feb 20 17:40:16 UTC 2019, build type: release, build system: Linux-2.6.32-754.6.3.el6.x86_64, C compiler: gcc 4.4.7, C++ compiler: g++ 4.4.7',)]"
]
},
- "execution_count": 67,
+ "execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
@@ -102,7 +99,7 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 5,
"metadata": {},
"outputs": [
{
@@ -183,7 +180,7 @@
" (9, [1, 111], 0)]"
]
},
- "execution_count": 3,
+ "execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
@@ -221,7 +218,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 6,
"metadata": {},
"outputs": [
{
@@ -302,7 +299,7 @@
" (9, [1, 111], 0.0)]"
]
},
- "execution_count": 4,
+ "execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
@@ -340,7 +337,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 22,
"metadata": {},
"outputs": [
{
@@ -396,7 +393,7 @@
" (6, [50, 45])]"
]
},
- "execution_count": 5,
+ "execution_count": 22,
"metadata": {},
"output_type": "execute_result"
}
@@ -431,7 +428,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 15,
"metadata": {},
"outputs": [
{
@@ -500,7 +497,7 @@
" (6, [50, 45], 0.0, [6, 7, 8])]"
]
},
- "execution_count": 6,
+ "execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
@@ -535,7 +532,7 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 16,
"metadata": {},
"outputs": [
{
@@ -561,7 +558,7 @@
" <td>1</td>\n",
" <td>[2, 1]</td>\n",
" <td>1.0</td>\n",
- " <td>[1, 2, 3]</td>\n",
+ " <td>[2, 1, 3]</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
@@ -596,7 +593,7 @@
"</table>"
],
"text/plain": [
- "[(1, [2, 1], 1.0, [1, 2, 3]),\n",
+ "[(1, [2, 1], 1.0, [2, 1, 3]),\n",
" (2, [2, 6], 1.0, [5, 4, 3]),\n",
" (3, [15, 40], 0.333333333333333, [7, 6, 5]),\n",
" (4, [12, 1], 1.0, [4, 5, 3]),\n",
@@ -604,7 +601,7 @@
" (6, [50, 45], 0.0, [6, 7, 8])]"
]
},
- "execution_count": 7,
+ "execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
@@ -640,7 +637,7 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 17,
"metadata": {},
"outputs": [
{
@@ -702,7 +699,7 @@
" (6, [50, 45], [6, 7, 8])]"
]
},
- "execution_count": 8,
+ "execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
@@ -736,7 +733,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 18,
"metadata": {},
"outputs": [
{
@@ -762,7 +759,7 @@
" <td>1</td>\n",
" <td>[2, 1]</td>\n",
" <td>1</td>\n",
- " <td>[2, 1, 3]</td>\n",
+ " <td>[1, 2, 3]</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
@@ -797,7 +794,7 @@
"</table>"
],
"text/plain": [
- "[(1, [2, 1], 1, [2, 1, 3]),\n",
+ "[(1, [2, 1], 1, [1, 2, 3]),\n",
" (2, [2, 6], 1, [5, 4, 3]),\n",
" (3, [15, 40], 0, [7, 6, 5]),\n",
" (4, [12, 1], 1, [4, 5, 3]),\n",
@@ -805,7 +802,7 @@
" (6, [50, 45], 0, [6, 7, 8])]"
]
},
- "execution_count": 9,
+ "execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
@@ -831,6 +828,201 @@
"\n",
"SELECT * FROM knn_result_classification ORDER BY id;"
]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 8. Use kd-tree algorithm \n",
+ "Here we build a kd-tree to depth 4 and search half (8) of the 16 leaf nodes (i.e., 2^4)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "6 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>data</th>\n",
+ " <th>k_nearest_neighbours</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>[2, 1]</td>\n",
+ " <td>[1, 2, 3]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>[2, 6]</td>\n",
+ " <td>[5, 4, 3]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>[15, 40]</td>\n",
+ " <td>[7, 6, 5]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>[12, 1]</td>\n",
+ " <td>[4, 5, 3]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>[2, 90]</td>\n",
+ " <td>[9, 6, 7]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>[50, 45]</td>\n",
+ " <td>[6, 7, 8]</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, [2, 1], [1, 2, 3]),\n",
+ " (2, [2, 6], [5, 4, 3]),\n",
+ " (3, [15, 40], [7, 6, 5]),\n",
+ " (4, [12, 1], [4, 5, 3]),\n",
+ " (5, [2, 90], [9, 6, 7]),\n",
+ " (6, [50, 45], [6, 7, 8])]"
+ ]
+ },
+ "execution_count": 19,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS knn_result_classification_kd;\n",
+ "\n",
+ "SELECT madlib.knn(\n",
+ " 'knn_train_data', -- Table of training data\n",
+ " 'data', -- Col name of training data\n",
+ " 'id', -- Col name of id in train data\n",
+ " NULL, -- Training labels\n",
+ " 'knn_test_data', -- Table of test data\n",
+ " 'data', -- Col name of test data\n",
+ " 'id', -- Col name of id in test data\n",
+ " 'knn_result_classification_kd', -- Output table\n",
+ " 3, -- Number of nearest neighbors\n",
+ " True, -- True to list nearest-neighbors by id\n",
+ " 'madlib.squared_dist_norm2', -- Distance function\n",
+ " False, -- For weighted average\n",
+ " 'kd_tree', -- Use kd-tree\n",
+ " 'depth=4, leaf_nodes=8' -- Kd-tree options\n",
+ " \n",
+ " );\n",
+ "SELECT * FROM knn_result_classification_kd ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "The result above is the same as brute force. If we search just 1 leaf node, run-time will be faster but accuracy will be lower. This shows up in this very small data set by not being able to find 3 nearest neighbors for all test points:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 31,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "5 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>data</th>\n",
+ " <th>k_nearest_neighbours</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>[2, 1]</td>\n",
+ " <td>[1]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>[2, 6]</td>\n",
+ " <td>[3, 2]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>[15, 40]</td>\n",
+ " <td>[7]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>[2, 90]</td>\n",
... 104 lines suppressed ...