You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@madlib.apache.org by fm...@apache.org on 2020/01/28 18:35:24 UTC
[madlib-site] branch asf-site updated: updated Jupyter notebooks
for 1.17
This is an automated email from the ASF dual-hosted git repository.
fmcquillan pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/madlib-site.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 8039211 updated Jupyter notebooks for 1.17
8039211 is described below
commit 8039211758b2e3f0a032ec51847b2d9e12668a5a
Author: Frank McQuillan <fm...@pivotal.io>
AuthorDate: Tue Jan 28 10:35:07 2020 -0800
updated Jupyter notebooks for 1.17
---
.../Deep-learning/Load-model-architecture-v1.ipynb | 752 ---
.../Deep-learning/MADlib-Keras-MLP-v1.ipynb | 3206 ------------
.../MADlib-Keras-cifar10-cnn-v2.ipynb | 1233 -----
.../MADlib-Keras-transfer-learning-v2.ipynb | 1624 -------
.../Deep-learning/Preprocessor-for-images-v1.ipynb | 1625 -------
.../automl/hyperband-diag-cifar10-v1.ipynb | 5094 ++++++++++++++++++++
.../Supervised-learning/KNN-v5.ipynb | 1049 ----
.../Supervised-learning/SVM-v1.ipynb | 2806 -----------
.../Unsupervised-learning/Kmeans-v2.ipynb | 1102 -----
.../Unsupervised-learning/LDA-v1.ipynb | 2034 --------
10 files changed, 5094 insertions(+), 15431 deletions(-)
diff --git a/community-artifacts/Deep-learning/Load-model-architecture-v1.ipynb b/community-artifacts/Deep-learning/Load-model-architecture-v1.ipynb
deleted file mode 100644
index 8e04b3c..0000000
--- a/community-artifacts/Deep-learning/Load-model-architecture-v1.ipynb
+++ /dev/null
@@ -1,752 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Load model architecture\n",
- "This utility function loads model architectures and weights into a table for use by deep learning algorithms in Keras. \n",
- "\n",
- "The model architecture loader was added in MADlib 1.16.\n",
- "\n",
- "## Table of contents\n",
- "\n",
- "<a href=\"#define_model_arch\">1. Define model architecture</a>\n",
- "\n",
- "<a href=\"#load_model_arch\">2. Load model architecture</a>\n",
- "\n",
- "<a href=\"#load_model_wts\">3. Load model weights</a>\n",
- "* <a href=\"#load_model_wts_madlib\">3a. Load weights from previous MADlib run</a>\n",
- "* <a href=\"#load_model_wts_keras1\">3b. Load weights from Keras using a PL/Python function</a>\n",
- "* <a href=\"#load_model_wts_keras2\">3c. Load weights from Keras using psycopg2</a>\n",
- "\n",
- "<a href=\"#delete_model\">4. Delete model</a>"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/config.py:13: ShimWarning: The `IPython.config` package has been deprecated since IPython 4.0. You should import from traitlets.config instead.\n",
- " \"You should import from traitlets.config instead.\", ShimWarning)\n",
- "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/utils/traitlets.py:5: UserWarning: IPython.utils.traitlets has moved to a top-level traitlets package.\n",
- " warn(\"IPython.utils.traitlets has moved to a top-level traitlets package.\")\n"
- ]
- }
- ],
- "source": [
- "%load_ext sql"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "u'Connected: fmcquillan@madlib'"
- ]
- },
- "execution_count": 2,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "# Greenplum Database 5.x on GCP for deep learning (PM demo machine)\n",
- "#%sql postgresql://gpadmin@35.239.240.26:5432/madlib\n",
- " \n",
- "# PostgreSQL local\n",
- "%sql postgresql://fmcquillan@localhost:5432/madlib"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>version</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>MADlib version: 1.16, git revision: rc/1.16-rc1, cmake configuration time: Mon Jul 1 17:45:09 UTC 2019, build type: Release, build system: Darwin-16.7.0, C compiler: Clang, C++ compiler: Clang</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'MADlib version: 1.16, git revision: rc/1.16-rc1, cmake configuration time: Mon Jul 1 17:45:09 UTC 2019, build type: Release, build system: Darwin-16.7.0, C compiler: Clang, C++ compiler: Clang',)]"
- ]
- },
- "execution_count": 3,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%sql select madlib.version();\n",
- "#%sql select version();"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"define_model_arch\"></a>\n",
- "# 1. Define model architecture\n",
- "\n",
- "Import Keras libraries"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Using TensorFlow backend.\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Couldn't import dot_parser, loading of dot files will not be possible.\n"
- ]
- }
- ],
- "source": [
- "import keras\n",
- "from keras.models import Sequential\n",
- "from keras.layers import Dense"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Define model architecture"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "_________________________________________________________________\n",
- "Layer (type) Output Shape Param # \n",
- "=================================================================\n",
- "dense_1 (Dense) (None, 10) 50 \n",
- "_________________________________________________________________\n",
- "dense_2 (Dense) (None, 10) 110 \n",
- "_________________________________________________________________\n",
- "dense_3 (Dense) (None, 3) 33 \n",
- "=================================================================\n",
- "Total params: 193\n",
- "Trainable params: 193\n",
- "Non-trainable params: 0\n",
- "_________________________________________________________________\n"
- ]
- }
- ],
- "source": [
- "model = Sequential()\n",
- "model.add(Dense(10, activation='relu', input_shape=(4,)))\n",
- "model.add(Dense(10, activation='relu'))\n",
- "model.add(Dense(3, activation='softmax'))\n",
- " \n",
- "model.summary()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 6,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "'{\"class_name\": \"Sequential\", \"keras_version\": \"2.1.6\", \"config\": [{\"class_name\": \"Dense\", \"config\": {\"kernel_initializer\": {\"class_name\": \"VarianceScaling\", \"config\": {\"distribution\": \"uniform\", \"scale\": 1.0, \"seed\": null, \"mode\": \"fan_avg\"}}, \"name\": \"dense_1\", \"kernel_constraint\": null, \"bias_regularizer\": null, \"bias_constraint\": null, \"dtype\": \"float32\", \"activation\": \"relu\", \"trainable\": true, \"kernel_regularizer\": n [...]
- ]
- },
- "execution_count": 6,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "model.to_json()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "'{\"class_name\": \"Sequential\", \"keras_version\": \"2.1.6\", \"config\": [{\"class_name\": \"Dense\", \"config\": {\"kernel_initializer\": {\"class_name\": \"VarianceScaling\", \"config\": {\"distribution\": \"uniform\", \"scale\": 1.0, \"seed\": null, \"mode\": \"fan_avg\"}}, \"name\": \"dense_1\", \"kernel_constraint\": null, \"bias_regularizer\": null, \"bias_constraint\": null, \"dtype\": \"float32\", \"activation\": \"linear\", \"trainable\": true, \"kernel_regularizer\": nu [...]
- " "
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"load_model_arch\"></a>\n",
- "# 2. Load model architecture\n",
- "\n",
- "Load into model architecture table:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 7,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>model_id</th>\n",
- " <th>model_arch</th>\n",
- " <th>model_weights</th>\n",
- " <th>name</th>\n",
- " <th>description</th>\n",
- " <th>__internal_madlib_id__</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1</td>\n",
- " <td>{u'class_name': u'Sequential', u'keras_version': u'2.1.6', u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer': {u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform', u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1', u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint': None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True, u'kernel_regularizer': None, u'bias_initia [...]
- " <td>None</td>\n",
- " <td>Sophie</td>\n",
- " <td>A simple model</td>\n",
- " <td>__madlib_temp_86175082_1566932966_2506754__</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(1, {u'class_name': u'Sequential', u'keras_version': u'2.1.6', u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer': {u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform', u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1', u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint': None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True, u'kernel_regularizer': None, u'bias_initializer': [...]
- ]
- },
- "execution_count": 7,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS model_arch_library;\n",
- "SELECT madlib.load_keras_model('model_arch_library', -- Output table,\n",
- " \n",
- "$$\n",
- "{\"class_name\": \"Sequential\", \"keras_version\": \"2.1.6\", \"config\": [{\"class_name\": \"Dense\", \"config\": {\"kernel_initializer\": {\"class_name\": \"VarianceScaling\", \"config\": {\"distribution\": \"uniform\", \"scale\": 1.0, \"seed\": null, \"mode\": \"fan_avg\"}}, \"name\": \"dense_1\", \"kernel_constraint\": null, \"bias_regularizer\": null, \"bias_constraint\": null, \"dtype\": \"float32\", \"activation\": \"relu\", \"trainable\": true, \"kernel_regularizer\": null, [...]
- "$$\n",
- "::json, -- JSON blob\n",
- " NULL, -- Weights\n",
- " 'Sophie', -- Name\n",
- " 'A simple model' -- Descr\n",
- ");\n",
- "\n",
- "SELECT * FROM model_arch_library;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Load another model architecture:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 8,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n",
- "2 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>model_id</th>\n",
- " <th>model_arch</th>\n",
- " <th>model_weights</th>\n",
- " <th>name</th>\n",
- " <th>description</th>\n",
- " <th>__internal_madlib_id__</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1</td>\n",
- " <td>{u'class_name': u'Sequential', u'keras_version': u'2.1.6', u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer': {u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform', u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1', u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint': None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True, u'kernel_regularizer': None, u'bias_initia [...]
- " <td>None</td>\n",
- " <td>Sophie</td>\n",
- " <td>A simple model</td>\n",
- " <td>__madlib_temp_86175082_1566932966_2506754__</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>2</td>\n",
- " <td>{u'class_name': u'Sequential', u'keras_version': u'2.1.6', u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer': {u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform', u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1', u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint': None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True, u'kernel_regularizer': None, u'bias_initia [...]
- " <td>None</td>\n",
- " <td>Maria</td>\n",
- " <td>Also a simple model</td>\n",
- " <td>__madlib_temp_82870028_1566932968_1094668__</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(1, {u'class_name': u'Sequential', u'keras_version': u'2.1.6', u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer': {u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform', u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1', u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint': None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True, u'kernel_regularizer': None, u'bias_initializer': [...]
- " (2, {u'class_name': u'Sequential', u'keras_version': u'2.1.6', u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer': {u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform', u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1', u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint': None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True, u'kernel_regularizer': None, u'bias_initializer': [...]
- ]
- },
- "execution_count": 8,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT madlib.load_keras_model('model_arch_library', -- Output table,\n",
- " \n",
- "$$\n",
- "{\"class_name\": \"Sequential\", \"keras_version\": \"2.1.6\", \"config\": [{\"class_name\": \"Dense\", \"config\": {\"kernel_initializer\": {\"class_name\": \"VarianceScaling\", \"config\": {\"distribution\": \"uniform\", \"scale\": 1.0, \"seed\": null, \"mode\": \"fan_avg\"}}, \"name\": \"dense_1\", \"kernel_constraint\": null, \"bias_regularizer\": null, \"bias_constraint\": null, \"dtype\": \"float32\", \"activation\": \"relu\", \"trainable\": true, \"kernel_regularizer\": null, [...]
- "$$\n",
- "::json, -- JSON blob\n",
- " NULL, -- Weights\n",
- " 'Maria', -- Name\n",
- " 'Also a simple model' -- Descr\n",
- ");\n",
- "\n",
- "SELECT * FROM model_arch_library;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"load_model_wts\"></a>\n",
- "# 3. Load model weights\n",
- "\n",
- "<a id=\"load_model_wts_madlib\"></a>\n",
- "## 3a. Load weights from previous MADlib run\n",
- "\n",
- "Use UPDATE to load directly into the table. For example, if 'model_data' are the weights in the output table 'iris_model' from a previous run of 'madlib_keras_fit()' :"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 9,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>count</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(1L,)]"
- ]
- },
- "execution_count": 9,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "UPDATE model_arch_library SET model_weights = model_data FROM iris_model WHERE model_id = 2;\n",
- "\n",
- "-- Check weights loaded OK\n",
- "SELECT COUNT(*) FROM model_arch_library WHERE model_weights IS NOT NULL;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"load_model_wts_keras1\"></a>\n",
- "## 3b. Load weights from Keras using a PL/Python function \n",
- "We need to flatten then serialize the weights to store as a PostgreSQL binary data type. Byte format is more efficient on space and memory compared to a numeric array. The model weights will be de-serialized when passed to Keras functions."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 13,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>count</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>2</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(2L,)]"
- ]
- },
- "execution_count": 13,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "CREATE OR REPLACE FUNCTION load_weights() RETURNS VOID AS\n",
- "$$\n",
- "from keras.layers import *\n",
- "from keras import Sequential\n",
- "import numpy as np\n",
- "import plpy\n",
- "\n",
- "# create model\n",
- "model = Sequential()\n",
- "model.add(Dense(10, activation='relu', input_shape=(4,)))\n",
- "model.add(Dense(10, activation='relu'))\n",
- "model.add(Dense(3, activation='softmax'))\n",
- "\n",
- "# get weights, flatten and serialize\n",
- "weights = model.get_weights()\n",
- "weights_flat = [w.flatten() for w in weights]\n",
- "weights1d = np.concatenate(weights_flat).ravel()\n",
- "weights_bytea = weights1d.tostring()\n",
- "\n",
- "# load query\n",
- "load_query = plpy.prepare(\"\"\"SELECT madlib.load_keras_model(\n",
- " 'model_arch_library',\n",
- " $1, $2, $3, $4)\n",
- " \"\"\", ['json','bytea', 'text', 'text'])\n",
- "plpy.execute(load_query, [model.to_json(), weights_bytea, \"Ella\", \"Model x\"])\n",
- "$$ language plpythonu;\n",
- "\n",
- "-- Call load function\n",
- "SELECT load_weights();\n",
- "\n",
- "-- Check weights loaded OK\n",
- "SELECT COUNT(*) FROM model_arch_library WHERE model_weights IS NOT NULL;"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 14,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "3 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>model_id</th>\n",
- " <th>name</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1</td>\n",
- " <td>Sophie</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>2</td>\n",
- " <td>Maria</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>3</td>\n",
- " <td>Ella</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(1, u'Sophie'), (2, u'Maria'), (3, u'Ella')]"
- ]
- },
- "execution_count": 14,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT model_id, name from model_arch_library ORDER BY model_id;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"load_model_wts_keras2\"></a>\n",
- "## 3c. Load weights from Keras using psycopg2\n",
- "Psycopg is a PostgreSQL database adapter for the Python programming language. As above we need to flatten then serialize the weights to store as a PostgreSQL binary data type.\n",
- "\n",
- "Note that the psycopg2.Binary function will increase the size of the Python object for the weights, so if your model is large it might be better to use a PL/Python function as in 3b. above."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 15,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>count</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>3</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(3L,)]"
- ]
- },
- "execution_count": 15,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "import psycopg2 as p2\n",
- "#conn = p2.connect('postgresql://gpadmin@35.239.240.26:5432/madlib')\n",
- "conn = p2.connect('postgresql://fmcquillan@localhost:5432/madlib')\n",
- "cur = conn.cursor()\n",
- "\n",
- "from keras.layers import *\n",
- "from keras import Sequential\n",
- "import numpy as np\n",
- "\n",
- "# create model\n",
- "model = Sequential()\n",
- "model.add(Dense(10, activation='relu', input_shape=(4,)))\n",
- "model.add(Dense(10, activation='relu'))\n",
- "model.add(Dense(3, activation='softmax'))\n",
- "\n",
- "# get weights, flatten and serialize\n",
- "weights = model.get_weights()\n",
- "weights_flat = [w.flatten() for w in weights]\n",
- "weights1d = np.concatenate(weights_flat).ravel()\n",
- "weights_bytea = p2.Binary(weights1d.tostring())\n",
- "\n",
- "query = \"SELECT madlib.load_keras_model('model_arch_library', %s,%s,%s,%s)\"\n",
- "cur.execute(query,[model.to_json(), weights_bytea, \"Grace\", \"Model y\"])\n",
- "conn.commit()\n",
- "\n",
- "# check weights loaded OK\n",
- "%sql SELECT COUNT(*) FROM model_arch_library WHERE model_weights IS NOT NULL;"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 16,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "4 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>model_id</th>\n",
- " <th>name</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1</td>\n",
- " <td>Sophie</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>2</td>\n",
- " <td>Maria</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>3</td>\n",
- " <td>Ella</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>4</td>\n",
- " <td>Grace</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(1, u'Sophie'), (2, u'Maria'), (3, u'Ella'), (4, u'Grace')]"
- ]
- },
- "execution_count": 16,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT model_id, name from model_arch_library ORDER BY model_id;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"delete_model\"></a>\n",
- "# 4. Delete model"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 17,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n",
- "3 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>model_id</th>\n",
- " <th>name</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>2</td>\n",
- " <td>Maria</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>3</td>\n",
- " <td>Ella</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>4</td>\n",
- " <td>Grace</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(2, u'Maria'), (3, u'Ella'), (4, u'Grace')]"
- ]
- },
- "execution_count": 17,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT madlib.delete_keras_model('model_arch_library', -- Output table\n",
- " 1 -- Model id\n",
- " );\n",
- "SELECT model_id, name from model_arch_library ORDER BY model_id;"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 2",
- "language": "python",
- "name": "python2"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 2
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython2",
- "version": "2.7.10"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 1
-}
diff --git a/community-artifacts/Deep-learning/MADlib-Keras-MLP-v1.ipynb b/community-artifacts/Deep-learning/MADlib-Keras-MLP-v1.ipynb
deleted file mode 100644
index aa61065..0000000
--- a/community-artifacts/Deep-learning/MADlib-Keras-MLP-v1.ipynb
+++ /dev/null
@@ -1,3206 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Multilayer Perceptron Using Keras and MADlib\n",
- "\n",
- "E2E classification example using MADlib calling a Keras MLP.\n",
- "\n",
- "Deep learning works best on very large datasets, but that is not convenient for a quick introduction to the syntax. So in this workbook we use the well known iris data set from https://archive.ics.uci.edu/ml/datasets/iris to help get you started. It is similar to the example in user docs http://madlib.apache.org/docs/latest/index.html\n",
- "\n",
- "For more realistic examples with images please refer to the deep learning notebooks at\n",
- "https://github.com/apache/madlib-site/tree/asf-site/community-artifacts\n",
- "\n",
- "## Table of contents\n",
- "\n",
- "<a href=\"#class\">Classification</a>\n",
- "\n",
- "* <a href=\"#create_input_data\">1. Create input data</a>\n",
- "\n",
- "* <a href=\"#pp\">2. Call preprocessor for deep learning</a>\n",
- "\n",
- "* <a href=\"#load\">3. Define and load model architecture</a>\n",
- "\n",
- "* <a href=\"#train\">4. Train</a>\n",
- "\n",
- "* <a href=\"#eval\">5. Evaluate</a>\n",
- "\n",
- "* <a href=\"#pred\">6. Predict</a>\n",
- "\n",
- "<a href=\"#class2\">Classification with Other Parameters</a>\n",
- "\n",
- "* <a href=\"#val_dataset\">1. Validation dataset</a>\n",
- "\n",
- "* <a href=\"#pred_prob\">2. Predict probabilities</a>\n",
- "\n",
- "* <a href=\"#warm_start\">3. Warm start</a>\n",
- "\n",
- "<a href=\"#transfer_learn\">Transfer learning</a>\n",
- "\n",
- "* <a href=\"#load2\">1. Define and load model architecture with some layers frozen</a>\n",
- "\n",
- "* <a href=\"#train2\">2. Train transfer model</a>"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {
- "scrolled": false
- },
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/config.py:13: ShimWarning: The `IPython.config` package has been deprecated since IPython 4.0. You should import from traitlets.config instead.\n",
- " \"You should import from traitlets.config instead.\", ShimWarning)\n",
- "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/utils/traitlets.py:5: UserWarning: IPython.utils.traitlets has moved to a top-level traitlets package.\n",
- " warn(\"IPython.utils.traitlets has moved to a top-level traitlets package.\")\n"
- ]
- }
- ],
- "source": [
- "%load_ext sql"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "u'Connected: fmcquillan@madlib'"
- ]
- },
- "execution_count": 2,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "# Greenplum Database 5.x on GCP (PM demo machine)\n",
- "#%sql postgresql://gpadmin@35.184.232.200:5432/madlib\n",
- " \n",
- "# Greenplum Database 5.x on GCP for deep learning (PM demo machine)\n",
- "#%sql postgresql://gpadmin@35.239.240.26:5432/madlib\n",
- " \n",
- "# PostgreSQL local\n",
- "%sql postgresql://fmcquillan@localhost:5432/madlib"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 30,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>version</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>PostgreSQL 11.4 on x86_64-apple-darwin16.7.0, compiled by gcc (Homebrew gcc 5.2.0) 5.2.0, 64-bit</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'PostgreSQL 11.4 on x86_64-apple-darwin16.7.0, compiled by gcc (Homebrew gcc 5.2.0) 5.2.0, 64-bit',)]"
- ]
- },
- "execution_count": 30,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "#%sql select madlib.version();\n",
- "%sql select version();"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"class\"></a>\n",
- "# Classification\n",
- "\n",
- "<a id=\"create_input_data\"></a>\n",
- "# 1. Create input data\n",
- "\n",
- "Load iris data set."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "Done.\n",
- "150 rows affected.\n",
- "150 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>id</th>\n",
- " <th>attributes</th>\n",
- " <th>class_text</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1</td>\n",
- " <td>[Decimal('5.1'), Decimal('3.5'), Decimal('1.4'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>2</td>\n",
- " <td>[Decimal('4.9'), Decimal('3.0'), Decimal('1.4'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>3</td>\n",
- " <td>[Decimal('4.7'), Decimal('3.2'), Decimal('1.3'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>4</td>\n",
- " <td>[Decimal('4.6'), Decimal('3.1'), Decimal('1.5'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>5</td>\n",
- " <td>[Decimal('5.0'), Decimal('3.6'), Decimal('1.4'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>6</td>\n",
- " <td>[Decimal('5.4'), Decimal('3.9'), Decimal('1.7'), Decimal('0.4')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>7</td>\n",
- " <td>[Decimal('4.6'), Decimal('3.4'), Decimal('1.4'), Decimal('0.3')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>8</td>\n",
- " <td>[Decimal('5.0'), Decimal('3.4'), Decimal('1.5'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>9</td>\n",
- " <td>[Decimal('4.4'), Decimal('2.9'), Decimal('1.4'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>10</td>\n",
- " <td>[Decimal('4.9'), Decimal('3.1'), Decimal('1.5'), Decimal('0.1')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>11</td>\n",
- " <td>[Decimal('5.4'), Decimal('3.7'), Decimal('1.5'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>12</td>\n",
- " <td>[Decimal('4.8'), Decimal('3.4'), Decimal('1.6'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>13</td>\n",
- " <td>[Decimal('4.8'), Decimal('3.0'), Decimal('1.4'), Decimal('0.1')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>14</td>\n",
- " <td>[Decimal('4.3'), Decimal('3.0'), Decimal('1.1'), Decimal('0.1')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>15</td>\n",
- " <td>[Decimal('5.8'), Decimal('4.0'), Decimal('1.2'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>16</td>\n",
- " <td>[Decimal('5.7'), Decimal('4.4'), Decimal('1.5'), Decimal('0.4')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>17</td>\n",
- " <td>[Decimal('5.4'), Decimal('3.9'), Decimal('1.3'), Decimal('0.4')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>18</td>\n",
- " <td>[Decimal('5.1'), Decimal('3.5'), Decimal('1.4'), Decimal('0.3')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>19</td>\n",
- " <td>[Decimal('5.7'), Decimal('3.8'), Decimal('1.7'), Decimal('0.3')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>20</td>\n",
- " <td>[Decimal('5.1'), Decimal('3.8'), Decimal('1.5'), Decimal('0.3')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>21</td>\n",
- " <td>[Decimal('5.4'), Decimal('3.4'), Decimal('1.7'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>22</td>\n",
- " <td>[Decimal('5.1'), Decimal('3.7'), Decimal('1.5'), Decimal('0.4')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>23</td>\n",
- " <td>[Decimal('4.6'), Decimal('3.6'), Decimal('1.0'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>24</td>\n",
- " <td>[Decimal('5.1'), Decimal('3.3'), Decimal('1.7'), Decimal('0.5')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>25</td>\n",
- " <td>[Decimal('4.8'), Decimal('3.4'), Decimal('1.9'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>26</td>\n",
- " <td>[Decimal('5.0'), Decimal('3.0'), Decimal('1.6'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>27</td>\n",
- " <td>[Decimal('5.0'), Decimal('3.4'), Decimal('1.6'), Decimal('0.4')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>28</td>\n",
- " <td>[Decimal('5.2'), Decimal('3.5'), Decimal('1.5'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>29</td>\n",
- " <td>[Decimal('5.2'), Decimal('3.4'), Decimal('1.4'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>30</td>\n",
- " <td>[Decimal('4.7'), Decimal('3.2'), Decimal('1.6'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>31</td>\n",
- " <td>[Decimal('4.8'), Decimal('3.1'), Decimal('1.6'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>32</td>\n",
- " <td>[Decimal('5.4'), Decimal('3.4'), Decimal('1.5'), Decimal('0.4')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>33</td>\n",
- " <td>[Decimal('5.2'), Decimal('4.1'), Decimal('1.5'), Decimal('0.1')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>34</td>\n",
- " <td>[Decimal('5.5'), Decimal('4.2'), Decimal('1.4'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>35</td>\n",
- " <td>[Decimal('4.9'), Decimal('3.1'), Decimal('1.5'), Decimal('0.1')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>36</td>\n",
- " <td>[Decimal('5.0'), Decimal('3.2'), Decimal('1.2'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>37</td>\n",
- " <td>[Decimal('5.5'), Decimal('3.5'), Decimal('1.3'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>38</td>\n",
- " <td>[Decimal('4.9'), Decimal('3.1'), Decimal('1.5'), Decimal('0.1')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>39</td>\n",
- " <td>[Decimal('4.4'), Decimal('3.0'), Decimal('1.3'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>40</td>\n",
- " <td>[Decimal('5.1'), Decimal('3.4'), Decimal('1.5'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>41</td>\n",
- " <td>[Decimal('5.0'), Decimal('3.5'), Decimal('1.3'), Decimal('0.3')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>42</td>\n",
- " <td>[Decimal('4.5'), Decimal('2.3'), Decimal('1.3'), Decimal('0.3')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>43</td>\n",
- " <td>[Decimal('4.4'), Decimal('3.2'), Decimal('1.3'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>44</td>\n",
- " <td>[Decimal('5.0'), Decimal('3.5'), Decimal('1.6'), Decimal('0.6')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>45</td>\n",
- " <td>[Decimal('5.1'), Decimal('3.8'), Decimal('1.9'), Decimal('0.4')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>46</td>\n",
- " <td>[Decimal('4.8'), Decimal('3.0'), Decimal('1.4'), Decimal('0.3')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>47</td>\n",
- " <td>[Decimal('5.1'), Decimal('3.8'), Decimal('1.6'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>48</td>\n",
- " <td>[Decimal('4.6'), Decimal('3.2'), Decimal('1.4'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>49</td>\n",
- " <td>[Decimal('5.3'), Decimal('3.7'), Decimal('1.5'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>50</td>\n",
- " <td>[Decimal('5.0'), Decimal('3.3'), Decimal('1.4'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>51</td>\n",
- " <td>[Decimal('7.0'), Decimal('3.2'), Decimal('4.7'), Decimal('1.4')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>52</td>\n",
- " <td>[Decimal('6.4'), Decimal('3.2'), Decimal('4.5'), Decimal('1.5')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>53</td>\n",
- " <td>[Decimal('6.9'), Decimal('3.1'), Decimal('4.9'), Decimal('1.5')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>54</td>\n",
- " <td>[Decimal('5.5'), Decimal('2.3'), Decimal('4.0'), Decimal('1.3')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>55</td>\n",
- " <td>[Decimal('6.5'), Decimal('2.8'), Decimal('4.6'), Decimal('1.5')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>56</td>\n",
- " <td>[Decimal('5.7'), Decimal('2.8'), Decimal('4.5'), Decimal('1.3')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>57</td>\n",
- " <td>[Decimal('6.3'), Decimal('3.3'), Decimal('4.7'), Decimal('1.6')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>58</td>\n",
- " <td>[Decimal('4.9'), Decimal('2.4'), Decimal('3.3'), Decimal('1.0')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>59</td>\n",
- " <td>[Decimal('6.6'), Decimal('2.9'), Decimal('4.6'), Decimal('1.3')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>60</td>\n",
- " <td>[Decimal('5.2'), Decimal('2.7'), Decimal('3.9'), Decimal('1.4')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>61</td>\n",
- " <td>[Decimal('5.0'), Decimal('2.0'), Decimal('3.5'), Decimal('1.0')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>62</td>\n",
- " <td>[Decimal('5.9'), Decimal('3.0'), Decimal('4.2'), Decimal('1.5')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>63</td>\n",
- " <td>[Decimal('6.0'), Decimal('2.2'), Decimal('4.0'), Decimal('1.0')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>64</td>\n",
- " <td>[Decimal('6.1'), Decimal('2.9'), Decimal('4.7'), Decimal('1.4')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>65</td>\n",
- " <td>[Decimal('5.6'), Decimal('2.9'), Decimal('3.6'), Decimal('1.3')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>66</td>\n",
- " <td>[Decimal('6.7'), Decimal('3.1'), Decimal('4.4'), Decimal('1.4')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>67</td>\n",
- " <td>[Decimal('5.6'), Decimal('3.0'), Decimal('4.5'), Decimal('1.5')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>68</td>\n",
- " <td>[Decimal('5.8'), Decimal('2.7'), Decimal('4.1'), Decimal('1.0')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>69</td>\n",
- " <td>[Decimal('6.2'), Decimal('2.2'), Decimal('4.5'), Decimal('1.5')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>70</td>\n",
- " <td>[Decimal('5.6'), Decimal('2.5'), Decimal('3.9'), Decimal('1.1')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>71</td>\n",
- " <td>[Decimal('5.9'), Decimal('3.2'), Decimal('4.8'), Decimal('1.8')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>72</td>\n",
- " <td>[Decimal('6.1'), Decimal('2.8'), Decimal('4.0'), Decimal('1.3')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>73</td>\n",
- " <td>[Decimal('6.3'), Decimal('2.5'), Decimal('4.9'), Decimal('1.5')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>74</td>\n",
- " <td>[Decimal('6.1'), Decimal('2.8'), Decimal('4.7'), Decimal('1.2')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>75</td>\n",
- " <td>[Decimal('6.4'), Decimal('2.9'), Decimal('4.3'), Decimal('1.3')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>76</td>\n",
- " <td>[Decimal('6.6'), Decimal('3.0'), Decimal('4.4'), Decimal('1.4')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>77</td>\n",
- " <td>[Decimal('6.8'), Decimal('2.8'), Decimal('4.8'), Decimal('1.4')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>78</td>\n",
- " <td>[Decimal('6.7'), Decimal('3.0'), Decimal('5.0'), Decimal('1.7')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>79</td>\n",
- " <td>[Decimal('6.0'), Decimal('2.9'), Decimal('4.5'), Decimal('1.5')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>80</td>\n",
- " <td>[Decimal('5.7'), Decimal('2.6'), Decimal('3.5'), Decimal('1.0')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>81</td>\n",
- " <td>[Decimal('5.5'), Decimal('2.4'), Decimal('3.8'), Decimal('1.1')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>82</td>\n",
- " <td>[Decimal('5.5'), Decimal('2.4'), Decimal('3.7'), Decimal('1.0')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>83</td>\n",
- " <td>[Decimal('5.8'), Decimal('2.7'), Decimal('3.9'), Decimal('1.2')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>84</td>\n",
- " <td>[Decimal('6.0'), Decimal('2.7'), Decimal('5.1'), Decimal('1.6')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>85</td>\n",
- " <td>[Decimal('5.4'), Decimal('3.0'), Decimal('4.5'), Decimal('1.5')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>86</td>\n",
- " <td>[Decimal('6.0'), Decimal('3.4'), Decimal('4.5'), Decimal('1.6')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>87</td>\n",
- " <td>[Decimal('6.7'), Decimal('3.1'), Decimal('4.7'), Decimal('1.5')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>88</td>\n",
- " <td>[Decimal('6.3'), Decimal('2.3'), Decimal('4.4'), Decimal('1.3')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>89</td>\n",
- " <td>[Decimal('5.6'), Decimal('3.0'), Decimal('4.1'), Decimal('1.3')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>90</td>\n",
- " <td>[Decimal('5.5'), Decimal('2.5'), Decimal('4.0'), Decimal('1.3')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>91</td>\n",
- " <td>[Decimal('5.5'), Decimal('2.6'), Decimal('4.4'), Decimal('1.2')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>92</td>\n",
- " <td>[Decimal('6.1'), Decimal('3.0'), Decimal('4.6'), Decimal('1.4')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>93</td>\n",
- " <td>[Decimal('5.8'), Decimal('2.6'), Decimal('4.0'), Decimal('1.2')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>94</td>\n",
- " <td>[Decimal('5.0'), Decimal('2.3'), Decimal('3.3'), Decimal('1.0')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>95</td>\n",
- " <td>[Decimal('5.6'), Decimal('2.7'), Decimal('4.2'), Decimal('1.3')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>96</td>\n",
- " <td>[Decimal('5.7'), Decimal('3.0'), Decimal('4.2'), Decimal('1.2')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>97</td>\n",
- " <td>[Decimal('5.7'), Decimal('2.9'), Decimal('4.2'), Decimal('1.3')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>98</td>\n",
- " <td>[Decimal('6.2'), Decimal('2.9'), Decimal('4.3'), Decimal('1.3')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>99</td>\n",
- " <td>[Decimal('5.1'), Decimal('2.5'), Decimal('3.0'), Decimal('1.1')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>100</td>\n",
- " <td>[Decimal('5.7'), Decimal('2.8'), Decimal('4.1'), Decimal('1.3')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>101</td>\n",
- " <td>[Decimal('6.3'), Decimal('3.3'), Decimal('6.0'), Decimal('2.5')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>102</td>\n",
- " <td>[Decimal('5.8'), Decimal('2.7'), Decimal('5.1'), Decimal('1.9')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>103</td>\n",
- " <td>[Decimal('7.1'), Decimal('3.0'), Decimal('5.9'), Decimal('2.1')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>104</td>\n",
- " <td>[Decimal('6.3'), Decimal('2.9'), Decimal('5.6'), Decimal('1.8')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>105</td>\n",
- " <td>[Decimal('6.5'), Decimal('3.0'), Decimal('5.8'), Decimal('2.2')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>106</td>\n",
- " <td>[Decimal('7.6'), Decimal('3.0'), Decimal('6.6'), Decimal('2.1')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>107</td>\n",
- " <td>[Decimal('4.9'), Decimal('2.5'), Decimal('4.5'), Decimal('1.7')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>108</td>\n",
- " <td>[Decimal('7.3'), Decimal('2.9'), Decimal('6.3'), Decimal('1.8')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>109</td>\n",
- " <td>[Decimal('6.7'), Decimal('2.5'), Decimal('5.8'), Decimal('1.8')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>110</td>\n",
- " <td>[Decimal('7.2'), Decimal('3.6'), Decimal('6.1'), Decimal('2.5')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>111</td>\n",
- " <td>[Decimal('6.5'), Decimal('3.2'), Decimal('5.1'), Decimal('2.0')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>112</td>\n",
- " <td>[Decimal('6.4'), Decimal('2.7'), Decimal('5.3'), Decimal('1.9')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>113</td>\n",
- " <td>[Decimal('6.8'), Decimal('3.0'), Decimal('5.5'), Decimal('2.1')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>114</td>\n",
- " <td>[Decimal('5.7'), Decimal('2.5'), Decimal('5.0'), Decimal('2.0')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>115</td>\n",
- " <td>[Decimal('5.8'), Decimal('2.8'), Decimal('5.1'), Decimal('2.4')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>116</td>\n",
- " <td>[Decimal('6.4'), Decimal('3.2'), Decimal('5.3'), Decimal('2.3')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>117</td>\n",
- " <td>[Decimal('6.5'), Decimal('3.0'), Decimal('5.5'), Decimal('1.8')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>118</td>\n",
- " <td>[Decimal('7.7'), Decimal('3.8'), Decimal('6.7'), Decimal('2.2')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>119</td>\n",
- " <td>[Decimal('7.7'), Decimal('2.6'), Decimal('6.9'), Decimal('2.3')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>120</td>\n",
- " <td>[Decimal('6.0'), Decimal('2.2'), Decimal('5.0'), Decimal('1.5')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>121</td>\n",
- " <td>[Decimal('6.9'), Decimal('3.2'), Decimal('5.7'), Decimal('2.3')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>122</td>\n",
- " <td>[Decimal('5.6'), Decimal('2.8'), Decimal('4.9'), Decimal('2.0')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>123</td>\n",
- " <td>[Decimal('7.7'), Decimal('2.8'), Decimal('6.7'), Decimal('2.0')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>124</td>\n",
- " <td>[Decimal('6.3'), Decimal('2.7'), Decimal('4.9'), Decimal('1.8')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>125</td>\n",
- " <td>[Decimal('6.7'), Decimal('3.3'), Decimal('5.7'), Decimal('2.1')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>126</td>\n",
- " <td>[Decimal('7.2'), Decimal('3.2'), Decimal('6.0'), Decimal('1.8')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>127</td>\n",
- " <td>[Decimal('6.2'), Decimal('2.8'), Decimal('4.8'), Decimal('1.8')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>128</td>\n",
- " <td>[Decimal('6.1'), Decimal('3.0'), Decimal('4.9'), Decimal('1.8')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>129</td>\n",
- " <td>[Decimal('6.4'), Decimal('2.8'), Decimal('5.6'), Decimal('2.1')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>130</td>\n",
- " <td>[Decimal('7.2'), Decimal('3.0'), Decimal('5.8'), Decimal('1.6')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>131</td>\n",
- " <td>[Decimal('7.4'), Decimal('2.8'), Decimal('6.1'), Decimal('1.9')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>132</td>\n",
- " <td>[Decimal('7.9'), Decimal('3.8'), Decimal('6.4'), Decimal('2.0')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>133</td>\n",
- " <td>[Decimal('6.4'), Decimal('2.8'), Decimal('5.6'), Decimal('2.2')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>134</td>\n",
- " <td>[Decimal('6.3'), Decimal('2.8'), Decimal('5.1'), Decimal('1.5')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>135</td>\n",
- " <td>[Decimal('6.1'), Decimal('2.6'), Decimal('5.6'), Decimal('1.4')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>136</td>\n",
- " <td>[Decimal('7.7'), Decimal('3.0'), Decimal('6.1'), Decimal('2.3')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>137</td>\n",
- " <td>[Decimal('6.3'), Decimal('3.4'), Decimal('5.6'), Decimal('2.4')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>138</td>\n",
- " <td>[Decimal('6.4'), Decimal('3.1'), Decimal('5.5'), Decimal('1.8')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>139</td>\n",
- " <td>[Decimal('6.0'), Decimal('3.0'), Decimal('4.8'), Decimal('1.8')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>140</td>\n",
- " <td>[Decimal('6.9'), Decimal('3.1'), Decimal('5.4'), Decimal('2.1')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>141</td>\n",
- " <td>[Decimal('6.7'), Decimal('3.1'), Decimal('5.6'), Decimal('2.4')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>142</td>\n",
- " <td>[Decimal('6.9'), Decimal('3.1'), Decimal('5.1'), Decimal('2.3')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>143</td>\n",
- " <td>[Decimal('5.8'), Decimal('2.7'), Decimal('5.1'), Decimal('1.9')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>144</td>\n",
- " <td>[Decimal('6.8'), Decimal('3.2'), Decimal('5.9'), Decimal('2.3')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>145</td>\n",
- " <td>[Decimal('6.7'), Decimal('3.3'), Decimal('5.7'), Decimal('2.5')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>146</td>\n",
- " <td>[Decimal('6.7'), Decimal('3.0'), Decimal('5.2'), Decimal('2.3')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>147</td>\n",
- " <td>[Decimal('6.3'), Decimal('2.5'), Decimal('5.0'), Decimal('1.9')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>148</td>\n",
- " <td>[Decimal('6.5'), Decimal('3.0'), Decimal('5.2'), Decimal('2.0')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>149</td>\n",
- " <td>[Decimal('6.2'), Decimal('3.4'), Decimal('5.4'), Decimal('2.3')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>150</td>\n",
- " <td>[Decimal('5.9'), Decimal('3.0'), Decimal('5.1'), Decimal('1.8')]</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(1, [Decimal('5.1'), Decimal('3.5'), Decimal('1.4'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (2, [Decimal('4.9'), Decimal('3.0'), Decimal('1.4'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (3, [Decimal('4.7'), Decimal('3.2'), Decimal('1.3'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (4, [Decimal('4.6'), Decimal('3.1'), Decimal('1.5'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (5, [Decimal('5.0'), Decimal('3.6'), Decimal('1.4'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (6, [Decimal('5.4'), Decimal('3.9'), Decimal('1.7'), Decimal('0.4')], u'Iris-setosa'),\n",
- " (7, [Decimal('4.6'), Decimal('3.4'), Decimal('1.4'), Decimal('0.3')], u'Iris-setosa'),\n",
- " (8, [Decimal('5.0'), Decimal('3.4'), Decimal('1.5'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (9, [Decimal('4.4'), Decimal('2.9'), Decimal('1.4'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (10, [Decimal('4.9'), Decimal('3.1'), Decimal('1.5'), Decimal('0.1')], u'Iris-setosa'),\n",
- " (11, [Decimal('5.4'), Decimal('3.7'), Decimal('1.5'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (12, [Decimal('4.8'), Decimal('3.4'), Decimal('1.6'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (13, [Decimal('4.8'), Decimal('3.0'), Decimal('1.4'), Decimal('0.1')], u'Iris-setosa'),\n",
- " (14, [Decimal('4.3'), Decimal('3.0'), Decimal('1.1'), Decimal('0.1')], u'Iris-setosa'),\n",
- " (15, [Decimal('5.8'), Decimal('4.0'), Decimal('1.2'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (16, [Decimal('5.7'), Decimal('4.4'), Decimal('1.5'), Decimal('0.4')], u'Iris-setosa'),\n",
- " (17, [Decimal('5.4'), Decimal('3.9'), Decimal('1.3'), Decimal('0.4')], u'Iris-setosa'),\n",
- " (18, [Decimal('5.1'), Decimal('3.5'), Decimal('1.4'), Decimal('0.3')], u'Iris-setosa'),\n",
- " (19, [Decimal('5.7'), Decimal('3.8'), Decimal('1.7'), Decimal('0.3')], u'Iris-setosa'),\n",
- " (20, [Decimal('5.1'), Decimal('3.8'), Decimal('1.5'), Decimal('0.3')], u'Iris-setosa'),\n",
- " (21, [Decimal('5.4'), Decimal('3.4'), Decimal('1.7'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (22, [Decimal('5.1'), Decimal('3.7'), Decimal('1.5'), Decimal('0.4')], u'Iris-setosa'),\n",
- " (23, [Decimal('4.6'), Decimal('3.6'), Decimal('1.0'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (24, [Decimal('5.1'), Decimal('3.3'), Decimal('1.7'), Decimal('0.5')], u'Iris-setosa'),\n",
- " (25, [Decimal('4.8'), Decimal('3.4'), Decimal('1.9'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (26, [Decimal('5.0'), Decimal('3.0'), Decimal('1.6'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (27, [Decimal('5.0'), Decimal('3.4'), Decimal('1.6'), Decimal('0.4')], u'Iris-setosa'),\n",
- " (28, [Decimal('5.2'), Decimal('3.5'), Decimal('1.5'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (29, [Decimal('5.2'), Decimal('3.4'), Decimal('1.4'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (30, [Decimal('4.7'), Decimal('3.2'), Decimal('1.6'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (31, [Decimal('4.8'), Decimal('3.1'), Decimal('1.6'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (32, [Decimal('5.4'), Decimal('3.4'), Decimal('1.5'), Decimal('0.4')], u'Iris-setosa'),\n",
- " (33, [Decimal('5.2'), Decimal('4.1'), Decimal('1.5'), Decimal('0.1')], u'Iris-setosa'),\n",
- " (34, [Decimal('5.5'), Decimal('4.2'), Decimal('1.4'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (35, [Decimal('4.9'), Decimal('3.1'), Decimal('1.5'), Decimal('0.1')], u'Iris-setosa'),\n",
- " (36, [Decimal('5.0'), Decimal('3.2'), Decimal('1.2'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (37, [Decimal('5.5'), Decimal('3.5'), Decimal('1.3'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (38, [Decimal('4.9'), Decimal('3.1'), Decimal('1.5'), Decimal('0.1')], u'Iris-setosa'),\n",
- " (39, [Decimal('4.4'), Decimal('3.0'), Decimal('1.3'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (40, [Decimal('5.1'), Decimal('3.4'), Decimal('1.5'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (41, [Decimal('5.0'), Decimal('3.5'), Decimal('1.3'), Decimal('0.3')], u'Iris-setosa'),\n",
- " (42, [Decimal('4.5'), Decimal('2.3'), Decimal('1.3'), Decimal('0.3')], u'Iris-setosa'),\n",
- " (43, [Decimal('4.4'), Decimal('3.2'), Decimal('1.3'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (44, [Decimal('5.0'), Decimal('3.5'), Decimal('1.6'), Decimal('0.6')], u'Iris-setosa'),\n",
- " (45, [Decimal('5.1'), Decimal('3.8'), Decimal('1.9'), Decimal('0.4')], u'Iris-setosa'),\n",
- " (46, [Decimal('4.8'), Decimal('3.0'), Decimal('1.4'), Decimal('0.3')], u'Iris-setosa'),\n",
- " (47, [Decimal('5.1'), Decimal('3.8'), Decimal('1.6'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (48, [Decimal('4.6'), Decimal('3.2'), Decimal('1.4'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (49, [Decimal('5.3'), Decimal('3.7'), Decimal('1.5'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (50, [Decimal('5.0'), Decimal('3.3'), Decimal('1.4'), Decimal('0.2')], u'Iris-setosa'),\n",
- " (51, [Decimal('7.0'), Decimal('3.2'), Decimal('4.7'), Decimal('1.4')], u'Iris-versicolor'),\n",
- " (52, [Decimal('6.4'), Decimal('3.2'), Decimal('4.5'), Decimal('1.5')], u'Iris-versicolor'),\n",
- " (53, [Decimal('6.9'), Decimal('3.1'), Decimal('4.9'), Decimal('1.5')], u'Iris-versicolor'),\n",
- " (54, [Decimal('5.5'), Decimal('2.3'), Decimal('4.0'), Decimal('1.3')], u'Iris-versicolor'),\n",
- " (55, [Decimal('6.5'), Decimal('2.8'), Decimal('4.6'), Decimal('1.5')], u'Iris-versicolor'),\n",
- " (56, [Decimal('5.7'), Decimal('2.8'), Decimal('4.5'), Decimal('1.3')], u'Iris-versicolor'),\n",
- " (57, [Decimal('6.3'), Decimal('3.3'), Decimal('4.7'), Decimal('1.6')], u'Iris-versicolor'),\n",
- " (58, [Decimal('4.9'), Decimal('2.4'), Decimal('3.3'), Decimal('1.0')], u'Iris-versicolor'),\n",
- " (59, [Decimal('6.6'), Decimal('2.9'), Decimal('4.6'), Decimal('1.3')], u'Iris-versicolor'),\n",
- " (60, [Decimal('5.2'), Decimal('2.7'), Decimal('3.9'), Decimal('1.4')], u'Iris-versicolor'),\n",
- " (61, [Decimal('5.0'), Decimal('2.0'), Decimal('3.5'), Decimal('1.0')], u'Iris-versicolor'),\n",
- " (62, [Decimal('5.9'), Decimal('3.0'), Decimal('4.2'), Decimal('1.5')], u'Iris-versicolor'),\n",
- " (63, [Decimal('6.0'), Decimal('2.2'), Decimal('4.0'), Decimal('1.0')], u'Iris-versicolor'),\n",
- " (64, [Decimal('6.1'), Decimal('2.9'), Decimal('4.7'), Decimal('1.4')], u'Iris-versicolor'),\n",
- " (65, [Decimal('5.6'), Decimal('2.9'), Decimal('3.6'), Decimal('1.3')], u'Iris-versicolor'),\n",
- " (66, [Decimal('6.7'), Decimal('3.1'), Decimal('4.4'), Decimal('1.4')], u'Iris-versicolor'),\n",
- " (67, [Decimal('5.6'), Decimal('3.0'), Decimal('4.5'), Decimal('1.5')], u'Iris-versicolor'),\n",
- " (68, [Decimal('5.8'), Decimal('2.7'), Decimal('4.1'), Decimal('1.0')], u'Iris-versicolor'),\n",
- " (69, [Decimal('6.2'), Decimal('2.2'), Decimal('4.5'), Decimal('1.5')], u'Iris-versicolor'),\n",
- " (70, [Decimal('5.6'), Decimal('2.5'), Decimal('3.9'), Decimal('1.1')], u'Iris-versicolor'),\n",
- " (71, [Decimal('5.9'), Decimal('3.2'), Decimal('4.8'), Decimal('1.8')], u'Iris-versicolor'),\n",
- " (72, [Decimal('6.1'), Decimal('2.8'), Decimal('4.0'), Decimal('1.3')], u'Iris-versicolor'),\n",
- " (73, [Decimal('6.3'), Decimal('2.5'), Decimal('4.9'), Decimal('1.5')], u'Iris-versicolor'),\n",
- " (74, [Decimal('6.1'), Decimal('2.8'), Decimal('4.7'), Decimal('1.2')], u'Iris-versicolor'),\n",
- " (75, [Decimal('6.4'), Decimal('2.9'), Decimal('4.3'), Decimal('1.3')], u'Iris-versicolor'),\n",
- " (76, [Decimal('6.6'), Decimal('3.0'), Decimal('4.4'), Decimal('1.4')], u'Iris-versicolor'),\n",
- " (77, [Decimal('6.8'), Decimal('2.8'), Decimal('4.8'), Decimal('1.4')], u'Iris-versicolor'),\n",
- " (78, [Decimal('6.7'), Decimal('3.0'), Decimal('5.0'), Decimal('1.7')], u'Iris-versicolor'),\n",
- " (79, [Decimal('6.0'), Decimal('2.9'), Decimal('4.5'), Decimal('1.5')], u'Iris-versicolor'),\n",
- " (80, [Decimal('5.7'), Decimal('2.6'), Decimal('3.5'), Decimal('1.0')], u'Iris-versicolor'),\n",
- " (81, [Decimal('5.5'), Decimal('2.4'), Decimal('3.8'), Decimal('1.1')], u'Iris-versicolor'),\n",
- " (82, [Decimal('5.5'), Decimal('2.4'), Decimal('3.7'), Decimal('1.0')], u'Iris-versicolor'),\n",
- " (83, [Decimal('5.8'), Decimal('2.7'), Decimal('3.9'), Decimal('1.2')], u'Iris-versicolor'),\n",
- " (84, [Decimal('6.0'), Decimal('2.7'), Decimal('5.1'), Decimal('1.6')], u'Iris-versicolor'),\n",
- " (85, [Decimal('5.4'), Decimal('3.0'), Decimal('4.5'), Decimal('1.5')], u'Iris-versicolor'),\n",
- " (86, [Decimal('6.0'), Decimal('3.4'), Decimal('4.5'), Decimal('1.6')], u'Iris-versicolor'),\n",
- " (87, [Decimal('6.7'), Decimal('3.1'), Decimal('4.7'), Decimal('1.5')], u'Iris-versicolor'),\n",
- " (88, [Decimal('6.3'), Decimal('2.3'), Decimal('4.4'), Decimal('1.3')], u'Iris-versicolor'),\n",
- " (89, [Decimal('5.6'), Decimal('3.0'), Decimal('4.1'), Decimal('1.3')], u'Iris-versicolor'),\n",
- " (90, [Decimal('5.5'), Decimal('2.5'), Decimal('4.0'), Decimal('1.3')], u'Iris-versicolor'),\n",
- " (91, [Decimal('5.5'), Decimal('2.6'), Decimal('4.4'), Decimal('1.2')], u'Iris-versicolor'),\n",
- " (92, [Decimal('6.1'), Decimal('3.0'), Decimal('4.6'), Decimal('1.4')], u'Iris-versicolor'),\n",
- " (93, [Decimal('5.8'), Decimal('2.6'), Decimal('4.0'), Decimal('1.2')], u'Iris-versicolor'),\n",
- " (94, [Decimal('5.0'), Decimal('2.3'), Decimal('3.3'), Decimal('1.0')], u'Iris-versicolor'),\n",
- " (95, [Decimal('5.6'), Decimal('2.7'), Decimal('4.2'), Decimal('1.3')], u'Iris-versicolor'),\n",
- " (96, [Decimal('5.7'), Decimal('3.0'), Decimal('4.2'), Decimal('1.2')], u'Iris-versicolor'),\n",
- " (97, [Decimal('5.7'), Decimal('2.9'), Decimal('4.2'), Decimal('1.3')], u'Iris-versicolor'),\n",
- " (98, [Decimal('6.2'), Decimal('2.9'), Decimal('4.3'), Decimal('1.3')], u'Iris-versicolor'),\n",
- " (99, [Decimal('5.1'), Decimal('2.5'), Decimal('3.0'), Decimal('1.1')], u'Iris-versicolor'),\n",
- " (100, [Decimal('5.7'), Decimal('2.8'), Decimal('4.1'), Decimal('1.3')], u'Iris-versicolor'),\n",
- " (101, [Decimal('6.3'), Decimal('3.3'), Decimal('6.0'), Decimal('2.5')], u'Iris-virginica'),\n",
- " (102, [Decimal('5.8'), Decimal('2.7'), Decimal('5.1'), Decimal('1.9')], u'Iris-virginica'),\n",
- " (103, [Decimal('7.1'), Decimal('3.0'), Decimal('5.9'), Decimal('2.1')], u'Iris-virginica'),\n",
- " (104, [Decimal('6.3'), Decimal('2.9'), Decimal('5.6'), Decimal('1.8')], u'Iris-virginica'),\n",
- " (105, [Decimal('6.5'), Decimal('3.0'), Decimal('5.8'), Decimal('2.2')], u'Iris-virginica'),\n",
- " (106, [Decimal('7.6'), Decimal('3.0'), Decimal('6.6'), Decimal('2.1')], u'Iris-virginica'),\n",
- " (107, [Decimal('4.9'), Decimal('2.5'), Decimal('4.5'), Decimal('1.7')], u'Iris-virginica'),\n",
- " (108, [Decimal('7.3'), Decimal('2.9'), Decimal('6.3'), Decimal('1.8')], u'Iris-virginica'),\n",
- " (109, [Decimal('6.7'), Decimal('2.5'), Decimal('5.8'), Decimal('1.8')], u'Iris-virginica'),\n",
- " (110, [Decimal('7.2'), Decimal('3.6'), Decimal('6.1'), Decimal('2.5')], u'Iris-virginica'),\n",
- " (111, [Decimal('6.5'), Decimal('3.2'), Decimal('5.1'), Decimal('2.0')], u'Iris-virginica'),\n",
- " (112, [Decimal('6.4'), Decimal('2.7'), Decimal('5.3'), Decimal('1.9')], u'Iris-virginica'),\n",
- " (113, [Decimal('6.8'), Decimal('3.0'), Decimal('5.5'), Decimal('2.1')], u'Iris-virginica'),\n",
- " (114, [Decimal('5.7'), Decimal('2.5'), Decimal('5.0'), Decimal('2.0')], u'Iris-virginica'),\n",
- " (115, [Decimal('5.8'), Decimal('2.8'), Decimal('5.1'), Decimal('2.4')], u'Iris-virginica'),\n",
- " (116, [Decimal('6.4'), Decimal('3.2'), Decimal('5.3'), Decimal('2.3')], u'Iris-virginica'),\n",
- " (117, [Decimal('6.5'), Decimal('3.0'), Decimal('5.5'), Decimal('1.8')], u'Iris-virginica'),\n",
- " (118, [Decimal('7.7'), Decimal('3.8'), Decimal('6.7'), Decimal('2.2')], u'Iris-virginica'),\n",
- " (119, [Decimal('7.7'), Decimal('2.6'), Decimal('6.9'), Decimal('2.3')], u'Iris-virginica'),\n",
- " (120, [Decimal('6.0'), Decimal('2.2'), Decimal('5.0'), Decimal('1.5')], u'Iris-virginica'),\n",
- " (121, [Decimal('6.9'), Decimal('3.2'), Decimal('5.7'), Decimal('2.3')], u'Iris-virginica'),\n",
- " (122, [Decimal('5.6'), Decimal('2.8'), Decimal('4.9'), Decimal('2.0')], u'Iris-virginica'),\n",
- " (123, [Decimal('7.7'), Decimal('2.8'), Decimal('6.7'), Decimal('2.0')], u'Iris-virginica'),\n",
- " (124, [Decimal('6.3'), Decimal('2.7'), Decimal('4.9'), Decimal('1.8')], u'Iris-virginica'),\n",
- " (125, [Decimal('6.7'), Decimal('3.3'), Decimal('5.7'), Decimal('2.1')], u'Iris-virginica'),\n",
- " (126, [Decimal('7.2'), Decimal('3.2'), Decimal('6.0'), Decimal('1.8')], u'Iris-virginica'),\n",
- " (127, [Decimal('6.2'), Decimal('2.8'), Decimal('4.8'), Decimal('1.8')], u'Iris-virginica'),\n",
- " (128, [Decimal('6.1'), Decimal('3.0'), Decimal('4.9'), Decimal('1.8')], u'Iris-virginica'),\n",
- " (129, [Decimal('6.4'), Decimal('2.8'), Decimal('5.6'), Decimal('2.1')], u'Iris-virginica'),\n",
- " (130, [Decimal('7.2'), Decimal('3.0'), Decimal('5.8'), Decimal('1.6')], u'Iris-virginica'),\n",
- " (131, [Decimal('7.4'), Decimal('2.8'), Decimal('6.1'), Decimal('1.9')], u'Iris-virginica'),\n",
- " (132, [Decimal('7.9'), Decimal('3.8'), Decimal('6.4'), Decimal('2.0')], u'Iris-virginica'),\n",
- " (133, [Decimal('6.4'), Decimal('2.8'), Decimal('5.6'), Decimal('2.2')], u'Iris-virginica'),\n",
- " (134, [Decimal('6.3'), Decimal('2.8'), Decimal('5.1'), Decimal('1.5')], u'Iris-virginica'),\n",
- " (135, [Decimal('6.1'), Decimal('2.6'), Decimal('5.6'), Decimal('1.4')], u'Iris-virginica'),\n",
- " (136, [Decimal('7.7'), Decimal('3.0'), Decimal('6.1'), Decimal('2.3')], u'Iris-virginica'),\n",
- " (137, [Decimal('6.3'), Decimal('3.4'), Decimal('5.6'), Decimal('2.4')], u'Iris-virginica'),\n",
- " (138, [Decimal('6.4'), Decimal('3.1'), Decimal('5.5'), Decimal('1.8')], u'Iris-virginica'),\n",
- " (139, [Decimal('6.0'), Decimal('3.0'), Decimal('4.8'), Decimal('1.8')], u'Iris-virginica'),\n",
- " (140, [Decimal('6.9'), Decimal('3.1'), Decimal('5.4'), Decimal('2.1')], u'Iris-virginica'),\n",
- " (141, [Decimal('6.7'), Decimal('3.1'), Decimal('5.6'), Decimal('2.4')], u'Iris-virginica'),\n",
- " (142, [Decimal('6.9'), Decimal('3.1'), Decimal('5.1'), Decimal('2.3')], u'Iris-virginica'),\n",
- " (143, [Decimal('5.8'), Decimal('2.7'), Decimal('5.1'), Decimal('1.9')], u'Iris-virginica'),\n",
- " (144, [Decimal('6.8'), Decimal('3.2'), Decimal('5.9'), Decimal('2.3')], u'Iris-virginica'),\n",
- " (145, [Decimal('6.7'), Decimal('3.3'), Decimal('5.7'), Decimal('2.5')], u'Iris-virginica'),\n",
- " (146, [Decimal('6.7'), Decimal('3.0'), Decimal('5.2'), Decimal('2.3')], u'Iris-virginica'),\n",
- " (147, [Decimal('6.3'), Decimal('2.5'), Decimal('5.0'), Decimal('1.9')], u'Iris-virginica'),\n",
- " (148, [Decimal('6.5'), Decimal('3.0'), Decimal('5.2'), Decimal('2.0')], u'Iris-virginica'),\n",
- " (149, [Decimal('6.2'), Decimal('3.4'), Decimal('5.4'), Decimal('2.3')], u'Iris-virginica'),\n",
- " (150, [Decimal('5.9'), Decimal('3.0'), Decimal('5.1'), Decimal('1.8')], u'Iris-virginica')]"
- ]
- },
- "execution_count": 4,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql \n",
- "DROP TABLE IF EXISTS iris_data;\n",
- "\n",
- "CREATE TABLE iris_data(\n",
- " id serial,\n",
- " attributes numeric[],\n",
- " class_text varchar\n",
- ");\n",
- "\n",
- "INSERT INTO iris_data(id, attributes, class_text) VALUES\n",
- "(1,ARRAY[5.1,3.5,1.4,0.2],'Iris-setosa'),\n",
- "(2,ARRAY[4.9,3.0,1.4,0.2],'Iris-setosa'),\n",
- "(3,ARRAY[4.7,3.2,1.3,0.2],'Iris-setosa'),\n",
- "(4,ARRAY[4.6,3.1,1.5,0.2],'Iris-setosa'),\n",
- "(5,ARRAY[5.0,3.6,1.4,0.2],'Iris-setosa'),\n",
- "(6,ARRAY[5.4,3.9,1.7,0.4],'Iris-setosa'),\n",
- "(7,ARRAY[4.6,3.4,1.4,0.3],'Iris-setosa'),\n",
- "(8,ARRAY[5.0,3.4,1.5,0.2],'Iris-setosa'),\n",
- "(9,ARRAY[4.4,2.9,1.4,0.2],'Iris-setosa'),\n",
- "(10,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa'),\n",
- "(11,ARRAY[5.4,3.7,1.5,0.2],'Iris-setosa'),\n",
- "(12,ARRAY[4.8,3.4,1.6,0.2],'Iris-setosa'),\n",
- "(13,ARRAY[4.8,3.0,1.4,0.1],'Iris-setosa'),\n",
- "(14,ARRAY[4.3,3.0,1.1,0.1],'Iris-setosa'),\n",
- "(15,ARRAY[5.8,4.0,1.2,0.2],'Iris-setosa'),\n",
- "(16,ARRAY[5.7,4.4,1.5,0.4],'Iris-setosa'),\n",
- "(17,ARRAY[5.4,3.9,1.3,0.4],'Iris-setosa'),\n",
- "(18,ARRAY[5.1,3.5,1.4,0.3],'Iris-setosa'),\n",
- "(19,ARRAY[5.7,3.8,1.7,0.3],'Iris-setosa'),\n",
- "(20,ARRAY[5.1,3.8,1.5,0.3],'Iris-setosa'),\n",
- "(21,ARRAY[5.4,3.4,1.7,0.2],'Iris-setosa'),\n",
- "(22,ARRAY[5.1,3.7,1.5,0.4],'Iris-setosa'),\n",
- "(23,ARRAY[4.6,3.6,1.0,0.2],'Iris-setosa'),\n",
- "(24,ARRAY[5.1,3.3,1.7,0.5],'Iris-setosa'),\n",
- "(25,ARRAY[4.8,3.4,1.9,0.2],'Iris-setosa'),\n",
- "(26,ARRAY[5.0,3.0,1.6,0.2],'Iris-setosa'),\n",
- "(27,ARRAY[5.0,3.4,1.6,0.4],'Iris-setosa'),\n",
- "(28,ARRAY[5.2,3.5,1.5,0.2],'Iris-setosa'),\n",
- "(29,ARRAY[5.2,3.4,1.4,0.2],'Iris-setosa'),\n",
- "(30,ARRAY[4.7,3.2,1.6,0.2],'Iris-setosa'),\n",
- "(31,ARRAY[4.8,3.1,1.6,0.2],'Iris-setosa'),\n",
- "(32,ARRAY[5.4,3.4,1.5,0.4],'Iris-setosa'),\n",
- "(33,ARRAY[5.2,4.1,1.5,0.1],'Iris-setosa'),\n",
- "(34,ARRAY[5.5,4.2,1.4,0.2],'Iris-setosa'),\n",
- "(35,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa'),\n",
- "(36,ARRAY[5.0,3.2,1.2,0.2],'Iris-setosa'),\n",
- "(37,ARRAY[5.5,3.5,1.3,0.2],'Iris-setosa'),\n",
- "(38,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa'),\n",
- "(39,ARRAY[4.4,3.0,1.3,0.2],'Iris-setosa'),\n",
- "(40,ARRAY[5.1,3.4,1.5,0.2],'Iris-setosa'),\n",
- "(41,ARRAY[5.0,3.5,1.3,0.3],'Iris-setosa'),\n",
- "(42,ARRAY[4.5,2.3,1.3,0.3],'Iris-setosa'),\n",
- "(43,ARRAY[4.4,3.2,1.3,0.2],'Iris-setosa'),\n",
- "(44,ARRAY[5.0,3.5,1.6,0.6],'Iris-setosa'),\n",
- "(45,ARRAY[5.1,3.8,1.9,0.4],'Iris-setosa'),\n",
- "(46,ARRAY[4.8,3.0,1.4,0.3],'Iris-setosa'),\n",
- "(47,ARRAY[5.1,3.8,1.6,0.2],'Iris-setosa'),\n",
- "(48,ARRAY[4.6,3.2,1.4,0.2],'Iris-setosa'),\n",
- "(49,ARRAY[5.3,3.7,1.5,0.2],'Iris-setosa'),\n",
- "(50,ARRAY[5.0,3.3,1.4,0.2],'Iris-setosa'),\n",
- "(51,ARRAY[7.0,3.2,4.7,1.4],'Iris-versicolor'),\n",
- "(52,ARRAY[6.4,3.2,4.5,1.5],'Iris-versicolor'),\n",
- "(53,ARRAY[6.9,3.1,4.9,1.5],'Iris-versicolor'),\n",
- "(54,ARRAY[5.5,2.3,4.0,1.3],'Iris-versicolor'),\n",
- "(55,ARRAY[6.5,2.8,4.6,1.5],'Iris-versicolor'),\n",
- "(56,ARRAY[5.7,2.8,4.5,1.3],'Iris-versicolor'),\n",
- "(57,ARRAY[6.3,3.3,4.7,1.6],'Iris-versicolor'),\n",
- "(58,ARRAY[4.9,2.4,3.3,1.0],'Iris-versicolor'),\n",
- "(59,ARRAY[6.6,2.9,4.6,1.3],'Iris-versicolor'),\n",
- "(60,ARRAY[5.2,2.7,3.9,1.4],'Iris-versicolor'),\n",
- "(61,ARRAY[5.0,2.0,3.5,1.0],'Iris-versicolor'),\n",
- "(62,ARRAY[5.9,3.0,4.2,1.5],'Iris-versicolor'),\n",
- "(63,ARRAY[6.0,2.2,4.0,1.0],'Iris-versicolor'),\n",
- "(64,ARRAY[6.1,2.9,4.7,1.4],'Iris-versicolor'),\n",
- "(65,ARRAY[5.6,2.9,3.6,1.3],'Iris-versicolor'),\n",
- "(66,ARRAY[6.7,3.1,4.4,1.4],'Iris-versicolor'),\n",
- "(67,ARRAY[5.6,3.0,4.5,1.5],'Iris-versicolor'),\n",
- "(68,ARRAY[5.8,2.7,4.1,1.0],'Iris-versicolor'),\n",
- "(69,ARRAY[6.2,2.2,4.5,1.5],'Iris-versicolor'),\n",
- "(70,ARRAY[5.6,2.5,3.9,1.1],'Iris-versicolor'),\n",
- "(71,ARRAY[5.9,3.2,4.8,1.8],'Iris-versicolor'),\n",
- "(72,ARRAY[6.1,2.8,4.0,1.3],'Iris-versicolor'),\n",
- "(73,ARRAY[6.3,2.5,4.9,1.5],'Iris-versicolor'),\n",
- "(74,ARRAY[6.1,2.8,4.7,1.2],'Iris-versicolor'),\n",
- "(75,ARRAY[6.4,2.9,4.3,1.3],'Iris-versicolor'),\n",
- "(76,ARRAY[6.6,3.0,4.4,1.4],'Iris-versicolor'),\n",
- "(77,ARRAY[6.8,2.8,4.8,1.4],'Iris-versicolor'),\n",
- "(78,ARRAY[6.7,3.0,5.0,1.7],'Iris-versicolor'),\n",
- "(79,ARRAY[6.0,2.9,4.5,1.5],'Iris-versicolor'),\n",
- "(80,ARRAY[5.7,2.6,3.5,1.0],'Iris-versicolor'),\n",
- "(81,ARRAY[5.5,2.4,3.8,1.1],'Iris-versicolor'),\n",
- "(82,ARRAY[5.5,2.4,3.7,1.0],'Iris-versicolor'),\n",
- "(83,ARRAY[5.8,2.7,3.9,1.2],'Iris-versicolor'),\n",
- "(84,ARRAY[6.0,2.7,5.1,1.6],'Iris-versicolor'),\n",
- "(85,ARRAY[5.4,3.0,4.5,1.5],'Iris-versicolor'),\n",
- "(86,ARRAY[6.0,3.4,4.5,1.6],'Iris-versicolor'),\n",
- "(87,ARRAY[6.7,3.1,4.7,1.5],'Iris-versicolor'),\n",
- "(88,ARRAY[6.3,2.3,4.4,1.3],'Iris-versicolor'),\n",
- "(89,ARRAY[5.6,3.0,4.1,1.3],'Iris-versicolor'),\n",
- "(90,ARRAY[5.5,2.5,4.0,1.3],'Iris-versicolor'),\n",
- "(91,ARRAY[5.5,2.6,4.4,1.2],'Iris-versicolor'),\n",
- "(92,ARRAY[6.1,3.0,4.6,1.4],'Iris-versicolor'),\n",
- "(93,ARRAY[5.8,2.6,4.0,1.2],'Iris-versicolor'),\n",
- "(94,ARRAY[5.0,2.3,3.3,1.0],'Iris-versicolor'),\n",
- "(95,ARRAY[5.6,2.7,4.2,1.3],'Iris-versicolor'),\n",
- "(96,ARRAY[5.7,3.0,4.2,1.2],'Iris-versicolor'),\n",
- "(97,ARRAY[5.7,2.9,4.2,1.3],'Iris-versicolor'),\n",
- "(98,ARRAY[6.2,2.9,4.3,1.3],'Iris-versicolor'),\n",
- "(99,ARRAY[5.1,2.5,3.0,1.1],'Iris-versicolor'),\n",
- "(100,ARRAY[5.7,2.8,4.1,1.3],'Iris-versicolor'),\n",
- "(101,ARRAY[6.3,3.3,6.0,2.5],'Iris-virginica'),\n",
- "(102,ARRAY[5.8,2.7,5.1,1.9],'Iris-virginica'),\n",
- "(103,ARRAY[7.1,3.0,5.9,2.1],'Iris-virginica'),\n",
- "(104,ARRAY[6.3,2.9,5.6,1.8],'Iris-virginica'),\n",
- "(105,ARRAY[6.5,3.0,5.8,2.2],'Iris-virginica'),\n",
- "(106,ARRAY[7.6,3.0,6.6,2.1],'Iris-virginica'),\n",
- "(107,ARRAY[4.9,2.5,4.5,1.7],'Iris-virginica'),\n",
- "(108,ARRAY[7.3,2.9,6.3,1.8],'Iris-virginica'),\n",
- "(109,ARRAY[6.7,2.5,5.8,1.8],'Iris-virginica'),\n",
- "(110,ARRAY[7.2,3.6,6.1,2.5],'Iris-virginica'),\n",
- "(111,ARRAY[6.5,3.2,5.1,2.0],'Iris-virginica'),\n",
- "(112,ARRAY[6.4,2.7,5.3,1.9],'Iris-virginica'),\n",
- "(113,ARRAY[6.8,3.0,5.5,2.1],'Iris-virginica'),\n",
- "(114,ARRAY[5.7,2.5,5.0,2.0],'Iris-virginica'),\n",
- "(115,ARRAY[5.8,2.8,5.1,2.4],'Iris-virginica'),\n",
- "(116,ARRAY[6.4,3.2,5.3,2.3],'Iris-virginica'),\n",
- "(117,ARRAY[6.5,3.0,5.5,1.8],'Iris-virginica'),\n",
- "(118,ARRAY[7.7,3.8,6.7,2.2],'Iris-virginica'),\n",
- "(119,ARRAY[7.7,2.6,6.9,2.3],'Iris-virginica'),\n",
- "(120,ARRAY[6.0,2.2,5.0,1.5],'Iris-virginica'),\n",
- "(121,ARRAY[6.9,3.2,5.7,2.3],'Iris-virginica'),\n",
- "(122,ARRAY[5.6,2.8,4.9,2.0],'Iris-virginica'),\n",
- "(123,ARRAY[7.7,2.8,6.7,2.0],'Iris-virginica'),\n",
- "(124,ARRAY[6.3,2.7,4.9,1.8],'Iris-virginica'),\n",
- "(125,ARRAY[6.7,3.3,5.7,2.1],'Iris-virginica'),\n",
- "(126,ARRAY[7.2,3.2,6.0,1.8],'Iris-virginica'),\n",
- "(127,ARRAY[6.2,2.8,4.8,1.8],'Iris-virginica'),\n",
- "(128,ARRAY[6.1,3.0,4.9,1.8],'Iris-virginica'),\n",
- "(129,ARRAY[6.4,2.8,5.6,2.1],'Iris-virginica'),\n",
- "(130,ARRAY[7.2,3.0,5.8,1.6],'Iris-virginica'),\n",
- "(131,ARRAY[7.4,2.8,6.1,1.9],'Iris-virginica'),\n",
- "(132,ARRAY[7.9,3.8,6.4,2.0],'Iris-virginica'),\n",
- "(133,ARRAY[6.4,2.8,5.6,2.2],'Iris-virginica'),\n",
- "(134,ARRAY[6.3,2.8,5.1,1.5],'Iris-virginica'),\n",
- "(135,ARRAY[6.1,2.6,5.6,1.4],'Iris-virginica'),\n",
- "(136,ARRAY[7.7,3.0,6.1,2.3],'Iris-virginica'),\n",
- "(137,ARRAY[6.3,3.4,5.6,2.4],'Iris-virginica'),\n",
- "(138,ARRAY[6.4,3.1,5.5,1.8],'Iris-virginica'),\n",
- "(139,ARRAY[6.0,3.0,4.8,1.8],'Iris-virginica'),\n",
- "(140,ARRAY[6.9,3.1,5.4,2.1],'Iris-virginica'),\n",
- "(141,ARRAY[6.7,3.1,5.6,2.4],'Iris-virginica'),\n",
- "(142,ARRAY[6.9,3.1,5.1,2.3],'Iris-virginica'),\n",
- "(143,ARRAY[5.8,2.7,5.1,1.9],'Iris-virginica'),\n",
- "(144,ARRAY[6.8,3.2,5.9,2.3],'Iris-virginica'),\n",
- "(145,ARRAY[6.7,3.3,5.7,2.5],'Iris-virginica'),\n",
- "(146,ARRAY[6.7,3.0,5.2,2.3],'Iris-virginica'),\n",
- "(147,ARRAY[6.3,2.5,5.0,1.9],'Iris-virginica'),\n",
- "(148,ARRAY[6.5,3.0,5.2,2.0],'Iris-virginica'),\n",
- "(149,ARRAY[6.2,3.4,5.4,2.3],'Iris-virginica'),\n",
- "(150,ARRAY[5.9,3.0,5.1,1.8],'Iris-virginica');\n",
- "\n",
- "SELECT * FROM iris_data ORDER BY id;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Create a test/validation dataset from the training data"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "1 rows affected.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>count</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>120</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(120L,)]"
- ]
- },
- "execution_count": 5,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS iris_train, iris_test;\n",
- "\n",
- "-- Set seed so results are reproducible\n",
- "SELECT setseed(0);\n",
- "\n",
- "SELECT madlib.train_test_split('iris_data', -- Source table\n",
- " 'iris', -- Output table root name\n",
- " 0.8, -- Train proportion\n",
- " NULL, -- Test proportion (0.2)\n",
- " NULL, -- Strata definition\n",
- " NULL, -- Output all columns\n",
- " NULL, -- Sample without replacement\n",
- " TRUE -- Separate output tables\n",
- " );\n",
- "\n",
- "SELECT COUNT(*) FROM iris_train;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"pp\"></a>\n",
- "# 2. Call preprocessor for deep learning\n",
- "Training dataset (uses training preprocessor):"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 6,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>independent_var</th>\n",
- " <th>dependent_var</th>\n",
- " <th>buffer_id</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[5.3, 3.7, 1.5, 0.2], [6.3, 3.4, 5.6, 2.4], [7.2, 3.2, 6.0, 1.8], [5.7, 4.4, 1.5, 0.4], [5.4, 3.7, 1.5, 0.2], [5.0, 3.0, 1.6, 0.2], [4.7, 3.2, 1.3, 0.2], [5.0, 2.3, 3.3, 1.0], [5.7, 2.8, 4.5, 1.3], [4.8, 3.0, 1.4, 0.1], [5.7, 3.8, 1.7, 0.3], [5.5, 4.2, 1.4, 0.2], [7.7, 3.8, 6.7, 2.2], [6.0, 2.2, 5.0, 1.5], [6.5, 3.0, 5.2, 2.0], [4.6, 3.6, 1.0, 0.2], [6.9, 3.1, 4.9, 1.5], [5.1, 3.8, 1.9, 0.4], [6.7, 2.5, 5.8, 1.8], [6.0, 2.7, 5.1, 1.6], [6.3, 3.3, 4.7, 1.6], [6.7, 3.1 [...]
- " <td>[[1, 0, 0], [0, 0, 1], [0, 0, 1], [1, 0, 0], [1, 0, 0], [1, 0, 0], [1, 0, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [1, 0, 0], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 0, 1], [1, 0, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 0, 1], [1, 0, 0], [0, 0, 1], [1, 0, 0], [1, 0, 0], [0, 0, 1] [...]
- " <td>0</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[([[5.3, 3.7, 1.5, 0.2], [6.3, 3.4, 5.6, 2.4], [7.2, 3.2, 6.0, 1.8], [5.7, 4.4, 1.5, 0.4], [5.4, 3.7, 1.5, 0.2], [5.0, 3.0, 1.6, 0.2], [4.7, 3.2, 1.3, 0.2], [5.0, 2.3, 3.3, 1.0], [5.7, 2.8, 4.5, 1.3], [4.8, 3.0, 1.4, 0.1], [5.7, 3.8, 1.7, 0.3], [5.5, 4.2, 1.4, 0.2], [7.7, 3.8, 6.7, 2.2], [6.0, 2.2, 5.0, 1.5], [6.5, 3.0, 5.2, 2.0], [4.6, 3.6, 1.0, 0.2], [6.9, 3.1, 4.9, 1.5], [5.1, 3.8, 1.9, 0.4], [6.7, 2.5, 5.8, 1.8], [6.0, 2.7, 5.1, 1.6], [6.3, 3.3, 4.7, 1.6], [6.7, 3.1, 4.7, 1.5 [...]
- ]
- },
- "execution_count": 6,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS iris_train_packed, iris_train_packed_summary;\n",
- "\n",
- "SELECT madlib.training_preprocessor_dl('iris_train', -- Source table\n",
- " 'iris_train_packed', -- Output table\n",
- " 'class_text', -- Dependent variable\n",
- " 'attributes' -- Independent variable\n",
- " ); \n",
- "\n",
- "SELECT * FROM iris_train_packed ORDER BY buffer_id;"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 7,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>source_table</th>\n",
- " <th>output_table</th>\n",
- " <th>dependent_varname</th>\n",
- " <th>independent_varname</th>\n",
- " <th>dependent_vartype</th>\n",
- " <th>class_values</th>\n",
- " <th>buffer_size</th>\n",
- " <th>normalizing_const</th>\n",
- " <th>num_classes</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>iris_train</td>\n",
- " <td>iris_train_packed</td>\n",
- " <td>class_text</td>\n",
- " <td>attributes</td>\n",
- " <td>character varying</td>\n",
- " <td>[u'Iris-setosa', u'Iris-versicolor', u'Iris-virginica']</td>\n",
- " <td>120</td>\n",
- " <td>1.0</td>\n",
- " <td>3</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'iris_train', u'iris_train_packed', u'class_text', u'attributes', u'character varying', [u'Iris-setosa', u'Iris-versicolor', u'Iris-virginica'], 120, 1.0, 3)]"
- ]
- },
- "execution_count": 7,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT * FROM iris_train_packed_summary;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Validation dataset (uses validation preprocessor):"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 8,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>independent_var</th>\n",
- " <th>dependent_var</th>\n",
- " <th>buffer_id</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[5.1, 3.4, 1.5, 0.2], [5.7, 2.8, 4.1, 1.3], [6.6, 3.0, 4.4, 1.4], [5.4, 3.9, 1.3, 0.4], [5.6, 2.7, 4.2, 1.3], [6.9, 3.1, 5.4, 2.1], [6.5, 3.0, 5.5, 1.8], [6.1, 3.0, 4.6, 1.4], [4.4, 3.2, 1.3, 0.2], [4.9, 2.5, 4.5, 1.7], [6.5, 2.8, 4.6, 1.5], [4.6, 3.1, 1.5, 0.2], [6.3, 2.3, 4.4, 1.3], [5.0, 3.3, 1.4, 0.2], [6.3, 2.9, 5.6, 1.8], [6.5, 3.2, 5.1, 2.0], [6.3, 2.8, 5.1, 1.5], [5.8, 2.7, 5.1, 1.9], [5.8, 2.7, 3.9, 1.2], [6.4, 3.2, 4.5, 1.5], [6.0, 3.4, 4.5, 1.6], [5.1, 3.3 [...]
- " <td>[[1, 0, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [1, 0, 0]]</td>\n",
- " <td>0</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[([[5.1, 3.4, 1.5, 0.2], [5.7, 2.8, 4.1, 1.3], [6.6, 3.0, 4.4, 1.4], [5.4, 3.9, 1.3, 0.4], [5.6, 2.7, 4.2, 1.3], [6.9, 3.1, 5.4, 2.1], [6.5, 3.0, 5.5, 1.8], [6.1, 3.0, 4.6, 1.4], [4.4, 3.2, 1.3, 0.2], [4.9, 2.5, 4.5, 1.7], [6.5, 2.8, 4.6, 1.5], [4.6, 3.1, 1.5, 0.2], [6.3, 2.3, 4.4, 1.3], [5.0, 3.3, 1.4, 0.2], [6.3, 2.9, 5.6, 1.8], [6.5, 3.2, 5.1, 2.0], [6.3, 2.8, 5.1, 1.5], [5.8, 2.7, 5.1, 1.9], [5.8, 2.7, 3.9, 1.2], [6.4, 3.2, 4.5, 1.5], [6.0, 3.4, 4.5, 1.6], [5.1, 3.3, 1.7, 0.5 [...]
- ]
- },
- "execution_count": 8,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS iris_test_packed, iris_test_packed_summary;\n",
- "\n",
- "SELECT madlib.validation_preprocessor_dl('iris_test', -- Source table\n",
- " 'iris_test_packed', -- Output table\n",
- " 'class_text', -- Dependent variable\n",
- " 'attributes', -- Independent variable\n",
- " 'iris_train_packed' -- From training preprocessor step\n",
- " ); \n",
- "\n",
- "SELECT * FROM iris_test_packed ORDER BY buffer_id;"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 9,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>source_table</th>\n",
- " <th>output_table</th>\n",
- " <th>dependent_varname</th>\n",
- " <th>independent_varname</th>\n",
- " <th>dependent_vartype</th>\n",
- " <th>class_values</th>\n",
- " <th>buffer_size</th>\n",
- " <th>normalizing_const</th>\n",
- " <th>num_classes</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>iris_test</td>\n",
- " <td>iris_test_packed</td>\n",
- " <td>class_text</td>\n",
- " <td>attributes</td>\n",
- " <td>character varying</td>\n",
- " <td>[u'Iris-setosa', u'Iris-versicolor', u'Iris-virginica']</td>\n",
- " <td>30</td>\n",
- " <td>1.0</td>\n",
- " <td>3</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'iris_test', u'iris_test_packed', u'class_text', u'attributes', u'character varying', [u'Iris-setosa', u'Iris-versicolor', u'Iris-virginica'], 30, 1.0, 3)]"
- ]
- },
- "execution_count": 9,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT * FROM iris_test_packed_summary;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"load\"></a>\n",
- "# 3. Define and load model architecture\n",
- "Import Keras libraries"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 10,
- "metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Using TensorFlow backend.\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Couldn't import dot_parser, loading of dot files will not be possible.\n"
- ]
- }
- ],
- "source": [
- "import keras\n",
- "from keras.models import Sequential\n",
- "from keras.layers import Dense"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Define model architecture"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 11,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "_________________________________________________________________\n",
- "Layer (type) Output Shape Param # \n",
- "=================================================================\n",
- "dense_1 (Dense) (None, 10) 50 \n",
- "_________________________________________________________________\n",
- "dense_2 (Dense) (None, 10) 110 \n",
- "_________________________________________________________________\n",
- "dense_3 (Dense) (None, 3) 33 \n",
- "=================================================================\n",
- "Total params: 193\n",
- "Trainable params: 193\n",
- "Non-trainable params: 0\n",
- "_________________________________________________________________\n"
- ]
- }
- ],
- "source": [
- "model_simple = Sequential()\n",
- "model_simple.add(Dense(10, activation='relu', input_shape=(4,)))\n",
- "model_simple.add(Dense(10, activation='relu'))\n",
- "model_simple.add(Dense(3, activation='softmax'))\n",
- " \n",
- "model_simple.summary()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 10,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "'{\"class_name\": \"Sequential\", \"keras_version\": \"2.1.6\", \"config\": [{\"class_name\": \"Dense\", \"config\": {\"kernel_initializer\": {\"class_name\": \"VarianceScaling\", \"config\": {\"distribution\": \"uniform\", \"scale\": 1.0, \"seed\": null, \"mode\": \"fan_avg\"}}, \"name\": \"dense_1\", \"kernel_constraint\": null, \"bias_regularizer\": null, \"bias_constraint\": null, \"dtype\": \"float32\", \"activation\": \"relu\", \"trainable\": true, \"kernel_regularizer\": n [...]
- ]
- },
- "execution_count": 10,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "model_simple.to_json()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Load into model architecture table"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 12,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>model_id</th>\n",
- " <th>model_arch</th>\n",
- " <th>model_weights</th>\n",
- " <th>name</th>\n",
- " <th>description</th>\n",
- " <th>__internal_madlib_id__</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1</td>\n",
- " <td>{u'class_name': u'Sequential', u'keras_version': u'2.1.6', u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer': {u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform', u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1', u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint': None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True, u'kernel_regularizer': None, u'bias_initia [...]
- " <td>None</td>\n",
- " <td>Sophie</td>\n",
- " <td>A simple model</td>\n",
- " <td>__madlib_temp_62550369_1562173248_86696923__</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(1, {u'class_name': u'Sequential', u'keras_version': u'2.1.6', u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer': {u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform', u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1', u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint': None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True, u'kernel_regularizer': None, u'bias_initializer': [...]
- ]
- },
- "execution_count": 12,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS model_arch_library;\n",
- "SELECT madlib.load_keras_model('model_arch_library', -- Output table,\n",
- " \n",
- "$$\n",
- "{\"class_name\": \"Sequential\", \"keras_version\": \"2.1.6\", \"config\": [{\"class_name\": \"Dense\", \"config\": {\"kernel_initializer\": {\"class_name\": \"VarianceScaling\", \"config\": {\"distribution\": \"uniform\", \"scale\": 1.0, \"seed\": null, \"mode\": \"fan_avg\"}}, \"name\": \"dense_1\", \"kernel_constraint\": null, \"bias_regularizer\": null, \"bias_constraint\": null, \"dtype\": \"float32\", \"activation\": \"relu\", \"trainable\": true, \"kernel_regularizer\": null, [...]
- "$$\n",
- "::json, -- JSON blob\n",
- " NULL, -- Weights\n",
- " 'Sophie', -- Name\n",
- " 'A simple model' -- Descr\n",
- ");\n",
- "\n",
- "SELECT * FROM model_arch_library;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"train\"></a>\n",
- "# 4. Train\n",
- "Train the model:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 13,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>madlib_keras_fit</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td></td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[('',)]"
- ]
- },
- "execution_count": 13,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS iris_model, iris_model_summary;\n",
- "\n",
- "SELECT madlib.madlib_keras_fit('iris_train_packed', -- source table\n",
- " 'iris_model', -- model output table\n",
- " 'model_arch_library', -- model arch table\n",
- " 1, -- model arch id\n",
- " $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$, -- compile_params\n",
- " $$ batch_size=5, epochs=3 $$, -- fit_params\n",
- " 10 -- num_iterations\n",
- " );"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "View the model summary:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 14,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>source_table</th>\n",
- " <th>model</th>\n",
- " <th>dependent_varname</th>\n",
- " <th>independent_varname</th>\n",
- " <th>model_arch_table</th>\n",
- " <th>model_arch_id</th>\n",
- " <th>compile_params</th>\n",
- " <th>fit_params</th>\n",
- " <th>num_iterations</th>\n",
- " <th>validation_table</th>\n",
- " <th>metrics_compute_frequency</th>\n",
- " <th>name</th>\n",
- " <th>description</th>\n",
- " <th>model_type</th>\n",
- " <th>model_size</th>\n",
- " <th>start_training_time</th>\n",
- " <th>end_training_time</th>\n",
- " <th>metrics_elapsed_time</th>\n",
- " <th>madlib_version</th>\n",
- " <th>num_classes</th>\n",
- " <th>class_values</th>\n",
- " <th>dependent_vartype</th>\n",
- " <th>normalizing_const</th>\n",
- " <th>metrics_type</th>\n",
- " <th>training_metrics_final</th>\n",
- " <th>training_loss_final</th>\n",
- " <th>training_metrics</th>\n",
- " <th>training_loss</th>\n",
- " <th>validation_metrics_final</th>\n",
- " <th>validation_loss_final</th>\n",
- " <th>validation_metrics</th>\n",
- " <th>validation_loss</th>\n",
- " <th>metrics_iters</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>iris_train_packed</td>\n",
- " <td>iris_model</td>\n",
- " <td>class_text</td>\n",
- " <td>attributes</td>\n",
- " <td>model_arch_library</td>\n",
- " <td>1</td>\n",
- " <td> loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] </td>\n",
- " <td> batch_size=5, epochs=3 </td>\n",
- " <td>10</td>\n",
- " <td>None</td>\n",
- " <td>10</td>\n",
- " <td>None</td>\n",
- " <td>None</td>\n",
- " <td>madlib_keras</td>\n",
- " <td>0.7900390625</td>\n",
- " <td>2019-07-03 10:00:52.477704</td>\n",
- " <td>2019-07-03 10:00:58.552077</td>\n",
- " <td>[6.07431507110596]</td>\n",
- " <td>1.16</td>\n",
- " <td>3</td>\n",
- " <td>[u'Iris-setosa', u'Iris-versicolor', u'Iris-virginica']</td>\n",
- " <td>character varying</td>\n",
- " <td>1.0</td>\n",
- " <td>[u'accuracy']</td>\n",
- " <td>0.883333325386</td>\n",
- " <td>0.347580313683</td>\n",
- " <td>[0.883333325386047]</td>\n",
- " <td>[0.347580313682556]</td>\n",
- " <td>None</td>\n",
- " <td>None</td>\n",
- " <td>None</td>\n",
- " <td>None</td>\n",
- " <td>[10]</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'iris_train_packed', u'iris_model', u'class_text', u'attributes', u'model_arch_library', 1, u\" loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] \", u' batch_size=5, epochs=3 ', 10, None, 10, None, None, u'madlib_keras', 0.7900390625, datetime.datetime(2019, 7, 3, 10, 0, 52, 477704), datetime.datetime(2019, 7, 3, 10, 0, 58, 552077), [6.07431507110596], u'1.16', 3, [u'Iris-setosa', u'Iris-versicolor', u'Iris-virginica'], u'character varying', 1.0, [u'accu [...]
- ]
- },
- "execution_count": 14,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT * FROM iris_model_summary;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"eval\"></a>\n",
- "# 5. Evaluate\n",
- "\n",
- "Now run evaluate using model we built above:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 15,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>loss</th>\n",
- " <th>metric</th>\n",
- " <th>metrics_type</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>0.43478807807</td>\n",
- " <td>0.899999976158</td>\n",
- " <td>[u'accuracy']</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(0.434788078069687, 0.899999976158142, [u'accuracy'])]"
- ]
- },
- "execution_count": 15,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS iris_validate;\n",
- "\n",
- "SELECT madlib.madlib_keras_evaluate('iris_model', -- model\n",
- " 'iris_test_packed', -- test table\n",
- " 'iris_validate' -- output table\n",
- " );\n",
- "\n",
- "SELECT * FROM iris_validate;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"pred\"></a>\n",
- "# 6. Predict\n",
- "\n",
- "Now predict using model we built. We will use the validation data set for prediction as well, which is not usual but serves to show the syntax. The prediction is in the estimated_class_text column:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 16,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "30 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>id</th>\n",
- " <th>estimated_class_text</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>4</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>17</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>24</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>30</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>39</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>40</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>43</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>50</td>\n",
- " <td>Iris-setosa</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>52</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>54</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>55</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>59</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>61</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>76</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>81</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>83</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>86</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>88</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>92</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>93</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>95</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>100</td>\n",
- " <td>Iris-versicolor</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>104</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>107</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>111</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>117</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>134</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>140</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>143</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>150</td>\n",
- " <td>Iris-virginica</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(4, u'Iris-setosa'),\n",
- " (17, u'Iris-setosa'),\n",
- " (24, u'Iris-setosa'),\n",
- " (30, u'Iris-setosa'),\n",
- " (39, u'Iris-setosa'),\n",
- " (40, u'Iris-setosa'),\n",
- " (43, u'Iris-setosa'),\n",
- " (50, u'Iris-setosa'),\n",
- " (52, u'Iris-versicolor'),\n",
- " (54, u'Iris-virginica'),\n",
- " (55, u'Iris-virginica'),\n",
- " (59, u'Iris-versicolor'),\n",
- " (61, u'Iris-versicolor'),\n",
- " (76, u'Iris-versicolor'),\n",
- " (81, u'Iris-versicolor'),\n",
- " (83, u'Iris-versicolor'),\n",
- " (86, u'Iris-versicolor'),\n",
- " (88, u'Iris-virginica'),\n",
- " (92, u'Iris-versicolor'),\n",
- " (93, u'Iris-versicolor'),\n",
- " (95, u'Iris-versicolor'),\n",
- " (100, u'Iris-versicolor'),\n",
- " (104, u'Iris-virginica'),\n",
- " (107, u'Iris-virginica'),\n",
- " (111, u'Iris-virginica'),\n",
- " (117, u'Iris-virginica'),\n",
- " (134, u'Iris-virginica'),\n",
- " (140, u'Iris-virginica'),\n",
- " (143, u'Iris-virginica'),\n",
- " (150, u'Iris-virginica')]"
- ]
- },
- "execution_count": 16,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS iris_predict;\n",
- "\n",
- "SELECT madlib.madlib_keras_predict('iris_model', -- model\n",
- " 'iris_test', -- test_table\n",
- " 'id', -- id column\n",
- " 'attributes', -- independent var\n",
- " 'iris_predict' -- output table\n",
- " );\n",
- "\n",
- "SELECT * FROM iris_predict ORDER BY id;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Count missclassifications"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 17,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>count</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>3</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(3L,)]"
- ]
- },
- "execution_count": 17,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT COUNT(*) FROM iris_predict JOIN iris_test USING (id) \n",
- "WHERE iris_predict.estimated_class_text != iris_test.class_text;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Percent missclassifications"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 18,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>test_accuracy_percent</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>90.00</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(Decimal('90.00'),)]"
- ]
- },
- "execution_count": 18,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT round(count(*)*100/(150*0.2),2) as test_accuracy_percent from\n",
- " (select iris_test.class_text as actual, iris_predict.estimated_class_text as estimated\n",
- " from iris_predict inner join iris_test\n",
- " on iris_test.id=iris_predict.id) q\n",
- "WHERE q.actual=q.estimated;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"class2\"></a>\n",
- "# Classification with Other Parameters\n",
- "\n",
- "<a id=\"val_dataset\"></a>\n",
- "# 1. Validation dataset\n",
- "Now use a validation dataset and compute metrics every 5th iteration using the 'metrics_compute_frequency' parameter. This can help reduce run time if you do not need metrics computed at every iteration."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 19,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>madlib_keras_fit</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td></td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[('',)]"
- ]
- },
- "execution_count": 19,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS iris_model, iris_model_summary;\n",
- "\n",
- "SELECT madlib.madlib_keras_fit('iris_train_packed', -- source table\n",
- " 'iris_model', -- model output table\n",
- " 'model_arch_library', -- model arch table\n",
- " 1, -- model arch id\n",
- " $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$, -- compile_params\n",
- " $$ batch_size=5, epochs=3 $$, -- fit_params\n",
- " 10, -- num_iterations\n",
- " 0, -- GPUs per host\n",
- " 'iris_test_packed', -- validation dataset\n",
- " 3, -- metrics compute frequency\n",
- " FALSE, -- warm start\n",
- " 'Sophie L.', -- name\n",
- " 'Simple MLP for iris dataset' -- description\n",
- " );"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "View the model summary:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 20,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>source_table</th>\n",
- " <th>model</th>\n",
- " <th>dependent_varname</th>\n",
- " <th>independent_varname</th>\n",
- " <th>model_arch_table</th>\n",
- " <th>model_arch_id</th>\n",
- " <th>compile_params</th>\n",
- " <th>fit_params</th>\n",
- " <th>num_iterations</th>\n",
- " <th>validation_table</th>\n",
- " <th>metrics_compute_frequency</th>\n",
- " <th>name</th>\n",
- " <th>description</th>\n",
- " <th>model_type</th>\n",
- " <th>model_size</th>\n",
- " <th>start_training_time</th>\n",
- " <th>end_training_time</th>\n",
- " <th>metrics_elapsed_time</th>\n",
- " <th>madlib_version</th>\n",
- " <th>num_classes</th>\n",
- " <th>class_values</th>\n",
- " <th>dependent_vartype</th>\n",
- " <th>normalizing_const</th>\n",
- " <th>metrics_type</th>\n",
- " <th>training_metrics_final</th>\n",
- " <th>training_loss_final</th>\n",
- " <th>training_metrics</th>\n",
- " <th>training_loss</th>\n",
- " <th>validation_metrics_final</th>\n",
- " <th>validation_loss_final</th>\n",
- " <th>validation_metrics</th>\n",
- " <th>validation_loss</th>\n",
- " <th>metrics_iters</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>iris_train_packed</td>\n",
- " <td>iris_model</td>\n",
- " <td>class_text</td>\n",
- " <td>attributes</td>\n",
- " <td>model_arch_library</td>\n",
- " <td>1</td>\n",
- " <td> loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] </td>\n",
- " <td> batch_size=5, epochs=3 </td>\n",
- " <td>10</td>\n",
- " <td>iris_test_packed</td>\n",
- " <td>3</td>\n",
- " <td>Sophie L.</td>\n",
- " <td>Simple MLP for iris dataset</td>\n",
- " <td>madlib_keras</td>\n",
- " <td>0.7900390625</td>\n",
- " <td>2019-07-03 10:01:36.883439</td>\n",
- " <td>2019-07-03 10:01:43.631505</td>\n",
- " <td>[2.01867508888245, 3.89846897125244, 5.95652198791504, 6.74802613258362]</td>\n",
- " <td>1.16</td>\n",
- " <td>3</td>\n",
- " <td>[u'Iris-setosa', u'Iris-versicolor', u'Iris-virginica']</td>\n",
- " <td>character varying</td>\n",
- " <td>1.0</td>\n",
- " <td>[u'accuracy']</td>\n",
- " <td>0.941666662693</td>\n",
- " <td>0.371823817492</td>\n",
- " <td>[0.741666674613953, 0.800000011920929, 0.916666686534882, 0.941666662693024]</td>\n",
- " <td>[0.829422652721405, 0.578483581542969, 0.405027717351913, 0.371823817491531]</td>\n",
- " <td>0.866666674614</td>\n",
- " <td>0.492831081152</td>\n",
- " <td>[0.566666662693024, 0.566666662693024, 0.833333313465118, 0.866666674613953]</td>\n",
- " <td>[0.945741951465607, 0.735191941261292, 0.535544157028198, 0.492831081151962]</td>\n",
- " <td>[3, 6, 9, 10]</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'iris_train_packed', u'iris_model', u'class_text', u'attributes', u'model_arch_library', 1, u\" loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] \", u' batch_size=5, epochs=3 ', 10, u'iris_test_packed', 3, u'Sophie L.', u'Simple MLP for iris dataset', u'madlib_keras', 0.7900390625, datetime.datetime(2019, 7, 3, 10, 1, 36, 883439), datetime.datetime(2019, 7, 3, 10, 1, 43, 631505), [2.01867508888245, 3.89846897125244, 5.95652198791504, 6.74802613258362], u [...]
- ]
- },
- "execution_count": 20,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT * FROM iris_model_summary;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"pred_prob\"></a>\n",
- "# 2. Predict probabilities\n",
- "Predict with probabilities for each class:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 21,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "30 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>id</th>\n",
- " <th>prob_Iris-setosa</th>\n",
- " <th>prob_Iris-versicolor</th>\n",
- " <th>prob_Iris-virginica</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>4</td>\n",
- " <td>0.89618546</td>\n",
- " <td>0.10063652</td>\n",
- " <td>0.0031779788</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>17</td>\n",
- " <td>0.9432447</td>\n",
- " <td>0.055964436</td>\n",
- " <td>0.00079082645</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>24</td>\n",
- " <td>0.87476385</td>\n",
- " <td>0.121528275</td>\n",
- " <td>0.0037078795</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>30</td>\n",
- " <td>0.8998875</td>\n",
- " <td>0.09730224</td>\n",
- " <td>0.0028102635</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>39</td>\n",
- " <td>0.89937997</td>\n",
- " <td>0.09733549</td>\n",
- " <td>0.003284483</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>40</td>\n",
- " <td>0.9124368</td>\n",
- " <td>0.08554972</td>\n",
- " <td>0.0020134845</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>43</td>\n",
- " <td>0.91849124</td>\n",
- " <td>0.079369105</td>\n",
- " <td>0.0021397525</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>50</td>\n",
- " <td>0.909191</td>\n",
- " <td>0.08853077</td>\n",
- " <td>0.002278217</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>52</td>\n",
- " <td>0.087838314</td>\n",
- " <td>0.5821711</td>\n",
- " <td>0.3299906</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>54</td>\n",
- " <td>0.040643755</td>\n",
- " <td>0.41653973</td>\n",
- " <td>0.5428166</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>55</td>\n",
- " <td>0.028924335</td>\n",
- " <td>0.42645925</td>\n",
- " <td>0.54461634</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>59</td>\n",
- " <td>0.043529026</td>\n",
- " <td>0.48816994</td>\n",
- " <td>0.468301</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>61</td>\n",
- " <td>0.06540182</td>\n",
- " <td>0.44180837</td>\n",
- " <td>0.4927898</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>76</td>\n",
- " <td>0.059951168</td>\n",
- " <td>0.5297594</td>\n",
- " <td>0.41028935</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>81</td>\n",
- " <td>0.0791236</td>\n",
- " <td>0.49948806</td>\n",
- " <td>0.42138833</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>83</td>\n",
- " <td>0.103119485</td>\n",
- " <td>0.54959977</td>\n",
- " <td>0.34728083</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>86</td>\n",
- " <td>0.15161447</td>\n",
- " <td>0.6227044</td>\n",
- " <td>0.22568108</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>88</td>\n",
- " <td>0.013850493</td>\n",
- " <td>0.3046302</td>\n",
- " <td>0.6815193</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>92</td>\n",
- " <td>0.070333414</td>\n",
- " <td>0.545334</td>\n",
- " <td>0.3843325</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>93</td>\n",
- " <td>0.07561784</td>\n",
- " <td>0.5140931</td>\n",
- " <td>0.41028905</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>95</td>\n",
- " <td>0.08128024</td>\n",
- " <td>0.52868783</td>\n",
- " <td>0.39003193</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>100</td>\n",
- " <td>0.10329097</td>\n",
- " <td>0.5575698</td>\n",
- " <td>0.33913925</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>104</td>\n",
- " <td>0.010344159</td>\n",
- " <td>0.31838065</td>\n",
- " <td>0.6712752</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>107</td>\n",
- " <td>0.039944757</td>\n",
- " <td>0.4255802</td>\n",
- " <td>0.5344751</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>111</td>\n",
- " <td>0.026599577</td>\n",
- " <td>0.44653893</td>\n",
- " <td>0.52686155</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>117</td>\n",
- " <td>0.012989115</td>\n",
- " <td>0.35122624</td>\n",
- " <td>0.6357847</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>134</td>\n",
- " <td>0.019927882</td>\n",
- " <td>0.38532582</td>\n",
- " <td>0.5947463</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>140</td>\n",
- " <td>0.009378748</td>\n",
- " <td>0.31938007</td>\n",
- " <td>0.6712412</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>143</td>\n",
- " <td>0.013671607</td>\n",
- " <td>0.3261078</td>\n",
- " <td>0.66022056</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>150</td>\n",
- " <td>0.031985275</td>\n",
- " <td>0.45162654</td>\n",
- " <td>0.51638824</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(4, 0.89618546, 0.10063652, 0.0031779788),\n",
- " (17, 0.9432447, 0.055964436, 0.00079082645),\n",
- " (24, 0.87476385, 0.121528275, 0.0037078795),\n",
- " (30, 0.8998875, 0.09730224, 0.0028102635),\n",
- " (39, 0.89937997, 0.09733549, 0.003284483),\n",
- " (40, 0.9124368, 0.08554972, 0.0020134845),\n",
- " (43, 0.91849124, 0.079369105, 0.0021397525),\n",
- " (50, 0.909191, 0.08853077, 0.002278217),\n",
- " (52, 0.087838314, 0.5821711, 0.3299906),\n",
- " (54, 0.040643755, 0.41653973, 0.5428166),\n",
- " (55, 0.028924335, 0.42645925, 0.54461634),\n",
- " (59, 0.043529026, 0.48816994, 0.468301),\n",
- " (61, 0.06540182, 0.44180837, 0.4927898),\n",
- " (76, 0.059951168, 0.5297594, 0.41028935),\n",
- " (81, 0.0791236, 0.49948806, 0.42138833),\n",
- " (83, 0.103119485, 0.54959977, 0.34728083),\n",
- " (86, 0.15161447, 0.6227044, 0.22568108),\n",
- " (88, 0.013850493, 0.3046302, 0.6815193),\n",
- " (92, 0.070333414, 0.545334, 0.3843325),\n",
- " (93, 0.07561784, 0.5140931, 0.41028905),\n",
- " (95, 0.08128024, 0.52868783, 0.39003193),\n",
- " (100, 0.10329097, 0.5575698, 0.33913925),\n",
- " (104, 0.010344159, 0.31838065, 0.6712752),\n",
- " (107, 0.039944757, 0.4255802, 0.5344751),\n",
- " (111, 0.026599577, 0.44653893, 0.52686155),\n",
- " (117, 0.012989115, 0.35122624, 0.6357847),\n",
- " (134, 0.019927882, 0.38532582, 0.5947463),\n",
- " (140, 0.009378748, 0.31938007, 0.6712412),\n",
- " (143, 0.013671607, 0.3261078, 0.66022056),\n",
- " (150, 0.031985275, 0.45162654, 0.51638824)]"
- ]
- },
- "execution_count": 21,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS iris_predict;\n",
- "\n",
- "SELECT madlib.madlib_keras_predict('iris_model', -- model\n",
- " 'iris_test', -- test_table\n",
- " 'id', -- id column\n",
- " 'attributes', -- independent var\n",
- " 'iris_predict', -- output table\n",
- " 'prob' -- response type\n",
- " );\n",
- "\n",
- "SELECT * FROM iris_predict ORDER BY id;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"warm_start\"></a>\n",
- "# 3. Warm start\n",
- "Next, use the warm_start parameter to continue learning, using the coefficients from the run above. Note that we don't drop the model table or model summary table:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 22,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>madlib_keras_fit</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td></td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[('',)]"
- ]
- },
- "execution_count": 22,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT madlib.madlib_keras_fit('iris_train_packed', -- source table\n",
- " 'iris_model', -- model output table\n",
- " 'model_arch_library', -- model arch table\n",
- " 1, -- model arch id\n",
- " $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$, -- compile_params\n",
- " $$ batch_size=5, epochs=3 $$, -- fit_params\n",
- " 5, -- num_iterations\n",
- " 0, -- GPUs per host\n",
- " 'iris_test_packed', -- validation dataset\n",
- " 1, -- metrics compute frequency\n",
- " TRUE, -- warm start\n",
- " 'Sophie L.', -- name \n",
- " 'Simple MLP for iris dataset' -- description\n",
- " );"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "In the summary table note that the loss and accuracy values pick up from where the previous run left off:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 23,
- "metadata": {
- "scrolled": true
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>source_table</th>\n",
- " <th>model</th>\n",
- " <th>dependent_varname</th>\n",
- " <th>independent_varname</th>\n",
- " <th>model_arch_table</th>\n",
- " <th>model_arch_id</th>\n",
- " <th>compile_params</th>\n",
- " <th>fit_params</th>\n",
- " <th>num_iterations</th>\n",
- " <th>validation_table</th>\n",
- " <th>metrics_compute_frequency</th>\n",
- " <th>name</th>\n",
- " <th>description</th>\n",
- " <th>model_type</th>\n",
- " <th>model_size</th>\n",
- " <th>start_training_time</th>\n",
- " <th>end_training_time</th>\n",
- " <th>metrics_elapsed_time</th>\n",
- " <th>madlib_version</th>\n",
- " <th>num_classes</th>\n",
- " <th>class_values</th>\n",
- " <th>dependent_vartype</th>\n",
- " <th>normalizing_const</th>\n",
- " <th>metrics_type</th>\n",
- " <th>training_metrics_final</th>\n",
- " <th>training_loss_final</th>\n",
- " <th>training_metrics</th>\n",
- " <th>training_loss</th>\n",
- " <th>validation_metrics_final</th>\n",
- " <th>validation_loss_final</th>\n",
- " <th>validation_metrics</th>\n",
- " <th>validation_loss</th>\n",
- " <th>metrics_iters</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>iris_train_packed</td>\n",
- " <td>iris_model</td>\n",
- " <td>class_text</td>\n",
- " <td>attributes</td>\n",
- " <td>model_arch_library</td>\n",
- " <td>1</td>\n",
- " <td> loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] </td>\n",
- " <td> batch_size=5, epochs=3 </td>\n",
- " <td>5</td>\n",
- " <td>iris_test_packed</td>\n",
- " <td>1</td>\n",
- " <td>Sophie L.</td>\n",
- " <td>Simple MLP for iris dataset</td>\n",
- " <td>madlib_keras</td>\n",
- " <td>0.7900390625</td>\n",
- " <td>2019-07-03 10:01:52.495132</td>\n",
- " <td>2019-07-03 10:01:56.620862</td>\n",
- " <td>[0.777317047119141, 1.62117600440979, 2.46934199333191, 3.33750104904175, 4.12567687034607]</td>\n",
- " <td>1.16</td>\n",
- " <td>3</td>\n",
- " <td>[u'Iris-setosa', u'Iris-versicolor', u'Iris-virginica']</td>\n",
- " <td>character varying</td>\n",
- " <td>1.0</td>\n",
- " <td>[u'accuracy']</td>\n",
- " <td>0.941666662693</td>\n",
- " <td>0.256453901529</td>\n",
- " <td>[0.941666662693024, 0.941666662693024, 0.941666662693024, 0.941666662693024, 0.941666662693024]</td>\n",
- " <td>[0.341408282518387, 0.313415616750717, 0.293370455503464, 0.273623049259186, 0.256453901529312]</td>\n",
- " <td>0.899999976158</td>\n",
- " <td>0.36899459362</td>\n",
- " <td>[0.833333313465118, 0.866666674613953, 0.899999976158142, 0.899999976158142, 0.899999976158142]</td>\n",
- " <td>[0.468038767576218, 0.429012358188629, 0.406685948371887, 0.386049389839172, 0.3689945936203]</td>\n",
- " <td>[1, 2, 3, 4, 5]</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'iris_train_packed', u'iris_model', u'class_text', u'attributes', u'model_arch_library', 1, u\" loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] \", u' batch_size=5, epochs=3 ', 5, u'iris_test_packed', 1, u'Sophie L.', u'Simple MLP for iris dataset', u'madlib_keras', 0.7900390625, datetime.datetime(2019, 7, 3, 10, 1, 52, 495132), datetime.datetime(2019, 7, 3, 10, 1, 56, 620862), [0.777317047119141, 1.62117600440979, 2.46934199333191, 3.33750104904175, 4. [...]
- ]
- },
- "execution_count": 23,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT * FROM iris_model_summary;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"transfer_learn\"></a>\n",
- "# Transfer learning\n",
- "\n",
- "<a id=\"load2\"></a>\n",
- "# 1. Define and load model architecture with some layers frozen\n",
- "Here we want to start with initial weights from a pre-trained model rather than training from scratch. We also want to use a model architecture with the earlier feature layer(s) frozen to save on training time. The example below is somewhat contrived but gives you the idea of the steps.\n",
- "\n",
- "First define a model architecture with the 1st hidden layer frozen:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 24,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "_________________________________________________________________\n",
- "Layer (type) Output Shape Param # \n",
- "=================================================================\n",
- "dense_4 (Dense) (None, 10) 50 \n",
- "_________________________________________________________________\n",
- "dense_5 (Dense) (None, 10) 110 \n",
- "_________________________________________________________________\n",
- "dense_6 (Dense) (None, 3) 33 \n",
- "=================================================================\n",
- "Total params: 193\n",
- "Trainable params: 143\n",
- "Non-trainable params: 50\n",
- "_________________________________________________________________\n"
- ]
- }
- ],
- "source": [
- "model_transfer = Sequential()\n",
- "model_transfer.add(Dense(10, activation='relu', input_shape=(4,), trainable=False))\n",
- "model_transfer.add(Dense(10, activation='relu'))\n",
- "model_transfer.add(Dense(3, activation='softmax'))\n",
- " \n",
- "model_transfer.summary()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 25,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "'{\"class_name\": \"Sequential\", \"keras_version\": \"2.1.6\", \"config\": [{\"class_name\": \"Dense\", \"config\": {\"kernel_initializer\": {\"class_name\": \"VarianceScaling\", \"config\": {\"distribution\": \"uniform\", \"scale\": 1.0, \"seed\": null, \"mode\": \"fan_avg\"}}, \"name\": \"dense_4\", \"kernel_constraint\": null, \"bias_regularizer\": null, \"bias_constraint\": null, \"dtype\": \"float32\", \"activation\": \"relu\", \"trainable\": false, \"kernel_regularizer\": [...]
- ]
- },
- "execution_count": 25,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "model_transfer.to_json()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Load transfer model into model architecture table"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 26,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n",
- "2 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>model_id</th>\n",
- " <th>model_arch</th>\n",
- " <th>model_weights</th>\n",
- " <th>name</th>\n",
- " <th>description</th>\n",
- " <th>__internal_madlib_id__</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1</td>\n",
- " <td>{u'class_name': u'Sequential', u'keras_version': u'2.1.6', u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer': {u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform', u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1', u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint': None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True, u'kernel_regularizer': None, u'bias_initia [...]
- " <td>None</td>\n",
- " <td>Sophie</td>\n",
- " <td>A simple model</td>\n",
- " <td>__madlib_temp_62550369_1562173248_86696923__</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>2</td>\n",
- " <td>{u'class_name': u'Sequential', u'keras_version': u'2.1.6', u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer': {u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform', u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_2', u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint': None, u'dtype': u'float32', u'activation': u'relu', u'trainable': False, u'kernel_regularizer': None, u'bias_initi [...]
- " <td>None</td>\n",
- " <td>Maria</td>\n",
- " <td>A transfer model</td>\n",
- " <td>__madlib_temp_71661491_1562173329_6849629__</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(1, {u'class_name': u'Sequential', u'keras_version': u'2.1.6', u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer': {u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform', u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1', u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint': None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True, u'kernel_regularizer': None, u'bias_initializer': [...]
- " (2, {u'class_name': u'Sequential', u'keras_version': u'2.1.6', u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer': {u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform', u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_2', u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint': None, u'dtype': u'float32', u'activation': u'relu', u'trainable': False, u'kernel_regularizer': None, u'bias_initializer' [...]
- ]
- },
- "execution_count": 26,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT madlib.load_keras_model('model_arch_library', -- Output table,\n",
- " \n",
- "$$\n",
- "{\"class_name\": \"Sequential\", \"keras_version\": \"2.1.6\", \"config\": [{\"class_name\": \"Dense\", \"config\": {\"kernel_initializer\": {\"class_name\": \"VarianceScaling\", \"config\": {\"distribution\": \"uniform\", \"scale\": 1.0, \"seed\": null, \"mode\": \"fan_avg\"}}, \"name\": \"dense_2\", \"kernel_constraint\": null, \"bias_regularizer\": null, \"bias_constraint\": null, \"dtype\": \"float32\", \"activation\": \"relu\", \"trainable\": false, \"kernel_regularizer\": null [...]
- "$$\n",
- "::json, -- JSON blob\n",
- " NULL, -- Weights\n",
- " 'Maria', -- Name\n",
- " 'A transfer model' -- Descr\n",
- ");\n",
- "\n",
- "SELECT * FROM model_arch_library;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"train2\"></a>\n",
- "# 2. Train transfer model\n",
- "\n",
- "Fetch the weights from a previous MADlib run. (Normally these would be downloaded from a source that trained the same model architecture on a related dataset.)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 27,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "[]"
- ]
- },
- "execution_count": 27,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "UPDATE model_arch_library SET model_weights = model_data FROM iris_model WHERE model_id = 2;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Now train the model using the transfer model and the pre-trained weights:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 28,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>madlib_keras_fit</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td></td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[('',)]"
- ]
- },
- "execution_count": 28,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS iris_model, iris_model_summary;\n",
- "\n",
- "SELECT madlib.madlib_keras_fit('iris_train_packed', -- source table\n",
- " 'iris_model', -- model output table\n",
- " 'model_arch_library', -- model arch table\n",
- " 2, -- model arch id\n",
- " $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$, -- compile_params\n",
- " $$ batch_size=5, epochs=3 $$, -- fit_params\n",
- " 10 -- num_iterations\n",
- " );"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 29,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>source_table</th>\n",
- " <th>model</th>\n",
- " <th>dependent_varname</th>\n",
- " <th>independent_varname</th>\n",
- " <th>model_arch_table</th>\n",
- " <th>model_arch_id</th>\n",
- " <th>compile_params</th>\n",
- " <th>fit_params</th>\n",
- " <th>num_iterations</th>\n",
- " <th>validation_table</th>\n",
- " <th>metrics_compute_frequency</th>\n",
- " <th>name</th>\n",
- " <th>description</th>\n",
- " <th>model_type</th>\n",
- " <th>model_size</th>\n",
- " <th>start_training_time</th>\n",
- " <th>end_training_time</th>\n",
- " <th>metrics_elapsed_time</th>\n",
- " <th>madlib_version</th>\n",
- " <th>num_classes</th>\n",
- " <th>class_values</th>\n",
- " <th>dependent_vartype</th>\n",
- " <th>normalizing_const</th>\n",
- " <th>metrics_type</th>\n",
- " <th>training_metrics_final</th>\n",
- " <th>training_loss_final</th>\n",
- " <th>training_metrics</th>\n",
- " <th>training_loss</th>\n",
- " <th>validation_metrics_final</th>\n",
- " <th>validation_loss_final</th>\n",
- " <th>validation_metrics</th>\n",
- " <th>validation_loss</th>\n",
- " <th>metrics_iters</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>iris_train_packed</td>\n",
- " <td>iris_model</td>\n",
- " <td>class_text</td>\n",
- " <td>attributes</td>\n",
- " <td>model_arch_library</td>\n",
- " <td>2</td>\n",
- " <td> loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] </td>\n",
- " <td> batch_size=5, epochs=3 </td>\n",
- " <td>10</td>\n",
- " <td>None</td>\n",
- " <td>10</td>\n",
- " <td>None</td>\n",
- " <td>None</td>\n",
- " <td>madlib_keras</td>\n",
- " <td>0.7900390625</td>\n",
- " <td>2019-07-03 10:02:15.366826</td>\n",
- " <td>2019-07-03 10:02:20.437775</td>\n",
- " <td>[5.07090902328491]</td>\n",
- " <td>1.16</td>\n",
- " <td>3</td>\n",
- " <td>[u'Iris-setosa', u'Iris-versicolor', u'Iris-virginica']</td>\n",
- " <td>character varying</td>\n",
- " <td>1.0</td>\n",
- " <td>[u'accuracy']</td>\n",
- " <td>0.933333337307</td>\n",
- " <td>0.188783079386</td>\n",
- " <td>[0.933333337306976]</td>\n",
- " <td>[0.188783079385757]</td>\n",
- " <td>None</td>\n",
- " <td>None</td>\n",
- " <td>None</td>\n",
- " <td>None</td>\n",
- " <td>[10]</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'iris_train_packed', u'iris_model', u'class_text', u'attributes', u'model_arch_library', 2, u\" loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] \", u' batch_size=5, epochs=3 ', 10, None, 10, None, None, u'madlib_keras', 0.7900390625, datetime.datetime(2019, 7, 3, 10, 2, 15, 366826), datetime.datetime(2019, 7, 3, 10, 2, 20, 437775), [5.07090902328491], u'1.16', 3, [u'Iris-setosa', u'Iris-versicolor', u'Iris-virginica'], u'character varying', 1.0, [u'accu [...]
- ]
- },
- "execution_count": 29,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT * FROM iris_model_summary;"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 2",
- "language": "python",
- "name": "python2"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 2
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython2",
- "version": "2.7.10"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 1
-}
diff --git a/community-artifacts/Deep-learning/MADlib-Keras-cifar10-cnn-v2.ipynb b/community-artifacts/Deep-learning/MADlib-Keras-cifar10-cnn-v2.ipynb
deleted file mode 100644
index faa322e..0000000
--- a/community-artifacts/Deep-learning/MADlib-Keras-cifar10-cnn-v2.ipynb
+++ /dev/null
@@ -1,1233 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# CNN Using Keras and MADlib\n",
- "\n",
- "E2E classification example using MADlib calling a Keras CNN. Based on model architecture in https://keras.io/examples/cifar10_cnn/\n",
- "\n",
- "To load images into tables we use the script called <em>madlib_image_loader.py</em> located at https://github.com/apache/madlib-site/tree/asf-site/community-artifacts/Deep-learning which uses the Python Imaging Library so supports multiple formats\n",
- "http://www.pythonware.com/products/pil/\n",
- "\n",
- "\n",
- "## Table of contents\n",
- "<a href=\"#import_libraries\">1. Import libraries</a>\n",
- "\n",
- "<a href=\"#load_and_prepare_data\">2. Load dataset into table</a>\n",
- "\n",
- "<a href=\"#image_preproc\">3. Call image preprocessor</a>\n",
- "\n",
- "<a href=\"#define_and_load_model\">4. Define and load model architecture</a>\n",
- "\n",
- "<a href=\"#train\">5. Train</a>\n",
- "\n",
- "<a href=\"#plot\">6. Plots by iteration and time</a>"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {
- "scrolled": true
- },
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/config.py:13: ShimWarning: The `IPython.config` package has been deprecated since IPython 4.0. You should import from traitlets.config instead.\n",
- " \"You should import from traitlets.config instead.\", ShimWarning)\n",
- "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/utils/traitlets.py:5: UserWarning: IPython.utils.traitlets has moved to a top-level traitlets package.\n",
- " warn(\"IPython.utils.traitlets has moved to a top-level traitlets package.\")\n"
- ]
- }
- ],
- "source": [
- "%load_ext sql"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "u'Connected: fmcquillan@madlib'"
- ]
- },
- "execution_count": 2,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "# Greenplum Database 5.x on GCP for deep learning (PM demo machine)\n",
- "#%sql postgresql://gpadmin@35.239.240.26:5432/madlib\n",
- " \n",
- "# PostgreSQL local\n",
- "%sql postgresql://fmcquillan@localhost:5432/madlib"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>version</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>MADlib version: 1.16, git revision: rc/1.16-rc1, cmake configuration time: Mon Jul 1 17:45:09 UTC 2019, build type: Release, build system: Darwin-16.7.0, C compiler: Clang, C++ compiler: Clang</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'MADlib version: 1.16, git revision: rc/1.16-rc1, cmake configuration time: Mon Jul 1 17:45:09 UTC 2019, build type: Release, build system: Darwin-16.7.0, C compiler: Clang, C++ compiler: Clang',)]"
- ]
- },
- "execution_count": 3,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%sql select madlib.version();\n",
- "#%sql select version();"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"import_libraries\"></a>\n",
- "# 1. Import libraries\n",
- "From https://keras.io/examples/mnist_transfer_cnn/ import libraries and define some params"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Using TensorFlow backend.\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Couldn't import dot_parser, loading of dot files will not be possible.\n"
- ]
- }
- ],
- "source": [
- "from __future__ import print_function\n",
- "import keras\n",
- "from keras.datasets import cifar10\n",
- "from keras.preprocessing.image import ImageDataGenerator\n",
- "from keras.models import Sequential\n",
- "from keras.layers import Dense, Dropout, Activation, Flatten\n",
- "from keras.layers import Conv2D, MaxPooling2D\n",
- "import os\n",
- "\n",
- "batch_size = 32\n",
- "num_classes = 10\n",
- "epochs = 100"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Others needed in this workbook"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {},
- "outputs": [],
- "source": [
- "import pandas as pd\n",
- "import numpy as np\n",
- "import sys\n",
- "import os\n",
- "from matplotlib import pyplot as plt"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"load_and_prepare_data\"></a>\n",
- "# 2. Set up image loader and load dataset into table\n",
- "\n",
- "First set up image loader using the script called <em>madlib_image_loader.py</em> located at https://github.com/apache/madlib-site/tree/asf-site/community-artifacts/Deep-learning"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 6,
- "metadata": {},
- "outputs": [],
- "source": [
- "madlib_site_dir = '/Users/fmcquillan/Documents/Product/MADlib/Demos/data'\n",
- "sys.path.append(madlib_site_dir)\n",
- "\n",
- "# Import image loader module\n",
- "from madlib_image_loader import ImageLoader, DbCredentials\n",
- "\n",
- "# Specify database credentials, for connecting to db\n",
- "db_creds = DbCredentials(user='fmcquillan',\n",
- " host='localhost',\n",
- " port='5432',\n",
- " password='')\n",
- "\n",
- "# Specify database credentials, for connecting to db\n",
- "#db_creds = DbCredentials(user='gpadmin', \n",
- "# db_name='madlib',\n",
- "# host='35.239.240.26',\n",
- "# port='5432',\n",
- "# password='')\n",
- "\n",
- "# Initialize ImageLoader (increase num_workers to run faster)\n",
- "iloader = ImageLoader(num_workers=5, db_creds=db_creds)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "First load CIFAR-10 data from Keras consisting of 50,000 32x32 color training images, labeled over 10 categories, and 10,000 test images."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 7,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "MainProcess: Connected to madlib db.\n",
- "Executing: CREATE TABLE cifar_10_train_data (id SERIAL, x REAL[], y TEXT)\n",
- "CREATE TABLE\n",
- "Created table cifar_10_train_data in madlib db\n",
- "Spawning 5 workers...\n",
- "Initializing PoolWorker-1 [pid 28054]\n",
- "PoolWorker-1: Created temporary directory /tmp/madlib_tdv3zEFPL1\n",
- "Initializing PoolWorker-2 [pid 28055]\n",
- "PoolWorker-2: Created temporary directory /tmp/madlib_bWb3jWWKsY\n",
- "Initializing PoolWorker-3 [pid 28056]\n",
- "PoolWorker-1: Connected to madlib db.\n",
- "PoolWorker-3: Created temporary directory /tmp/madlib_KetBMAbjq5\n",
- "Initializing PoolWorker-4 [pid 28057]\n",
- "PoolWorker-2: Connected to madlib db.\n",
- "PoolWorker-4: Created temporary directory /tmp/madlib_sME12BQHb1\n",
- "Initializing PoolWorker-5 [pid 28059]\n",
- "PoolWorker-3: Connected to madlib db.\n",
- "PoolWorker-5: Created temporary directory /tmp/madlib_i6LP0aJJDY\n",
- "PoolWorker-4: Connected to madlib db.\n",
- "PoolWorker-5: Connected to madlib db.\n",
- "PoolWorker-1: Wrote 1000 images to /tmp/madlib_tdv3zEFPL1/cifar_10_train_data0000.tmp\n",
- "PoolWorker-2: Wrote 1000 images to /tmp/madlib_bWb3jWWKsY/cifar_10_train_data0000.tmp\n",
- "PoolWorker-3: Wrote 1000 images to /tmp/madlib_KetBMAbjq5/cifar_10_train_data0000.tmp\n",
- "PoolWorker-4: Wrote 1000 images to /tmp/madlib_sME12BQHb1/cifar_10_train_data0000.tmp\n",
- "PoolWorker-5: Wrote 1000 images to /tmp/madlib_i6LP0aJJDY/cifar_10_train_data0000.tmp\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to /tmp/madlib_tdv3zEFPL1/cifar_10_train_data0001.tmp\n",
- "PoolWorker-2: Wrote 1000 images to /tmp/madlib_bWb3jWWKsY/cifar_10_train_data0001.tmp\n",
- "PoolWorker-3: Wrote 1000 images to /tmp/madlib_KetBMAbjq5/cifar_10_train_data0001.tmp\n",
- "PoolWorker-4: Wrote 1000 images to /tmp/madlib_sME12BQHb1/cifar_10_train_data0001.tmp\n",
- "PoolWorker-5: Wrote 1000 images to /tmp/madlib_i6LP0aJJDY/cifar_10_train_data0001.tmp\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to /tmp/madlib_tdv3zEFPL1/cifar_10_train_data0002.tmp\n",
- "PoolWorker-2: Wrote 1000 images to /tmp/madlib_bWb3jWWKsY/cifar_10_train_data0002.tmp\n",
- "PoolWorker-4: Wrote 1000 images to /tmp/madlib_sME12BQHb1/cifar_10_train_data0002.tmp\n",
- "PoolWorker-5: Wrote 1000 images to /tmp/madlib_i6LP0aJJDY/cifar_10_train_data0002.tmp\n",
- "PoolWorker-3: Wrote 1000 images to /tmp/madlib_KetBMAbjq5/cifar_10_train_data0002.tmp\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to /tmp/madlib_tdv3zEFPL1/cifar_10_train_data0003.tmp\n",
- "PoolWorker-2: Wrote 1000 images to /tmp/madlib_bWb3jWWKsY/cifar_10_train_data0003.tmp\n",
- "PoolWorker-4: Wrote 1000 images to /tmp/madlib_sME12BQHb1/cifar_10_train_data0003.tmp\n",
- "PoolWorker-5: Wrote 1000 images to /tmp/madlib_i6LP0aJJDY/cifar_10_train_data0003.tmp\n",
- "PoolWorker-3: Wrote 1000 images to /tmp/madlib_KetBMAbjq5/cifar_10_train_data0003.tmp\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to /tmp/madlib_tdv3zEFPL1/cifar_10_train_data0004.tmp\n",
- "PoolWorker-2: Wrote 1000 images to /tmp/madlib_bWb3jWWKsY/cifar_10_train_data0004.tmp\n",
- "PoolWorker-4: Wrote 1000 images to /tmp/madlib_sME12BQHb1/cifar_10_train_data0004.tmp\n",
- "PoolWorker-5: Wrote 1000 images to /tmp/madlib_i6LP0aJJDY/cifar_10_train_data0004.tmp\n",
- "PoolWorker-3: Wrote 1000 images to /tmp/madlib_KetBMAbjq5/cifar_10_train_data0004.tmp\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to /tmp/madlib_tdv3zEFPL1/cifar_10_train_data0005.tmp\n",
- "PoolWorker-2: Wrote 1000 images to /tmp/madlib_bWb3jWWKsY/cifar_10_train_data0005.tmp\n",
- "PoolWorker-4: Wrote 1000 images to /tmp/madlib_sME12BQHb1/cifar_10_train_data0005.tmp\n",
- "PoolWorker-5: Wrote 1000 images to /tmp/madlib_i6LP0aJJDY/cifar_10_train_data0005.tmp\n",
- "PoolWorker-3: Wrote 1000 images to /tmp/madlib_KetBMAbjq5/cifar_10_train_data0005.tmp\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to /tmp/madlib_tdv3zEFPL1/cifar_10_train_data0006.tmp\n",
- "PoolWorker-2: Wrote 1000 images to /tmp/madlib_bWb3jWWKsY/cifar_10_train_data0006.tmp\n",
- "PoolWorker-4: Wrote 1000 images to /tmp/madlib_sME12BQHb1/cifar_10_train_data0006.tmp\n",
- "PoolWorker-5: Wrote 1000 images to /tmp/madlib_i6LP0aJJDY/cifar_10_train_data0006.tmp\n",
- "PoolWorker-3: Wrote 1000 images to /tmp/madlib_KetBMAbjq5/cifar_10_train_data0006.tmp\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to /tmp/madlib_tdv3zEFPL1/cifar_10_train_data0007.tmp\n",
- "PoolWorker-5: Wrote 1000 images to /tmp/madlib_i6LP0aJJDY/cifar_10_train_data0007.tmp\n",
- "PoolWorker-2: Wrote 1000 images to /tmp/madlib_bWb3jWWKsY/cifar_10_train_data0007.tmp\n",
- "PoolWorker-4: Wrote 1000 images to /tmp/madlib_sME12BQHb1/cifar_10_train_data0007.tmp\n",
- "PoolWorker-3: Wrote 1000 images to /tmp/madlib_KetBMAbjq5/cifar_10_train_data0007.tmp\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to /tmp/madlib_tdv3zEFPL1/cifar_10_train_data0008.tmp\n",
- "PoolWorker-4: Wrote 1000 images to /tmp/madlib_sME12BQHb1/cifar_10_train_data0008.tmp\n",
- "PoolWorker-5: Wrote 1000 images to /tmp/madlib_i6LP0aJJDY/cifar_10_train_data0008.tmp\n",
- "PoolWorker-2: Wrote 1000 images to /tmp/madlib_bWb3jWWKsY/cifar_10_train_data0008.tmp\n",
- "PoolWorker-3: Wrote 1000 images to /tmp/madlib_KetBMAbjq5/cifar_10_train_data0008.tmp\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to /tmp/madlib_tdv3zEFPL1/cifar_10_train_data0009.tmp\n",
- "PoolWorker-4: Wrote 1000 images to /tmp/madlib_sME12BQHb1/cifar_10_train_data0009.tmp\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to /tmp/madlib_tdv3zEFPL1/cifar_10_train_data0010.tmp\n",
- "PoolWorker-4: Wrote 1000 images to /tmp/madlib_sME12BQHb1/cifar_10_train_data0010.tmp\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to /tmp/madlib_tdv3zEFPL1/cifar_10_train_data0011.tmp\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-3: Removed temporary directory /tmp/madlib_KetBMAbjq5\n",
- "PoolWorker-2: Removed temporary directory /tmp/madlib_bWb3jWWKsY\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "PoolWorker-4: Removed temporary directory /tmp/madlib_sME12BQHb1\n",
- "PoolWorker-1: Removed temporary directory /tmp/madlib_tdv3zEFPL1\n",
- "PoolWorker-5: Removed temporary directory /tmp/madlib_i6LP0aJJDY\n",
- "Done! Loaded 50000 images in 24.227011919s\n",
- "5 workers terminated.\n",
- "MainProcess: Connected to madlib db.\n",
- "Executing: CREATE TABLE cifar_10_test_data (id SERIAL, x REAL[], y TEXT)\n",
- "CREATE TABLE\n",
- "Created table cifar_10_test_data in madlib db\n",
- "Spawning 5 workers...\n",
- "Initializing PoolWorker-6 [pid 28066]\n",
- "PoolWorker-6: Created temporary directory /tmp/madlib_yKNKBHEc3G\n",
- "Initializing PoolWorker-7 [pid 28067]\n",
- "PoolWorker-7: Created temporary directory /tmp/madlib_hb8ESuQLva\n",
- "Initializing PoolWorker-8 [pid 28068]\n",
- "PoolWorker-8: Created temporary directory /tmp/madlib_PmtDmYhSBj\n",
- "PoolWorker-6: Connected to madlib db.\n",
- "Initializing PoolWorker-9 [pid 28069]\n",
- "PoolWorker-7: Connected to madlib db.\n",
- "PoolWorker-9: Created temporary directory /tmp/madlib_h7oUVpBwyZ\n",
- "Initializing PoolWorker-10 [pid 28071]\n",
- "PoolWorker-8: Connected to madlib db.\n",
- "PoolWorker-10: Created temporary directory /tmp/madlib_9TZoE98hbn\n",
- "PoolWorker-9: Connected to madlib db.\n",
- "PoolWorker-10: Connected to madlib db.\n",
- "PoolWorker-8: Wrote 1000 images to /tmp/madlib_PmtDmYhSBj/cifar_10_test_data0000.tmp\n",
- "PoolWorker-6: Wrote 1000 images to /tmp/madlib_yKNKBHEc3G/cifar_10_test_data0000.tmp\n",
- "PoolWorker-7: Wrote 1000 images to /tmp/madlib_hb8ESuQLva/cifar_10_test_data0000.tmp\n",
- "PoolWorker-9: Wrote 1000 images to /tmp/madlib_h7oUVpBwyZ/cifar_10_test_data0000.tmp\n",
- "PoolWorker-10: Wrote 1000 images to /tmp/madlib_9TZoE98hbn/cifar_10_test_data0000.tmp\n",
- "PoolWorker-8: Loaded 1000 images into cifar_10_test_data\n",
- "PoolWorker-7: Loaded 1000 images into cifar_10_test_data\n",
- "PoolWorker-6: Loaded 1000 images into cifar_10_test_data\n",
- "PoolWorker-10: Loaded 1000 images into cifar_10_test_data\n",
- "PoolWorker-9: Loaded 1000 images into cifar_10_test_data\n",
- "PoolWorker-8: Wrote 1000 images to /tmp/madlib_PmtDmYhSBj/cifar_10_test_data0001.tmp\n",
- "PoolWorker-7: Wrote 1000 images to /tmp/madlib_hb8ESuQLva/cifar_10_test_data0001.tmp\n",
- "PoolWorker-6: Wrote 1000 images to /tmp/madlib_yKNKBHEc3G/cifar_10_test_data0001.tmp\n",
- "PoolWorker-10: Wrote 1000 images to /tmp/madlib_9TZoE98hbn/cifar_10_test_data0001.tmp\n",
- "PoolWorker-9: Wrote 1000 images to /tmp/madlib_h7oUVpBwyZ/cifar_10_test_data0001.tmp\n",
- "PoolWorker-8: Loaded 1000 images into cifar_10_test_data\n",
- "PoolWorker-7: Loaded 1000 images into cifar_10_test_data\n",
- "PoolWorker-6: Loaded 1000 images into cifar_10_test_data\n",
- "PoolWorker-10: Loaded 1000 images into cifar_10_test_data\n",
- "PoolWorker-9: Loaded 1000 images into cifar_10_test_data\n",
- "PoolWorker-8: Removed temporary directory /tmp/madlib_PmtDmYhSBj\n",
- "PoolWorker-7: Removed temporary directory /tmp/madlib_hb8ESuQLva\n",
- "PoolWorker-6: Removed temporary directory /tmp/madlib_yKNKBHEc3G\n",
- "PoolWorker-9: Removed temporary directory /tmp/madlib_h7oUVpBwyZ\n",
- "PoolWorker-10: Removed temporary directory /tmp/madlib_9TZoE98hbn\n",
- "Done! Loaded 10000 images in 4.54620194435s\n",
- "5 workers terminated.\n"
- ]
- }
- ],
- "source": [
- "# Load dataset into np array\n",
- "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
- "\n",
- "%sql DROP TABLE IF EXISTS cifar_10_train_data, cifar_10_test_data;\n",
- "\n",
- "# Save images to temporary directories and load into database\n",
- "iloader.load_dataset_from_np(x_train, y_train, 'cifar_10_train_data', append=False)\n",
- "iloader.load_dataset_from_np(x_test, y_test, 'cifar_10_test_data', append=False)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 10,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>count</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>50000</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(50000L,)]"
- ]
- },
- "execution_count": 10,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%sql select count(*) from cifar_10_train_data;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"image_preproc\"></a>\n",
- "# 3. Call image preprocessor\n",
- "\n",
- "Transforms from one image per row to multiple images per row for batch optimization. Also normalizes and one-hot encodes.\n",
- "\n",
- "Training data"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 9,
- "metadata": {
- "scrolled": true
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>source_table</th>\n",
- " <th>output_table</th>\n",
- " <th>dependent_varname</th>\n",
- " <th>independent_varname</th>\n",
- " <th>dependent_vartype</th>\n",
- " <th>class_values</th>\n",
- " <th>buffer_size</th>\n",
- " <th>normalizing_const</th>\n",
- " <th>num_classes</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>cifar_10_train_data</td>\n",
- " <td>cifar_10_train_data_packed</td>\n",
- " <td>y</td>\n",
- " <td>x</td>\n",
- " <td>text</td>\n",
- " <td>[u'0', u'1', u'2', u'3', u'4', u'5', u'6', u'7', u'8', u'9']</td>\n",
- " <td>1000</td>\n",
- " <td>255.0</td>\n",
- " <td>10</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'cifar_10_train_data', u'cifar_10_train_data_packed', u'y', u'x', u'text', [u'0', u'1', u'2', u'3', u'4', u'5', u'6', u'7', u'8', u'9'], 1000, 255.0, 10)]"
- ]
- },
- "execution_count": 9,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS cifar_10_train_data_packed, cifar_10_train_data_packed_summary;\n",
- "\n",
- "SELECT madlib.training_preprocessor_dl('cifar_10_train_data', -- Source table\n",
- " 'cifar_10_train_data_packed', -- Output table\n",
- " 'y', -- Dependent variable\n",
- " 'x', -- Independent variable\n",
- " 1000, -- Buffer size\n",
- " 255 -- Normalizing constant\n",
- " );\n",
- "\n",
- "SELECT * FROM cifar_10_train_data_packed_summary;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Test data"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 11,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>source_table</th>\n",
- " <th>output_table</th>\n",
- " <th>dependent_varname</th>\n",
- " <th>independent_varname</th>\n",
- " <th>dependent_vartype</th>\n",
- " <th>class_values</th>\n",
- " <th>buffer_size</th>\n",
- " <th>normalizing_const</th>\n",
- " <th>num_classes</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>cifar_10_test_data</td>\n",
- " <td>cifar_10_test_data_packed</td>\n",
- " <td>y</td>\n",
- " <td>x</td>\n",
- " <td>text</td>\n",
- " <td>[u'0', u'1', u'2', u'3', u'4', u'5', u'6', u'7', u'8', u'9']</td>\n",
- " <td>1000</td>\n",
- " <td>255.0</td>\n",
- " <td>10</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'cifar_10_test_data', u'cifar_10_test_data_packed', u'y', u'x', u'text', [u'0', u'1', u'2', u'3', u'4', u'5', u'6', u'7', u'8', u'9'], 1000, 255.0, 10)]"
- ]
- },
- "execution_count": 11,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS cifar_10_test_data_packed, cifar_10_test_data_packed_summary;\n",
- "\n",
- "SELECT madlib.validation_preprocessor_dl('cifar_10_test_data', -- Source table\n",
- " 'cifar_10_test_data_packed', -- Output table\n",
- " 'y', -- Dependent variable\n",
- " 'x', -- Independent variable\n",
- " 'cifar_10_train_data_packed', -- Training preproc table\n",
- " 1000 -- Buffer size\n",
- " );\n",
- "\n",
- "SELECT * FROM cifar_10_test_data_packed_summary;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"define_and_load_model\"></a>\n",
- "# 4. Define and load model architecture\n",
- "\n",
- "Model architecture from https://keras.io/examples/cifar10_cnn/"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 12,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "_________________________________________________________________\n",
- "Layer (type) Output Shape Param # \n",
- "=================================================================\n",
- "conv2d_1 (Conv2D) (None, 32, 32, 32) 896 \n",
- "_________________________________________________________________\n",
- "activation_1 (Activation) (None, 32, 32, 32) 0 \n",
- "_________________________________________________________________\n",
- "conv2d_2 (Conv2D) (None, 30, 30, 32) 9248 \n",
- "_________________________________________________________________\n",
- "activation_2 (Activation) (None, 30, 30, 32) 0 \n",
- "_________________________________________________________________\n",
- "max_pooling2d_1 (MaxPooling2 (None, 15, 15, 32) 0 \n",
- "_________________________________________________________________\n",
- "dropout_1 (Dropout) (None, 15, 15, 32) 0 \n",
- "_________________________________________________________________\n",
- "conv2d_3 (Conv2D) (None, 15, 15, 64) 18496 \n",
- "_________________________________________________________________\n",
- "activation_3 (Activation) (None, 15, 15, 64) 0 \n",
- "_________________________________________________________________\n",
- "conv2d_4 (Conv2D) (None, 13, 13, 64) 36928 \n",
- "_________________________________________________________________\n",
- "activation_4 (Activation) (None, 13, 13, 64) 0 \n",
- "_________________________________________________________________\n",
- "max_pooling2d_2 (MaxPooling2 (None, 6, 6, 64) 0 \n",
- "_________________________________________________________________\n",
- "dropout_2 (Dropout) (None, 6, 6, 64) 0 \n",
- "_________________________________________________________________\n",
- "flatten_1 (Flatten) (None, 2304) 0 \n",
- "_________________________________________________________________\n",
- "dense_1 (Dense) (None, 512) 1180160 \n",
- "_________________________________________________________________\n",
- "activation_5 (Activation) (None, 512) 0 \n",
- "_________________________________________________________________\n",
- "dropout_3 (Dropout) (None, 512) 0 \n",
- "_________________________________________________________________\n",
- "dense_2 (Dense) (None, 10) 5130 \n",
- "_________________________________________________________________\n",
- "activation_6 (Activation) (None, 10) 0 \n",
- "=================================================================\n",
- "Total params: 1,250,858\n",
- "Trainable params: 1,250,858\n",
- "Non-trainable params: 0\n",
- "_________________________________________________________________\n"
- ]
- }
- ],
- "source": [
- "model = Sequential()\n",
- "model.add(Conv2D(32, (3, 3), padding='same',\n",
- " input_shape=x_train.shape[1:]))\n",
- "model.add(Activation('relu'))\n",
- "model.add(Conv2D(32, (3, 3)))\n",
- "model.add(Activation('relu'))\n",
- "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
- "model.add(Dropout(0.25))\n",
- "\n",
- "model.add(Conv2D(64, (3, 3), padding='same'))\n",
- "model.add(Activation('relu'))\n",
- "model.add(Conv2D(64, (3, 3)))\n",
- "model.add(Activation('relu'))\n",
- "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
- "model.add(Dropout(0.25))\n",
- "\n",
- "model.add(Flatten())\n",
- "model.add(Dense(512))\n",
- "model.add(Activation('relu'))\n",
- "model.add(Dropout(0.5))\n",
- "model.add(Dense(num_classes))\n",
- "model.add(Activation('softmax'))\n",
- "\n",
- "model.summary()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Load into model architecture table using psycopg2"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 13,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>model_id</th>\n",
- " <th>name</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1</td>\n",
- " <td>CNN from Keras docs for CIFAR-10</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(1, u'CNN from Keras docs for CIFAR-10')]"
- ]
- },
- "execution_count": 13,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "import psycopg2 as p2\n",
- "#conn = p2.connect('postgresql://gpadmin@35.239.240.26:5432/madlib')\n",
- "conn = p2.connect('postgresql://fmcquillan@localhost:5432/madlib')\n",
- "cur = conn.cursor()\n",
- "\n",
- "%sql DROP TABLE IF EXISTS model_arch_library;\n",
- "query = \"SELECT madlib.load_keras_model('model_arch_library', %s, NULL, %s)\"\n",
- "cur.execute(query,[model.to_json(), \"CNN from Keras docs for CIFAR-10\"])\n",
- "conn.commit()\n",
- "\n",
- "# check model loaded OK\n",
- "%sql SELECT model_id, name FROM model_arch_library;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"train\"></a>\n",
- "# 5. Train"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n"
- ]
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS cifar_10_model, cifar_10_model_summary;\n",
- "\n",
- "SELECT madlib.madlib_keras_fit('cifar_10_train_data_packed', -- source table\n",
- " 'cifar_10_model', -- model output table\n",
- " 'model_arch_library', -- model arch table\n",
- " 1, -- model arch id\n",
- " $$ loss='categorical_crossentropy', optimizer='rmsprop(lr=0.0001, decay=1e-6)', metrics=['accuracy']$$, -- compile_params\n",
- " $$ batch_size=32, epochs=3 $$, -- fit_params\n",
- " 3, -- num_iterations\n",
- " 0, -- GPUs per host\n",
- " 'cifar_10_test_data_packed', -- validation dataset\n",
- " 2 -- metrics compute frequency\n",
- " );"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "View the model summary:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 110,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>source_table</th>\n",
- " <th>model</th>\n",
- " <th>dependent_varname</th>\n",
- " <th>independent_varname</th>\n",
- " <th>model_arch_table</th>\n",
- " <th>model_arch_id</th>\n",
- " <th>compile_params</th>\n",
- " <th>fit_params</th>\n",
- " <th>num_iterations</th>\n",
- " <th>validation_table</th>\n",
- " <th>metrics_compute_frequency</th>\n",
- " <th>name</th>\n",
- " <th>description</th>\n",
- " <th>model_type</th>\n",
- " <th>model_size</th>\n",
- " <th>start_training_time</th>\n",
- " <th>end_training_time</th>\n",
- " <th>metrics_elapsed_time</th>\n",
- " <th>madlib_version</th>\n",
- " <th>num_classes</th>\n",
- " <th>class_values</th>\n",
- " <th>dependent_vartype</th>\n",
- " <th>normalizing_const</th>\n",
- " <th>metrics_type</th>\n",
- " <th>training_metrics_final</th>\n",
- " <th>training_loss_final</th>\n",
- " <th>training_metrics</th>\n",
- " <th>training_loss</th>\n",
- " <th>validation_metrics_final</th>\n",
- " <th>validation_loss_final</th>\n",
- " <th>validation_metrics</th>\n",
- " <th>validation_loss</th>\n",
- " <th>metrics_iters</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>cifar_10_train_data_packed</td>\n",
- " <td>cifar_10_model</td>\n",
- " <td>y</td>\n",
- " <td>x</td>\n",
- " <td>model_arch_library</td>\n",
- " <td>1</td>\n",
- " <td> loss='categorical_crossentropy', optimizer='rmsprop(lr=0.0001, decay=1e-6)', metrics=['accuracy']</td>\n",
- " <td> batch_size=32, epochs=3 </td>\n",
- " <td>20</td>\n",
- " <td>cifar_10_test_data_packed</td>\n",
- " <td>2</td>\n",
- " <td>None</td>\n",
- " <td>None</td>\n",
- " <td>madlib_keras</td>\n",
- " <td>4886.20019531</td>\n",
- " <td>2019-06-25 05:40:29.287703</td>\n",
- " <td>2019-06-25 07:59:52.961506</td>\n",
- " <td>[798.95044708252, 1616.68976902962, 2447.13853096962, 3273.68762302399, 4116.44566893578, 4962.07483291626, 5805.66080999374, 6665.33687210083, 7526.0603749752, 8363.67366909981]</td>\n",
- " <td>1.16-dev</td>\n",
- " <td>10</td>\n",
- " <td>[u'0', u'1', u'2', u'3', u'4', u'5', u'6', u'7', u'8', u'9']</td>\n",
- " <td>text</td>\n",
- " <td>255.0</td>\n",
- " <td>[u'accuracy']</td>\n",
- " <td>0.836480021477</td>\n",
- " <td>0.500134825706</td>\n",
- " <td>[0.579240024089813, 0.672980010509491, 0.723999977111816, 0.75764000415802, 0.783959984779358, 0.79475998878479, 0.811240017414093, 0.822780013084412, 0.829559981822968, 0.836480021476746]</td>\n",
- " <td>[1.19081699848175, 0.940543830394745, 0.800645172595978, 0.700933694839478, 0.636690974235535, 0.599389910697937, 0.556614756584167, 0.53840559720993, 0.517430067062378, 0.500134825706482]</td>\n",
- " <td>0.778900027275</td>\n",
- " <td>0.661625564098</td>\n",
- " <td>[0.57150000333786, 0.653800010681152, 0.692200005054474, 0.721300005912781, 0.740000009536743, 0.751299977302551, 0.756099998950958, 0.769999980926514, 0.77240002155304, 0.778900027275085]</td>\n",
- " <td>[1.20945084095001, 0.987037718296051, 0.871006071567535, 0.800125658512115, 0.751632690429688, 0.72808450460434, 0.704570233821869, 0.684175074100494, 0.675221920013428, 0.661625564098358]</td>\n",
- " <td>[2, 4, 6, 8, 10, 12, 14, 16, 18, 20]</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'cifar_10_train_data_packed', u'cifar_10_model', u'y', u'x', u'model_arch_library', 1, u\" loss='categorical_crossentropy', optimizer='rmsprop(lr=0.0001, decay=1e-6)', metrics=['accuracy']\", u' batch_size=32, epochs=3 ', 20, u'cifar_10_test_data_packed', 2, None, None, u'madlib_keras', 4886.20019531, datetime.datetime(2019, 6, 25, 5, 40, 29, 287703), datetime.datetime(2019, 6, 25, 7, 59, 52, 961506), [798.95044708252, 1616.68976902962, 2447.13853096962, 3273.68762302399, 4116 [...]
- ]
- },
- "execution_count": 110,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT * FROM cifar_10_model_summary;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Evaluate using test data (same values as last iteration from the fit output summary above)."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 8,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>loss</th>\n",
- " <th>metric</th>\n",
- " <th>metrics_type</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>0.661625564098</td>\n",
- " <td>0.778900027275</td>\n",
- " <td>[u'accuracy']</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(0.661625564098358, 0.778900027275085, [u'accuracy'])]"
- ]
- },
- "execution_count": 8,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS cifar10_validate;\n",
- "\n",
- "SELECT madlib.madlib_keras_evaluate('cifar_10_model', -- model\n",
- " 'cifar_10_test_data_packed', -- test table\n",
- " 'cifar10_validate' -- output table\n",
- " );\n",
- "\n",
- "SELECT * FROM cifar10_validate;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"plot\"></a>\n",
- "# 6. Plots by iteration and by time\n",
- "Accuracy by iteration"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 100,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n",
- "1 rows affected.\n",
- "1 rows affected.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "<matplotlib.legend.Legend at 0x1195ef9d0>"
- ]
- },
- "execution_count": 100,
- "metadata": {},
- "output_type": "execute_result"
- },
- {
- "data": {
- "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzs3Xd4FFX3wPHvSSD0Jr0TqVKkikZBg6gUXxEVESuK/rCLIio2xIIdrKivr6KiCHZEBBExUZCIgHSQ3pEWaoAQkj2/P+4kLCEN2M2mnM/z7LO7M3dmzk42c/bOnXtHVBVjjDEmK2GhDsAYY0zeZ8nCGGNMtixZGGOMyZYlC2OMMdmyZGGMMSZbliyMMcZky5KFMQWAiFwkIusCtK4PROSxQKzrFGJYLiIdQxmDOZYli0JMRK4TkTkikiAi/4rIZBHp4M0bKiKf+ZVVETng [...]
- "text/plain": [
- "<Figure size 432x288 with 1 Axes>"
- ]
- },
- "metadata": {
- "needs_background": "light"
- },
- "output_type": "display_data"
- }
- ],
- "source": [
- "# get accuracy and iteration number\n",
- "iters_proxy = %sql SELECT metrics_iters FROM cifar_10_model_summary;\n",
- "train_accuracy_proxy = %sql SELECT training_metrics FROM cifar_10_model_summary;\n",
- "test_accuracy_proxy = %sql SELECT validation_metrics FROM cifar_10_model_summary;\n",
- "\n",
- "# get number of points\n",
- "num_points_proxy = %sql SELECT array_length(metrics_iters,1) FROM cifar_10_model_summary;\n",
- "num_points = num_points_proxy[0]\n",
- "\n",
- "# reshape to np arrays\n",
- "iters = np.array(iters_proxy).reshape(num_points)\n",
- "train_accuracy = np.array(train_accuracy_proxy).reshape(num_points)\n",
- "test_accuracy = np.array(test_accuracy_proxy).reshape(num_points)\n",
- "\n",
- "#plot\n",
- "plt.title('CIFAR-10 accuracy by iteration')\n",
- "plt.xlabel('Iteration number')\n",
- "plt.ylabel('Accuracy')\n",
- "plt.grid(True)\n",
- "plt.plot(iters, train_accuracy, 'g.-', label='Train')\n",
- "plt.plot(iters, test_accuracy, 'r.-', label='Test')\n",
- "plt.legend()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Loss by iteration"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 101,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "<matplotlib.legend.Legend at 0x119279910>"
- ]
- },
- "execution_count": 101,
- "metadata": {},
- "output_type": "execute_result"
- },
- {
- "data": {
- "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzs3XmcTfX/wPHXe3aMdWjIjOy7yNqEDCqiaJEsUSStX60qUUkSpW/1LeUbSvxoSqlUKmUJX6MQMZQlyR7GNpRlzPv3x7kzc41ZmTN3lvfz8TiPe+85n3POe+7cue/5fD7nfD6iqhhjjDEAfr4OwBhjTP5hScEYY0wKSwrGGGNSWFIwxhiTwpKCMcaYFJYUjDHGpLCkYIo8ERkkIovy6Fw1RSRXrgMXkadFZGJuHOsCYpgnIn19GYPJXZYUTKZEpI+IrBSRYyKyR0S+FpE2 [...]
- "text/plain": [
- "<Figure size 432x288 with 1 Axes>"
- ]
- },
- "metadata": {
- "needs_background": "light"
- },
- "output_type": "display_data"
- }
- ],
- "source": [
- "# get loss\n",
- "train_loss_proxy = %sql SELECT training_loss FROM cifar_10_model_summary;\n",
- "test_loss_proxy = %sql SELECT validation_loss FROM cifar_10_model_summary;\n",
- "\n",
- "# reshape to np arrays\n",
- "train_loss = np.array(train_loss_proxy).reshape(num_points)\n",
- "test_loss = np.array(test_loss_proxy).reshape(num_points)\n",
- "\n",
- "#plot\n",
- "plt.title('CIFAR-10 loss by iteration')\n",
- "plt.xlabel('Iteration number')\n",
- "plt.ylabel('Loss')\n",
- "plt.grid(True)\n",
- "plt.plot(iters, train_loss, 'g.-', label='Train')\n",
- "plt.plot(iters, test_loss, 'r.-', label='Test')\n",
- "plt.legend()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Accuracy by time"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 108,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "<matplotlib.legend.Legend at 0x119664410>"
- ]
- },
- "execution_count": 108,
- "metadata": {},
- "output_type": "execute_result"
- },
- {
- "data": {
- "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xd4FFX3wPHvSSD0DqLShdCUJghEUKNYaBYUFRBULPx8bSgWxMqrrwj6+trAilgARQFFUEAUiYAgIFJUpEvvLRBa2vn9cSdhE1KWsJtNOZ/n2Sc7M3dmzu7Cnr1z79wrqooxxhiTlbBQB2CMMSbvs2RhjDEmW5YsjDHGZMuShTHGmGxZsjDGGJMtSxbGGGOyZcnCmEJIRC4XkQ1BPP4qEbkoWMc3uc+ShfGbiPQSkd9EJE5EtovINBFp720bLCJjfMqqiBz2ysaJyIF0 [...]
- "text/plain": [
- "<Figure size 432x288 with 1 Axes>"
- ]
- },
- "metadata": {
- "needs_background": "light"
- },
- "output_type": "display_data"
- }
- ],
- "source": [
- "# get time\n",
- "time_proxy = %sql SELECT metrics_elapsed_time FROM cifar_10_model_summary;\n",
- "\n",
- "# reshape to np arrays\n",
- "time = np.array(time_proxy).reshape(num_points)/60.0\n",
- "\n",
- "#plot\n",
- "plt.title('CIFAR-10 accuracy by time')\n",
- "plt.xlabel('Time (min)')\n",
- "plt.ylabel('Accuracy')\n",
- "plt.grid(True)\n",
- "plt.plot(time, train_accuracy, 'g.-', label='Train')\n",
- "plt.plot(time, test_accuracy, 'r.-', label='Test')\n",
- "plt.legend()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Time to achieve a given accuracy"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 109,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "<matplotlib.legend.Legend at 0x119628690>"
- ]
- },
- "execution_count": 109,
- "metadata": {},
- "output_type": "execute_result"
- },
- {
- "data": {
- "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEWCAYAAABIVsEJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xd4VFX6wPHvm0AIJRCkC2iwAFKkBEuQXYPIWhbFgoJIUVyxrWXRFcR1cXFta+/tJ2tgEcKCBVkUBQ2KREqA0KX33hIChJDk/f1xLzFAICGZmTuTeT/Pk2dm7r1z73syMG/OOfecI6qKMcYYc7oivA7AGGNMaLIEYowxplQsgRhjjCkVSyDGGGNKxRKIMcaYUrEEYowxplQsgRhzHBH5PxEZGoDrXCki6/x9HWP8xRKIOSkR6S0ic0UkS0S2isjXItLJ3fe0iPyn0LEq [...]
- "text/plain": [
- "<Figure size 432x288 with 1 Axes>"
- ]
- },
- "metadata": {
- "needs_background": "light"
- },
- "output_type": "display_data"
- }
- ],
- "source": [
- "#plot\n",
- "plt.title('CIFAR-10 time by accuracy')\n",
- "plt.xlabel('Accuracy')\n",
- "plt.ylabel('Time (min)')\n",
- "plt.grid(True)\n",
- "plt.plot(train_accuracy, time, 'g.-', label='Train')\n",
- "plt.plot(test_accuracy, time, 'r.-', label='Test')\n",
- "plt.legend()"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 2",
- "language": "python",
- "name": "python2"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 2
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython2",
- "version": "2.7.10"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 1
-}
diff --git a/community-artifacts/Deep-learning/MADlib-Keras-transfer-learning-v2.ipynb b/community-artifacts/Deep-learning/MADlib-Keras-transfer-learning-v2.ipynb
deleted file mode 100644
index d8f46ed..0000000
--- a/community-artifacts/Deep-learning/MADlib-Keras-transfer-learning-v2.ipynb
+++ /dev/null
@@ -1,1624 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Transfer Learning Using Keras and MADlib\n",
- "\n",
- "This is a transfer learning example based on https://keras.io/examples/mnist_transfer_cnn/ \n",
- "\n",
- "To load images into tables we use the script called <em>madlib_image_loader.py</em> located at https://github.com/apache/madlib-site/tree/asf-site/community-artifacts/Deep-learning which uses the Python Imaging Library so supports multiple formats http://www.pythonware.com/products/pil/\n",
- "\n",
- "## Table of contents\n",
- "<a href=\"#import_libraries\">1. Import libraries</a>\n",
- "\n",
- "<a href=\"#load_and_prepare_data\">2. Load and prepare data</a>\n",
- "\n",
- "<a href=\"#image_preproc\">3. Call image preprocessor</a>\n",
- "\n",
- "<a href=\"#define_and_load_model\">4. Define and load model architecture</a>\n",
- "\n",
- "<a href=\"#train\">5. Train</a>\n",
- "\n",
- "<a href=\"#transfer_learning\">6. Transfer learning</a>"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {
- "scrolled": true
- },
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/config.py:13: ShimWarning: The `IPython.config` package has been deprecated since IPython 4.0. You should import from traitlets.config instead.\n",
- " \"You should import from traitlets.config instead.\", ShimWarning)\n",
- "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/utils/traitlets.py:5: UserWarning: IPython.utils.traitlets has moved to a top-level traitlets package.\n",
- " warn(\"IPython.utils.traitlets has moved to a top-level traitlets package.\")\n"
- ]
- }
- ],
- "source": [
- "%load_ext sql"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "u'Connected: gpadmin@madlib'"
- ]
- },
- "execution_count": 2,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "# Greenplum Database 5.x on GCP for deep learning (PM demo machine)\n",
- "%sql postgresql://gpadmin@35.239.240.26:5432/madlib\n",
- " \n",
- "# PostgreSQL local\n",
- "#%sql postgresql://fmcquillan@localhost:5432/madlib"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>version</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>MADlib version: 1.17-dev, git revision: rel/v1.16-2-g8a612fe, cmake configuration time: Wed Jul 17 18:49:47 UTC 2019, build type: release, build system: Linux-3.10.0-957.21.3.el7.x86_64, C compiler: gcc 4.8.5, C++ compiler: g++ 4.8.5</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'MADlib version: 1.17-dev, git revision: rel/v1.16-2-g8a612fe, cmake configuration time: Wed Jul 17 18:49:47 UTC 2019, build type: release, build system: Linux-3.10.0-957.21.3.el7.x86_64, C compiler: gcc 4.8.5, C++ compiler: g++ 4.8.5',)]"
- ]
- },
- "execution_count": 3,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%sql select madlib.version();\n",
- "#%sql select version();"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"import_libraries\"></a>\n",
- "# 1. Import libraries\n",
- "From https://keras.io/examples/mnist_transfer_cnn/ import libraries and define some params"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Using TensorFlow backend.\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Couldn't import dot_parser, loading of dot files will not be possible.\n"
- ]
- }
- ],
- "source": [
- "from __future__ import print_function\n",
- "\n",
- "import datetime\n",
- "import keras\n",
- "from keras.datasets import mnist\n",
- "from keras.models import Sequential\n",
- "from keras.layers import Dense, Dropout, Activation, Flatten\n",
- "from keras.layers import Conv2D, MaxPooling2D\n",
- "from keras import backend as K\n",
- "\n",
- "now = datetime.datetime.now\n",
- "\n",
- "batch_size = 128\n",
- "num_classes = 5\n",
- "epochs = 5\n",
- "\n",
- "# input image dimensions\n",
- "img_rows, img_cols = 28, 28\n",
- "# number of convolutional filters to use\n",
- "filters = 32\n",
- "# size of pooling area for max pooling\n",
- "pool_size = 2\n",
- "# convolution kernel size\n",
- "kernel_size = 3\n",
- "\n",
- "if K.image_data_format() == 'channels_first':\n",
- " input_shape = (1, img_rows, img_cols)\n",
- "else:\n",
- " input_shape = (img_rows, img_cols, 1)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Others needed in this workbook"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {},
- "outputs": [],
- "source": [
- "import pandas as pd\n",
- "import numpy as np"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"load_and_prepare_data\"></a>\n",
- "# 2. Load and prepare data\n",
- "\n",
- "First load MNIST data from Keras, consisting of 60,000 28x28 grayscale images of the 10 digits, along with a test set of 10,000 images."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 6,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(4861, 28, 28)\n",
- "(4861, 28, 28, 1)\n"
- ]
- }
- ],
- "source": [
- "# the data, split between train and test sets\n",
- "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
- "\n",
- "# create two datasets one with digits below 5 and one with 5 and above\n",
- "x_train_lt5 = x_train[y_train < 5]\n",
- "y_train_lt5 = y_train[y_train < 5]\n",
- "x_test_lt5 = x_test[y_test < 5]\n",
- "y_test_lt5 = y_test[y_test < 5]\n",
- "\n",
- "x_train_gte5 = x_train[y_train >= 5]\n",
- "y_train_gte5 = y_train[y_train >= 5] - 5\n",
- "x_test_gte5 = x_test[y_test >= 5]\n",
- "y_test_gte5 = y_test[y_test >= 5] - 5\n",
- "\n",
- "# reshape to match model architecture\n",
- "print(x_test_gte5.shape)\n",
- "x_train_lt5=x_train_lt5.reshape(len(x_train_lt5), *input_shape)\n",
- "x_test_lt5 = x_test_lt5.reshape(len(x_test_lt5), *input_shape)\n",
- "x_train_gte5=x_train_gte5.reshape(len(x_train_gte5), *input_shape)\n",
- "x_test_gte5 = x_test_gte5.reshape(len(x_test_gte5), *input_shape)\n",
- "print(x_test_gte5.shape)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Load datasets into tables using image loader scripts called <em>madlib_image_loader.py</em> located at https://github.com/apache/madlib-site/tree/asf-site/community-artifacts/Deep-learning"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 7,
- "metadata": {},
- "outputs": [],
- "source": [
- "# MADlib tools directory\n",
- "import sys\n",
- "import os\n",
- "madlib_site_dir = '/Users/fmcquillan/Documents/Product/MADlib/Demos/data'\n",
- "sys.path.append(madlib_site_dir)\n",
- "\n",
- "# Import image loader module\n",
- "from madlib_image_loader import ImageLoader, DbCredentials"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 8,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Specify database credentials, for connecting to db\n",
- "db_creds = DbCredentials(user='gpadmin',\n",
- " host='35.239.240.26',\n",
- " port='5432',\n",
- " password='')"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 9,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Initialize ImageLoader (increase num_workers to run faster)\n",
- "iloader = ImageLoader(num_workers=5, db_creds=db_creds)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 10,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "MainProcess: Connected to madlib db.\n",
- "Executing: CREATE TABLE train_lt5 (id SERIAL, x REAL[], y TEXT)\n",
- "CREATE TABLE\n",
- "Created table train_lt5 in madlib db\n",
- "Spawning 5 workers...\n",
- "Initializing PoolWorker-1 [pid 32452]\n",
- "PoolWorker-1: Created temporary directory /tmp/madlib_RqdNj6bF0W\n",
- "Initializing PoolWorker-2 [pid 32453]\n",
- "PoolWorker-2: Created temporary directory /tmp/madlib_2dVRSzAit5\n",
- "Initializing PoolWorker-3 [pid 32454]\n",
- "PoolWorker-3: Created temporary directory /tmp/madlib_wAZd4zRnM5\n",
- "Initializing PoolWorker-4 [pid 32455]\n",
- "PoolWorker-4: Created temporary directory /tmp/madlib_fGfGIO2M6v\n",
- "Initializing PoolWorker-5 [pid 32456]\n",
- "PoolWorker-5: Created temporary directory /tmp/madlib_wJYreK1smG\n",
- "PoolWorker-1: Connected to madlib db.\n",
- "PoolWorker-2: Connected to madlib db.\n",
- "PoolWorker-3: Connected to madlib db.\n",
- "PoolWorker-4: Connected to madlib db.\n",
- "PoolWorker-5: Connected to madlib db.\n",
- "PoolWorker-1: Wrote 1000 images to /tmp/madlib_RqdNj6bF0W/train_lt50000.tmp\n",
- "PoolWorker-2: Wrote 1000 images to /tmp/madlib_2dVRSzAit5/train_lt50000.tmp\n",
- "PoolWorker-4: Wrote 1000 images to /tmp/madlib_fGfGIO2M6v/train_lt50000.tmp\n",
- "PoolWorker-3: Wrote 1000 images to /tmp/madlib_wAZd4zRnM5/train_lt50000.tmp\n",
- "PoolWorker-5: Wrote 1000 images to /tmp/madlib_wJYreK1smG/train_lt50000.tmp\n",
- "PoolWorker-1: Loaded 1000 images into train_lt5\n",
- "PoolWorker-4: Loaded 1000 images into train_lt5\n",
- "PoolWorker-5: Loaded 1000 images into train_lt5\n",
- "PoolWorker-3: Loaded 1000 images into train_lt5\n",
- "PoolWorker-1: Wrote 1000 images to /tmp/madlib_RqdNj6bF0W/train_lt50001.tmp\n",
- "PoolWorker-5: Wrote 1000 images to /tmp/madlib_wJYreK1smG/train_lt50001.tmp\n",
- "PoolWorker-4: Wrote 1000 images to /tmp/madlib_fGfGIO2M6v/train_lt50001.tmp\n",
- "PoolWorker-2: Loaded 1000 images into train_lt5\n",
- "PoolWorker-3: Wrote 1000 images to /tmp/madlib_wAZd4zRnM5/train_lt50001.tmp\n",
- "PoolWorker-1: Loaded 1000 images into train_lt5\n",
- "PoolWorker-2: Wrote 1000 images to /tmp/madlib_2dVRSzAit5/train_lt50001.tmp\n",
- "PoolWorker-5: Loaded 1000 images into train_lt5\n",
- "PoolWorker-3: Loaded 1000 images into train_lt5\n",
- "PoolWorker-4: Loaded 1000 images into train_lt5\n",
- "PoolWorker-1: Wrote 1000 images to /tmp/madlib_RqdNj6bF0W/train_lt50002.tmp\n",
- "PoolWorker-5: Wrote 1000 images to /tmp/madlib_wJYreK1smG/train_lt50002.tmp\n",
- "PoolWorker-3: Wrote 1000 images to /tmp/madlib_wAZd4zRnM5/train_lt50002.tmp\n",
- "PoolWorker-4: Wrote 1000 images to /tmp/madlib_fGfGIO2M6v/train_lt50002.tmp\n",
- "PoolWorker-2: Loaded 1000 images into train_lt5\n",
- "PoolWorker-1: Loaded 1000 images into train_lt5\n",
- "PoolWorker-5: Loaded 1000 images into train_lt5\n",
- "PoolWorker-4: Loaded 1000 images into train_lt5\n",
- "PoolWorker-3: Loaded 1000 images into train_lt5\n",
- "PoolWorker-2: Wrote 1000 images to /tmp/madlib_2dVRSzAit5/train_lt50002.tmp\n",
- "PoolWorker-1: Wrote 1000 images to /tmp/madlib_RqdNj6bF0W/train_lt50003.tmp\n",
- "PoolWorker-5: Wrote 1000 images to /tmp/madlib_wJYreK1smG/train_lt50003.tmp\n",
- "PoolWorker-4: Wrote 1000 images to /tmp/madlib_fGfGIO2M6v/train_lt50003.tmp\n",
- "PoolWorker-3: Wrote 1000 images to /tmp/madlib_wAZd4zRnM5/train_lt50003.tmp\n",
- "PoolWorker-2: Loaded 1000 images into train_lt5\n",
- "PoolWorker-1: Loaded 1000 images into train_lt5\n",
- "PoolWorker-4: Loaded 1000 images into train_lt5\n",
- "PoolWorker-2: Wrote 1000 images to /tmp/madlib_2dVRSzAit5/train_lt50003.tmp\n",
- "PoolWorker-3: Loaded 1000 images into train_lt5\n",
- "PoolWorker-1: Wrote 1000 images to /tmp/madlib_RqdNj6bF0W/train_lt50004.tmp\n",
- "PoolWorker-4: Wrote 1000 images to /tmp/madlib_fGfGIO2M6v/train_lt50004.tmp\n",
- "PoolWorker-3: Wrote 1000 images to /tmp/madlib_wAZd4zRnM5/train_lt50004.tmp\n",
- "PoolWorker-2: Loaded 1000 images into train_lt5\n",
- "PoolWorker-4: Loaded 1000 images into train_lt5\n",
- "PoolWorker-1: Loaded 1000 images into train_lt5\n",
- "PoolWorker-3: Loaded 1000 images into train_lt5\n",
- "PoolWorker-2: Wrote 1000 images to /tmp/madlib_2dVRSzAit5/train_lt50004.tmp\n",
- "PoolWorker-4: Wrote 1000 images to /tmp/madlib_fGfGIO2M6v/train_lt50005.tmp\n",
- "PoolWorker-1: Wrote 1000 images to /tmp/madlib_RqdNj6bF0W/train_lt50005.tmp\n",
- "PoolWorker-5: Loaded 1000 images into train_lt5\n",
- "PoolWorker-3: Wrote 1000 images to /tmp/madlib_wAZd4zRnM5/train_lt50005.tmp\n",
- "PoolWorker-4: Loaded 1000 images into train_lt5\n",
- "PoolWorker-5: Wrote 1000 images to /tmp/madlib_wJYreK1smG/train_lt50004.tmp\n",
- "PoolWorker-2: Loaded 1000 images into train_lt5\n",
- "PoolWorker-1: Loaded 1000 images into train_lt5\n",
- "PoolWorker-4: Wrote 596 images to /tmp/madlib_fGfGIO2M6v/train_lt50006.tmp\n",
- "PoolWorker-3: Loaded 1000 images into train_lt5\n",
- "PoolWorker-2: Wrote 1000 images to /tmp/madlib_2dVRSzAit5/train_lt50005.tmp\n",
- "PoolWorker-4: Loaded 596 images into train_lt5\n",
- "PoolWorker-2: Loaded 1000 images into train_lt5\n",
- "PoolWorker-5: Loaded 1000 images into train_lt5\n",
- "PoolWorker-5: Wrote 1000 images to /tmp/madlib_wJYreK1smG/train_lt50005.tmp\n",
- "PoolWorker-5: Loaded 1000 images into train_lt5\n",
- "PoolWorker-1: Removed temporary directory /tmp/madlib_RqdNj6bF0W\n",
- "PoolWorker-5: Removed temporary directory /tmp/madlib_wJYreK1smG\n",
- "PoolWorker-4: Removed temporary directory /tmp/madlib_fGfGIO2M6v\n",
- "PoolWorker-3: Removed temporary directory /tmp/madlib_wAZd4zRnM5\n",
- "PoolWorker-2: Removed temporary directory /tmp/madlib_2dVRSzAit5\n",
- "Done! Loaded 30596 images in 14.2171461582s\n",
- "5 workers terminated.\n",
- "MainProcess: Connected to madlib db.\n",
- "Executing: CREATE TABLE test_lt5 (id SERIAL, x REAL[], y TEXT)\n",
- "CREATE TABLE\n",
- "Created table test_lt5 in madlib db\n",
- "Spawning 5 workers...\n",
- "Initializing PoolWorker-6 [pid 32457]\n",
- "PoolWorker-6: Created temporary directory /tmp/madlib_4n6hb9xfIi\n",
- "Initializing PoolWorker-7 [pid 32458]\n",
- "PoolWorker-7: Created temporary directory /tmp/madlib_F1kcKRJNq0\n",
- "Initializing PoolWorker-8 [pid 32459]\n",
- "PoolWorker-8: Created temporary directory /tmp/madlib_XpsmAPt8MP\n",
- "Initializing PoolWorker-9 [pid 32460]\n",
- "PoolWorker-10: Created temporary directory /tmp/madlib_ZcWfxD9Vqm\n",
- "PoolWorker-9: Created temporary directory /tmp/madlib_7TQeRq7vm6\n",
- "Initializing PoolWorker-10 [pid 32461]\n",
- "PoolWorker-6: Connected to madlib db.\n",
- "PoolWorker-7: Connected to madlib db.\n",
- "PoolWorker-8: Connected to madlib db.\n",
- "PoolWorker-9: Connected to madlib db.\n",
- "PoolWorker-10: Connected to madlib db.\n",
- "PoolWorker-9: Wrote 1000 images to /tmp/madlib_7TQeRq7vm6/test_lt50000.tmp\n",
- "PoolWorker-6: Wrote 1000 images to /tmp/madlib_4n6hb9xfIi/test_lt50000.tmp\n",
- "PoolWorker-7: Wrote 1000 images to /tmp/madlib_F1kcKRJNq0/test_lt50000.tmp\n",
- "PoolWorker-8: Wrote 1000 images to /tmp/madlib_XpsmAPt8MP/test_lt50000.tmp\n",
- "PoolWorker-10: Wrote 1000 images to /tmp/madlib_ZcWfxD9Vqm/test_lt50000.tmp\n",
- "PoolWorker-6: Loaded 1000 images into test_lt5\n",
- "PoolWorker-6: Wrote 139 images to /tmp/madlib_4n6hb9xfIi/test_lt50001.tmp\n",
- "PoolWorker-7: Loaded 1000 images into test_lt5\n",
- "PoolWorker-8: Loaded 1000 images into test_lt5\n",
- "PoolWorker-9: Loaded 1000 images into test_lt5\n",
- "PoolWorker-10: Loaded 1000 images into test_lt5\n",
- "PoolWorker-6: Loaded 139 images into test_lt5\n",
- "PoolWorker-9: Removed temporary directory /tmp/madlib_7TQeRq7vm6\n",
- "PoolWorker-10: Removed temporary directory /tmp/madlib_ZcWfxD9Vqm\n",
- "PoolWorker-8: Removed temporary directory /tmp/madlib_XpsmAPt8MP\n",
- "PoolWorker-7: Removed temporary directory /tmp/madlib_F1kcKRJNq0\n",
- "PoolWorker-6: Removed temporary directory /tmp/madlib_4n6hb9xfIi\n",
- "Done! Loaded 5139 images in 3.46985602379s\n",
- "5 workers terminated.\n",
- "MainProcess: Connected to madlib db.\n",
- "Executing: CREATE TABLE train_gte5 (id SERIAL, x REAL[], y TEXT)\n",
- "CREATE TABLE\n",
- "Created table train_gte5 in madlib db\n",
- "Spawning 5 workers...\n",
- "Initializing PoolWorker-11 [pid 32462]\n",
- "PoolWorker-11: Created temporary directory /tmp/madlib_F1VrR8QcQB\n",
- "Initializing PoolWorker-12 [pid 32463]\n",
- "PoolWorker-12: Created temporary directory /tmp/madlib_8dZVe2Lowc\n",
- "Initializing PoolWorker-13 [pid 32464]\n",
- "PoolWorker-13: Created temporary directory /tmp/madlib_V3jNZmDpr3\n",
- "Initializing PoolWorker-14 [pid 32465]\n",
- "PoolWorker-14: Created temporary directory /tmp/madlib_EtHaTZ0PtA\n",
- "Initializing PoolWorker-15 [pid 32466]\n",
- "PoolWorker-15: Created temporary directory /tmp/madlib_XtW6GudAE0\n",
- "PoolWorker-11: Connected to madlib db.\n",
- "PoolWorker-12: Connected to madlib db.\n",
- "PoolWorker-13: Connected to madlib db.\n",
- "PoolWorker-14: Connected to madlib db.\n",
- "PoolWorker-15: Connected to madlib db.\n",
- "PoolWorker-11: Wrote 1000 images to /tmp/madlib_F1VrR8QcQB/train_gte50000.tmp\n",
- "PoolWorker-12: Wrote 1000 images to /tmp/madlib_8dZVe2Lowc/train_gte50000.tmp\n",
- "PoolWorker-13: Wrote 1000 images to /tmp/madlib_V3jNZmDpr3/train_gte50000.tmp\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "PoolWorker-15: Wrote 1000 images to /tmp/madlib_XtW6GudAE0/train_gte50000.tmp\n",
- "PoolWorker-14: Wrote 1000 images to /tmp/madlib_EtHaTZ0PtA/train_gte50000.tmp\n",
- "PoolWorker-15: Loaded 1000 images into train_gte5\n",
- "PoolWorker-14: Loaded 1000 images into train_gte5\n",
- "PoolWorker-15: Wrote 1000 images to /tmp/madlib_XtW6GudAE0/train_gte50001.tmp\n",
- "PoolWorker-14: Wrote 1000 images to /tmp/madlib_EtHaTZ0PtA/train_gte50001.tmp\n",
- "PoolWorker-15: Loaded 1000 images into train_gte5\n",
- "PoolWorker-15: Wrote 1000 images to /tmp/madlib_XtW6GudAE0/train_gte50002.tmp\n",
- "PoolWorker-14: Loaded 1000 images into train_gte5\n",
- "PoolWorker-14: Wrote 1000 images to /tmp/madlib_EtHaTZ0PtA/train_gte50002.tmp\n",
- "PoolWorker-15: Loaded 1000 images into train_gte5\n",
- "PoolWorker-11: Loaded 1000 images into train_gte5\n",
- "PoolWorker-12: Loaded 1000 images into train_gte5\n",
- "PoolWorker-14: Loaded 1000 images into train_gte5\n",
- "PoolWorker-15: Wrote 1000 images to /tmp/madlib_XtW6GudAE0/train_gte50003.tmp\n",
- "PoolWorker-11: Wrote 1000 images to /tmp/madlib_F1VrR8QcQB/train_gte50001.tmp\n",
- "PoolWorker-13: Loaded 1000 images into train_gte5\n",
- "PoolWorker-12: Wrote 1000 images to /tmp/madlib_8dZVe2Lowc/train_gte50001.tmp\n",
- "PoolWorker-14: Wrote 1000 images to /tmp/madlib_EtHaTZ0PtA/train_gte50003.tmp\n",
- "PoolWorker-13: Wrote 1000 images to /tmp/madlib_V3jNZmDpr3/train_gte50001.tmp\n",
- "PoolWorker-14: Loaded 1000 images into train_gte5\n",
- "PoolWorker-15: Loaded 1000 images into train_gte5\n",
- "PoolWorker-14: Wrote 1000 images to /tmp/madlib_EtHaTZ0PtA/train_gte50004.tmp\n",
- "PoolWorker-15: Wrote 1000 images to /tmp/madlib_XtW6GudAE0/train_gte50004.tmp\n",
- "PoolWorker-12: Loaded 1000 images into train_gte5\n",
- "PoolWorker-13: Loaded 1000 images into train_gte5\n",
- "PoolWorker-14: Loaded 1000 images into train_gte5\n",
- "PoolWorker-12: Wrote 1000 images to /tmp/madlib_8dZVe2Lowc/train_gte50002.tmp\n",
- "PoolWorker-13: Wrote 1000 images to /tmp/madlib_V3jNZmDpr3/train_gte50002.tmp\n",
- "PoolWorker-14: Wrote 1000 images to /tmp/madlib_EtHaTZ0PtA/train_gte50005.tmp\n",
- "PoolWorker-11: Loaded 1000 images into train_gte5\n",
- "PoolWorker-11: Wrote 1000 images to /tmp/madlib_F1VrR8QcQB/train_gte50002.tmp\n",
- "PoolWorker-13: Loaded 1000 images into train_gte5\n",
- "PoolWorker-14: Loaded 1000 images into train_gte5\n",
- "PoolWorker-13: Wrote 1000 images to /tmp/madlib_V3jNZmDpr3/train_gte50003.tmp\n",
- "PoolWorker-14: Wrote 1000 images to /tmp/madlib_EtHaTZ0PtA/train_gte50006.tmp\n",
- "PoolWorker-11: Loaded 1000 images into train_gte5\n",
- "PoolWorker-12: Loaded 1000 images into train_gte5\n",
- "PoolWorker-11: Wrote 1000 images to /tmp/madlib_F1VrR8QcQB/train_gte50003.tmp\n",
- "PoolWorker-12: Wrote 1000 images to /tmp/madlib_8dZVe2Lowc/train_gte50003.tmp\n",
- "PoolWorker-15: Loaded 1000 images into train_gte5\n",
- "PoolWorker-13: Loaded 1000 images into train_gte5\n",
- "PoolWorker-14: Loaded 1000 images into train_gte5\n",
- "PoolWorker-15: Wrote 1000 images to /tmp/madlib_XtW6GudAE0/train_gte50005.tmp\n",
- "PoolWorker-13: Wrote 1000 images to /tmp/madlib_V3jNZmDpr3/train_gte50004.tmp\n",
- "PoolWorker-14: Wrote 1000 images to /tmp/madlib_EtHaTZ0PtA/train_gte50007.tmp\n",
- "PoolWorker-11: Loaded 1000 images into train_gte5\n",
- "PoolWorker-14: Loaded 1000 images into train_gte5\n",
- "PoolWorker-13: Loaded 1000 images into train_gte5\n",
- "PoolWorker-12: Loaded 1000 images into train_gte5\n",
- "PoolWorker-11: Wrote 1000 images to /tmp/madlib_F1VrR8QcQB/train_gte50004.tmp\n",
- "PoolWorker-13: Wrote 1000 images to /tmp/madlib_V3jNZmDpr3/train_gte50005.tmp\n",
- "PoolWorker-15: Loaded 1000 images into train_gte5\n",
- "PoolWorker-11: Loaded 1000 images into train_gte5\n",
- "PoolWorker-13: Loaded 1000 images into train_gte5\n",
- "PoolWorker-11: Wrote 404 images to /tmp/madlib_F1VrR8QcQB/train_gte50005.tmp\n",
- "PoolWorker-11: Loaded 404 images into train_gte5\n",
- "PoolWorker-13: Removed temporary directory /tmp/madlib_V3jNZmDpr3\n",
- "PoolWorker-14: Removed temporary directory /tmp/madlib_EtHaTZ0PtA\n",
- "PoolWorker-12: Removed temporary directory /tmp/madlib_8dZVe2Lowc\n",
- "PoolWorker-15: Removed temporary directory /tmp/madlib_XtW6GudAE0\n",
- "PoolWorker-11: Removed temporary directory /tmp/madlib_F1VrR8QcQB\n",
- "Done! Loaded 29404 images in 18.3531939983s\n",
- "5 workers terminated.\n",
- "MainProcess: Connected to madlib db.\n",
- "Executing: CREATE TABLE test_gte5 (id SERIAL, x REAL[], y TEXT)\n",
- "CREATE TABLE\n",
- "Created table test_gte5 in madlib db\n",
- "Spawning 5 workers...\n",
- "Initializing PoolWorker-16 [pid 32468]\n",
- "PoolWorker-16: Created temporary directory /tmp/madlib_yQApveqAcA\n",
- "Initializing PoolWorker-17 [pid 32469]\n",
- "PoolWorker-17: Created temporary directory /tmp/madlib_2LKVOaFY1A\n",
- "Initializing PoolWorker-18 [pid 32470]\n",
- "PoolWorker-18: Created temporary directory /tmp/madlib_qicxNf38wn\n",
- "Initializing PoolWorker-19 [pid 32471]\n",
- "PoolWorker-19: Created temporary directory /tmp/madlib_cD0CFY0uOR\n",
- "Initializing PoolWorker-20 [pid 32472]\n",
- "PoolWorker-20: Created temporary directory /tmp/madlib_yNG8SjRSWz\n",
- "PoolWorker-16: Connected to madlib db.\n",
- "PoolWorker-17: Connected to madlib db.\n",
- "PoolWorker-18: Connected to madlib db.\n",
- "PoolWorker-19: Connected to madlib db.\n",
- "PoolWorker-20: Connected to madlib db.\n",
- "PoolWorker-20: Wrote 861 images to /tmp/madlib_yNG8SjRSWz/test_gte50000.tmp\n",
- "PoolWorker-17: Wrote 1000 images to /tmp/madlib_2LKVOaFY1A/test_gte50000.tmp\n",
- "PoolWorker-18: Wrote 1000 images to /tmp/madlib_qicxNf38wn/test_gte50000.tmp\n",
- "PoolWorker-16: Wrote 1000 images to /tmp/madlib_yQApveqAcA/test_gte50000.tmp\n",
- "PoolWorker-19: Wrote 1000 images to /tmp/madlib_cD0CFY0uOR/test_gte50000.tmp\n",
- "PoolWorker-17: Loaded 1000 images into test_gte5\n",
- "PoolWorker-18: Loaded 1000 images into test_gte5\n",
- "PoolWorker-19: Loaded 1000 images into test_gte5\n",
- "PoolWorker-16: Loaded 1000 images into test_gte5\n",
- "PoolWorker-20: Loaded 861 images into test_gte5\n",
- "PoolWorker-17: Removed temporary directory /tmp/madlib_2LKVOaFY1A\n",
- "PoolWorker-16: Removed temporary directory /tmp/madlib_yQApveqAcA\n",
- "PoolWorker-18: Removed temporary directory /tmp/madlib_qicxNf38wn\n",
- "PoolWorker-20: Removed temporary directory /tmp/madlib_yNG8SjRSWz\n",
- "PoolWorker-19: Removed temporary directory /tmp/madlib_cD0CFY0uOR\n",
- "Done! Loaded 4861 images in 3.39312386513s\n",
- "5 workers terminated.\n"
- ]
- }
- ],
- "source": [
- "# Drop tables\n",
- "%sql DROP TABLE IF EXISTS train_lt5, test_lt5, train_gte5, test_gte5\n",
- "\n",
- "# Save images to temporary directories and load into database\n",
- "iloader.load_dataset_from_np(x_train_lt5, y_train_lt5, 'train_lt5', append=False, no_temp_files=False)\n",
- "iloader.load_dataset_from_np(x_test_lt5, y_test_lt5, 'test_lt5', append=False, no_temp_files=False)\n",
- "iloader.load_dataset_from_np(x_train_gte5, y_train_gte5, 'train_gte5', append=False, no_temp_files=False)\n",
- "iloader.load_dataset_from_np(x_test_gte5, y_test_gte5, 'test_gte5', append=False, no_temp_files=False)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"image_preproc\"></a>\n",
- "# 3. Call image preprocessor\n",
- "\n",
- "Transforms from one image per row to multiple images per row for batch optimization. Also normalizes and one-hot encodes.\n",
- "\n",
- "Training dataset < 5"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 11,
- "metadata": {
- "scrolled": true
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>source_table</th>\n",
- " <th>output_table</th>\n",
- " <th>dependent_varname</th>\n",
- " <th>independent_varname</th>\n",
- " <th>dependent_vartype</th>\n",
- " <th>class_values</th>\n",
- " <th>buffer_size</th>\n",
- " <th>normalizing_const</th>\n",
- " <th>num_classes</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>train_lt5</td>\n",
- " <td>train_lt5_packed</td>\n",
- " <td>y</td>\n",
- " <td>x</td>\n",
- " <td>text</td>\n",
- " <td>[u'0', u'1', u'2', u'3', u'4']</td>\n",
- " <td>1000</td>\n",
- " <td>255.0</td>\n",
- " <td>5</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'train_lt5', u'train_lt5_packed', u'y', u'x', u'text', [u'0', u'1', u'2', u'3', u'4'], 1000, 255.0, 5)]"
- ]
- },
- "execution_count": 11,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS train_lt5_packed, train_lt5_packed_summary;\n",
- "\n",
- "SELECT madlib.training_preprocessor_dl('train_lt5', -- Source table\n",
- " 'train_lt5_packed', -- Output table\n",
- " 'y', -- Dependent variable\n",
- " 'x', -- Independent variable\n",
- " 1000, -- Buffer size\n",
- " 255 -- Normalizing constant\n",
- " );\n",
- "\n",
- "SELECT * FROM train_lt5_packed_summary;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Test dataset < 5"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 12,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>source_table</th>\n",
- " <th>output_table</th>\n",
- " <th>dependent_varname</th>\n",
- " <th>independent_varname</th>\n",
- " <th>dependent_vartype</th>\n",
- " <th>class_values</th>\n",
- " <th>buffer_size</th>\n",
- " <th>normalizing_const</th>\n",
- " <th>num_classes</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>test_lt5</td>\n",
- " <td>test_lt5_packed</td>\n",
- " <td>y</td>\n",
- " <td>x</td>\n",
- " <td>text</td>\n",
- " <td>[u'0', u'1', u'2', u'3', u'4']</td>\n",
- " <td>2570</td>\n",
- " <td>255.0</td>\n",
- " <td>5</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'test_lt5', u'test_lt5_packed', u'y', u'x', u'text', [u'0', u'1', u'2', u'3', u'4'], 2570, 255.0, 5)]"
- ]
- },
- "execution_count": 12,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS test_lt5_packed, test_lt5_packed_summary;\n",
- "\n",
- "SELECT madlib.validation_preprocessor_dl('test_lt5', -- Source table\n",
- " 'test_lt5_packed', -- Output table\n",
- " 'y', -- Dependent variable\n",
- " 'x', -- Independent variable\n",
- " 'train_lt5_packed' -- Training preproc table\n",
- " );\n",
- "\n",
- "SELECT * FROM test_lt5_packed_summary;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Training dataset >= 5"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 13,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>source_table</th>\n",
- " <th>output_table</th>\n",
- " <th>dependent_varname</th>\n",
- " <th>independent_varname</th>\n",
- " <th>dependent_vartype</th>\n",
- " <th>class_values</th>\n",
- " <th>buffer_size</th>\n",
- " <th>normalizing_const</th>\n",
- " <th>num_classes</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>train_gte5</td>\n",
- " <td>train_gte5_packed</td>\n",
- " <td>y</td>\n",
- " <td>x</td>\n",
- " <td>text</td>\n",
- " <td>[u'0', u'1', u'2', u'3', u'4']</td>\n",
- " <td>1000</td>\n",
- " <td>255.0</td>\n",
- " <td>5</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'train_gte5', u'train_gte5_packed', u'y', u'x', u'text', [u'0', u'1', u'2', u'3', u'4'], 1000, 255.0, 5)]"
- ]
- },
- "execution_count": 13,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS train_gte5_packed, train_gte5_packed_summary;\n",
- "\n",
- "SELECT madlib.training_preprocessor_dl('train_gte5', -- Source table\n",
- " 'train_gte5_packed', -- Output table\n",
- " 'y', -- Dependent variable\n",
- " 'x', -- Independent variable\n",
- " 1000, -- Buffer size\n",
- " 255 -- Normalizing constant\n",
- " );\n",
- "\n",
- "SELECT * FROM train_gte5_packed_summary;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Test dataset >= 5"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 14,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>source_table</th>\n",
- " <th>output_table</th>\n",
- " <th>dependent_varname</th>\n",
- " <th>independent_varname</th>\n",
- " <th>dependent_vartype</th>\n",
- " <th>class_values</th>\n",
- " <th>buffer_size</th>\n",
- " <th>normalizing_const</th>\n",
- " <th>num_classes</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>test_gte5</td>\n",
- " <td>test_gte5_packed</td>\n",
- " <td>y</td>\n",
- " <td>x</td>\n",
- " <td>text</td>\n",
- " <td>[u'0', u'1', u'2', u'3', u'4']</td>\n",
- " <td>2431</td>\n",
- " <td>255.0</td>\n",
- " <td>5</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'test_gte5', u'test_gte5_packed', u'y', u'x', u'text', [u'0', u'1', u'2', u'3', u'4'], 2431, 255.0, 5)]"
- ]
- },
- "execution_count": 14,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS test_gte5_packed, test_gte5_packed_summary;\n",
- "\n",
- "SELECT madlib.validation_preprocessor_dl('test_gte5', -- Source table\n",
- " 'test_gte5_packed', -- Output table\n",
- " 'y', -- Dependent variable\n",
- " 'x', -- Independent variable\n",
- " 'train_gte5_packed' -- Training preproc table\n",
- " );\n",
- "\n",
- "SELECT * FROM test_gte5_packed_summary;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"define_and_load_model\"></a>\n",
- "# 4. Define and load model architecture\n",
- "\n",
- "Model with feature and classification layers trainable"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 15,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "_________________________________________________________________\n",
- "Layer (type) Output Shape Param # \n",
- "=================================================================\n",
- "conv2d_1 (Conv2D) (None, 26, 26, 32) 320 \n",
- "_________________________________________________________________\n",
- "activation_1 (Activation) (None, 26, 26, 32) 0 \n",
- "_________________________________________________________________\n",
- "conv2d_2 (Conv2D) (None, 24, 24, 32) 9248 \n",
- "_________________________________________________________________\n",
- "activation_2 (Activation) (None, 24, 24, 32) 0 \n",
- "_________________________________________________________________\n",
- "max_pooling2d_1 (MaxPooling2 (None, 12, 12, 32) 0 \n",
- "_________________________________________________________________\n",
- "dropout_1 (Dropout) (None, 12, 12, 32) 0 \n",
- "_________________________________________________________________\n",
- "flatten_1 (Flatten) (None, 4608) 0 \n",
- "_________________________________________________________________\n",
- "dense_1 (Dense) (None, 128) 589952 \n",
- "_________________________________________________________________\n",
- "activation_3 (Activation) (None, 128) 0 \n",
- "_________________________________________________________________\n",
- "dropout_2 (Dropout) (None, 128) 0 \n",
- "_________________________________________________________________\n",
- "dense_2 (Dense) (None, 5) 645 \n",
- "_________________________________________________________________\n",
- "activation_4 (Activation) (None, 5) 0 \n",
- "=================================================================\n",
- "Total params: 600,165\n",
- "Trainable params: 600,165\n",
- "Non-trainable params: 0\n",
- "_________________________________________________________________\n"
- ]
- }
- ],
- "source": [
- "# define two groups of layers: feature (convolutions) and classification (dense)\n",
- "feature_layers = [\n",
- " Conv2D(filters, kernel_size,\n",
- " padding='valid',\n",
- " input_shape=input_shape),\n",
- " Activation('relu'),\n",
- " Conv2D(filters, kernel_size),\n",
- " Activation('relu'),\n",
- " MaxPooling2D(pool_size=pool_size),\n",
- " Dropout(0.25),\n",
- " Flatten(),\n",
- "]\n",
- "\n",
- "classification_layers = [\n",
- " Dense(128),\n",
- " Activation('relu'),\n",
- " Dropout(0.5),\n",
- " Dense(num_classes),\n",
- " Activation('softmax')\n",
- "]\n",
- "\n",
- "# create complete model\n",
- "model = Sequential(feature_layers + classification_layers)\n",
- "\n",
- "model.summary()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Load into model architecture table using psycopg2"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 16,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>model_id</th>\n",
- " <th>name</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1</td>\n",
- " <td>feature + classification layers trainable</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(1, u'feature + classification layers trainable')]"
- ]
- },
- "execution_count": 16,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "import psycopg2 as p2\n",
- "conn = p2.connect('postgresql://gpadmin@35.239.240.26:5432/madlib')\n",
- "cur = conn.cursor()\n",
- "\n",
- "%sql DROP TABLE IF EXISTS model_arch_library;\n",
- "query = \"SELECT madlib.load_keras_model('model_arch_library', %s, NULL, %s)\"\n",
- "cur.execute(query,[model.to_json(), \"feature + classification layers trainable\"])\n",
- "conn.commit()\n",
- "\n",
- "# check model loaded OK\n",
- "%sql SELECT model_id, name FROM model_arch_library;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Model with feature layers frozen"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 17,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "_________________________________________________________________\n",
- "Layer (type) Output Shape Param # \n",
- "=================================================================\n",
- "conv2d_1 (Conv2D) (None, 26, 26, 32) 320 \n",
- "_________________________________________________________________\n",
- "activation_1 (Activation) (None, 26, 26, 32) 0 \n",
- "_________________________________________________________________\n",
- "conv2d_2 (Conv2D) (None, 24, 24, 32) 9248 \n",
- "_________________________________________________________________\n",
- "activation_2 (Activation) (None, 24, 24, 32) 0 \n",
- "_________________________________________________________________\n",
- "max_pooling2d_1 (MaxPooling2 (None, 12, 12, 32) 0 \n",
- "_________________________________________________________________\n",
- "dropout_1 (Dropout) (None, 12, 12, 32) 0 \n",
- "_________________________________________________________________\n",
- "flatten_1 (Flatten) (None, 4608) 0 \n",
- "_________________________________________________________________\n",
- "dense_1 (Dense) (None, 128) 589952 \n",
- "_________________________________________________________________\n",
- "activation_3 (Activation) (None, 128) 0 \n",
- "_________________________________________________________________\n",
- "dropout_2 (Dropout) (None, 128) 0 \n",
- "_________________________________________________________________\n",
- "dense_2 (Dense) (None, 5) 645 \n",
- "_________________________________________________________________\n",
- "activation_4 (Activation) (None, 5) 0 \n",
- "=================================================================\n",
- "Total params: 600,165\n",
- "Trainable params: 590,597\n",
- "Non-trainable params: 9,568\n",
- "_________________________________________________________________\n"
- ]
- }
- ],
- "source": [
- "# freeze feature layers\n",
- "for l in feature_layers:\n",
- " l.trainable = False\n",
- "\n",
- "model.summary()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Load into transfer model architecture table using psycopg2"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 18,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "2 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>model_id</th>\n",
- " <th>name</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1</td>\n",
- " <td>feature + classification layers trainable</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>2</td>\n",
- " <td>only classification layers trainable</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(1, u'feature + classification layers trainable'),\n",
- " (2, u'only classification layers trainable')]"
- ]
- },
- "execution_count": 18,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "cur.execute(query,[model.to_json(), \"only classification layers trainable\"])\n",
- "conn.commit()\n",
- "\n",
- "# check model loaded OK\n",
- "%sql SELECT model_id, name FROM model_arch_library ORDER BY model_id;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"train\"></a>\n",
- "# 5. Train\n",
- "Train the model for 5-digit classification [0..4] "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 22,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>madlib_keras_fit</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td></td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[('',)]"
- ]
- },
- "execution_count": 22,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS mnist_model, mnist_model_summary;\n",
- "\n",
- "SELECT madlib.madlib_keras_fit('train_lt5_packed', -- source table\n",
- " 'mnist_model', -- model output table\n",
- " 'model_arch_library', -- model arch table\n",
- " 1, -- model arch id\n",
- " $$ loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy']$$, -- compile_params\n",
- " $$ batch_size=128, epochs=1 $$, -- fit_params\n",
- " 5 -- num_iterations\n",
- " );"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "View the model summary:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 23,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>source_table</th>\n",
- " <th>model</th>\n",
- " <th>dependent_varname</th>\n",
- " <th>independent_varname</th>\n",
- " <th>model_arch_table</th>\n",
- " <th>model_arch_id</th>\n",
- " <th>compile_params</th>\n",
- " <th>fit_params</th>\n",
- " <th>num_iterations</th>\n",
- " <th>validation_table</th>\n",
- " <th>metrics_compute_frequency</th>\n",
- " <th>name</th>\n",
- " <th>description</th>\n",
- " <th>model_type</th>\n",
- " <th>model_size</th>\n",
- " <th>start_training_time</th>\n",
- " <th>end_training_time</th>\n",
- " <th>metrics_elapsed_time</th>\n",
- " <th>madlib_version</th>\n",
- " <th>num_classes</th>\n",
- " <th>class_values</th>\n",
- " <th>dependent_vartype</th>\n",
- " <th>normalizing_const</th>\n",
- " <th>metrics_type</th>\n",
- " <th>training_metrics_final</th>\n",
- " <th>training_loss_final</th>\n",
- " <th>training_metrics</th>\n",
- " <th>training_loss</th>\n",
- " <th>validation_metrics_final</th>\n",
- " <th>validation_loss_final</th>\n",
- " <th>validation_metrics</th>\n",
- " <th>validation_loss</th>\n",
- " <th>metrics_iters</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>train_lt5_packed</td>\n",
- " <td>mnist_model</td>\n",
- " <td>y</td>\n",
- " <td>x</td>\n",
- " <td>model_arch_library</td>\n",
- " <td>1</td>\n",
- " <td> loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy']</td>\n",
- " <td> batch_size=128, epochs=1 </td>\n",
- " <td>5</td>\n",
- " <td>None</td>\n",
- " <td>5</td>\n",
- " <td>None</td>\n",
- " <td>None</td>\n",
- " <td>madlib_keras</td>\n",
- " <td>2344.43066406</td>\n",
- " <td>2019-06-24 19:08:31.328530</td>\n",
- " <td>2019-06-24 19:13:50.944601</td>\n",
- " <td>[319.616029977798]</td>\n",
- " <td>1.16-dev</td>\n",
- " <td>5</td>\n",
- " <td>[u'0', u'1', u'2', u'3', u'4']</td>\n",
- " <td>text</td>\n",
- " <td>255.0</td>\n",
- " <td>[u'accuracy']</td>\n",
- " <td>0.996045231819</td>\n",
- " <td>0.0139331035316</td>\n",
- " <td>[0.996045231819153]</td>\n",
- " <td>[0.013933103531599]</td>\n",
- " <td>None</td>\n",
- " <td>None</td>\n",
- " <td>None</td>\n",
- " <td>None</td>\n",
- " <td>[5]</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'train_lt5_packed', u'mnist_model', u'y', u'x', u'model_arch_library', 1, u\" loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy']\", u' batch_size=128, epochs=1 ', 5, None, 5, None, None, u'madlib_keras', 2344.43066406, datetime.datetime(2019, 6, 24, 19, 8, 31, 328530), datetime.datetime(2019, 6, 24, 19, 13, 50, 944601), [319.616029977798], u'1.16-dev', 5, [u'0', u'1', u'2', u'3', u'4'], u'text', 255.0, [u'accuracy'], 0.996045231819, 0.0139331035316, [0 [...]
- ]
- },
- "execution_count": 23,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT * FROM mnist_model_summary;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Evaluate using test data"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 24,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>loss</th>\n",
- " <th>metric</th>\n",
- " <th>metrics_type</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>0.00919340737164</td>\n",
- " <td>0.997081160545</td>\n",
- " <td>[u'accuracy']</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(0.00919340737164021, 0.997081160545349, [u'accuracy'])]"
- ]
- },
- "execution_count": 24,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS mnist_validate;\n",
- "\n",
- "SELECT madlib.madlib_keras_evaluate('mnist_model', -- model\n",
- " 'test_lt5_packed', -- test table\n",
- " 'mnist_validate' -- output table\n",
- " );\n",
- "\n",
- "SELECT * FROM mnist_validate;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"transfer_learning\"></a>\n",
- "# 6. Transfer learning"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Use UPDATE to load trained weights from previous run into the model library table:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 25,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "[]"
- ]
- },
- "execution_count": 25,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "UPDATE model_arch_library SET model_weights = model_data FROM mnist_model WHERE model_id = 2;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Transfer: train dense layers for new classification task [5..9]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 26,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>madlib_keras_fit</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td></td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[('',)]"
- ]
- },
- "execution_count": 26,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS mnist_transfer_model, mnist_transfer_model_summary;\n",
- "\n",
- "SELECT madlib.madlib_keras_fit('train_gte5_packed', -- source table\n",
- " 'mnist_transfer_model',-- model output table\n",
- " 'model_arch_library', -- model arch table\n",
- " 2, -- model arch id\n",
- " $$ loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy']$$, -- compile_params\n",
- " $$ batch_size=128, epochs=1 $$, -- fit_params\n",
- " 5 -- num_iterations\n",
- " );"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "View the model summary"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 27,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>source_table</th>\n",
- " <th>model</th>\n",
- " <th>dependent_varname</th>\n",
- " <th>independent_varname</th>\n",
- " <th>model_arch_table</th>\n",
- " <th>model_arch_id</th>\n",
- " <th>compile_params</th>\n",
- " <th>fit_params</th>\n",
- " <th>num_iterations</th>\n",
- " <th>validation_table</th>\n",
- " <th>metrics_compute_frequency</th>\n",
- " <th>name</th>\n",
- " <th>description</th>\n",
- " <th>model_type</th>\n",
- " <th>model_size</th>\n",
- " <th>start_training_time</th>\n",
- " <th>end_training_time</th>\n",
- " <th>metrics_elapsed_time</th>\n",
- " <th>madlib_version</th>\n",
- " <th>num_classes</th>\n",
- " <th>class_values</th>\n",
- " <th>dependent_vartype</th>\n",
- " <th>normalizing_const</th>\n",
- " <th>metrics_type</th>\n",
- " <th>training_metrics_final</th>\n",
- " <th>training_loss_final</th>\n",
- " <th>training_metrics</th>\n",
- " <th>training_loss</th>\n",
- " <th>validation_metrics_final</th>\n",
- " <th>validation_loss_final</th>\n",
- " <th>validation_metrics</th>\n",
- " <th>validation_loss</th>\n",
- " <th>metrics_iters</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>train_gte5_packed</td>\n",
- " <td>mnist_transfer_model</td>\n",
- " <td>y</td>\n",
- " <td>x</td>\n",
- " <td>model_arch_library</td>\n",
- " <td>2</td>\n",
- " <td> loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy']</td>\n",
- " <td> batch_size=128, epochs=1 </td>\n",
- " <td>5</td>\n",
- " <td>None</td>\n",
- " <td>5</td>\n",
- " <td>None</td>\n",
- " <td>None</td>\n",
- " <td>madlib_keras</td>\n",
- " <td>2344.43066406</td>\n",
- " <td>2019-06-24 19:16:55.336042</td>\n",
- " <td>2019-06-24 19:19:53.589704</td>\n",
- " <td>[178.253571987152]</td>\n",
- " <td>1.16-dev</td>\n",
- " <td>5</td>\n",
- " <td>[u'0', u'1', u'2', u'3', u'4']</td>\n",
- " <td>text</td>\n",
- " <td>255.0</td>\n",
- " <td>[u'accuracy']</td>\n",
- " <td>0.991429746151</td>\n",
- " <td>0.0280887652189</td>\n",
- " <td>[0.99142974615097]</td>\n",
- " <td>[0.028088765218854]</td>\n",
- " <td>None</td>\n",
- " <td>None</td>\n",
- " <td>None</td>\n",
- " <td>None</td>\n",
- " <td>[5]</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'train_gte5_packed', u'mnist_transfer_model', u'y', u'x', u'model_arch_library', 2, u\" loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy']\", u' batch_size=128, epochs=1 ', 5, None, 5, None, None, u'madlib_keras', 2344.43066406, datetime.datetime(2019, 6, 24, 19, 16, 55, 336042), datetime.datetime(2019, 6, 24, 19, 19, 53, 589704), [178.253571987152], u'1.16-dev', 5, [u'0', u'1', u'2', u'3', u'4'], u'text', 255.0, [u'accuracy'], 0.991429746151, 0.028088 [...]
- ]
- },
- "execution_count": 27,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT * FROM mnist_transfer_model_summary;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Evaluate using test data"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 30,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>loss</th>\n",
- " <th>metric</th>\n",
- " <th>metrics_type</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>0.0312170274556</td>\n",
- " <td>0.989714026451</td>\n",
- " <td>[u'accuracy']</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(0.0312170274555683, 0.989714026451111, [u'accuracy'])]"
- ]
- },
- "execution_count": 30,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS mnist_transfer_validate;\n",
- "\n",
- "SELECT madlib.madlib_keras_evaluate('mnist_transfer_model', -- model\n",
- " 'test_gte5_packed', -- test table\n",
- " 'mnist_transfer_validate' -- output table\n",
- " );\n",
- "\n",
- "SELECT * FROM mnist_transfer_validate;"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 2",
- "language": "python",
- "name": "python2"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 2
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython2",
- "version": "2.7.10"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 1
-}
diff --git a/community-artifacts/Deep-learning/Preprocessor-for-images-v1.ipynb b/community-artifacts/Deep-learning/Preprocessor-for-images-v1.ipynb
deleted file mode 100644
index 2ab4b91..0000000
--- a/community-artifacts/Deep-learning/Preprocessor-for-images-v1.ipynb
+++ /dev/null
@@ -1,1625 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Preprocessor for image data\n",
- "This is a mini-batch preprocessor utility for image data:\n",
- "* training_preprocessor_dl() for training datasets\n",
- "* validation_preprocessor_dl() for validation datasets\n",
- "\n",
- "Note that there is a separate mini-batch preprocessor utility for general use cases\n",
- "http://madlib.apache.org/docs/latest/group__grp__minibatch__preprocessing.html\n",
- "\n",
- "The preprocessor for image data was added in MADlib 1.16.\n",
- "\n",
- "## Table of contents\n",
- "\n",
- "<a href=\"#load_data\">1. Load data</a>\n",
- "\n",
- "<a href=\"#pp_train\">2. Run preprocessor for training image data</a>\n",
- "\n",
- "<a href=\"#pp_val\">3. Run preprocessor for validation image data</a>\n",
- "\n",
- "<a href=\"#load_data2\">4. Load data, another format</a>\n",
- "\n",
- "<a href=\"#pp_train2\">5. Run preprocessor for training image data</a>\n",
- "\n",
- "<a href=\"#pp_val2\">6. Run preprocessor for validation image data</a>\n",
- "\n",
- "<a href=\"#change_buffer\">7. Change buffer size</a>\n",
- "\n",
- "<a href=\"#set_num_classes\">8. Setting number of classes</a>"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/config.py:13: ShimWarning: The `IPython.config` package has been deprecated since IPython 4.0. You should import from traitlets.config instead.\n",
- " \"You should import from traitlets.config instead.\", ShimWarning)\n",
- "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/utils/traitlets.py:5: UserWarning: IPython.utils.traitlets has moved to a top-level traitlets package.\n",
- " warn(\"IPython.utils.traitlets has moved to a top-level traitlets package.\")\n"
- ]
- }
- ],
- "source": [
- "%load_ext sql"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "u'Connected: gpadmin@madlib'"
- ]
- },
- "execution_count": 2,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "# Greenplum Database 5.x on GCP (PM demo machine)\n",
- "%sql postgresql://gpadmin@35.239.240.26:5432/madlib\n",
- " \n",
- "# PostgreSQL local\n",
- "#%sql postgresql://fmcquillan@localhost:5432/madlib"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>version</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>MADlib version: 1.16-dev, git revision: rel/v1.15.1-98-g544a8e5, cmake configuration time: Mon May 20 16:40:50 UTC 2019, build type: release, build system: Linux-3.10.0-957.12.1.el7.x86_64, C compiler: gcc 4.8.5, C++ compiler: g++ 4.8.5</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'MADlib version: 1.16-dev, git revision: rel/v1.15.1-98-g544a8e5, cmake configuration time: Mon May 20 16:40:50 UTC 2019, build type: release, build system: Linux-3.10.0-957.12.1.el7.x86_64, C compiler: gcc 4.8.5, C++ compiler: g++ 4.8.5',)]"
- ]
- },
- "execution_count": 3,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%sql select madlib.version();\n",
- "#%sql select version();"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"load_data\"></a>\n",
- "# 1. Load data\n",
- "\n",
- "Create an artificial 2x2 resolution color image data set with 3 possible classifications. The RGB values are per-pixel arrays:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "52 rows affected.\n",
- "52 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>rgb</th>\n",
- " <th>species</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[76, 125, 240], [191, 13, 20]], [[153, 77, 7], [41, 143, 172]]]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[15, 126, 174], [246, 129, 81]], [[143, 220, 157], [96, 207, 223]]]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[84, 24, 1], [201, 28, 77]], [[70, 12, 11], [83, 33, 165]]]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[40, 206, 47], [211, 138, 62]], [[82, 56, 52], [210, 137, 195]]]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[38, 35, 125], [5, 18, 209]], [[29, 19, 153], [57, 95, 223]]]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[107, 50, 102], [15, 210, 142]], [[222, 1, 97], [103, 63, 179]]]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[115, 133, 40], [55, 51, 78]], [[89, 176, 83], [108, 129, 112]]]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[27, 169, 222], [249, 239, 73]], [[43, 85, 88], [253, 227, 54]]]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[68, 157, 61], [246, 60, 176]], [[123, 100, 230], [175, 178, 64]]]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[4, 172, 224], [116, 42, 251]], [[30, 8, 244], [12, 81, 31]]]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[169, 28, 68], [223, 26, 136]], [[124, 87, 126], [184, 7, 250]]]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[237, 168, 205], [45, 7, 210]], [[217, 231, 70], [3, 226, 100]]]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[214, 112, 91], [246, 209, 4]], [[18, 21, 227], [44, 157, 95]]]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[27, 22, 138], [21, 50, 119]], [[189, 255, 164], [196, 209, 125]]]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[23, 128, 141], [123, 138, 99]], [[236, 230, 88], [189, 234, 106]]]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[205, 151, 111], [44, 26, 139]], [[66, 163, 159], [116, 26, 92]]]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[191, 32, 68], [60, 203, 92]], [[188, 88, 215], [70, 186, 195]]]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[19, 128, 22], [125, 82, 227]], [[20, 193, 14], [45, 76, 80]]]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[235, 196, 235], [71, 55, 170]], [[103, 123, 230], [50, 215, 161]]]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[174, 231, 68], [112, 19, 87]], [[240, 41, 212], [66, 12, 232]]]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[26, 21, 79], [106, 230, 59]], [[46, 209, 130], [101, 123, 233]]]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[96, 27, 183], [1, 164, 100]], [[232, 232, 213], [251, 62, 197]]]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[18, 7, 47], [250, 10, 73]], [[15, 89, 180], [244, 148, 226]]]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[22, 71, 64], [255, 39, 160]], [[26, 222, 161], [190, 66, 137]]]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[23, 132, 228], [220, 168, 247]], [[226, 215, 241], [236, 32, 255]]]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[212, 244, 217], [182, 185, 239]], [[253, 249, 238], [36, 153, 7]]]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[202, 170, 34], [234, 24, 7]], [[99, 34, 11], [185, 160, 246]]]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[231, 138, 156], [250, 92, 165]], [[215, 8, 125], [201, 61, 208]]]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[6, 175, 244], [189, 184, 190]], [[103, 218, 167], [127, 225, 10]]]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[236, 195, 65], [226, 86, 41]], [[108, 242, 35], [200, 150, 250]]]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[19, 196, 13], [228, 219, 19]], [[147, 207, 208], [75, 141, 54]]]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[52, 181, 6], [63, 87, 243]], [[2, 152, 212], [88, 193, 64]]]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[227, 8, 224], [222, 216, 243]], [[161, 229, 215], [125, 248, 106]]]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[200, 181, 217], [254, 218, 13]], [[179, 224, 76], [10, 210, 78]]]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[167, 166, 99], [231, 239, 70]], [[239, 207, 36], [200, 194, 197]]]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[152, 66, 164], [2, 142, 108]], [[182, 102, 106], [144, 116, 29]]]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[191, 39, 66], [13, 202, 233]], [[179, 44, 209], [162, 114, 192]]]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[150, 136, 50], [91, 52, 202]], [[157, 217, 204], [43, 68, 130]]]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[174, 18, 5], [204, 130, 196]], [[243, 197, 210], [189, 174, 133]]]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[127, 38, 91], [63, 151, 242]], [[198, 201, 77], [250, 147, 234]]]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[95, 21, 24], [226, 167, 198]], [[244, 172, 146], [119, 113, 133]]]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[66, 66, 234], [199, 43, 105]], [[237, 134, 168], [132, 120, 110]]]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[197, 104, 225], [175, 59, 64]], [[197, 83, 34], [108, 25, 22]]]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[171, 141, 136], [48, 201, 203]], [[113, 179, 145], [156, 27, 127]]]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[195, 3, 155], [49, 80, 96]], [[153, 49, 15], [212, 113, 212]]]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[147, 63, 64], [169, 87, 235]], [[54, 223, 26], [254, 170, 139]]]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[59, 40, 204], [186, 74, 143]], [[189, 229, 192], [14, 69, 89]]]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[83, 45, 175], [39, 84, 66]], [[102, 149, 235], [189, 127, 32]]]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[153, 31, 71], [37, 207, 130]], [[76, 155, 61], [151, 42, 250]]]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[234, 7, 193], [67, 70, 20]], [[112, 245, 59], [196, 55, 161]]]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[34, 95, 216], [67, 252, 113]], [[97, 67, 150], [49, 197, 226]]]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[2, 121, 246], [252, 245, 224]], [[3, 182, 35], [73, 202, 147]]]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[([[[76, 125, 240], [191, 13, 20]], [[153, 77, 7], [41, 143, 172]]], u'bird'),\n",
- " ([[[15, 126, 174], [246, 129, 81]], [[143, 220, 157], [96, 207, 223]]], u'bird'),\n",
- " ([[[84, 24, 1], [201, 28, 77]], [[70, 12, 11], [83, 33, 165]]], u'dog'),\n",
- " ([[[40, 206, 47], [211, 138, 62]], [[82, 56, 52], [210, 137, 195]]], u'bird'),\n",
- " ([[[38, 35, 125], [5, 18, 209]], [[29, 19, 153], [57, 95, 223]]], u'cat'),\n",
- " ([[[107, 50, 102], [15, 210, 142]], [[222, 1, 97], [103, 63, 179]]], u'dog'),\n",
- " ([[[115, 133, 40], [55, 51, 78]], [[89, 176, 83], [108, 129, 112]]], u'dog'),\n",
- " ([[[27, 169, 222], [249, 239, 73]], [[43, 85, 88], [253, 227, 54]]], u'bird'),\n",
- " ([[[68, 157, 61], [246, 60, 176]], [[123, 100, 230], [175, 178, 64]]], u'dog'),\n",
- " ([[[4, 172, 224], [116, 42, 251]], [[30, 8, 244], [12, 81, 31]]], u'dog'),\n",
- " ([[[169, 28, 68], [223, 26, 136]], [[124, 87, 126], [184, 7, 250]]], u'cat'),\n",
- " ([[[237, 168, 205], [45, 7, 210]], [[217, 231, 70], [3, 226, 100]]], u'cat'),\n",
- " ([[[214, 112, 91], [246, 209, 4]], [[18, 21, 227], [44, 157, 95]]], u'dog'),\n",
- " ([[[27, 22, 138], [21, 50, 119]], [[189, 255, 164], [196, 209, 125]]], u'bird'),\n",
- " ([[[23, 128, 141], [123, 138, 99]], [[236, 230, 88], [189, 234, 106]]], u'bird'),\n",
- " ([[[205, 151, 111], [44, 26, 139]], [[66, 163, 159], [116, 26, 92]]], u'dog'),\n",
- " ([[[191, 32, 68], [60, 203, 92]], [[188, 88, 215], [70, 186, 195]]], u'cat'),\n",
- " ([[[19, 128, 22], [125, 82, 227]], [[20, 193, 14], [45, 76, 80]]], u'bird'),\n",
- " ([[[235, 196, 235], [71, 55, 170]], [[103, 123, 230], [50, 215, 161]]], u'dog'),\n",
- " ([[[174, 231, 68], [112, 19, 87]], [[240, 41, 212], [66, 12, 232]]], u'cat'),\n",
- " ([[[26, 21, 79], [106, 230, 59]], [[46, 209, 130], [101, 123, 233]]], u'bird'),\n",
- " ([[[96, 27, 183], [1, 164, 100]], [[232, 232, 213], [251, 62, 197]]], u'cat'),\n",
- " ([[[18, 7, 47], [250, 10, 73]], [[15, 89, 180], [244, 148, 226]]], u'bird'),\n",
- " ([[[22, 71, 64], [255, 39, 160]], [[26, 222, 161], [190, 66, 137]]], u'dog'),\n",
- " ([[[23, 132, 228], [220, 168, 247]], [[226, 215, 241], [236, 32, 255]]], u'cat'),\n",
- " ([[[212, 244, 217], [182, 185, 239]], [[253, 249, 238], [36, 153, 7]]], u'cat'),\n",
- " ([[[202, 170, 34], [234, 24, 7]], [[99, 34, 11], [185, 160, 246]]], u'dog'),\n",
- " ([[[231, 138, 156], [250, 92, 165]], [[215, 8, 125], [201, 61, 208]]], u'cat'),\n",
- " ([[[6, 175, 244], [189, 184, 190]], [[103, 218, 167], [127, 225, 10]]], u'dog'),\n",
- " ([[[236, 195, 65], [226, 86, 41]], [[108, 242, 35], [200, 150, 250]]], u'bird'),\n",
- " ([[[19, 196, 13], [228, 219, 19]], [[147, 207, 208], [75, 141, 54]]], u'cat'),\n",
- " ([[[52, 181, 6], [63, 87, 243]], [[2, 152, 212], [88, 193, 64]]], u'cat'),\n",
- " ([[[227, 8, 224], [222, 216, 243]], [[161, 229, 215], [125, 248, 106]]], u'cat'),\n",
- " ([[[200, 181, 217], [254, 218, 13]], [[179, 224, 76], [10, 210, 78]]], u'dog'),\n",
- " ([[[167, 166, 99], [231, 239, 70]], [[239, 207, 36], [200, 194, 197]]], u'bird'),\n",
- " ([[[152, 66, 164], [2, 142, 108]], [[182, 102, 106], [144, 116, 29]]], u'dog'),\n",
- " ([[[191, 39, 66], [13, 202, 233]], [[179, 44, 209], [162, 114, 192]]], u'dog'),\n",
- " ([[[150, 136, 50], [91, 52, 202]], [[157, 217, 204], [43, 68, 130]]], u'dog'),\n",
- " ([[[174, 18, 5], [204, 130, 196]], [[243, 197, 210], [189, 174, 133]]], u'bird'),\n",
- " ([[[127, 38, 91], [63, 151, 242]], [[198, 201, 77], [250, 147, 234]]], u'bird'),\n",
- " ([[[95, 21, 24], [226, 167, 198]], [[244, 172, 146], [119, 113, 133]]], u'cat'),\n",
- " ([[[66, 66, 234], [199, 43, 105]], [[237, 134, 168], [132, 120, 110]]], u'cat'),\n",
- " ([[[197, 104, 225], [175, 59, 64]], [[197, 83, 34], [108, 25, 22]]], u'cat'),\n",
- " ([[[171, 141, 136], [48, 201, 203]], [[113, 179, 145], [156, 27, 127]]], u'cat'),\n",
- " ([[[195, 3, 155], [49, 80, 96]], [[153, 49, 15], [212, 113, 212]]], u'cat'),\n",
- " ([[[147, 63, 64], [169, 87, 235]], [[54, 223, 26], [254, 170, 139]]], u'bird'),\n",
- " ([[[59, 40, 204], [186, 74, 143]], [[189, 229, 192], [14, 69, 89]]], u'cat'),\n",
- " ([[[83, 45, 175], [39, 84, 66]], [[102, 149, 235], [189, 127, 32]]], u'dog'),\n",
- " ([[[153, 31, 71], [37, 207, 130]], [[76, 155, 61], [151, 42, 250]]], u'dog'),\n",
- " ([[[234, 7, 193], [67, 70, 20]], [[112, 245, 59], [196, 55, 161]]], u'dog'),\n",
- " ([[[34, 95, 216], [67, 252, 113]], [[97, 67, 150], [49, 197, 226]]], u'bird'),\n",
- " ([[[2, 121, 246], [252, 245, 224]], [[3, 182, 35], [73, 202, 147]]], u'cat')]"
- ]
- },
- "execution_count": 4,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS image_data;\n",
- "\n",
- "CREATE TABLE image_data AS (\n",
- " SELECT ARRAY[\n",
- " ARRAY[\n",
- " ARRAY[(random() * 256)::integer, -- pixel (1,1)\n",
- " (random() * 256)::integer,\n",
- " (random() * 256)::integer],\n",
- " ARRAY[(random() * 256)::integer, -- pixel (2,1)\n",
- " (random() * 256)::integer,\n",
- " (random() * 256)::integer]\n",
- " ],\n",
- " ARRAY[\n",
- " ARRAY[(random() * 256)::integer, -- pixel (1,2)\n",
- " (random() * 256)::integer,\n",
- " (random() * 256)::integer],\n",
- " ARRAY[(random() * 256)::integer, -- pixel (2,1)\n",
- " (random() * 256)::integer,\n",
- " (random() * 256)::integer]\n",
- " ]\n",
- " ] as rgb, ('{cat,dog,bird}'::text[])[ceil(random()*3)] as species\n",
- " FROM generate_series(1, 52)\n",
- ");\n",
- "\n",
- "SELECT * FROM image_data;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"pp_train\"></a>\n",
- "# 2. Run preprocessor for training image data\n",
- "\n",
- "Run the preprocessor to generate the packed output table:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "2 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>independent_var</th>\n",
- " <th>dependent_var</th>\n",
- " <th>buffer_id</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.0862745, 0.278431, 0.25098], [1.0, 0.152941, 0.627451]], [[0.101961, 0.870588, 0.631373], [0.745098, 0.258824, 0.537255]]], [[[0.0588235, 0.494118, 0.682353], [0.964706, 0.505882, 0.317647]], [[0.560784, 0.862745, 0.615686], [0.376471, 0.811765, 0.87451]]], [[[0.156863, 0.807843, 0.184314], [0.827451, 0.541176, 0.243137]], [[0.321569, 0.219608, 0.203922], [0.823529, 0.537255, 0.764706]]], [[[0.419608, 0.196078, 0.4], [0.0588235, 0.823529, 0.556863]], [[0.870588, [...]
- " <td>[[0, 0, 1], [1, 0, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [1, 0, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 0, 1], [1, 0, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0]]</td>\n",
- " <td>0</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.45098, 0.521569, 0.156863], [0.215686, 0.2, 0.305882]], [[0.34902, 0.690196, 0.32549], [0.423529, 0.505882, 0.439216]]], [[[0.0901961, 0.501961, 0.552941], [0.482353, 0.541176, 0.388235]], [[0.92549, 0.901961, 0.345098], [0.741176, 0.917647, 0.415686]]], [[[0.921569, 0.768628, 0.921569], [0.278431, 0.215686, 0.666667]], [[0.403922, 0.482353, 0.901961], [0.196078, 0.843137, 0.631373]]], [[[0.772549, 0.407843, 0.882353], [0.686275, 0.231373, 0.25098]], [[0.772549, [...]
- " <td>[[0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0]]</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[([[[[0.0862745, 0.278431, 0.25098], [1.0, 0.152941, 0.627451]], [[0.101961, 0.870588, 0.631373], [0.745098, 0.258824, 0.537255]]], [[[0.0588235, 0.494118, 0.682353], [0.964706, 0.505882, 0.317647]], [[0.560784, 0.862745, 0.615686], [0.376471, 0.811765, 0.87451]]], [[[0.156863, 0.807843, 0.184314], [0.827451, 0.541176, 0.243137]], [[0.321569, 0.219608, 0.203922], [0.823529, 0.537255, 0.764706]]], [[[0.419608, 0.196078, 0.4], [0.0588235, 0.823529, 0.556863]], [[0.870588, 0.0039215 [...]
- " ([[[[0.45098, 0.521569, 0.156863], [0.215686, 0.2, 0.305882]], [[0.34902, 0.690196, 0.32549], [0.423529, 0.505882, 0.439216]]], [[[0.0901961, 0.501961, 0.552941], [0.482353, 0.541176, 0.388235]], [[0.92549, 0.901961, 0.345098], [0.741176, 0.917647, 0.415686]]], [[[0.921569, 0.768628, 0.921569], [0.278431, 0.215686, 0.666667]], [[0.403922, 0.482353, 0.901961], [0.196078, 0.843137, 0.631373]]], [[[0.772549, 0.407843, 0.882353], [0.686275, 0.231373, 0.25098]], [[0.772549, 0.32549, [...]
- ]
- },
- "execution_count": 5,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS image_data_packed, image_data_packed_summary;\n",
- "\n",
- "SELECT madlib.training_preprocessor_dl('image_data', -- Source table\n",
- " 'image_data_packed', -- Output table\n",
- " 'species', -- Dependent variable\n",
- " 'rgb', -- Independent variable\n",
- " NULL, -- Buffer size\n",
- " 255 -- Normalizing constant\n",
- " );\n",
- "\n",
- "SELECT * FROM image_data_packed ORDER BY buffer_id;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "For small datasets like in this example, buffer size is mainly determined by the number of segments in the database. For a Greenplum database with 2 segments, there will be 2 rows with a buffer size of 26. For PostgresSQL, there would be only one row with a buffer size of 52 since it is a single node database. For larger data sets, other factors go into computing buffers size besides number of segments. \n",
- "\n",
- "Review the output summary table:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 6,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>source_table</th>\n",
- " <th>output_table</th>\n",
- " <th>dependent_varname</th>\n",
- " <th>independent_varname</th>\n",
- " <th>dependent_vartype</th>\n",
- " <th>class_values</th>\n",
- " <th>buffer_size</th>\n",
- " <th>normalizing_const</th>\n",
- " <th>num_classes</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>image_data</td>\n",
- " <td>image_data_packed</td>\n",
- " <td>species</td>\n",
- " <td>rgb</td>\n",
- " <td>text</td>\n",
- " <td>[u'bird', u'cat', u'dog']</td>\n",
- " <td>26</td>\n",
- " <td>255.0</td>\n",
- " <td>3</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'image_data', u'image_data_packed', u'species', u'rgb', u'text', [u'bird', u'cat', u'dog'], 26, Decimal('255.0'), 3)]"
- ]
- },
- "execution_count": 6,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT * FROM image_data_packed_summary;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"pp_val\"></a>\n",
- "# 3. Run preprocessor for validation image data\n",
- "\n",
- "Run the preprocessor for the validation dataset. In this example, we use the same images for validation to demonstrate, but normally validation data is different than training data:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 7,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "26 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>independent_var</th>\n",
- " <th>dependent_var</th>\n",
- " <th>buffer_id</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.290196, 0.105882, 0.152941], [0.47451, 0.670588, 0.356863]], [[0.101961, 0.458824, 0.188235], [0.584314, 0.921569, 0.854902]]], [[[0.941177, 0.92549, 0.34902], [0.137255, 0.360784, 0.411765]], [[0.0627451, 0.917647, 0.898039], [0.203922, 0.313726, 0.247059]]]]</td>\n",
- " <td>[[1, 0, 0], [0, 0, 1]]</td>\n",
- " <td>0</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.815686, 0.294118, 0.568627], [0.603922, 0.662745, 0.870588]], [[0.913726, 0.352941, 0.0745098], [0.0117647, 0.596078, 0.6]]], [[[0.0509804, 0.262745, 0.933333], [0.894118, 0.603922, 0.0901961]], [[0.643137, 0.12549, 0.623529], [0.0392157, 0.713726, 0.819608]]]]</td>\n",
- " <td>[[0, 1, 0], [0, 0, 1]]</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.14902, 0.52549, 0.6], [0.784314, 0.619608, 0.823529]], [[0.0823529, 0.0862745, 0.454902], [0.835294, 0.231373, 0.996078]]], [[[0.713726, 0.803922, 0.0156863], [0.678431, 0.415686, 0.470588]], [[0.156863, 0.85098, 0.941177], [0.27451, 0.141176, 0.72549]]]]</td>\n",
- " <td>[[1, 0, 0], [0, 0, 1]]</td>\n",
- " <td>2</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.584314, 0.890196, 0.94902], [0.419608, 0.709804, 0.760784]], [[0.713726, 0.270588, 0.360784], [0.372549, 0.141176, 0.270588]]], [[[0.819608, 0.823529, 0.0980392], [0.262745, 0.713726, 0.552941]], [[0.203922, 0.890196, 0.945098], [0.156863, 0.235294, 0.466667]]]]</td>\n",
- " <td>[[1, 0, 0], [0, 1, 0]]</td>\n",
- " <td>3</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.215686, 0.282353, 0.321569], [0.815686, 0.588235, 0.466667]], [[0.337255, 0.184314, 0.247059], [0.956863, 0.00392157, 0.329412]]], [[[0.129412, 0.388235, 0.270588], [0.980392, 0.623529, 0.984314]], [[0.780392, 0.639216, 0.658824], [0.192157, 0.105882, 0.815686]]]]</td>\n",
- " <td>[[0, 1, 0], [0, 1, 0]]</td>\n",
- " <td>4</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.458824, 0.160784, 0.270588], [0.45098, 0.94902, 0.854902]], [[0.337255, 0.894118, 0.27451], [0.0431373, 0.65098, 0.988235]]], [[[0.0431373, 0.0862745, 0.180392], [0.772549, 0.615686, 1.0]], [[0.588235, 0.713726, 0.254902], [0.298039, 0.262745, 0.458824]]]]</td>\n",
- " <td>[[0, 1, 0], [0, 1, 0]]</td>\n",
- " <td>5</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.00784314, 0.356863, 0.454902], [0.282353, 0.0784314, 0.670588]], [[0.564706, 0.4, 0.478431], [0.14902, 0.866667, 0.815686]]], [[[0.207843, 0.615686, 0.419608], [0.670588, 0.760784, 0.54902]], [[0.054902, 0.0313726, 0.52549], [0.678431, 0.0117647, 0.298039]]]]</td>\n",
- " <td>[[0, 1, 0], [0, 1, 0]]</td>\n",
- " <td>6</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.666667, 0.490196, 0.419608], [0.478431, 0.529412, 0.462745]], [[0.564706, 0.709804, 0.231373], [0.176471, 0.701961, 0.819608]]], [[[0.113725, 0.764706, 0.337255], [0.439216, 0.803922, 0.796079]], [[0.6, 0.0745098, 0.243137], [0.54902, 0.929412, 0.580392]]]]</td>\n",
- " <td>[[1, 0, 0], [0, 0, 1]]</td>\n",
- " <td>7</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.196078, 0.619608, 0.0862745], [0.180392, 0.933333, 0.0941177]], [[0.537255, 0.384314, 0.376471], [0.619608, 0.0509804, 0.941177]]], [[[0.960784, 0.113725, 0.14902], [0.415686, 0.301961, 0.356863]], [[0.027451, 0.721569, 0.0235294], [0.788235, 0.266667, 0.0784314]]]]</td>\n",
- " <td>[[0, 1, 0], [1, 0, 0]]</td>\n",
- " <td>8</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.529412, 0.0862745, 0.882353], [0.341176, 0.415686, 0.996078]], [[0.101961, 0.752941, 0.431373], [0.909804, 0.545098, 0.027451]]], [[[0.792157, 0.760784, 0.827451], [0.0862745, 0.0705882, 0.490196]], [[0.576471, 0.490196, 0.972549], [0.101961, 0.952941, 0.533333]]]]</td>\n",
- " <td>[[1, 0, 0], [1, 0, 0]]</td>\n",
- " <td>9</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.180392, 0.709804, 0.509804], [1.0, 0.592157, 0.466667]], [[0.113725, 0.741176, 0.882353], [0.415686, 0.0941177, 0.905882]]], [[[0.784314, 0.576471, 0.905882], [0.360784, 0.0117647, 0.0980392]], [[0.980392, 0.0980392, 0.282353], [0.913726, 0.196078, 0.819608]]]]</td>\n",
- " <td>[[0, 1, 0], [0, 1, 0]]</td>\n",
- " <td>10</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.572549, 0.431373, 0.345098], [0.509804, 0.447059, 0.87451]], [[0.592157, 0.32549, 0.211765], [0.00784314, 0.313726, 0.313726]]], [[[0.117647, 0.694118, 0.4], [0.196078, 0.505882, 0.188235]], [[0.956863, 0.329412, 0.27451], [0.0235294, 0.823529, 0.854902]]]]</td>\n",
- " <td>[[1, 0, 0], [0, 0, 1]]</td>\n",
- " <td>11</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.792157, 0.956863, 0.458824], [0.317647, 0.764706, 0.639216]], [[0.0235294, 0.270588, 0.635294], [0.615686, 0.737255, 0.74902]]], [[[0.745098, 0.219608, 0.301961], [0.776471, 0.196078, 0.0823529]], [[0.34902, 0.0980392, 0.443137], [0.360784, 0.196078, 0.419608]]]]</td>\n",
- " <td>[[0, 0, 1], [0, 0, 1]]</td>\n",
- " <td>12</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.478431, 0.329412, 0.654902], [0.290196, 0.623529, 0.223529]], [[0.721569, 0.964706, 0.729412], [0.164706, 0.835294, 0.321569]]], [[[0.615686, 0.156863, 0.447059], [0.521569, 0.290196, 0.564706]], [[0.207843, 0.690196, 0.760784], [0.717647, 0.878431, 0.713726]]]]</td>\n",
- " <td>[[0, 0, 1], [0, 1, 0]]</td>\n",
- " <td>13</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.0431373, 0.32549, 0.803922], [0.356863, 0.0823529, 0.54902]], [[0.572549, 0.384314, 0.321569], [0.768628, 0.466667, 0.670588]]], [[[0.14902, 0.737255, 0.866667], [0.0, 0.243137, 0.65098]], [[0.956863, 0.705882, 0.972549], [0.721569, 0.341176, 0.996078]]]]</td>\n",
- " <td>[[1, 0, 0], [1, 0, 0]]</td>\n",
- " <td>14</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.976471, 0.607843, 0.72549], [0.721569, 0.960784, 0.341176]], [[0.878431, 0.4, 0.858824], [0.164706, 0.964706, 0.0666667]]], [[[0.909804, 0.027451, 0.0588235], [0.32549, 0.486275, 0.537255]], [[0.658824, 0.137255, 0.827451], [0.27451, 0.360784, 0.545098]]]]</td>\n",
- " <td>[[1, 0, 0], [0, 1, 0]]</td>\n",
- " <td>15</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.0862745, 0.709804, 0.0705882], [0.403922, 0.196078, 0.109804]], [[0.733333, 1.0, 0.466667], [0.815686, 0.541176, 0.0352941]]], [[[0.721569, 0.780392, 0.729412], [0.431373, 0.823529, 0.882353]], [[0.164706, 0.686275, 0.882353], [0.407843, 0.333333, 0.835294]]]]</td>\n",
- " <td>[[0, 1, 0], [0, 1, 0]]</td>\n",
- " <td>16</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.862745, 0.803922, 0.662745], [0.529412, 0.666667, 0.568627]], [[0.552941, 0.72549, 0.894118], [0.0352941, 0.254902, 0.54902]]], [[[0.301961, 0.552941, 0.447059], [0.294118, 0.541176, 0.419608]], [[0.898039, 0.266667, 0.137255], [0.854902, 0.603922, 0.0117647]]]]</td>\n",
- " <td>[[0, 1, 0], [0, 1, 0]]</td>\n",
- " <td>17</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.0784314, 0.823529, 0.533333], [0.623529, 0.0588235, 0.619608]], [[0.329412, 0.12549, 0.0196078], [0.52549, 0.235294, 0.752941]]], [[[0.462745, 0.180392, 0.211765], [0.52549, 0.0313726, 0.933333]], [[0.305882, 0.760784, 0.360784], [0.12549, 0.639216, 0.52549]]]]</td>\n",
- " <td>[[0, 0, 1], [1, 0, 0]]</td>\n",
- " <td>18</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.701961, 0.564706, 0.0588235], [0.737255, 0.760784, 0.921569]], [[0.537255, 0.415686, 0.447059], [0.2, 0.984314, 1.0]]], [[[0.517647, 0.933333, 0.141176], [0.352941, 0.0352941, 0.447059]], [[0.905882, 0.486275, 0.737255], [0.443137, 0.905882, 0.631373]]]]</td>\n",
- " <td>[[1, 0, 0], [1, 0, 0]]</td>\n",
- " <td>19</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.878431, 0.0352941, 0.176471], [0.419608, 0.207843, 0.258824]], [[0.243137, 0.741176, 0.882353], [0.298039, 0.356863, 0.207843]]], [[[0.0392157, 0.482353, 0.309804], [0.0509804, 0.737255, 0.768628]], [[0.231373, 0.94902, 0.290196], [0.262745, 0.878431, 0.596078]]]]</td>\n",
- " <td>[[0, 0, 1], [0, 1, 0]]</td>\n",
- " <td>20</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.376471, 0.733333, 0.662745], [0.121569, 0.25098, 0.360784]], [[0.686275, 0.309804, 0.0941177], [0.443137, 0.231373, 0.631373]]], [[[0.239216, 0.721569, 0.658824], [0.764706, 0.529412, 0.172549]], [[0.694118, 0.670588, 0.52549], [0.729412, 0.113725, 0.427451]]]]</td>\n",
- " <td>[[1, 0, 0], [0, 1, 0]]</td>\n",
- " <td>21</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.85098, 0.866667, 0.113725], [0.478431, 0.572549, 0.152941]], [[0.964706, 0.878431, 0.203922], [0.694118, 0.647059, 0.431373]]], [[[0.67451, 0.831373, 0.839216], [0.67451, 0.752941, 0.713726]], [[0.705882, 0.933333, 0.129412], [0.917647, 0.184314, 0.372549]]]]</td>\n",
- " <td>[[0, 0, 1], [0, 0, 1]]</td>\n",
- " <td>22</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.0627451, 0.670588, 0.00392157], [0.270588, 0.0941177, 0.380392]], [[0.0, 0.752941, 0.505882], [0.25098, 0.109804, 0.188235]]], [[[0.937255, 0.694118, 0.513726], [0.529412, 0.713726, 0.752941]], [[0.247059, 0.368627, 0.513726], [0.776471, 0.541176, 0.2]]]]</td>\n",
- " <td>[[0, 0, 1], [0, 0, 1]]</td>\n",
- " <td>23</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.203922, 0.627451, 0.792157], [0.835294, 0.482353, 0.462745]], [[0.662745, 0.321569, 0.133333], [0.411765, 0.027451, 0.839216]]], [[[0.0627451, 0.929412, 0.552941], [0.490196, 0.137255, 0.4]], [[0.352941, 0.25098, 0.882353], [0.92549, 0.403922, 0.839216]]]]</td>\n",
- " <td>[[0, 0, 1], [1, 0, 0]]</td>\n",
- " <td>24</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[[[0.603922, 0.529412, 0.443137], [0.0352941, 0.164706, 0.376471]], [[0.729412, 0.678431, 0.905882], [0.439216, 0.427451, 0.14902]]], [[[0.160784, 0.752941, 0.52549], [0.533333, 0.403922, 0.588235]], [[0.2, 0.407843, 0.858824], [0.290196, 0.788235, 0.858824]]]]</td>\n",
- " <td>[[1, 0, 0], [0, 1, 0]]</td>\n",
- " <td>25</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[([[[[0.290196, 0.105882, 0.152941], [0.47451, 0.670588, 0.356863]], [[0.101961, 0.458824, 0.188235], [0.584314, 0.921569, 0.854902]]], [[[0.941177, 0.92549, 0.34902], [0.137255, 0.360784, 0.411765]], [[0.0627451, 0.917647, 0.898039], [0.203922, 0.313726, 0.247059]]]], [[1, 0, 0], [0, 0, 1]], 0),\n",
- " ([[[[0.815686, 0.294118, 0.568627], [0.603922, 0.662745, 0.870588]], [[0.913726, 0.352941, 0.0745098], [0.0117647, 0.596078, 0.6]]], [[[0.0509804, 0.262745, 0.933333], [0.894118, 0.603922, 0.0901961]], [[0.643137, 0.12549, 0.623529], [0.0392157, 0.713726, 0.819608]]]], [[0, 1, 0], [0, 0, 1]], 1),\n",
- " ([[[[0.14902, 0.52549, 0.6], [0.784314, 0.619608, 0.823529]], [[0.0823529, 0.0862745, 0.454902], [0.835294, 0.231373, 0.996078]]], [[[0.713726, 0.803922, 0.0156863], [0.678431, 0.415686, 0.470588]], [[0.156863, 0.85098, 0.941177], [0.27451, 0.141176, 0.72549]]]], [[1, 0, 0], [0, 0, 1]], 2),\n",
- " ([[[[0.584314, 0.890196, 0.94902], [0.419608, 0.709804, 0.760784]], [[0.713726, 0.270588, 0.360784], [0.372549, 0.141176, 0.270588]]], [[[0.819608, 0.823529, 0.0980392], [0.262745, 0.713726, 0.552941]], [[0.203922, 0.890196, 0.945098], [0.156863, 0.235294, 0.466667]]]], [[1, 0, 0], [0, 1, 0]], 3),\n",
- " ([[[[0.215686, 0.282353, 0.321569], [0.815686, 0.588235, 0.466667]], [[0.337255, 0.184314, 0.247059], [0.956863, 0.00392157, 0.329412]]], [[[0.129412, 0.388235, 0.270588], [0.980392, 0.623529, 0.984314]], [[0.780392, 0.639216, 0.658824], [0.192157, 0.105882, 0.815686]]]], [[0, 1, 0], [0, 1, 0]], 4),\n",
- " ([[[[0.458824, 0.160784, 0.270588], [0.45098, 0.94902, 0.854902]], [[0.337255, 0.894118, 0.27451], [0.0431373, 0.65098, 0.988235]]], [[[0.0431373, 0.0862745, 0.180392], [0.772549, 0.615686, 1.0]], [[0.588235, 0.713726, 0.254902], [0.298039, 0.262745, 0.458824]]]], [[0, 1, 0], [0, 1, 0]], 5),\n",
- " ([[[[0.00784314, 0.356863, 0.454902], [0.282353, 0.0784314, 0.670588]], [[0.564706, 0.4, 0.478431], [0.14902, 0.866667, 0.815686]]], [[[0.207843, 0.615686, 0.419608], [0.670588, 0.760784, 0.54902]], [[0.054902, 0.0313726, 0.52549], [0.678431, 0.0117647, 0.298039]]]], [[0, 1, 0], [0, 1, 0]], 6),\n",
- " ([[[[0.666667, 0.490196, 0.419608], [0.478431, 0.529412, 0.462745]], [[0.564706, 0.709804, 0.231373], [0.176471, 0.701961, 0.819608]]], [[[0.113725, 0.764706, 0.337255], [0.439216, 0.803922, 0.796079]], [[0.6, 0.0745098, 0.243137], [0.54902, 0.929412, 0.580392]]]], [[1, 0, 0], [0, 0, 1]], 7),\n",
- " ([[[[0.196078, 0.619608, 0.0862745], [0.180392, 0.933333, 0.0941177]], [[0.537255, 0.384314, 0.376471], [0.619608, 0.0509804, 0.941177]]], [[[0.960784, 0.113725, 0.14902], [0.415686, 0.301961, 0.356863]], [[0.027451, 0.721569, 0.0235294], [0.788235, 0.266667, 0.0784314]]]], [[0, 1, 0], [1, 0, 0]], 8),\n",
- " ([[[[0.529412, 0.0862745, 0.882353], [0.341176, 0.415686, 0.996078]], [[0.101961, 0.752941, 0.431373], [0.909804, 0.545098, 0.027451]]], [[[0.792157, 0.760784, 0.827451], [0.0862745, 0.0705882, 0.490196]], [[0.576471, 0.490196, 0.972549], [0.101961, 0.952941, 0.533333]]]], [[1, 0, 0], [1, 0, 0]], 9),\n",
- " ([[[[0.180392, 0.709804, 0.509804], [1.0, 0.592157, 0.466667]], [[0.113725, 0.741176, 0.882353], [0.415686, 0.0941177, 0.905882]]], [[[0.784314, 0.576471, 0.905882], [0.360784, 0.0117647, 0.0980392]], [[0.980392, 0.0980392, 0.282353], [0.913726, 0.196078, 0.819608]]]], [[0, 1, 0], [0, 1, 0]], 10),\n",
- " ([[[[0.572549, 0.431373, 0.345098], [0.509804, 0.447059, 0.87451]], [[0.592157, 0.32549, 0.211765], [0.00784314, 0.313726, 0.313726]]], [[[0.117647, 0.694118, 0.4], [0.196078, 0.505882, 0.188235]], [[0.956863, 0.329412, 0.27451], [0.0235294, 0.823529, 0.854902]]]], [[1, 0, 0], [0, 0, 1]], 11),\n",
- " ([[[[0.792157, 0.956863, 0.458824], [0.317647, 0.764706, 0.639216]], [[0.0235294, 0.270588, 0.635294], [0.615686, 0.737255, 0.74902]]], [[[0.745098, 0.219608, 0.301961], [0.776471, 0.196078, 0.0823529]], [[0.34902, 0.0980392, 0.443137], [0.360784, 0.196078, 0.419608]]]], [[0, 0, 1], [0, 0, 1]], 12),\n",
- " ([[[[0.478431, 0.329412, 0.654902], [0.290196, 0.623529, 0.223529]], [[0.721569, 0.964706, 0.729412], [0.164706, 0.835294, 0.321569]]], [[[0.615686, 0.156863, 0.447059], [0.521569, 0.290196, 0.564706]], [[0.207843, 0.690196, 0.760784], [0.717647, 0.878431, 0.713726]]]], [[0, 0, 1], [0, 1, 0]], 13),\n",
- " ([[[[0.0431373, 0.32549, 0.803922], [0.356863, 0.0823529, 0.54902]], [[0.572549, 0.384314, 0.321569], [0.768628, 0.466667, 0.670588]]], [[[0.14902, 0.737255, 0.866667], [0.0, 0.243137, 0.65098]], [[0.956863, 0.705882, 0.972549], [0.721569, 0.341176, 0.996078]]]], [[1, 0, 0], [1, 0, 0]], 14),\n",
- " ([[[[0.976471, 0.607843, 0.72549], [0.721569, 0.960784, 0.341176]], [[0.878431, 0.4, 0.858824], [0.164706, 0.964706, 0.0666667]]], [[[0.909804, 0.027451, 0.0588235], [0.32549, 0.486275, 0.537255]], [[0.658824, 0.137255, 0.827451], [0.27451, 0.360784, 0.545098]]]], [[1, 0, 0], [0, 1, 0]], 15),\n",
- " ([[[[0.0862745, 0.709804, 0.0705882], [0.403922, 0.196078, 0.109804]], [[0.733333, 1.0, 0.466667], [0.815686, 0.541176, 0.0352941]]], [[[0.721569, 0.780392, 0.729412], [0.431373, 0.823529, 0.882353]], [[0.164706, 0.686275, 0.882353], [0.407843, 0.333333, 0.835294]]]], [[0, 1, 0], [0, 1, 0]], 16),\n",
- " ([[[[0.862745, 0.803922, 0.662745], [0.529412, 0.666667, 0.568627]], [[0.552941, 0.72549, 0.894118], [0.0352941, 0.254902, 0.54902]]], [[[0.301961, 0.552941, 0.447059], [0.294118, 0.541176, 0.419608]], [[0.898039, 0.266667, 0.137255], [0.854902, 0.603922, 0.0117647]]]], [[0, 1, 0], [0, 1, 0]], 17),\n",
- " ([[[[0.0784314, 0.823529, 0.533333], [0.623529, 0.0588235, 0.619608]], [[0.329412, 0.12549, 0.0196078], [0.52549, 0.235294, 0.752941]]], [[[0.462745, 0.180392, 0.211765], [0.52549, 0.0313726, 0.933333]], [[0.305882, 0.760784, 0.360784], [0.12549, 0.639216, 0.52549]]]], [[0, 0, 1], [1, 0, 0]], 18),\n",
- " ([[[[0.701961, 0.564706, 0.0588235], [0.737255, 0.760784, 0.921569]], [[0.537255, 0.415686, 0.447059], [0.2, 0.984314, 1.0]]], [[[0.517647, 0.933333, 0.141176], [0.352941, 0.0352941, 0.447059]], [[0.905882, 0.486275, 0.737255], [0.443137, 0.905882, 0.631373]]]], [[1, 0, 0], [1, 0, 0]], 19),\n",
- " ([[[[0.878431, 0.0352941, 0.176471], [0.419608, 0.207843, 0.258824]], [[0.243137, 0.741176, 0.882353], [0.298039, 0.356863, 0.207843]]], [[[0.0392157, 0.482353, 0.309804], [0.0509804, 0.737255, 0.768628]], [[0.231373, 0.94902, 0.290196], [0.262745, 0.878431, 0.596078]]]], [[0, 0, 1], [0, 1, 0]], 20),\n",
- " ([[[[0.376471, 0.733333, 0.662745], [0.121569, 0.25098, 0.360784]], [[0.686275, 0.309804, 0.0941177], [0.443137, 0.231373, 0.631373]]], [[[0.239216, 0.721569, 0.658824], [0.764706, 0.529412, 0.172549]], [[0.694118, 0.670588, 0.52549], [0.729412, 0.113725, 0.427451]]]], [[1, 0, 0], [0, 1, 0]], 21),\n",
- " ([[[[0.85098, 0.866667, 0.113725], [0.478431, 0.572549, 0.152941]], [[0.964706, 0.878431, 0.203922], [0.694118, 0.647059, 0.431373]]], [[[0.67451, 0.831373, 0.839216], [0.67451, 0.752941, 0.713726]], [[0.705882, 0.933333, 0.129412], [0.917647, 0.184314, 0.372549]]]], [[0, 0, 1], [0, 0, 1]], 22),\n",
- " ([[[[0.0627451, 0.670588, 0.00392157], [0.270588, 0.0941177, 0.380392]], [[0.0, 0.752941, 0.505882], [0.25098, 0.109804, 0.188235]]], [[[0.937255, 0.694118, 0.513726], [0.529412, 0.713726, 0.752941]], [[0.247059, 0.368627, 0.513726], [0.776471, 0.541176, 0.2]]]], [[0, 0, 1], [0, 0, 1]], 23),\n",
- " ([[[[0.203922, 0.627451, 0.792157], [0.835294, 0.482353, 0.462745]], [[0.662745, 0.321569, 0.133333], [0.411765, 0.027451, 0.839216]]], [[[0.0627451, 0.929412, 0.552941], [0.490196, 0.137255, 0.4]], [[0.352941, 0.25098, 0.882353], [0.92549, 0.403922, 0.839216]]]], [[0, 0, 1], [1, 0, 0]], 24),\n",
- " ([[[[0.603922, 0.529412, 0.443137], [0.0352941, 0.164706, 0.376471]], [[0.729412, 0.678431, 0.905882], [0.439216, 0.427451, 0.14902]]], [[[0.160784, 0.752941, 0.52549], [0.533333, 0.403922, 0.588235]], [[0.2, 0.407843, 0.858824], [0.290196, 0.788235, 0.858824]]]], [[1, 0, 0], [0, 1, 0]], 25)]"
- ]
- },
- "execution_count": 7,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS val_image_data_packed, val_image_data_packed_summary;\n",
- "SELECT madlib.validation_preprocessor_dl(\n",
- " 'image_data', -- Source table\n",
- " 'val_image_data_packed', -- Output table\n",
- " 'species', -- Dependent variable\n",
- " 'rgb', -- Independent variable\n",
- " 'image_data_packed', -- From training preprocessor step\n",
- " 2 -- Buffer size\n",
- " );\n",
- "SELECT * FROM val_image_data_packed ORDER BY buffer_id;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Review the output summary table:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 8,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>source_table</th>\n",
- " <th>output_table</th>\n",
- " <th>dependent_varname</th>\n",
- " <th>independent_varname</th>\n",
- " <th>dependent_vartype</th>\n",
- " <th>class_values</th>\n",
- " <th>buffer_size</th>\n",
- " <th>normalizing_const</th>\n",
- " <th>num_classes</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>image_data</td>\n",
- " <td>val_image_data_packed</td>\n",
- " <td>species</td>\n",
- " <td>rgb</td>\n",
- " <td>text</td>\n",
- " <td>[u'bird', u'cat', u'dog']</td>\n",
- " <td>2</td>\n",
- " <td>255.0</td>\n",
- " <td>3</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'image_data', u'val_image_data_packed', u'species', u'rgb', u'text', [u'bird', u'cat', u'dog'], 2, Decimal('255.0'), 3)]"
- ]
- },
- "execution_count": 8,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT * FROM val_image_data_packed_summary;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"load_data2\"></a>\n",
- "# 4. Load data, another format\n",
- "Create an artificial 2x2 resolution color image data set with 3 possible classifications. The RGB values are unrolled in to a flat array:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 6,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "52 rows affected.\n",
- "52 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>rgb</th>\n",
- " <th>species</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[87, 118, 166, 176, 152, 5, 135, 219, 1, 249, 60, 67]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[109, 9, 113, 57, 10, 234, 47, 6, 223, 16, 9, 148]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[82, 94, 197, 145, 99, 28, 7, 8, 203, 159, 13, 83]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[14, 75, 182, 81, 218, 36, 90, 74, 93, 100, 52, 140]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[19, 156, 115, 167, 206, 198, 5, 147, 86, 104, 175, 93]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[122, 252, 125, 205, 118, 140, 24, 44, 221, 242, 80, 55]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[173, 155, 112, 57, 6, 131, 212, 121, 42, 162, 63, 47]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[149, 150, 227, 242, 6, 93, 238, 132, 42, 100, 15, 66]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[236, 52, 225, 36, 112, 141, 191, 224, 198, 197, 98, 154]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[140, 60, 125, 187, 113, 18, 81, 84, 5, 88, 178, 243]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[220, 87, 235, 30, 232, 216, 82, 200, 251, 194, 85, 186]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[27, 127, 5, 181, 189, 145, 241, 58, 76, 97, 76, 157]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[80, 245, 103, 67, 209, 67, 154, 188, 97, 130, 148, 179]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[143, 117, 160, 74, 23, 187, 200, 28, 111, 133, 173, 96]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[249, 193, 10, 150, 119, 91, 139, 222, 158, 92, 33, 56]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[130, 186, 172, 53, 4, 59, 170, 164, 133, 193, 94, 77]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[206, 210, 138, 46, 145, 131, 239, 156, 24, 102, 246, 163]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[148, 255, 101, 204, 23, 231, 134, 195, 27, 138, 254, 197]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[131, 134, 140, 208, 100, 90, 162, 238, 136, 52, 112, 119]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[137, 221, 198, 44, 34, 90, 42, 135, 38, 65, 109, 171]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[136, 54, 1, 78, 99, 132, 212, 239, 84, 56, 73, 246]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[209, 42, 150, 72, 249, 30, 37, 191, 74, 71, 24, 116]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[62, 181, 59, 233, 185, 195, 31, 187, 17, 130, 63, 229]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[147, 28, 187, 137, 66, 140, 179, 215, 211, 172, 246, 249]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[64, 63, 131, 180, 13, 193, 105, 72, 170, 35, 11, 201]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[28, 75, 28, 0, 189, 175, 29, 120, 56, 94, 3, 235]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[215, 151, 44, 207, 2, 107, 15, 133, 31, 28, 71, 137]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[197, 68, 195, 107, 92, 71, 80, 55, 239, 70, 26, 198]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[6, 230, 54, 75, 186, 42, 36, 112, 227, 19, 109, 220]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[145, 117, 8, 147, 175, 205, 215, 113, 57, 51, 184, 136]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[167, 207, 132, 109, 6, 138, 83, 60, 213, 13, 102, 249]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[73, 12, 234, 37, 13, 123, 154, 21, 14, 72, 226, 229]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[26, 24, 114, 163, 130, 25, 114, 6, 134, 119, 144, 217]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[101, 230, 25, 94, 99, 99, 106, 77, 136, 119, 199, 34]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[213, 106, 109, 186, 36, 136, 210, 151, 43, 84, 176, 156]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[54, 20, 235, 15, 199, 80, 245, 224, 174, 87, 67, 24]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[203, 143, 107, 237, 26, 65, 87, 136, 251, 123, 16, 205]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[58, 34, 193, 214, 124, 248, 234, 103, 7, 177, 183, 251]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[101, 83, 212, 125, 247, 159, 12, 98, 139, 38, 163, 226]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[157, 93, 189, 107, 110, 248, 140, 48, 206, 8, 39, 184]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[46, 105, 38, 41, 250, 139, 124, 206, 8, 115, 109, 19]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[248, 57, 119, 218, 231, 21, 55, 164, 127, 166, 156, 11]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[106, 20, 253, 34, 131, 43, 139, 170, 84, 133, 53, 208]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[60, 67, 191, 79, 24, 184, 136, 143, 146, 111, 164, 201]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[35, 110, 175, 47, 68, 25, 67, 65, 59, 198, 107, 198]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[191, 74, 164, 144, 157, 224, 211, 92, 48, 234, 20, 184]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[166, 39, 29, 110, 59, 65, 221, 234, 111, 33, 3, 178]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[62, 120, 205, 4, 231, 140, 78, 139, 28, 235, 108, 238]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[155, 217, 91, 83, 82, 0, 122, 111, 110, 181, 176, 75]</td>\n",
- " <td>dog</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[31, 108, 162, 209, 205, 224, 73, 154, 228, 48, 38, 50]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[65, 29, 39, 48, 100, 194, 8, 190, 21, 90, 190, 144]</td>\n",
- " <td>bird</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[45, 69, 122, 120, 228, 153, 228, 134, 106, 177, 103, 179]</td>\n",
- " <td>cat</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[([87, 118, 166, 176, 152, 5, 135, 219, 1, 249, 60, 67], u'dog'),\n",
- " ([109, 9, 113, 57, 10, 234, 47, 6, 223, 16, 9, 148], u'cat'),\n",
- " ([82, 94, 197, 145, 99, 28, 7, 8, 203, 159, 13, 83], u'dog'),\n",
- " ([14, 75, 182, 81, 218, 36, 90, 74, 93, 100, 52, 140], u'dog'),\n",
- " ([19, 156, 115, 167, 206, 198, 5, 147, 86, 104, 175, 93], u'dog'),\n",
- " ([122, 252, 125, 205, 118, 140, 24, 44, 221, 242, 80, 55], u'cat'),\n",
- " ([173, 155, 112, 57, 6, 131, 212, 121, 42, 162, 63, 47], u'cat'),\n",
- " ([149, 150, 227, 242, 6, 93, 238, 132, 42, 100, 15, 66], u'dog'),\n",
- " ([236, 52, 225, 36, 112, 141, 191, 224, 198, 197, 98, 154], u'cat'),\n",
- " ([140, 60, 125, 187, 113, 18, 81, 84, 5, 88, 178, 243], u'bird'),\n",
- " ([220, 87, 235, 30, 232, 216, 82, 200, 251, 194, 85, 186], u'dog'),\n",
- " ([27, 127, 5, 181, 189, 145, 241, 58, 76, 97, 76, 157], u'bird'),\n",
- " ([80, 245, 103, 67, 209, 67, 154, 188, 97, 130, 148, 179], u'cat'),\n",
- " ([143, 117, 160, 74, 23, 187, 200, 28, 111, 133, 173, 96], u'bird'),\n",
- " ([249, 193, 10, 150, 119, 91, 139, 222, 158, 92, 33, 56], u'cat'),\n",
- " ([130, 186, 172, 53, 4, 59, 170, 164, 133, 193, 94, 77], u'bird'),\n",
- " ([206, 210, 138, 46, 145, 131, 239, 156, 24, 102, 246, 163], u'cat'),\n",
- " ([148, 255, 101, 204, 23, 231, 134, 195, 27, 138, 254, 197], u'cat'),\n",
- " ([131, 134, 140, 208, 100, 90, 162, 238, 136, 52, 112, 119], u'bird'),\n",
- " ([137, 221, 198, 44, 34, 90, 42, 135, 38, 65, 109, 171], u'cat'),\n",
- " ([136, 54, 1, 78, 99, 132, 212, 239, 84, 56, 73, 246], u'cat'),\n",
- " ([209, 42, 150, 72, 249, 30, 37, 191, 74, 71, 24, 116], u'bird'),\n",
- " ([62, 181, 59, 233, 185, 195, 31, 187, 17, 130, 63, 229], u'dog'),\n",
- " ([147, 28, 187, 137, 66, 140, 179, 215, 211, 172, 246, 249], u'dog'),\n",
- " ([64, 63, 131, 180, 13, 193, 105, 72, 170, 35, 11, 201], u'bird'),\n",
- " ([28, 75, 28, 0, 189, 175, 29, 120, 56, 94, 3, 235], u'cat'),\n",
- " ([215, 151, 44, 207, 2, 107, 15, 133, 31, 28, 71, 137], u'dog'),\n",
- " ([197, 68, 195, 107, 92, 71, 80, 55, 239, 70, 26, 198], u'cat'),\n",
- " ([6, 230, 54, 75, 186, 42, 36, 112, 227, 19, 109, 220], u'cat'),\n",
- " ([145, 117, 8, 147, 175, 205, 215, 113, 57, 51, 184, 136], u'dog'),\n",
- " ([167, 207, 132, 109, 6, 138, 83, 60, 213, 13, 102, 249], u'dog'),\n",
- " ([73, 12, 234, 37, 13, 123, 154, 21, 14, 72, 226, 229], u'bird'),\n",
- " ([26, 24, 114, 163, 130, 25, 114, 6, 134, 119, 144, 217], u'bird'),\n",
- " ([101, 230, 25, 94, 99, 99, 106, 77, 136, 119, 199, 34], u'dog'),\n",
- " ([213, 106, 109, 186, 36, 136, 210, 151, 43, 84, 176, 156], u'dog'),\n",
- " ([54, 20, 235, 15, 199, 80, 245, 224, 174, 87, 67, 24], u'dog'),\n",
- " ([203, 143, 107, 237, 26, 65, 87, 136, 251, 123, 16, 205], u'cat'),\n",
- " ([58, 34, 193, 214, 124, 248, 234, 103, 7, 177, 183, 251], u'dog'),\n",
- " ([101, 83, 212, 125, 247, 159, 12, 98, 139, 38, 163, 226], u'bird'),\n",
- " ([157, 93, 189, 107, 110, 248, 140, 48, 206, 8, 39, 184], u'dog'),\n",
- " ([46, 105, 38, 41, 250, 139, 124, 206, 8, 115, 109, 19], u'bird'),\n",
- " ([248, 57, 119, 218, 231, 21, 55, 164, 127, 166, 156, 11], u'bird'),\n",
- " ([106, 20, 253, 34, 131, 43, 139, 170, 84, 133, 53, 208], u'cat'),\n",
- " ([60, 67, 191, 79, 24, 184, 136, 143, 146, 111, 164, 201], u'cat'),\n",
- " ([35, 110, 175, 47, 68, 25, 67, 65, 59, 198, 107, 198], u'dog'),\n",
- " ([191, 74, 164, 144, 157, 224, 211, 92, 48, 234, 20, 184], u'dog'),\n",
- " ([166, 39, 29, 110, 59, 65, 221, 234, 111, 33, 3, 178], u'dog'),\n",
- " ([62, 120, 205, 4, 231, 140, 78, 139, 28, 235, 108, 238], u'cat'),\n",
- " ([155, 217, 91, 83, 82, 0, 122, 111, 110, 181, 176, 75], u'dog'),\n",
- " ([31, 108, 162, 209, 205, 224, 73, 154, 228, 48, 38, 50], u'bird'),\n",
- " ([65, 29, 39, 48, 100, 194, 8, 190, 21, 90, 190, 144], u'bird'),\n",
- " ([45, 69, 122, 120, 228, 153, 228, 134, 106, 177, 103, 179], u'cat')]"
- ]
- },
- "execution_count": 6,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS image_data;\n",
- "\n",
- "CREATE TABLE image_data AS (\n",
- "SELECT ARRAY[\n",
- " (random() * 256)::integer, -- R values\n",
- " (random() * 256)::integer,\n",
- " (random() * 256)::integer,\n",
- " (random() * 256)::integer,\n",
- " (random() * 256)::integer, -- G values\n",
- " (random() * 256)::integer,\n",
- " (random() * 256)::integer,\n",
- " (random() * 256)::integer,\n",
- " (random() * 256)::integer, -- B values\n",
- " (random() * 256)::integer,\n",
- " (random() * 256)::integer,\n",
- " (random() * 256)::integer\n",
- " ] as rgb, ('{cat,dog,bird}'::text[])[ceil(random()*3)] as species\n",
- "FROM generate_series(1, 52)\n",
- ");\n",
- "\n",
- "SELECT * FROM image_data;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"pp_train2\"></a>\n",
- "# 5. Run preprocessor for training image data\n",
- "\n",
- "Run the preprocessor to generate the packed output table:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 7,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "2 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>independent_var</th>\n",
- " <th>dependent_var</th>\n",
- " <th>buffer_id</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[0.105882, 0.498039, 0.0196078, 0.709804, 0.741176, 0.568627, 0.945098, 0.227451, 0.298039, 0.380392, 0.298039, 0.615686], [0.537255, 0.866667, 0.776471, 0.172549, 0.133333, 0.352941, 0.164706, 0.529412, 0.14902, 0.254902, 0.427451, 0.670588], [0.235294, 0.262745, 0.74902, 0.309804, 0.0941177, 0.721569, 0.533333, 0.560784, 0.572549, 0.435294, 0.643137, 0.788235], [0.560784, 0.458824, 0.627451, 0.290196, 0.0901961, 0.733333, 0.784314, 0.109804, 0.435294, 0.521569, 0.6 [...]
- " <td>[[1, 0, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [1, 0, 0], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 1, 0]]</td>\n",
- " <td>0</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[0.313726, 0.960784, 0.403922, 0.262745, 0.819608, 0.262745, 0.603922, 0.737255, 0.380392, 0.509804, 0.580392, 0.701961], [0.654902, 0.811765, 0.517647, 0.427451, 0.0235294, 0.541176, 0.32549, 0.235294, 0.835294, 0.0509804, 0.4, 0.976471], [0.180392, 0.411765, 0.14902, 0.160784, 0.980392, 0.545098, 0.486275, 0.807843, 0.0313726, 0.45098, 0.427451, 0.0745098], [0.0235294, 0.901961, 0.211765, 0.294118, 0.729412, 0.164706, 0.141176, 0.439216, 0.890196, 0.0745098, 0.4274 [...]
- " <td>[[0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 0, 1], [0, 0, 1], [1, 0, 0]]</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[([[0.105882, 0.498039, 0.0196078, 0.709804, 0.741176, 0.568627, 0.945098, 0.227451, 0.298039, 0.380392, 0.298039, 0.615686], [0.537255, 0.866667, 0.776471, 0.172549, 0.133333, 0.352941, 0.164706, 0.529412, 0.14902, 0.254902, 0.427451, 0.670588], [0.235294, 0.262745, 0.74902, 0.309804, 0.0941177, 0.721569, 0.533333, 0.560784, 0.572549, 0.435294, 0.643137, 0.788235], [0.560784, 0.458824, 0.627451, 0.290196, 0.0901961, 0.733333, 0.784314, 0.109804, 0.435294, 0.521569, 0.678431, 0.3 [...]
- " ([[0.313726, 0.960784, 0.403922, 0.262745, 0.819608, 0.262745, 0.603922, 0.737255, 0.380392, 0.509804, 0.580392, 0.701961], [0.654902, 0.811765, 0.517647, 0.427451, 0.0235294, 0.541176, 0.32549, 0.235294, 0.835294, 0.0509804, 0.4, 0.976471], [0.180392, 0.411765, 0.14902, 0.160784, 0.980392, 0.545098, 0.486275, 0.807843, 0.0313726, 0.45098, 0.427451, 0.0745098], [0.0235294, 0.901961, 0.211765, 0.294118, 0.729412, 0.164706, 0.141176, 0.439216, 0.890196, 0.0745098, 0.427451, 0.8627 [...]
- ]
- },
- "execution_count": 7,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS image_data_packed, image_data_packed_summary;\n",
- "\n",
- "SELECT madlib.training_preprocessor_dl('image_data', -- Source table\n",
- " 'image_data_packed', -- Output table\n",
- " 'species', -- Dependent variable\n",
- " 'rgb', -- Independent variable\n",
- " NULL, -- Buffer size\n",
- " 255 -- Normalizing constant\n",
- " );\n",
- "\n",
- "SELECT * FROM image_data_packed ORDER BY buffer_id;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"pp_val2\"></a>\n",
- "# 6. Run preprocessor for validation image data\n",
- "\n",
- "Run the preprocessor for the validation dataset. In this example, we use the same images for validation to demonstrate, but normally validation data is different than training data:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 11,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "2 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>independent_var</th>\n",
- " <th>dependent_var</th>\n",
- " <th>buffer_id</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[0.576471, 0.701961, 0.580392, 0.627451, 0.964706, 0.509804, 0.517647, 0.564706, 0.6, 0.152941, 0.690196, 0.215686], [0.258824, 0.478431, 0.772549, 0.105882, 0.152941, 0.345098, 0.803922, 0.729412, 0.972549, 0.764706, 0.235294, 0.482353], [0.72549, 0.682353, 0.109804, 0.105882, 0.796079, 0.368627, 0.584314, 0.564706, 0.47451, 0.733333, 0.909804, 0.27451], [0.152941, 0.870588, 0.623529, 0.917647, 0.384314, 0.345098, 0.596078, 0.494118, 0.45098, 0.388235, 0.862745, 0.0 [...]
- " <td>[[0, 1, 0], [0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [1, 0, 0], [1, 0, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 0, 1]]</td>\n",
- " <td>0</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[0.231373, 0.423529, 0.152941, 0.521569, 0.533333, 0.305882, 0.996078, 0.2, 0.662745, 0.0941177, 0.654902, 0.85098], [0.4, 0.0352941, 0.639216, 0.968628, 0.141176, 0.87451, 0.388235, 0.294118, 0.388235, 0.921569, 0.6, 0.384314], [0.831373, 0.635294, 0.0117647, 0.0470588, 0.831373, 0.411765, 0.0784314, 0.466667, 0.372549, 0.223529, 0.337255, 0.764706], [0.878431, 0.0392157, 0.694118, 0.356863, 0.364706, 0.52549, 0.996078, 0.372549, 0.568627, 0.823529, 0.784314, 0.6509 [...]
- " <td>[[1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 0, 1], [1, 0, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0]]</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[([[0.576471, 0.701961, 0.580392, 0.627451, 0.964706, 0.509804, 0.517647, 0.564706, 0.6, 0.152941, 0.690196, 0.215686], [0.258824, 0.478431, 0.772549, 0.105882, 0.152941, 0.345098, 0.803922, 0.729412, 0.972549, 0.764706, 0.235294, 0.482353], [0.72549, 0.682353, 0.109804, 0.105882, 0.796079, 0.368627, 0.584314, 0.564706, 0.47451, 0.733333, 0.909804, 0.27451], [0.152941, 0.870588, 0.623529, 0.917647, 0.384314, 0.345098, 0.596078, 0.494118, 0.45098, 0.388235, 0.862745, 0.0313726], [ [...]
- " ([[0.231373, 0.423529, 0.152941, 0.521569, 0.533333, 0.305882, 0.996078, 0.2, 0.662745, 0.0941177, 0.654902, 0.85098], [0.4, 0.0352941, 0.639216, 0.968628, 0.141176, 0.87451, 0.388235, 0.294118, 0.388235, 0.921569, 0.6, 0.384314], [0.831373, 0.635294, 0.0117647, 0.0470588, 0.831373, 0.411765, 0.0784314, 0.466667, 0.372549, 0.223529, 0.337255, 0.764706], [0.878431, 0.0392157, 0.694118, 0.356863, 0.364706, 0.52549, 0.996078, 0.372549, 0.568627, 0.823529, 0.784314, 0.65098], [0.333 [...]
- ]
- },
- "execution_count": 11,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS val_image_data_packed, val_image_data_packed_summary;\n",
- "\n",
- "SELECT madlib.validation_preprocessor_dl(\n",
- " 'image_data', -- Source table\n",
- " 'val_image_data_packed', -- Output table\n",
- " 'species', -- Dependent variable\n",
- " 'rgb', -- Independent variable\n",
- " 'image_data_packed', -- From training preprocessor step\n",
- " NULL -- Buffer size\n",
- " );\n",
- "\n",
- "SELECT * FROM val_image_data_packed ORDER BY buffer_id;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"change_buffer\"></a>\n",
- "# 7. Change buffer size \n",
- "\n",
- "Generally the default buffer size will work well, but if you have occasion to change it:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 12,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "6 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>independent_var</th>\n",
- " <th>dependent_var</th>\n",
- " <th>buffer_id</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[0.72549, 0.682353, 0.109804, 0.105882, 0.796079, 0.368627, 0.584314, 0.564706, 0.47451, 0.733333, 0.909804, 0.27451], [0.521569, 0.733333, 0.968628, 0.776471, 0.945098, 0.443137, 0.760784, 0.129412, 0.235294, 0.847059, 0.0392157, 0.635294], [0.380392, 0.866667, 0.529412, 0.760784, 0.541176, 0.647059, 0.407843, 0.54902, 0.0352941, 0.894118, 0.619608, 0.533333], [0.145098, 0.203922, 0.878431, 0.258824, 0.858824, 0.882353, 0.490196, 0.796079, 0.478431, 0.854902, 0.2156 [...]
- " <td>[[0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0]]</td>\n",
- " <td>0</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[0.192157, 0.0392157, 0.556863, 0.74902, 0.211765, 0.74902, 0.541176, 0.588235, 0.67451, 0.776471, 0.917647, 0.137255], [0.231373, 0.423529, 0.152941, 0.521569, 0.533333, 0.305882, 0.996078, 0.2, 0.662745, 0.0941177, 0.654902, 0.85098], [0.0, 0.192157, 0.478431, 0.905882, 0.901961, 0.980392, 0.180392, 0.533333, 0.192157, 0.631373, 0.564706, 0.976471], [0.0666667, 0.780392, 0.67451, 0.0901961, 0.894118, 0.839216, 0.431373, 0.254902, 0.454902, 0.960784, 0.784314, 0.929 [...]
- " <td>[[0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0]]</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[0.282353, 0.407843, 0.635294, 0.52549, 0.556863, 0.0117647, 0.384314, 0.0862745, 0.772549, 0.92549, 0.729412, 0.176471], [0.145098, 0.678431, 0.517647, 0.0235294, 0.470588, 0.0392157, 0.756863, 0.435294, 0.815686, 0.698039, 0.882353, 0.572549], [0.576471, 0.701961, 0.580392, 0.627451, 0.964706, 0.509804, 0.517647, 0.564706, 0.6, 0.152941, 0.690196, 0.215686], [0.152941, 0.870588, 0.623529, 0.917647, 0.384314, 0.345098, 0.596078, 0.494118, 0.45098, 0.388235, 0.862745 [...]
- " <td>[[0, 0, 1], [1, 0, 0], [0, 1, 0], [1, 0, 0], [1, 0, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 0, 1]]</td>\n",
- " <td>2</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[0.168627, 0.858824, 0.027451, 0.972549, 0.458824, 0.556863, 0.407843, 0.494118, 0.721569, 0.784314, 0.219608, 0.4], [0.113725, 0.419608, 0.862745, 0.74902, 0.560784, 0.443137, 0.509804, 0.788235, 0.478431, 0.831373, 0.478431, 0.109804], [0.960784, 0.545098, 0.396078, 0.521569, 0.254902, 0.458824, 0.298039, 0.933333, 0.54902, 0.192157, 0.768628, 0.980392], [0.403922, 0.564706, 0.384314, 0.690196, 0.658824, 0.341176, 0.521569, 0.717647, 0.207843, 0.623529, 0.380392, 0 [...]
- " <td>[[1, 0, 0], [0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0]]</td>\n",
- " <td>3</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[0.658824, 0.827451, 0.835294, 0.462745, 0.764706, 0.752941, 0.811765, 0.901961, 0.113725, 0.215686, 0.964706, 0.0235294], [0.00392157, 0.901961, 0.160784, 0.654902, 0.184314, 0.313726, 0.521569, 0.807843, 0.227451, 0.905882, 0.152941, 0.823529], [0.113725, 0.647059, 0.00392157, 0.396078, 0.486275, 0.0705882, 0.494118, 0.309804, 0.384314, 0.666667, 0.278431, 0.905882], [0.478431, 0.988235, 0.188235, 0.796079, 0.0901961, 0.913726, 0.4, 0.298039, 0.545098, 0.12549, 0.0 [...]
- " <td>[[0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1]]</td>\n",
- " <td>4</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[0.466667, 0.603922, 0.972549, 0.235294, 0.866667, 0.737255, 0.580392, 0.870588, 0.113725, 0.168627, 0.156863, 0.882353], [0.164706, 0.337255, 0.360784, 0.619608, 0.529412, 0.533333, 0.470588, 0.556863, 0.498039, 0.929412, 0.109804, 0.905882], [0.192157, 0.796079, 0.376471, 0.92549, 0.235294, 0.329412, 0.470588, 0.627451, 0.85098, 0.72549, 0.0823529, 0.14902], [0.509804, 0.815686, 0.270588, 0.768628, 0.843137, 0.623529, 0.00784314, 0.376471, 0.74902, 0.290196, 0.1019 [...]
- " <td>[[1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0]]</td>\n",
- " <td>5</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[([[0.72549, 0.682353, 0.109804, 0.105882, 0.796079, 0.368627, 0.584314, 0.564706, 0.47451, 0.733333, 0.909804, 0.27451], [0.521569, 0.733333, 0.968628, 0.776471, 0.945098, 0.443137, 0.760784, 0.129412, 0.235294, 0.847059, 0.0392157, 0.635294], [0.380392, 0.866667, 0.529412, 0.760784, 0.541176, 0.647059, 0.407843, 0.54902, 0.0352941, 0.894118, 0.619608, 0.533333], [0.145098, 0.203922, 0.878431, 0.258824, 0.858824, 0.882353, 0.490196, 0.796079, 0.478431, 0.854902, 0.215686, 0.2862 [...]
- " ([[0.192157, 0.0392157, 0.556863, 0.74902, 0.211765, 0.74902, 0.541176, 0.588235, 0.67451, 0.776471, 0.917647, 0.137255], [0.231373, 0.423529, 0.152941, 0.521569, 0.533333, 0.305882, 0.996078, 0.2, 0.662745, 0.0941177, 0.654902, 0.85098], [0.0, 0.192157, 0.478431, 0.905882, 0.901961, 0.980392, 0.180392, 0.533333, 0.192157, 0.631373, 0.564706, 0.976471], [0.0666667, 0.780392, 0.67451, 0.0901961, 0.894118, 0.839216, 0.431373, 0.254902, 0.454902, 0.960784, 0.784314, 0.929412], [0.2 [...]
- " ([[0.282353, 0.407843, 0.635294, 0.52549, 0.556863, 0.0117647, 0.384314, 0.0862745, 0.772549, 0.92549, 0.729412, 0.176471], [0.145098, 0.678431, 0.517647, 0.0235294, 0.470588, 0.0392157, 0.756863, 0.435294, 0.815686, 0.698039, 0.882353, 0.572549], [0.576471, 0.701961, 0.580392, 0.627451, 0.964706, 0.509804, 0.517647, 0.564706, 0.6, 0.152941, 0.690196, 0.215686], [0.152941, 0.870588, 0.623529, 0.917647, 0.384314, 0.345098, 0.596078, 0.494118, 0.45098, 0.388235, 0.862745, 0.031372 [...]
- " ([[0.168627, 0.858824, 0.027451, 0.972549, 0.458824, 0.556863, 0.407843, 0.494118, 0.721569, 0.784314, 0.219608, 0.4], [0.113725, 0.419608, 0.862745, 0.74902, 0.560784, 0.443137, 0.509804, 0.788235, 0.478431, 0.831373, 0.478431, 0.109804], [0.960784, 0.545098, 0.396078, 0.521569, 0.254902, 0.458824, 0.298039, 0.933333, 0.54902, 0.192157, 0.768628, 0.980392], [0.403922, 0.564706, 0.384314, 0.690196, 0.658824, 0.341176, 0.521569, 0.717647, 0.207843, 0.623529, 0.380392, 0.380392], [...]
- " ([[0.658824, 0.827451, 0.835294, 0.462745, 0.764706, 0.752941, 0.811765, 0.901961, 0.113725, 0.215686, 0.964706, 0.0235294], [0.00392157, 0.901961, 0.160784, 0.654902, 0.184314, 0.313726, 0.521569, 0.807843, 0.227451, 0.905882, 0.152941, 0.823529], [0.113725, 0.647059, 0.00392157, 0.396078, 0.486275, 0.0705882, 0.494118, 0.309804, 0.384314, 0.666667, 0.278431, 0.905882], [0.478431, 0.988235, 0.188235, 0.796079, 0.0901961, 0.913726, 0.4, 0.298039, 0.545098, 0.12549, 0.0823529, 0. [...]
- " ([[0.466667, 0.603922, 0.972549, 0.235294, 0.866667, 0.737255, 0.580392, 0.870588, 0.113725, 0.168627, 0.156863, 0.882353], [0.164706, 0.337255, 0.360784, 0.619608, 0.529412, 0.533333, 0.470588, 0.556863, 0.498039, 0.929412, 0.109804, 0.905882], [0.192157, 0.796079, 0.376471, 0.92549, 0.235294, 0.329412, 0.470588, 0.627451, 0.85098, 0.72549, 0.0823529, 0.14902], [0.509804, 0.815686, 0.270588, 0.768628, 0.843137, 0.623529, 0.00784314, 0.376471, 0.74902, 0.290196, 0.101961, 0.9098 [...]
- ]
- },
- "execution_count": 12,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS image_data_packed, image_data_packed_summary;\n",
- "\n",
- "SELECT madlib.training_preprocessor_dl('image_data', -- Source table\n",
- " 'image_data_packed', -- Output table\n",
- " 'species', -- Dependent variable\n",
- " 'rgb', -- Independent variable\n",
- " 10, -- Buffer size\n",
- " 255 -- Normalizing constant\n",
- " );\n",
- "\n",
- "SELECT * FROM image_data_packed ORDER BY buffer_id;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Review the output summary data:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 13,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>source_table</th>\n",
- " <th>output_table</th>\n",
- " <th>dependent_varname</th>\n",
- " <th>independent_varname</th>\n",
- " <th>dependent_vartype</th>\n",
- " <th>class_values</th>\n",
- " <th>buffer_size</th>\n",
- " <th>normalizing_const</th>\n",
- " <th>num_classes</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>image_data</td>\n",
- " <td>image_data_packed</td>\n",
- " <td>species</td>\n",
- " <td>rgb</td>\n",
- " <td>text</td>\n",
- " <td>[u'bird', u'cat', u'dog']</td>\n",
- " <td>10</td>\n",
- " <td>255.0</td>\n",
- " <td>3</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'image_data', u'image_data_packed', u'species', u'rgb', u'text', [u'bird', u'cat', u'dog'], 10, Decimal('255.0'), 3)]"
- ]
- },
- "execution_count": 13,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT * FROM image_data_packed_summary;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<a id=\"set_num_classes\"></a>\n",
- "# 8. Setting number of classes\n",
- "\n",
- "If want the 1-hot encoded vector to have more classes than present in the data, use the num_classes param "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 14,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "2 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>independent_var</th>\n",
- " <th>dependent_var</th>\n",
- " <th>buffer_id</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[0.258824, 0.478431, 0.772549, 0.105882, 0.152941, 0.345098, 0.803922, 0.729412, 0.972549, 0.764706, 0.235294, 0.482353], [0.482353, 0.211765, 0.886275, 0.32549, 0.745098, 0.72549, 0.172549, 0.717647, 0.647059, 0.4, 0.694118, 0.466667], [0.72549, 0.603922, 0.207843, 0.631373, 0.733333, 0.792157, 0.913726, 0.443137, 0.384314, 0.14902, 0.407843, 0.772549], [0.580392, 0.282353, 0.713726, 0.596078, 0.239216, 0.968628, 0.388235, 0.109804, 0.360784, 0.576471, 0.745098, 0.6 [...]
- " <td>[[0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [1, 0, 0, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [1, 0, 0, 0, 0], [0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [1, 0, 0, 0, 0]]</td>\n",
- " <td>0</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[[0.509804, 0.815686, 0.270588, 0.768628, 0.843137, 0.623529, 0.00784314, 0.376471, 0.74902, 0.290196, 0.101961, 0.909804], [0.831373, 0.635294, 0.0117647, 0.0470588, 0.831373, 0.411765, 0.0784314, 0.466667, 0.372549, 0.223529, 0.337255, 0.764706], [0.192157, 0.0392157, 0.556863, 0.74902, 0.211765, 0.74902, 0.541176, 0.588235, 0.67451, 0.776471, 0.917647, 0.137255], [0.164706, 0.337255, 0.360784, 0.619608, 0.529412, 0.533333, 0.470588, 0.556863, 0.498039, 0.929412, 0. [...]
- " <td>[[0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [0, 0, 1, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [1, 0, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [0, 0, 1, 0, 0], [1, 0, 0, 0, 0], [0, 0, 1, 0, 0], [1, 0, 0, 0, 0], [1, 0, 0, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [0, 0, 1, 0, 0], [0, 0, 1, 0, 0]]</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[([[0.258824, 0.478431, 0.772549, 0.105882, 0.152941, 0.345098, 0.803922, 0.729412, 0.972549, 0.764706, 0.235294, 0.482353], [0.482353, 0.211765, 0.886275, 0.32549, 0.745098, 0.72549, 0.172549, 0.717647, 0.647059, 0.4, 0.694118, 0.466667], [0.72549, 0.603922, 0.207843, 0.631373, 0.733333, 0.792157, 0.913726, 0.443137, 0.384314, 0.14902, 0.407843, 0.772549], [0.580392, 0.282353, 0.713726, 0.596078, 0.239216, 0.968628, 0.388235, 0.109804, 0.360784, 0.576471, 0.745098, 0.615686], [0 [...]
- " ([[0.509804, 0.815686, 0.270588, 0.768628, 0.843137, 0.623529, 0.00784314, 0.376471, 0.74902, 0.290196, 0.101961, 0.909804], [0.831373, 0.635294, 0.0117647, 0.0470588, 0.831373, 0.411765, 0.0784314, 0.466667, 0.372549, 0.223529, 0.337255, 0.764706], [0.192157, 0.0392157, 0.556863, 0.74902, 0.211765, 0.74902, 0.541176, 0.588235, 0.67451, 0.776471, 0.917647, 0.137255], [0.164706, 0.337255, 0.360784, 0.619608, 0.529412, 0.533333, 0.470588, 0.556863, 0.498039, 0.929412, 0.109804, 0. [...]
- ]
- },
- "execution_count": 14,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS image_data_packed, image_data_packed_summary;\n",
- "\n",
- "SELECT madlib.training_preprocessor_dl('image_data', -- Source table\n",
- " 'image_data_packed', -- Output table\n",
- " 'species', -- Dependent variable\n",
- " 'rgb', -- Independent variable\n",
- " NULL, -- Buffer size\n",
- " 255, -- Normalizing constant\n",
- " 5 -- Number of desired class values\n",
- " );\n",
- "\n",
- "SELECT * FROM image_data_packed ORDER BY buffer_id;"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 16,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>source_table</th>\n",
- " <th>output_table</th>\n",
- " <th>dependent_varname</th>\n",
- " <th>independent_varname</th>\n",
- " <th>dependent_vartype</th>\n",
- " <th>class_values</th>\n",
- " <th>buffer_size</th>\n",
- " <th>normalizing_const</th>\n",
- " <th>num_classes</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>image_data</td>\n",
- " <td>image_data_packed</td>\n",
- " <td>species</td>\n",
- " <td>rgb</td>\n",
- " <td>text</td>\n",
- " <td>[u'bird', u'cat', u'dog', None, None]</td>\n",
- " <td>26</td>\n",
- " <td>255.0</td>\n",
- " <td>5</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'image_data', u'image_data_packed', u'species', u'rgb', u'text', [u'bird', u'cat', u'dog', None, None], 26, Decimal('255.0'), 5)]"
- ]
- },
- "execution_count": 16,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT * FROM image_data_packed_summary;"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 2",
- "language": "python",
- "name": "python2"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 2
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython2",
- "version": "2.7.12"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 1
-}
diff --git a/community-artifacts/Deep-learning/automl/hyperband-diag-cifar10-v1.ipynb b/community-artifacts/Deep-learning/automl/hyperband-diag-cifar10-v1.ipynb
new file mode 100644
index 0000000..9f1a558
--- /dev/null
+++ b/community-artifacts/Deep-learning/automl/hyperband-diag-cifar10-v1.ipynb
@@ -0,0 +1,5094 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Hyperband diagonal using CIFAR-10\n",
+ "\n",
+ "Implemention of Hyperband https://arxiv.org/pdf/1603.06560.pdf for MPP with a synchronous barrier. Uses the Hyperband schedule but runs it on a diagonal across brackets, instead of one bracket at a time, to be more efficient with cluster resources. \n",
+ "\n",
+ "This is an initial implementation to show functionality; there is still work to do to improve usability.\n",
+ "\n",
+ "\n",
+ "## Table of contents \n",
+ "\n",
+ "<a href=\"#setup\">0. Setup</a>\n",
+ "\n",
+ "<a href=\"#load_dataset\">1. Load dataset into table</a>\n",
+ "\n",
+ "<a href=\"#distr\">2. Setup distribution rules and call preprocessor</a>\n",
+ "\n",
+ "<a href=\"#arch\">3. Define and load model architectures</a>\n",
+ "\n",
+ "<a href=\"#hyperband\">4. Hyperband diagonal</a>\n",
+ "\n",
+ "<a href=\"#plot\">5. Plot results</a>\n",
+ "\n",
+ "<a href=\"#print\">6. Pretty print schedules</a>\n",
+ "\n",
+ "<a href=\"#predict\">7. Inference</a>"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "The CIFAR-10 dataset consists of 60,000 32x32 colour images in 10 classes, with 6,000 images per class. There are 50,000 training images and 10,000 test images.\n",
+ "https://www.cs.toronto.edu/~kriz/cifar.html"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/jpeg": "/9j/4AAQSkZJRgABAQAAkACQAAD/4QB0RXhpZgAATU0AKgAAAAgABAEaAAUAAAABAAAAPgEbAAUAAAABAAAARgEoAAMAAAABAAIAAIdpAAQAAAABAAAATgAAAAAAAACQAAAAAQAAAJAAAAABAAKgAgAEAAAAAQAAA7igAwAEAAAAAQAAAuIAAAAA/+0AOFBob3Rvc2hvcCAzLjAAOEJJTQQEAAAAAAAAOEJJTQQlAAAAAAAQ1B2M2Y8AsgTpgAmY7PhCfv/AABEIAuIDuAMBIgACEQEDEQH/xAAfAAABBQEBAQEBAQAAAAAAAAAAAQIDBAUGBwgJCgv/xAC1EAACAQMDAgQDBQUEBAAAAX0BAgMABBEFEiExQQYTUWEHInEUMoGRoQgjQrHBFVLR8CQzYnKCCQoWFxgZGiUmJygpKjQ1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV [...]
+ "text/plain": [
+ "<IPython.core.display.Image object>"
+ ]
+ },
+ "execution_count": 2,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "from IPython.display import Image\n",
+ "Image(\"../images/cifar10.jpg\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"setup\"></a>\n",
+ "# 0. Setup"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/config.py:13: ShimWarning: The `IPython.config` package has been deprecated since IPython 4.0. You should import from traitlets.config instead.\n",
+ " \"You should import from traitlets.config instead.\", ShimWarning)\n",
+ "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/utils/traitlets.py:5: UserWarning: IPython.utils.traitlets has moved to a top-level traitlets package.\n",
+ " warn(\"IPython.utils.traitlets has moved to a top-level traitlets package.\")\n"
+ ]
+ }
+ ],
+ "source": [
+ "%load_ext sql"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "u'Connected: gpadmin@cifar_demo'"
+ ]
+ },
+ "execution_count": 2,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Greenplum Database 5.x on GCP - via tunnel\n",
+ "%sql postgresql://gpadmin@localhost:8000/cifar_demo\n",
+ "#%sql postgresql://gpadmin@35.230.53.21:5432/cifar_demo\n",
+ "\n",
+ "# PostgreSQL local\n",
+ "#%sql postgresql://fmcquillan@localhost:5432/madlib"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>version</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>MADlib version: 1.17-dev, git revision: rc/1.16-rc1-63-g7625ae0, cmake configuration time: Tue Jan 14 23:42:21 UTC 2020, build type: RelWithDebInfo, build system: Linux-2.6.32-754.6.3.el6.x86_64, C compiler: gcc 4.4.7, C++ compiler: g++ 4.4.7</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'MADlib version: 1.17-dev, git revision: rc/1.16-rc1-63-g7625ae0, cmake configuration time: Tue Jan 14 23:42:21 UTC 2020, build type: RelWithDebInfo, build system: Linux-2.6.32-754.6.3.el6.x86_64, C compiler: gcc 4.4.7, C++ compiler: g++ 4.4.7',)]"
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%sql select madlib.version();\n",
+ "#%sql select version();"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Import libraries and define some params"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Using TensorFlow backend.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Couldn't import dot_parser, loading of dot files will not be possible.\n"
+ ]
+ }
+ ],
+ "source": [
+ "from __future__ import print_function\n",
+ "import keras\n",
+ "from keras.datasets import cifar10\n",
+ "from keras.preprocessing.image import ImageDataGenerator\n",
+ "from keras.models import Sequential\n",
+ "from keras.layers import Dense, Dropout, Activation, Flatten, BatchNormalization\n",
+ "from keras.layers import Conv2D, MaxPooling2D\n",
+ "import os"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Others needed in this workbook"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "import numpy as np\n",
+ "import sys\n",
+ "import os\n",
+ "from matplotlib import pyplot as plt"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"load_dataset\"></a>\n",
+ "# 1. Load dataset into table"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "PXF to load iage data\n",
+ "\n",
+ "Alternatively just get the dataset from Keras in the usual way"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%sql\n",
+ "CREATE EXTERNAL TABLE cifar_external_batchsize_500 (\n",
+ " fullpaths TEXT[], \n",
+ " y TEXT[], \n",
+ " names TEXT[], \n",
+ " x INT[]\n",
+ ") \n",
+ "LOCATION ('pxf://madlib-datasets/cifar10/?\n",
+ "PROFILE=gs:image&SERVER=gs-aa&BATCH_SIZE=500&STREAM_FRAGMENTS=true') FORMAT 'csv';\n",
+ "\n",
+ "CREATE TABLE cifar10_train AS SELECT * FROM cifar_external_batchsize_500;"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>count</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>50000</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(50000L,)]"
+ ]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%sql SELECT COUNT(*) FROM cifar10_train;"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>count</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>10000</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(10000L,)]"
+ ]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%sql SELECT COUNT(*) FROM cifar10_val;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"distr\"></a>\n",
+ "# 2. Setup distribution rules and call preprocessor\n",
+ "\n",
+ "Get cluster configuration\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "20 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>hostname</th>\n",
+ " <th>gpu_descr</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>phoenix0</td>\n",
+ " <td>device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>phoenix0</td>\n",
+ " <td>device: 1, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:05.0, compute capability: 6.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>phoenix0</td>\n",
+ " <td>device: 2, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:06.0, compute capability: 6.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>phoenix0</td>\n",
+ " <td>device: 3, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:07.0, compute capability: 6.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>phoenix1</td>\n",
+ " <td>device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>phoenix1</td>\n",
+ " <td>device: 1, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:05.0, compute capability: 6.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>phoenix1</td>\n",
+ " <td>device: 2, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:06.0, compute capability: 6.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>phoenix1</td>\n",
+ " <td>device: 3, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:07.0, compute capability: 6.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>phoenix2</td>\n",
+ " <td>device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>phoenix2</td>\n",
+ " <td>device: 1, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:05.0, compute capability: 6.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>phoenix2</td>\n",
+ " <td>device: 2, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:06.0, compute capability: 6.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>phoenix2</td>\n",
+ " <td>device: 3, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:07.0, compute capability: 6.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>phoenix3</td>\n",
+ " <td>device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>phoenix3</td>\n",
+ " <td>device: 1, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:05.0, compute capability: 6.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>phoenix3</td>\n",
+ " <td>device: 2, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:06.0, compute capability: 6.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>phoenix3</td>\n",
+ " <td>device: 3, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:07.0, compute capability: 6.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>phoenix4</td>\n",
+ " <td>device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>phoenix4</td>\n",
+ " <td>device: 1, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:05.0, compute capability: 6.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>phoenix4</td>\n",
+ " <td>device: 2, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:06.0, compute capability: 6.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>phoenix4</td>\n",
+ " <td>device: 3, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:07.0, compute capability: 6.0</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'phoenix0', u'device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0'),\n",
+ " (u'phoenix0', u'device: 1, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:05.0, compute capability: 6.0'),\n",
+ " (u'phoenix0', u'device: 2, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:06.0, compute capability: 6.0'),\n",
+ " (u'phoenix0', u'device: 3, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:07.0, compute capability: 6.0'),\n",
+ " (u'phoenix1', u'device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0'),\n",
+ " (u'phoenix1', u'device: 1, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:05.0, compute capability: 6.0'),\n",
+ " (u'phoenix1', u'device: 2, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:06.0, compute capability: 6.0'),\n",
+ " (u'phoenix1', u'device: 3, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:07.0, compute capability: 6.0'),\n",
+ " (u'phoenix2', u'device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0'),\n",
+ " (u'phoenix2', u'device: 1, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:05.0, compute capability: 6.0'),\n",
+ " (u'phoenix2', u'device: 2, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:06.0, compute capability: 6.0'),\n",
+ " (u'phoenix2', u'device: 3, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:07.0, compute capability: 6.0'),\n",
+ " (u'phoenix3', u'device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0'),\n",
+ " (u'phoenix3', u'device: 1, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:05.0, compute capability: 6.0'),\n",
+ " (u'phoenix3', u'device: 2, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:06.0, compute capability: 6.0'),\n",
+ " (u'phoenix3', u'device: 3, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:07.0, compute capability: 6.0'),\n",
+ " (u'phoenix4', u'device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0'),\n",
+ " (u'phoenix4', u'device: 1, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:05.0, compute capability: 6.0'),\n",
+ " (u'phoenix4', u'device: 2, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:06.0, compute capability: 6.0'),\n",
+ " (u'phoenix4', u'device: 3, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:07.0, compute capability: 6.0')]"
+ ]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS host_gpu_mapping_tf;\n",
+ "SELECT * FROM madlib.gpu_configuration('host_gpu_mapping_tf');\n",
+ "SELECT * FROM host_gpu_mapping_tf ORDER BY hostname, gpu_descr;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Build distribution rules table for 4 VMs"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS segments_to_use_4VMs;\n",
+ "CREATE TABLE segments_to_use_4VMs AS\n",
+ " SELECT DISTINCT dbid, hostname FROM gp_segment_configuration JOIN host_gpu_mapping_tf USING (hostname)\n",
+ " WHERE role='p' AND content>=0 AND hostname!='phoenix4';\n",
+ "SELECT * FROM segments_to_use_4VMs ORDER BY hostname, dbid;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Build distribution rules table for 2 VMs"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS segments_to_use_2VMs;\n",
+ "CREATE TABLE segments_to_use_2VMs AS\n",
+ " SELECT DISTINCT dbid, hostname FROM gp_segment_configuration JOIN host_gpu_mapping_tf USING (hostname)\n",
+ " WHERE role='p' AND content>=0 AND (hostname='phoenix0' OR hostname='phoenix1');\n",
+ "SELECT * FROM segments_to_use_2VMs ORDER BY hostname, dbid;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Build distribution rules table for 1 VMs"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS segments_to_use_1VM;\n",
+ "CREATE TABLE segments_to_use_1VM AS\n",
+ " SELECT DISTINCT dbid, hostname FROM gp_segment_configuration JOIN host_gpu_mapping_tf USING (hostname)\n",
+ " WHERE role='p' AND content>=0 AND hostname='phoenix0';\n",
+ "SELECT * FROM segments_to_use_1VM ORDER BY hostname, dbid;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Build distribution rules table for 1 segment"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "5 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>dbid</th>\n",
+ " <th>content</th>\n",
+ " <th>role</th>\n",
+ " <th>preferred_role</th>\n",
+ " <th>mode</th>\n",
+ " <th>status</th>\n",
+ " <th>port</th>\n",
+ " <th>hostname</th>\n",
+ " <th>address</th>\n",
+ " <th>replication_port</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>-1</td>\n",
+ " <td>p</td>\n",
+ " <td>p</td>\n",
+ " <td>s</td>\n",
+ " <td>u</td>\n",
+ " <td>5432</td>\n",
+ " <td>phoenix0</td>\n",
+ " <td>phoenix0</td>\n",
+ " <td>None</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>0</td>\n",
+ " <td>p</td>\n",
+ " <td>p</td>\n",
+ " <td>c</td>\n",
+ " <td>u</td>\n",
+ " <td>40000</td>\n",
+ " <td>phoenix0</td>\n",
+ " <td>phoenix0</td>\n",
+ " <td>70000</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>1</td>\n",
+ " <td>p</td>\n",
+ " <td>p</td>\n",
+ " <td>c</td>\n",
+ " <td>u</td>\n",
+ " <td>40001</td>\n",
+ " <td>phoenix0</td>\n",
+ " <td>phoenix0</td>\n",
+ " <td>70001</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>2</td>\n",
+ " <td>p</td>\n",
+ " <td>p</td>\n",
+ " <td>c</td>\n",
+ " <td>u</td>\n",
+ " <td>40002</td>\n",
+ " <td>phoenix0</td>\n",
+ " <td>phoenix0</td>\n",
+ " <td>70002</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>3</td>\n",
+ " <td>p</td>\n",
+ " <td>p</td>\n",
+ " <td>c</td>\n",
+ " <td>u</td>\n",
+ " <td>40003</td>\n",
+ " <td>phoenix0</td>\n",
+ " <td>phoenix0</td>\n",
+ " <td>70003</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, -1, u'p', u'p', u's', u'u', 5432, u'phoenix0', u'phoenix0', None),\n",
+ " (2, 0, u'p', u'p', u'c', u'u', 40000, u'phoenix0', u'phoenix0', 70000),\n",
+ " (3, 1, u'p', u'p', u'c', u'u', 40001, u'phoenix0', u'phoenix0', 70001),\n",
+ " (4, 2, u'p', u'p', u'c', u'u', 40002, u'phoenix0', u'phoenix0', 70002),\n",
+ " (5, 3, u'p', u'p', u'c', u'u', 40003, u'phoenix0', u'phoenix0', 70003)]"
+ ]
+ },
+ "execution_count": 26,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM gp_segment_configuration WHERE role='p' AND hostname='phoenix0' ORDER BY dbid;"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>dbid</th>\n",
+ " <th>hostname</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>phoenix0</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(2, u'phoenix0')]"
+ ]
+ },
+ "execution_count": 27,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS segments_to_use_1seg;\n",
+ "CREATE TABLE segments_to_use_1seg AS\n",
+ " SELECT DISTINCT dbid, hostname FROM gp_segment_configuration JOIN host_gpu_mapping_tf USING (hostname)\n",
+ " WHERE dbid=2;\n",
+ "SELECT * FROM segments_to_use_1seg ORDER BY hostname, dbid;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Training dataset (uses training preprocessor):"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "16 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>independent_var_shape</th>\n",
+ " <th>dependent_var_shape</th>\n",
+ " <th>buffer_id</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[3125, 32, 32, 3]</td>\n",
+ " <td>[3125, 10]</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[3125, 32, 32, 3]</td>\n",
+ " <td>[3125, 10]</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[3125, 32, 32, 3]</td>\n",
+ " <td>[3125, 10]</td>\n",
+ " <td>2</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[3125, 32, 32, 3]</td>\n",
+ " <td>[3125, 10]</td>\n",
+ " <td>3</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[3125, 32, 32, 3]</td>\n",
+ " <td>[3125, 10]</td>\n",
+ " <td>4</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[3125, 32, 32, 3]</td>\n",
+ " <td>[3125, 10]</td>\n",
+ " <td>5</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[3125, 32, 32, 3]</td>\n",
+ " <td>[3125, 10]</td>\n",
+ " <td>6</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[3125, 32, 32, 3]</td>\n",
+ " <td>[3125, 10]</td>\n",
+ " <td>7</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[3125, 32, 32, 3]</td>\n",
+ " <td>[3125, 10]</td>\n",
+ " <td>8</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[3125, 32, 32, 3]</td>\n",
+ " <td>[3125, 10]</td>\n",
+ " <td>9</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[3125, 32, 32, 3]</td>\n",
+ " <td>[3125, 10]</td>\n",
+ " <td>10</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[3125, 32, 32, 3]</td>\n",
+ " <td>[3125, 10]</td>\n",
+ " <td>11</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[3125, 32, 32, 3]</td>\n",
+ " <td>[3125, 10]</td>\n",
+ " <td>12</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[3125, 32, 32, 3]</td>\n",
+ " <td>[3125, 10]</td>\n",
+ " <td>13</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[3125, 32, 32, 3]</td>\n",
+ " <td>[3125, 10]</td>\n",
+ " <td>14</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[3125, 32, 32, 3]</td>\n",
+ " <td>[3125, 10]</td>\n",
+ " <td>15</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[([3125, 32, 32, 3], [3125, 10], 0),\n",
+ " ([3125, 32, 32, 3], [3125, 10], 1),\n",
+ " ([3125, 32, 32, 3], [3125, 10], 2),\n",
+ " ([3125, 32, 32, 3], [3125, 10], 3),\n",
+ " ([3125, 32, 32, 3], [3125, 10], 4),\n",
+ " ([3125, 32, 32, 3], [3125, 10], 5),\n",
+ " ([3125, 32, 32, 3], [3125, 10], 6),\n",
+ " ([3125, 32, 32, 3], [3125, 10], 7),\n",
+ " ([3125, 32, 32, 3], [3125, 10], 8),\n",
+ " ([3125, 32, 32, 3], [3125, 10], 9),\n",
+ " ([3125, 32, 32, 3], [3125, 10], 10),\n",
+ " ([3125, 32, 32, 3], [3125, 10], 11),\n",
+ " ([3125, 32, 32, 3], [3125, 10], 12),\n",
+ " ([3125, 32, 32, 3], [3125, 10], 13),\n",
+ " ([3125, 32, 32, 3], [3125, 10], 14),\n",
+ " ([3125, 32, 32, 3], [3125, 10], 15)]"
+ ]
+ },
+ "execution_count": 7,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS cifar10_train_packed, cifar10_train_packed_summary;\n",
+ "\n",
+ "SELECT madlib.training_preprocessor_dl('cifar10_train', -- Source table\n",
+ " 'cifar10_train_packed', -- Output table\n",
+ " 'y', -- Dependent variable\n",
+ " 'x', -- Independent variable\n",
+ " NULL, -- Buffer size\n",
+ " 256.0, -- Normalizing constant\n",
+ " NULL, -- Number of classes\n",
+ " 'segments_to_use_4VMs' -- Distribution rules\n",
+ " );\n",
+ "\n",
+ "SELECT independent_var_shape, dependent_var_shape, buffer_id FROM cifar10_train_packed ORDER BY buffer_id;"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>source_table</th>\n",
+ " <th>output_table</th>\n",
+ " <th>dependent_varname</th>\n",
+ " <th>independent_varname</th>\n",
+ " <th>dependent_vartype</th>\n",
+ " <th>class_values</th>\n",
+ " <th>buffer_size</th>\n",
+ " <th>normalizing_const</th>\n",
+ " <th>num_classes</th>\n",
+ " <th>distribution_rules</th>\n",
+ " <th>__internal_gpu_config__</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>cifar10_train</td>\n",
+ " <td>cifar10_train_packed</td>\n",
+ " <td>y</td>\n",
+ " <td>x</td>\n",
+ " <td>smallint</td>\n",
+ " <td>[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]</td>\n",
+ " <td>3125</td>\n",
+ " <td>256.0</td>\n",
+ " <td>10</td>\n",
+ " <td>[2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]</td>\n",
+ " <td>[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'cifar10_train', u'cifar10_train_packed', u'y', u'x', u'smallint', [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 3125, 256.0, 10, [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15])]"
+ ]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM cifar10_train_packed_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Validation dataset (uses validation preprocessor):"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "16 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>independent_var_shape</th>\n",
+ " <th>dependent_var_shape</th>\n",
+ " <th>buffer_id</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[625, 32, 32, 3]</td>\n",
+ " <td>[625, 10]</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[625, 32, 32, 3]</td>\n",
+ " <td>[625, 10]</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[625, 32, 32, 3]</td>\n",
+ " <td>[625, 10]</td>\n",
+ " <td>2</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[625, 32, 32, 3]</td>\n",
+ " <td>[625, 10]</td>\n",
+ " <td>3</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[625, 32, 32, 3]</td>\n",
+ " <td>[625, 10]</td>\n",
+ " <td>4</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[625, 32, 32, 3]</td>\n",
+ " <td>[625, 10]</td>\n",
+ " <td>5</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[625, 32, 32, 3]</td>\n",
+ " <td>[625, 10]</td>\n",
+ " <td>6</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[625, 32, 32, 3]</td>\n",
+ " <td>[625, 10]</td>\n",
+ " <td>7</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[625, 32, 32, 3]</td>\n",
+ " <td>[625, 10]</td>\n",
+ " <td>8</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[625, 32, 32, 3]</td>\n",
+ " <td>[625, 10]</td>\n",
+ " <td>9</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[625, 32, 32, 3]</td>\n",
+ " <td>[625, 10]</td>\n",
+ " <td>10</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[625, 32, 32, 3]</td>\n",
+ " <td>[625, 10]</td>\n",
+ " <td>11</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[625, 32, 32, 3]</td>\n",
+ " <td>[625, 10]</td>\n",
+ " <td>12</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[625, 32, 32, 3]</td>\n",
+ " <td>[625, 10]</td>\n",
+ " <td>13</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[625, 32, 32, 3]</td>\n",
+ " <td>[625, 10]</td>\n",
+ " <td>14</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[625, 32, 32, 3]</td>\n",
+ " <td>[625, 10]</td>\n",
+ " <td>15</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[([625, 32, 32, 3], [625, 10], 0),\n",
+ " ([625, 32, 32, 3], [625, 10], 1),\n",
+ " ([625, 32, 32, 3], [625, 10], 2),\n",
+ " ([625, 32, 32, 3], [625, 10], 3),\n",
+ " ([625, 32, 32, 3], [625, 10], 4),\n",
+ " ([625, 32, 32, 3], [625, 10], 5),\n",
+ " ([625, 32, 32, 3], [625, 10], 6),\n",
+ " ([625, 32, 32, 3], [625, 10], 7),\n",
+ " ([625, 32, 32, 3], [625, 10], 8),\n",
+ " ([625, 32, 32, 3], [625, 10], 9),\n",
+ " ([625, 32, 32, 3], [625, 10], 10),\n",
+ " ([625, 32, 32, 3], [625, 10], 11),\n",
+ " ([625, 32, 32, 3], [625, 10], 12),\n",
+ " ([625, 32, 32, 3], [625, 10], 13),\n",
+ " ([625, 32, 32, 3], [625, 10], 14),\n",
+ " ([625, 32, 32, 3], [625, 10], 15)]"
+ ]
+ },
+ "execution_count": 9,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS cifar10_val_packed, cifar10_val_packed_summary;\n",
+ "\n",
+ "SELECT madlib.validation_preprocessor_dl('cifar10_val', -- Source table\n",
+ " 'cifar10_val_packed', -- Output table\n",
+ " 'y', -- Dependent variable\n",
+ " 'x', -- Independent variable\n",
+ " 'cifar10_train_packed', -- From training preprocessor step\n",
+ " NULL, -- Buffer size\n",
+ " 'segments_to_use_4VMs' -- Distribution rules\n",
+ " ); \n",
+ "\n",
+ "SELECT independent_var_shape, dependent_var_shape, buffer_id FROM cifar10_val_packed ORDER BY buffer_id;"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>source_table</th>\n",
+ " <th>output_table</th>\n",
+ " <th>dependent_varname</th>\n",
+ " <th>independent_varname</th>\n",
+ " <th>dependent_vartype</th>\n",
+ " <th>class_values</th>\n",
+ " <th>buffer_size</th>\n",
+ " <th>normalizing_const</th>\n",
+ " <th>num_classes</th>\n",
+ " <th>distribution_rules</th>\n",
+ " <th>__internal_gpu_config__</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>cifar10_val</td>\n",
+ " <td>cifar10_val_packed</td>\n",
+ " <td>y</td>\n",
+ " <td>x</td>\n",
+ " <td>smallint</td>\n",
+ " <td>[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]</td>\n",
+ " <td>625</td>\n",
+ " <td>256.0</td>\n",
+ " <td>10</td>\n",
+ " <td>[2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]</td>\n",
+ " <td>[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'cifar10_val', u'cifar10_val_packed', u'y', u'x', u'smallint', [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 625, 256.0, 10, [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15])]"
+ ]
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM cifar10_val_packed_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"arch\"></a>\n",
+ "# 3. Define and load model architectures\n",
+ "\n",
+ "Model architecture from https://keras.io/examples/cifar10_cnn/"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "num_classes = 10\n",
+ "\n",
+ "#to be removed\n",
+ "#do this just to get shape for model architecture \n",
+ "(x_train, y_train), (x_test, y_test) = cifar10.load_data()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "_________________________________________________________________\n",
+ "Layer (type) Output Shape Param # \n",
+ "=================================================================\n",
+ "conv2d_1 (Conv2D) (None, 32, 32, 32) 896 \n",
+ "_________________________________________________________________\n",
+ "activation_1 (Activation) (None, 32, 32, 32) 0 \n",
+ "_________________________________________________________________\n",
+ "conv2d_2 (Conv2D) (None, 30, 30, 32) 9248 \n",
+ "_________________________________________________________________\n",
+ "activation_2 (Activation) (None, 30, 30, 32) 0 \n",
+ "_________________________________________________________________\n",
+ "max_pooling2d_1 (MaxPooling2 (None, 15, 15, 32) 0 \n",
+ "_________________________________________________________________\n",
+ "dropout_1 (Dropout) (None, 15, 15, 32) 0 \n",
+ "_________________________________________________________________\n",
+ "conv2d_3 (Conv2D) (None, 15, 15, 64) 18496 \n",
+ "_________________________________________________________________\n",
+ "activation_3 (Activation) (None, 15, 15, 64) 0 \n",
+ "_________________________________________________________________\n",
+ "conv2d_4 (Conv2D) (None, 13, 13, 64) 36928 \n",
+ "_________________________________________________________________\n",
+ "activation_4 (Activation) (None, 13, 13, 64) 0 \n",
+ "_________________________________________________________________\n",
+ "max_pooling2d_2 (MaxPooling2 (None, 6, 6, 64) 0 \n",
+ "_________________________________________________________________\n",
+ "dropout_2 (Dropout) (None, 6, 6, 64) 0 \n",
+ "_________________________________________________________________\n",
+ "flatten_1 (Flatten) (None, 2304) 0 \n",
+ "_________________________________________________________________\n",
+ "dense_1 (Dense) (None, 512) 1180160 \n",
+ "_________________________________________________________________\n",
+ "activation_5 (Activation) (None, 512) 0 \n",
+ "_________________________________________________________________\n",
+ "dropout_3 (Dropout) (None, 512) 0 \n",
+ "_________________________________________________________________\n",
+ "dense_2 (Dense) (None, 10) 5130 \n",
+ "_________________________________________________________________\n",
+ "activation_6 (Activation) (None, 10) 0 \n",
+ "=================================================================\n",
+ "Total params: 1,250,858\n",
+ "Trainable params: 1,250,858\n",
+ "Non-trainable params: 0\n",
+ "_________________________________________________________________\n"
+ ]
+ }
+ ],
+ "source": [
+ "model1 = Sequential()\n",
+ "\n",
+ "model1.add(Conv2D(32, (3, 3), padding='same',\n",
+ " input_shape=x_train.shape[1:]))\n",
+ "model1.add(Activation('relu'))\n",
+ "model1.add(Conv2D(32, (3, 3)))\n",
+ "model1.add(Activation('relu'))\n",
+ "model1.add(MaxPooling2D(pool_size=(2, 2)))\n",
+ "model1.add(Dropout(0.25))\n",
+ "\n",
+ "model1.add(Conv2D(64, (3, 3), padding='same'))\n",
+ "model1.add(Activation('relu'))\n",
+ "model1.add(Conv2D(64, (3, 3)))\n",
+ "model1.add(Activation('relu'))\n",
+ "model1.add(MaxPooling2D(pool_size=(2, 2)))\n",
+ "model1.add(Dropout(0.25))\n",
+ "\n",
+ "model1.add(Flatten())\n",
+ "model1.add(Dense(512))\n",
+ "model1.add(Activation('relu'))\n",
+ "model1.add(Dropout(0.5))\n",
+ "model1.add(Dense(num_classes))\n",
+ "model1.add(Activation('softmax'))\n",
+ "\n",
+ "model1.summary()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'{\"class_name\": \"Sequential\", \"keras_version\": \"2.1.6\", \"config\": [{\"class_name\": \"Conv2D\", \"config\": {\"kernel_initializer\": {\"class_name\": \"VarianceScaling\", \"config\": {\"distribution\": \"uniform\", \"scale\": 1.0, \"seed\": null, \"mode\": \"fan_avg\"}}, \"name\": \"conv2d_1\", \"kernel_constraint\": null, \"bias_regularizer\": null, \"bias_constraint\": null, \"dtype\": \"float32\", \"activation\": \"linear\", \"trainable\": true, \"data_format\": \"ch [...]
+ ]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "model1.to_json()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Model architecture from https://machinelearningmastery.com/how-to-develop-a-cnn-from-scratch-for-cifar-10-photo-classification/"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "_________________________________________________________________\n",
+ "Layer (type) Output Shape Param # \n",
+ "=================================================================\n",
+ "conv2d_5 (Conv2D) (None, 32, 32, 32) 896 \n",
+ "_________________________________________________________________\n",
+ "batch_normalization_1 (Batch (None, 32, 32, 32) 128 \n",
+ "_________________________________________________________________\n",
+ "conv2d_6 (Conv2D) (None, 32, 32, 32) 9248 \n",
+ "_________________________________________________________________\n",
+ "batch_normalization_2 (Batch (None, 32, 32, 32) 128 \n",
+ "_________________________________________________________________\n",
+ "max_pooling2d_3 (MaxPooling2 (None, 16, 16, 32) 0 \n",
+ "_________________________________________________________________\n",
+ "dropout_4 (Dropout) (None, 16, 16, 32) 0 \n",
+ "_________________________________________________________________\n",
+ "conv2d_7 (Conv2D) (None, 16, 16, 64) 18496 \n",
+ "_________________________________________________________________\n",
+ "batch_normalization_3 (Batch (None, 16, 16, 64) 256 \n",
+ "_________________________________________________________________\n",
+ "conv2d_8 (Conv2D) (None, 16, 16, 64) 36928 \n",
+ "_________________________________________________________________\n",
+ "batch_normalization_4 (Batch (None, 16, 16, 64) 256 \n",
+ "_________________________________________________________________\n",
+ "max_pooling2d_4 (MaxPooling2 (None, 8, 8, 64) 0 \n",
+ "_________________________________________________________________\n",
+ "dropout_5 (Dropout) (None, 8, 8, 64) 0 \n",
+ "_________________________________________________________________\n",
+ "conv2d_9 (Conv2D) (None, 8, 8, 128) 73856 \n",
+ "_________________________________________________________________\n",
+ "batch_normalization_5 (Batch (None, 8, 8, 128) 512 \n",
+ "_________________________________________________________________\n",
+ "conv2d_10 (Conv2D) (None, 8, 8, 128) 147584 \n",
+ "_________________________________________________________________\n",
+ "batch_normalization_6 (Batch (None, 8, 8, 128) 512 \n",
+ "_________________________________________________________________\n",
+ "max_pooling2d_5 (MaxPooling2 (None, 4, 4, 128) 0 \n",
+ "_________________________________________________________________\n",
+ "dropout_6 (Dropout) (None, 4, 4, 128) 0 \n",
+ "_________________________________________________________________\n",
+ "flatten_2 (Flatten) (None, 2048) 0 \n",
+ "_________________________________________________________________\n",
+ "dense_3 (Dense) (None, 128) 262272 \n",
+ "_________________________________________________________________\n",
+ "batch_normalization_7 (Batch (None, 128) 512 \n",
+ "_________________________________________________________________\n",
+ "dropout_7 (Dropout) (None, 128) 0 \n",
+ "_________________________________________________________________\n",
+ "dense_4 (Dense) (None, 10) 1290 \n",
+ "=================================================================\n",
+ "Total params: 552,874\n",
+ "Trainable params: 551,722\n",
+ "Non-trainable params: 1,152\n",
+ "_________________________________________________________________\n"
+ ]
+ }
+ ],
+ "source": [
+ "model2 = Sequential()\n",
+ "\n",
+ "model2.add(Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same', input_shape=(32, 32, 3)))\n",
+ "model2.add(BatchNormalization())\n",
+ "model2.add(Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))\n",
+ "model2.add(BatchNormalization())\n",
+ "model2.add(MaxPooling2D((2, 2)))\n",
+ "model2.add(Dropout(0.2))\n",
+ "\n",
+ "model2.add(Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))\n",
+ "model2.add(BatchNormalization())\n",
+ "model2.add(Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))\n",
+ "model2.add(BatchNormalization())\n",
+ "model2.add(MaxPooling2D((2, 2)))\n",
+ "model2.add(Dropout(0.3))\n",
+ "\n",
+ "model2.add(Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))\n",
+ "model2.add(BatchNormalization())\n",
+ "model2.add(Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))\n",
+ "model2.add(BatchNormalization())\n",
+ "model2.add(MaxPooling2D((2, 2)))\n",
+ "model2.add(Dropout(0.4))\n",
+ "\n",
+ "model2.add(Flatten())\n",
+ "model2.add(Dense(128, activation='relu', kernel_initializer='he_uniform'))\n",
+ "model2.add(BatchNormalization())\n",
+ "model2.add(Dropout(0.5))\n",
+ "model2.add(Dense(10, activation='softmax'))\n",
+ "\n",
+ "model2.summary()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'{\"class_name\": \"Sequential\", \"keras_version\": \"2.1.6\", \"config\": [{\"class_name\": \"Conv2D\", \"config\": {\"kernel_initializer\": {\"class_name\": \"VarianceScaling\", \"config\": {\"distribution\": \"uniform\", \"scale\": 2.0, \"seed\": null, \"mode\": \"fan_in\"}}, \"name\": \"conv2d_5\", \"kernel_constraint\": null, \"bias_regularizer\": null, \"bias_constraint\": null, \"dtype\": \"float32\", \"activation\": \"relu\", \"trainable\": true, \"data_format\": \"chann [...]
+ ]
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "model2.to_json()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Another model architecture from https://machinelearningmastery.com/how-to-develop-a-cnn-from-scratch-for-cifar-10-photo-classification/"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "_________________________________________________________________\n",
+ "Layer (type) Output Shape Param # \n",
+ "=================================================================\n",
+ "conv2d_11 (Conv2D) (None, 32, 32, 32) 896 \n",
+ "_________________________________________________________________\n",
+ "conv2d_12 (Conv2D) (None, 32, 32, 32) 9248 \n",
+ "_________________________________________________________________\n",
+ "max_pooling2d_6 (MaxPooling2 (None, 16, 16, 32) 0 \n",
+ "_________________________________________________________________\n",
+ "dropout_8 (Dropout) (None, 16, 16, 32) 0 \n",
+ "_________________________________________________________________\n",
+ "conv2d_13 (Conv2D) (None, 16, 16, 64) 18496 \n",
+ "_________________________________________________________________\n",
+ "conv2d_14 (Conv2D) (None, 16, 16, 64) 36928 \n",
+ "_________________________________________________________________\n",
+ "max_pooling2d_7 (MaxPooling2 (None, 8, 8, 64) 0 \n",
+ "_________________________________________________________________\n",
+ "dropout_9 (Dropout) (None, 8, 8, 64) 0 \n",
+ "_________________________________________________________________\n",
+ "conv2d_15 (Conv2D) (None, 8, 8, 128) 73856 \n",
+ "_________________________________________________________________\n",
+ "conv2d_16 (Conv2D) (None, 8, 8, 128) 147584 \n",
+ "_________________________________________________________________\n",
+ "max_pooling2d_8 (MaxPooling2 (None, 4, 4, 128) 0 \n",
+ "_________________________________________________________________\n",
+ "dropout_10 (Dropout) (None, 4, 4, 128) 0 \n",
+ "_________________________________________________________________\n",
+ "flatten_3 (Flatten) (None, 2048) 0 \n",
+ "_________________________________________________________________\n",
+ "dense_5 (Dense) (None, 128) 262272 \n",
+ "_________________________________________________________________\n",
+ "dropout_11 (Dropout) (None, 128) 0 \n",
+ "_________________________________________________________________\n",
+ "dense_6 (Dense) (None, 10) 1290 \n",
+ "=================================================================\n",
+ "Total params: 550,570\n",
+ "Trainable params: 550,570\n",
+ "Non-trainable params: 0\n",
+ "_________________________________________________________________\n"
+ ]
+ }
+ ],
+ "source": [
+ "model3 = Sequential()\n",
+ "\n",
+ "model3.add(Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same', input_shape=(32, 32, 3)))\n",
+ "model3.add(Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))\n",
+ "model3.add(MaxPooling2D((2, 2)))\n",
+ "model3.add(Dropout(0.2))\n",
+ "model3.add(Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))\n",
+ "model3.add(Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))\n",
+ "model3.add(MaxPooling2D((2, 2)))\n",
+ "model3.add(Dropout(0.3))\n",
+ "model3.add(Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))\n",
+ "model3.add(Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))\n",
+ "model3.add(MaxPooling2D((2, 2)))\n",
+ "model3.add(Dropout(0.4))\n",
+ "model3.add(Flatten())\n",
+ "model3.add(Dense(128, activation='relu', kernel_initializer='he_uniform'))\n",
+ "model3.add(Dropout(0.5))\n",
+ "model3.add(Dense(10, activation='softmax'))\n",
+ "\n",
+ "model3.summary()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Load into model architecture table using psycopg2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "3 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>model_id</th>\n",
+ " <th>name</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>CNN from Keras docs for CIFAR-10</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>CNN from Jason Brownlee blog post</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>CNN from Jason Brownlee blog post - no batch normalization</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, u'CNN from Keras docs for CIFAR-10'),\n",
+ " (2, u'CNN from Jason Brownlee blog post'),\n",
+ " (3, u'CNN from Jason Brownlee blog post - no batch normalization')]"
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
... 10607 lines suppressed ...