You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@madlib.apache.org by fm...@apache.org on 2017/12/08 20:55:11 UTC
[1/7] madlib-site git commit: add new workbooks for 1dot13
Repository: madlib-site
Updated Branches:
refs/heads/asf-site 4ddf1756b -> 95826612d
http://git-wip-us.apache.org/repos/asf/madlib-site/blob/95826612/community-artifacts/mlp-v2.ipynb
----------------------------------------------------------------------
diff --git a/community-artifacts/mlp-v2.ipynb b/community-artifacts/mlp-v2.ipynb
new file mode 100644
index 0000000..145b3e2
--- /dev/null
+++ b/community-artifacts/mlp-v2.ipynb
@@ -0,0 +1,3755 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Multilayer Perceptron"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 117,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "The sql extension is already loaded. To reload it, use:\n",
+ " %reload_ext sql\n"
+ ]
+ }
+ ],
+ "source": [
+ "%load_ext sql"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 118,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "u'Connected: gpdbchina@madlib'"
+ ]
+ },
+ "execution_count": 118,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Greenplum 4.3.10.0\n",
+ "%sql postgresql://gpdbchina@10.194.10.68:61000/madlib\n",
+ " \n",
+ "# PostgreSQL local\n",
+ "#%sql postgresql://fmcquillan@localhost:5432/madlib\n",
+ "\n",
+ "# Greenplum 4.2.3.0\n",
+ "#%sql postgresql://gpdbchina@10.194.10.68:55000/madlib"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 119,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>version</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>MADlib version: 2.0-dev, git revision: rel/v1.12-9-gf790a61, cmake configuration time: Tue Sep 19 17:56:02 UTC 2017, build type: Release, build system: Linux-2.6.18-238.27.1.el5.hotfix.bz516490, C compiler: gcc 4.4.0, C++ compiler: g++ 4.4.0</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'MADlib version: 2.0-dev, git revision: rel/v1.12-9-gf790a61, cmake configuration time: Tue Sep 19 17:56:02 UTC 2017, build type: Release, build system: Linux-2.6.18-238.27.1.el5.hotfix.bz516490, C compiler: gcc 4.4.0, C++ compiler: g++ 4.4.0',)]"
+ ]
+ },
+ "execution_count": 119,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%sql select madlib.version();\n",
+ "#%sql select version();"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 1. Create input table for classification"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 120,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "Done.\n",
+ "52 rows affected.\n",
+ "52 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>attributes</th>\n",
+ " <th>class_text</th>\n",
+ " <th>class</th>\n",
+ " <th>state</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>[Decimal('5.0'), Decimal('3.2'), Decimal('1.2'), Decimal('0.2')]</td>\n",
+ " <td>Iris_setosa</td>\n",
+ " <td>1</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>[Decimal('5.5'), Decimal('3.5'), Decimal('1.3'), Decimal('0.2')]</td>\n",
+ " <td>Iris_setosa</td>\n",
+ " <td>1</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>[Decimal('4.9'), Decimal('3.1'), Decimal('1.5'), Decimal('0.1')]</td>\n",
+ " <td>Iris_setosa</td>\n",
+ " <td>1</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>[Decimal('4.4'), Decimal('3.0'), Decimal('1.3'), Decimal('0.2')]</td>\n",
+ " <td>Iris_setosa</td>\n",
+ " <td>1</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>[Decimal('5.1'), Decimal('3.4'), Decimal('1.5'), Decimal('0.2')]</td>\n",
+ " <td>Iris_setosa</td>\n",
+ " <td>1</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>[Decimal('5.0'), Decimal('3.5'), Decimal('1.3'), Decimal('0.3')]</td>\n",
+ " <td>Iris_setosa</td>\n",
+ " <td>1</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>7</td>\n",
+ " <td>[Decimal('4.5'), Decimal('2.3'), Decimal('1.3'), Decimal('0.3')]</td>\n",
+ " <td>Iris_setosa</td>\n",
+ " <td>1</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>8</td>\n",
+ " <td>[Decimal('4.4'), Decimal('3.2'), Decimal('1.3'), Decimal('0.2')]</td>\n",
+ " <td>Iris_setosa</td>\n",
+ " <td>1</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>9</td>\n",
+ " <td>[Decimal('5.0'), Decimal('3.5'), Decimal('1.6'), Decimal('0.6')]</td>\n",
+ " <td>Iris_setosa</td>\n",
+ " <td>1</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>10</td>\n",
+ " <td>[Decimal('5.1'), Decimal('3.8'), Decimal('1.9'), Decimal('0.4')]</td>\n",
+ " <td>Iris_setosa</td>\n",
+ " <td>1</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>11</td>\n",
+ " <td>[Decimal('4.8'), Decimal('3.0'), Decimal('1.4'), Decimal('0.3')]</td>\n",
+ " <td>Iris_setosa</td>\n",
+ " <td>1</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>12</td>\n",
+ " <td>[Decimal('5.1'), Decimal('3.8'), Decimal('1.6'), Decimal('0.2')]</td>\n",
+ " <td>Iris_setosa</td>\n",
+ " <td>1</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>13</td>\n",
+ " <td>[Decimal('5.7'), Decimal('2.8'), Decimal('4.5'), Decimal('1.3')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>14</td>\n",
+ " <td>[Decimal('6.3'), Decimal('3.3'), Decimal('4.7'), Decimal('1.6')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>15</td>\n",
+ " <td>[Decimal('4.9'), Decimal('2.4'), Decimal('3.3'), Decimal('1.0')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>16</td>\n",
+ " <td>[Decimal('6.6'), Decimal('2.9'), Decimal('4.6'), Decimal('1.3')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>17</td>\n",
+ " <td>[Decimal('5.2'), Decimal('2.7'), Decimal('3.9'), Decimal('1.4')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>18</td>\n",
+ " <td>[Decimal('5.0'), Decimal('2.0'), Decimal('3.5'), Decimal('1.0')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>19</td>\n",
+ " <td>[Decimal('5.9'), Decimal('3.0'), Decimal('4.2'), Decimal('1.5')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>20</td>\n",
+ " <td>[Decimal('6.0'), Decimal('2.2'), Decimal('4.0'), Decimal('1.0')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>21</td>\n",
+ " <td>[Decimal('6.1'), Decimal('2.9'), Decimal('4.7'), Decimal('1.4')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>22</td>\n",
+ " <td>[Decimal('5.6'), Decimal('2.9'), Decimal('3.6'), Decimal('1.3')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>23</td>\n",
+ " <td>[Decimal('6.7'), Decimal('3.1'), Decimal('4.4'), Decimal('1.4')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>24</td>\n",
+ " <td>[Decimal('5.6'), Decimal('3.0'), Decimal('4.5'), Decimal('1.5')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>25</td>\n",
+ " <td>[Decimal('5.8'), Decimal('2.7'), Decimal('4.1'), Decimal('1.0')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>26</td>\n",
+ " <td>[Decimal('6.2'), Decimal('2.2'), Decimal('4.5'), Decimal('1.5')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>27</td>\n",
+ " <td>[Decimal('5.6'), Decimal('2.5'), Decimal('3.9'), Decimal('1.1')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Alaska</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>28</td>\n",
+ " <td>[Decimal('5.0'), Decimal('3.4'), Decimal('1.5'), Decimal('0.2')]</td>\n",
+ " <td>Iris_setosa</td>\n",
+ " <td>1</td>\n",
+ " <td>Tennessee</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>29</td>\n",
+ " <td>[Decimal('4.4'), Decimal('2.9'), Decimal('1.4'), Decimal('0.2')]</td>\n",
+ " <td>Iris_setosa</td>\n",
+ " <td>1</td>\n",
+ " <td>Tennessee</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>30</td>\n",
+ " <td>[Decimal('4.9'), Decimal('3.1'), Decimal('1.5'), Decimal('0.1')]</td>\n",
+ " <td>Iris_setosa</td>\n",
+ " <td>1</td>\n",
+ " <td>Tennessee</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>31</td>\n",
+ " <td>[Decimal('5.4'), Decimal('3.7'), Decimal('1.5'), Decimal('0.2')]</td>\n",
+ " <td>Iris_setosa</td>\n",
+ " <td>1</td>\n",
+ " <td>Tennessee</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>32</td>\n",
+ " <td>[Decimal('4.8'), Decimal('3.4'), Decimal('1.6'), Decimal('0.2')]</td>\n",
+ " <td>Iris_setosa</td>\n",
+ " <td>1</td>\n",
+ " <td>Tennessee</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>33</td>\n",
+ " <td>[Decimal('4.8'), Decimal('3.0'), Decimal('1.4'), Decimal('0.1')]</td>\n",
+ " <td>Iris_setosa</td>\n",
+ " <td>1</td>\n",
+ " <td>Tennessee</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>34</td>\n",
+ " <td>[Decimal('4.3'), Decimal('3.0'), Decimal('1.1'), Decimal('0.1')]</td>\n",
+ " <td>Iris_setosa</td>\n",
+ " <td>1</td>\n",
+ " <td>Tennessee</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>35</td>\n",
+ " <td>[Decimal('5.8'), Decimal('4.0'), Decimal('1.2'), Decimal('0.2')]</td>\n",
+ " <td>Iris_setosa</td>\n",
+ " <td>1</td>\n",
+ " <td>Tennessee</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>36</td>\n",
+ " <td>[Decimal('5.7'), Decimal('4.4'), Decimal('1.5'), Decimal('0.4')]</td>\n",
+ " <td>Iris_setosa</td>\n",
+ " <td>1</td>\n",
+ " <td>Tennessee</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>37</td>\n",
+ " <td>[Decimal('5.4'), Decimal('3.9'), Decimal('1.3'), Decimal('0.4')]</td>\n",
+ " <td>Iris_setosa</td>\n",
+ " <td>1</td>\n",
+ " <td>Tennessee</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>38</td>\n",
+ " <td>[Decimal('6.0'), Decimal('2.9'), Decimal('4.5'), Decimal('1.5')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Tennessee</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>39</td>\n",
+ " <td>[Decimal('5.7'), Decimal('2.6'), Decimal('3.5'), Decimal('1.0')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Tennessee</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>40</td>\n",
+ " <td>[Decimal('5.5'), Decimal('2.4'), Decimal('3.8'), Decimal('1.1')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Tennessee</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>41</td>\n",
+ " <td>[Decimal('5.5'), Decimal('2.4'), Decimal('3.7'), Decimal('1.0')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Tennessee</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>42</td>\n",
+ " <td>[Decimal('5.8'), Decimal('2.7'), Decimal('3.9'), Decimal('1.2')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Tennessee</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>43</td>\n",
+ " <td>[Decimal('6.0'), Decimal('2.7'), Decimal('5.1'), Decimal('1.6')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Tennessee</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>44</td>\n",
+ " <td>[Decimal('5.4'), Decimal('3.0'), Decimal('4.5'), Decimal('1.5')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Tennessee</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>45</td>\n",
+ " <td>[Decimal('6.0'), Decimal('3.4'), Decimal('4.5'), Decimal('1.6')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Tennessee</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>46</td>\n",
+ " <td>[Decimal('6.7'), Decimal('3.1'), Decimal('4.7'), Decimal('1.5')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Tennessee</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>47</td>\n",
+ " <td>[Decimal('6.3'), Decimal('2.3'), Decimal('4.4'), Decimal('1.3')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Tennessee</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>48</td>\n",
+ " <td>[Decimal('5.6'), Decimal('3.0'), Decimal('4.1'), Decimal('1.3')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Tennessee</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>49</td>\n",
+ " <td>[Decimal('5.5'), Decimal('2.5'), Decimal('4.0'), Decimal('1.3')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Tennessee</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>50</td>\n",
+ " <td>[Decimal('5.5'), Decimal('2.6'), Decimal('4.4'), Decimal('1.2')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Tennessee</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>51</td>\n",
+ " <td>[Decimal('6.1'), Decimal('3.0'), Decimal('4.6'), Decimal('1.4')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Tennessee</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>52</td>\n",
+ " <td>[Decimal('5.8'), Decimal('2.6'), Decimal('4.0'), Decimal('1.2')]</td>\n",
+ " <td>Iris_versicolor</td>\n",
+ " <td>2</td>\n",
+ " <td>Tennessee</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, [Decimal('5.0'), Decimal('3.2'), Decimal('1.2'), Decimal('0.2')], u'Iris_setosa', 1, u'Alaska'),\n",
+ " (2, [Decimal('5.5'), Decimal('3.5'), Decimal('1.3'), Decimal('0.2')], u'Iris_setosa', 1, u'Alaska'),\n",
+ " (3, [Decimal('4.9'), Decimal('3.1'), Decimal('1.5'), Decimal('0.1')], u'Iris_setosa', 1, u'Alaska'),\n",
+ " (4, [Decimal('4.4'), Decimal('3.0'), Decimal('1.3'), Decimal('0.2')], u'Iris_setosa', 1, u'Alaska'),\n",
+ " (5, [Decimal('5.1'), Decimal('3.4'), Decimal('1.5'), Decimal('0.2')], u'Iris_setosa', 1, u'Alaska'),\n",
+ " (6, [Decimal('5.0'), Decimal('3.5'), Decimal('1.3'), Decimal('0.3')], u'Iris_setosa', 1, u'Alaska'),\n",
+ " (7, [Decimal('4.5'), Decimal('2.3'), Decimal('1.3'), Decimal('0.3')], u'Iris_setosa', 1, u'Alaska'),\n",
+ " (8, [Decimal('4.4'), Decimal('3.2'), Decimal('1.3'), Decimal('0.2')], u'Iris_setosa', 1, u'Alaska'),\n",
+ " (9, [Decimal('5.0'), Decimal('3.5'), Decimal('1.6'), Decimal('0.6')], u'Iris_setosa', 1, u'Alaska'),\n",
+ " (10, [Decimal('5.1'), Decimal('3.8'), Decimal('1.9'), Decimal('0.4')], u'Iris_setosa', 1, u'Alaska'),\n",
+ " (11, [Decimal('4.8'), Decimal('3.0'), Decimal('1.4'), Decimal('0.3')], u'Iris_setosa', 1, u'Alaska'),\n",
+ " (12, [Decimal('5.1'), Decimal('3.8'), Decimal('1.6'), Decimal('0.2')], u'Iris_setosa', 1, u'Alaska'),\n",
+ " (13, [Decimal('5.7'), Decimal('2.8'), Decimal('4.5'), Decimal('1.3')], u'Iris_versicolor', 2, u'Alaska'),\n",
+ " (14, [Decimal('6.3'), Decimal('3.3'), Decimal('4.7'), Decimal('1.6')], u'Iris_versicolor', 2, u'Alaska'),\n",
+ " (15, [Decimal('4.9'), Decimal('2.4'), Decimal('3.3'), Decimal('1.0')], u'Iris_versicolor', 2, u'Alaska'),\n",
+ " (16, [Decimal('6.6'), Decimal('2.9'), Decimal('4.6'), Decimal('1.3')], u'Iris_versicolor', 2, u'Alaska'),\n",
+ " (17, [Decimal('5.2'), Decimal('2.7'), Decimal('3.9'), Decimal('1.4')], u'Iris_versicolor', 2, u'Alaska'),\n",
+ " (18, [Decimal('5.0'), Decimal('2.0'), Decimal('3.5'), Decimal('1.0')], u'Iris_versicolor', 2, u'Alaska'),\n",
+ " (19, [Decimal('5.9'), Decimal('3.0'), Decimal('4.2'), Decimal('1.5')], u'Iris_versicolor', 2, u'Alaska'),\n",
+ " (20, [Decimal('6.0'), Decimal('2.2'), Decimal('4.0'), Decimal('1.0')], u'Iris_versicolor', 2, u'Alaska'),\n",
+ " (21, [Decimal('6.1'), Decimal('2.9'), Decimal('4.7'), Decimal('1.4')], u'Iris_versicolor', 2, u'Alaska'),\n",
+ " (22, [Decimal('5.6'), Decimal('2.9'), Decimal('3.6'), Decimal('1.3')], u'Iris_versicolor', 2, u'Alaska'),\n",
+ " (23, [Decimal('6.7'), Decimal('3.1'), Decimal('4.4'), Decimal('1.4')], u'Iris_versicolor', 2, u'Alaska'),\n",
+ " (24, [Decimal('5.6'), Decimal('3.0'), Decimal('4.5'), Decimal('1.5')], u'Iris_versicolor', 2, u'Alaska'),\n",
+ " (25, [Decimal('5.8'), Decimal('2.7'), Decimal('4.1'), Decimal('1.0')], u'Iris_versicolor', 2, u'Alaska'),\n",
+ " (26, [Decimal('6.2'), Decimal('2.2'), Decimal('4.5'), Decimal('1.5')], u'Iris_versicolor', 2, u'Alaska'),\n",
+ " (27, [Decimal('5.6'), Decimal('2.5'), Decimal('3.9'), Decimal('1.1')], u'Iris_versicolor', 2, u'Alaska'),\n",
+ " (28, [Decimal('5.0'), Decimal('3.4'), Decimal('1.5'), Decimal('0.2')], u'Iris_setosa', 1, u'Tennessee'),\n",
+ " (29, [Decimal('4.4'), Decimal('2.9'), Decimal('1.4'), Decimal('0.2')], u'Iris_setosa', 1, u'Tennessee'),\n",
+ " (30, [Decimal('4.9'), Decimal('3.1'), Decimal('1.5'), Decimal('0.1')], u'Iris_setosa', 1, u'Tennessee'),\n",
+ " (31, [Decimal('5.4'), Decimal('3.7'), Decimal('1.5'), Decimal('0.2')], u'Iris_setosa', 1, u'Tennessee'),\n",
+ " (32, [Decimal('4.8'), Decimal('3.4'), Decimal('1.6'), Decimal('0.2')], u'Iris_setosa', 1, u'Tennessee'),\n",
+ " (33, [Decimal('4.8'), Decimal('3.0'), Decimal('1.4'), Decimal('0.1')], u'Iris_setosa', 1, u'Tennessee'),\n",
+ " (34, [Decimal('4.3'), Decimal('3.0'), Decimal('1.1'), Decimal('0.1')], u'Iris_setosa', 1, u'Tennessee'),\n",
+ " (35, [Decimal('5.8'), Decimal('4.0'), Decimal('1.2'), Decimal('0.2')], u'Iris_setosa', 1, u'Tennessee'),\n",
+ " (36, [Decimal('5.7'), Decimal('4.4'), Decimal('1.5'), Decimal('0.4')], u'Iris_setosa', 1, u'Tennessee'),\n",
+ " (37, [Decimal('5.4'), Decimal('3.9'), Decimal('1.3'), Decimal('0.4')], u'Iris_setosa', 1, u'Tennessee'),\n",
+ " (38, [Decimal('6.0'), Decimal('2.9'), Decimal('4.5'), Decimal('1.5')], u'Iris_versicolor', 2, u'Tennessee'),\n",
+ " (39, [Decimal('5.7'), Decimal('2.6'), Decimal('3.5'), Decimal('1.0')], u'Iris_versicolor', 2, u'Tennessee'),\n",
+ " (40, [Decimal('5.5'), Decimal('2.4'), Decimal('3.8'), Decimal('1.1')], u'Iris_versicolor', 2, u'Tennessee'),\n",
+ " (41, [Decimal('5.5'), Decimal('2.4'), Decimal('3.7'), Decimal('1.0')], u'Iris_versicolor', 2, u'Tennessee'),\n",
+ " (42, [Decimal('5.8'), Decimal('2.7'), Decimal('3.9'), Decimal('1.2')], u'Iris_versicolor', 2, u'Tennessee'),\n",
+ " (43, [Decimal('6.0'), Decimal('2.7'), Decimal('5.1'), Decimal('1.6')], u'Iris_versicolor', 2, u'Tennessee'),\n",
+ " (44, [Decimal('5.4'), Decimal('3.0'), Decimal('4.5'), Decimal('1.5')], u'Iris_versicolor', 2, u'Tennessee'),\n",
+ " (45, [Decimal('6.0'), Decimal('3.4'), Decimal('4.5'), Decimal('1.6')], u'Iris_versicolor', 2, u'Tennessee'),\n",
+ " (46, [Decimal('6.7'), Decimal('3.1'), Decimal('4.7'), Decimal('1.5')], u'Iris_versicolor', 2, u'Tennessee'),\n",
+ " (47, [Decimal('6.3'), Decimal('2.3'), Decimal('4.4'), Decimal('1.3')], u'Iris_versicolor', 2, u'Tennessee'),\n",
+ " (48, [Decimal('5.6'), Decimal('3.0'), Decimal('4.1'), Decimal('1.3')], u'Iris_versicolor', 2, u'Tennessee'),\n",
+ " (49, [Decimal('5.5'), Decimal('2.5'), Decimal('4.0'), Decimal('1.3')], u'Iris_versicolor', 2, u'Tennessee'),\n",
+ " (50, [Decimal('5.5'), Decimal('2.6'), Decimal('4.4'), Decimal('1.2')], u'Iris_versicolor', 2, u'Tennessee'),\n",
+ " (51, [Decimal('6.1'), Decimal('3.0'), Decimal('4.6'), Decimal('1.4')], u'Iris_versicolor', 2, u'Tennessee'),\n",
+ " (52, [Decimal('5.8'), Decimal('2.6'), Decimal('4.0'), Decimal('1.2')], u'Iris_versicolor', 2, u'Tennessee')]"
+ ]
+ },
+ "execution_count": 120,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql \n",
+ "DROP TABLE IF EXISTS iris_data;\n",
+ "\n",
+ "CREATE TABLE iris_data(\n",
+ " id serial,\n",
+ " attributes numeric[],\n",
+ " class_text varchar,\n",
+ " class integer,\n",
+ " state varchar\n",
+ ");\n",
+ "\n",
+ "INSERT INTO iris_data(id, attributes, class_text, class, state) VALUES\n",
+ "(1,ARRAY[5.0,3.2,1.2,0.2],'Iris_setosa',1,'Alaska'),\n",
+ "(2,ARRAY[5.5,3.5,1.3,0.2],'Iris_setosa',1,'Alaska'),\n",
+ "(3,ARRAY[4.9,3.1,1.5,0.1],'Iris_setosa',1,'Alaska'),\n",
+ "(4,ARRAY[4.4,3.0,1.3,0.2],'Iris_setosa',1,'Alaska'),\n",
+ "(5,ARRAY[5.1,3.4,1.5,0.2],'Iris_setosa',1,'Alaska'),\n",
+ "(6,ARRAY[5.0,3.5,1.3,0.3],'Iris_setosa',1,'Alaska'),\n",
+ "(7,ARRAY[4.5,2.3,1.3,0.3],'Iris_setosa',1,'Alaska'),\n",
+ "(8,ARRAY[4.4,3.2,1.3,0.2],'Iris_setosa',1,'Alaska'),\n",
+ "(9,ARRAY[5.0,3.5,1.6,0.6],'Iris_setosa',1,'Alaska'),\n",
+ "(10,ARRAY[5.1,3.8,1.9,0.4],'Iris_setosa',1,'Alaska'),\n",
+ "(11,ARRAY[4.8,3.0,1.4,0.3],'Iris_setosa',1,'Alaska'),\n",
+ "(12,ARRAY[5.1,3.8,1.6,0.2],'Iris_setosa',1,'Alaska'),\n",
+ "(13,ARRAY[5.7,2.8,4.5,1.3],'Iris_versicolor',2,'Alaska'),\n",
+ "(14,ARRAY[6.3,3.3,4.7,1.6],'Iris_versicolor',2,'Alaska'),\n",
+ "(15,ARRAY[4.9,2.4,3.3,1.0],'Iris_versicolor',2,'Alaska'),\n",
+ "(16,ARRAY[6.6,2.9,4.6,1.3],'Iris_versicolor',2,'Alaska'),\n",
+ "(17,ARRAY[5.2,2.7,3.9,1.4],'Iris_versicolor',2,'Alaska'),\n",
+ "(18,ARRAY[5.0,2.0,3.5,1.0],'Iris_versicolor',2,'Alaska'),\n",
+ "(19,ARRAY[5.9,3.0,4.2,1.5],'Iris_versicolor',2,'Alaska'),\n",
+ "(20,ARRAY[6.0,2.2,4.0,1.0],'Iris_versicolor',2,'Alaska'),\n",
+ "(21,ARRAY[6.1,2.9,4.7,1.4],'Iris_versicolor',2,'Alaska'),\n",
+ "(22,ARRAY[5.6,2.9,3.6,1.3],'Iris_versicolor',2,'Alaska'),\n",
+ "(23,ARRAY[6.7,3.1,4.4,1.4],'Iris_versicolor',2,'Alaska'),\n",
+ "(24,ARRAY[5.6,3.0,4.5,1.5],'Iris_versicolor',2,'Alaska'),\n",
+ "(25,ARRAY[5.8,2.7,4.1,1.0],'Iris_versicolor',2,'Alaska'),\n",
+ "(26,ARRAY[6.2,2.2,4.5,1.5],'Iris_versicolor',2,'Alaska'),\n",
+ "(27,ARRAY[5.6,2.5,3.9,1.1],'Iris_versicolor',2,'Alaska'),\n",
+ "(28,ARRAY[5.0,3.4,1.5,0.2],'Iris_setosa',1,'Tennessee'),\n",
+ "(29,ARRAY[4.4,2.9,1.4,0.2],'Iris_setosa',1,'Tennessee'),\n",
+ "(30,ARRAY[4.9,3.1,1.5,0.1],'Iris_setosa',1,'Tennessee'),\n",
+ "(31,ARRAY[5.4,3.7,1.5,0.2],'Iris_setosa',1,'Tennessee'),\n",
+ "(32,ARRAY[4.8,3.4,1.6,0.2],'Iris_setosa',1,'Tennessee'),\n",
+ "(33,ARRAY[4.8,3.0,1.4,0.1],'Iris_setosa',1,'Tennessee'),\n",
+ "(34,ARRAY[4.3,3.0,1.1,0.1],'Iris_setosa',1,'Tennessee'),\n",
+ "(35,ARRAY[5.8,4.0,1.2,0.2],'Iris_setosa',1,'Tennessee'),\n",
+ "(36,ARRAY[5.7,4.4,1.5,0.4],'Iris_setosa',1,'Tennessee'),\n",
+ "(37,ARRAY[5.4,3.9,1.3,0.4],'Iris_setosa',1,'Tennessee'),\n",
+ "(38,ARRAY[6.0,2.9,4.5,1.5],'Iris_versicolor',2,'Tennessee'),\n",
+ "(39,ARRAY[5.7,2.6,3.5,1.0],'Iris_versicolor',2,'Tennessee'),\n",
+ "(40,ARRAY[5.5,2.4,3.8,1.1],'Iris_versicolor',2,'Tennessee'),\n",
+ "(41,ARRAY[5.5,2.4,3.7,1.0],'Iris_versicolor',2,'Tennessee'),\n",
+ "(42,ARRAY[5.8,2.7,3.9,1.2],'Iris_versicolor',2,'Tennessee'),\n",
+ "(43,ARRAY[6.0,2.7,5.1,1.6],'Iris_versicolor',2,'Tennessee'),\n",
+ "(44,ARRAY[5.4,3.0,4.5,1.5],'Iris_versicolor',2,'Tennessee'),\n",
+ "(45,ARRAY[6.0,3.4,4.5,1.6],'Iris_versicolor',2,'Tennessee'),\n",
+ "(46,ARRAY[6.7,3.1,4.7,1.5],'Iris_versicolor',2,'Tennessee'),\n",
+ "(47,ARRAY[6.3,2.3,4.4,1.3],'Iris_versicolor',2,'Tennessee'),\n",
+ "(48,ARRAY[5.6,3.0,4.1,1.3],'Iris_versicolor',2,'Tennessee'),\n",
+ "(49,ARRAY[5.5,2.5,4.0,1.3],'Iris_versicolor',2,'Tennessee'),\n",
+ "(50,ARRAY[5.5,2.6,4.4,1.2],'Iris_versicolor',2,'Tennessee'),\n",
+ "(51,ARRAY[6.1,3.0,4.6,1.4],'Iris_versicolor',2,'Tennessee'),\n",
+ "(52,ARRAY[5.8,2.6,4.0,1.2],'Iris_versicolor',2,'Tennessee');\n",
+ "\n",
+ "SELECT * FROM iris_data ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 2. Classification model\n",
+ "\n",
+ "Generate a multilayer perceptron with a single hidden layer of 5 units. Use the attributes column as the independent variables, and use the class column as the classification. Set the tolerance to 0 so that 500 iterations will be run. Use a hyperbolic tangent activation function. The model will be written to mlp_model."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 121,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>mlp_classification</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td></td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[('',)]"
+ ]
+ },
+ "execution_count": 121,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS mlp_model, mlp_model_summary, mlp_model_standardization;\n",
+ "\n",
+ "-- Set seed so results are reproducible\n",
+ "SELECT setseed(0);\n",
+ "\n",
+ "SELECT madlib.mlp_classification(\n",
+ " 'iris_data', -- Source table\n",
+ " 'mlp_model', -- Destination table\n",
+ " 'attributes', -- Input features\n",
+ " 'class_text', -- Label\n",
+ " ARRAY[5], -- Number of units per layer\n",
+ " 'learning_rate_init=0.003,\n",
+ " n_iterations=500,\n",
+ " tolerance=0', -- Optimizer params\n",
+ " 'tanh', -- Activation function\n",
+ " NULL, -- Default weight (1)\n",
+ " FALSE, -- No warm start\n",
+ " FALSE -- Not verbose\n",
+ ");"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 3. View the classification model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 122,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>coeff</th>\n",
+ " <th>loss</th>\n",
+ " <th>num_iterations</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[0.418378455147, 0.0769385374681, -0.146527483484, 0.998888382223, 1.12987089952, -0.437290718473, -0.409273157364, 0.498986904082, -0.544260518783, -0.0914464021734, 0.0390525613337, 0.288450666036, -0.327040616284, 0.16574382417, 0.872753267348, 0.675971654762, 0.318207842514, 0.290765083843, -0.468849343505, 0.288355834648, -0.773220988598, -0.277246576994, -0.676925299898, 0.669871211106, -0.391874491507, 0.0738458598585, -1.85544863789, -0.0153495613681, -0.111982901002, -0.225221594881, -0.217849875552, 0.392310858189, 1.37852218837, -1.08667184295, 1.29050890203, -0.251494108369, 0.229914016792]</td>\n",
+ " <td>0.0105868627707</td>\n",
+ " <td>500</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[([0.418378455147, 0.0769385374681, -0.146527483484, 0.998888382223, 1.12987089952, -0.437290718473, -0.409273157364, 0.498986904082, -0.544260518783, -0.0914464021734, 0.0390525613337, 0.288450666036, -0.327040616284, 0.16574382417, 0.872753267348, 0.675971654762, 0.318207842514, 0.290765083843, -0.468849343505, 0.288355834648, -0.773220988598, -0.277246576994, -0.676925299898, 0.669871211106, -0.391874491507, 0.0738458598585, -1.85544863789, -0.0153495613681, -0.111982901002, -0.225221594881, -0.217849875552, 0.392310858189, 1.37852218837, -1.08667184295, 1.29050890203, -0.251494108369, 0.229914016792], 0.0105868627707, 500)]"
+ ]
+ },
+ "execution_count": 122,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM mlp_model;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 4. Classification with n_tries\n",
+ "Now, use the n_tries optimizer parameter to learn and choose the best model among n_tries number of models learned by the algorithm. Run only for 100 iterations and choose the best model from this short run."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 123,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>mlp_classification</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td></td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[('',)]"
+ ]
+ },
+ "execution_count": 123,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS mlp_model, mlp_model_summary, mlp_model_standardization;\n",
+ "\n",
+ "-- Set seed so results are reproducible\n",
+ "SELECT setseed(0);\n",
+ "\n",
+ "SELECT madlib.mlp_classification(\n",
+ " 'iris_data', -- Source table\n",
+ " 'mlp_model', -- Destination table\n",
+ " 'attributes', -- Input features\n",
+ " 'class_text', -- Label\n",
+ " ARRAY[5], -- Number of units per layer\n",
+ " 'learning_rate_init=0.003,\n",
+ " n_iterations=50,\n",
+ " tolerance=0,\n",
+ " n_tries=3', -- Optimizer params, with n_tries\n",
+ " 'tanh', -- Activation function\n",
+ " NULL, -- Default weight (1)\n",
+ " FALSE, -- No warm start\n",
+ " FALSE -- Not verbose\n",
+ ");"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 5. View the classification model with n_tries"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 124,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>coeff</th>\n",
+ " <th>loss</th>\n",
+ " <th>num_iterations</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[0.386129091572, 0.0230804544453, 0.0360373350657, 0.85734404404, 1.00624019669, -0.511165317414, -0.256735693588, 0.424597795202, -0.344940825214, 0.105701317173, -0.0239838319549, 0.19433093383, -0.198067795405, -0.00423615288994, 0.714494593542, 0.67620771672, 0.31692178694, 0.291634997959, -0.470338692309, 0.286903795447, -0.797640757541, -0.36833411352, -0.623711536473, 0.556232364597, -0.501937062704, 0.0664559736029, -1.29169165842, -0.39640660159, 0.349410636066, -0.266301858512, -0.12933320224, 0.399700744444, 0.814765208894, -0.705614802744, 0.829115365015, -0.210413844738, 0.14139734348]</td>\n",
+ " <td>0.169187585083</td>\n",
+ " <td>50</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[([0.386129091572, 0.0230804544453, 0.0360373350657, 0.85734404404, 1.00624019669, -0.511165317414, -0.256735693588, 0.424597795202, -0.344940825214, 0.105701317173, -0.0239838319549, 0.19433093383, -0.198067795405, -0.00423615288994, 0.714494593542, 0.67620771672, 0.31692178694, 0.291634997959, -0.470338692309, 0.286903795447, -0.797640757541, -0.36833411352, -0.623711536473, 0.556232364597, -0.501937062704, 0.0664559736029, -1.29169165842, -0.39640660159, 0.349410636066, -0.266301858512, -0.12933320224, 0.399700744444, 0.814765208894, -0.705614802744, 0.829115365015, -0.210413844738, 0.14139734348], 0.169187585083, 50)]"
+ ]
+ },
+ "execution_count": 124,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM mlp_model;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 6. Classification with warm start\n",
+ "Next, use the warm_start parameter to start learning a new model, using the coefficients already present in mlp_model. Note that we must not drop the mlp_model table, and cannot use the n_tries parameter if warm_start is used."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 126,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>mlp_classification</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td></td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[('',)]"
+ ]
+ },
+ "execution_count": 126,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT madlib.mlp_classification(\n",
+ " 'iris_data', -- Source table\n",
+ " 'mlp_model', -- Destination table\n",
+ " 'attributes', -- Input features\n",
+ " 'class_text', -- Label\n",
+ " ARRAY[5], -- Number of units per layer\n",
+ " 'learning_rate_init=0.003,\n",
+ " n_iterations=450,\n",
+ " tolerance=0', -- Optimizer params\n",
+ " 'tanh', -- Activation function\n",
+ " NULL, -- Default weight (1)\n",
+ " TRUE, -- Warm start\n",
+ " FALSE -- Not verbose\n",
+ ");"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 7. View the classification model with warm_start\n",
+ "Notice that the loss is lower compared to the previous example run of 500 iterations without n_tries, despite having the same values for every other parameter. This is because the algorithm learnt three different models starting with a different set of initial weights for the coefficients, and chose the best model among them."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 127,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>coeff</th>\n",
+ " <th>loss</th>\n",
+ " <th>num_iterations</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[0.43089489035, 0.0840880267225, -0.180463790458, 1.02521696861, 1.15327056306, -0.425505368961, -0.445320886704, 0.51333393744, -0.594555309569, -0.140856773835, 0.0616746727506, 0.306834150297, -0.359872964248, 0.206830852263, 0.911186393447, 0.675568040494, 0.316470613819, 0.291871023206, -0.471193550262, 0.286075681893, -0.759796508813, -0.252763378948, -0.690187333341, 0.70213228757, -0.360741864941, 0.0646874010451, -1.97963455591, 0.0877591861911, -0.220936503136, -0.222359882677, -0.244113625762, 0.401469317001, 1.50270810639, -1.18978059055, 1.39946250414, -0.254355820573, 0.256177767002]</td>\n",
+ " <td>0.00513279402477</td>\n",
+ " <td>450</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[([0.43089489035, 0.0840880267225, -0.180463790458, 1.02521696861, 1.15327056306, -0.425505368961, -0.445320886704, 0.51333393744, -0.594555309569, -0.140856773835, 0.0616746727506, 0.306834150297, -0.359872964248, 0.206830852263, 0.911186393447, 0.675568040494, 0.316470613819, 0.291871023206, -0.471193550262, 0.286075681893, -0.759796508813, -0.252763378948, -0.690187333341, 0.70213228757, -0.360741864941, 0.0646874010451, -1.97963455591, 0.0877591861911, -0.220936503136, -0.222359882677, -0.244113625762, 0.401469317001, 1.50270810639, -1.18978059055, 1.39946250414, -0.254355820573, 0.256177767002], 0.00513279402477, 450)]"
+ ]
+ },
+ "execution_count": 127,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM mlp_model;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 8. Classification with grouping\n",
+ "Next, group the training data by state, and learn a different model for each state."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 128,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>mlp_classification</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td></td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[('',)]"
+ ]
+ },
+ "execution_count": 128,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS mlp_model_group, mlp_model_group_summary, mlp_model_group_standardization;\n",
+ "\n",
+ "-- Set seed so results are reproducible\n",
+ "SELECT setseed(0);\n",
+ "\n",
+ "SELECT madlib.mlp_classification(\n",
+ " 'iris_data', -- Source table\n",
+ " 'mlp_model_group', -- Destination table\n",
+ " 'attributes', -- Input features\n",
+ " 'class_text', -- Label\n",
+ " ARRAY[5], -- Number of units per layer\n",
+ " 'learning_rate_init=0.003,\n",
+ " n_iterations=50,\n",
+ " tolerance=0,\n",
+ " n_tries=3', -- Optimizer params, with n_tries\n",
+ " 'tanh', -- Activation function\n",
+ " NULL, -- Default weight (1)\n",
+ " FALSE, -- No warm start\n",
+ " FALSE, -- Not verbose\n",
+ " 'state' -- Grouping column\n",
+ ");"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 9. View the classification model with grouping\n",
+ "A separate model is learnt for each state, and the result table displays the name of the state (grouping column) associated with the model."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 129,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "2 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>state</th>\n",
+ " <th>coeff</th>\n",
+ " <th>loss</th>\n",
+ " <th>num_iterations</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>Alaska</td>\n",
+ " <td>[0.361442205117, 0.0437833983169, 0.0228267239333, 0.86051976995, 1.00898866341, -0.509493068163, -0.259717661563, 0.423128001757, -0.348520276299, 0.102569227429, -0.023820253367, 0.194242907683, -0.194973841963, -0.00354279848888, 0.715171987578, 0.679803198566, 0.30085527398, 0.290398225819, -0.481339584921, 0.273581567191, -0.798021118384, -0.364181329645, -0.631718355895, 0.568550659826, -0.492243523831, 0.0678832140837, -1.29021370112, -0.396961890891, 0.347728139082, -0.247139454296, -0.165073883437, 0.398273503965, 0.813287251616, -0.705059513443, 0.830797861999, -0.229576248952, 0.177138024677]</td>\n",
+ " <td>0.169694760351</td>\n",
+ " <td>50</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>Tennessee</td>\n",
+ " <td>[0.286460719233, -0.391554641774, 0.647611140251, -0.670341290996, 0.111261651571, -0.323445227135, -0.362568050061, 0.556843820772, 0.542593756362, -0.849092718814, 0.639022610897, 0.289608013322, -0.771514626336, -0.225524049129, 0.55654170876, 0.699286372946, -0.0971044516431, -0.856859889972, -0.53385894952, 0.181718585915, -0.635491855087, -0.334708753092, 0.676444860354, 0.798824989025, 0.712555678327, -0.362929653511, 0.347434351706, 0.177418782808, 0.225111014926, 0.178241297519, -0.0229063368163, 0.0404201612098, -1.12689954392, -0.523963295766, 1.0563791555, 0.642674420838, 0.742594634114]</td>\n",
+ " <td>0.159575522026</td>\n",
+ " <td>50</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'Alaska', [0.361442205117, 0.0437833983169, 0.0228267239333, 0.86051976995, 1.00898866341, -0.509493068163, -0.259717661563, 0.423128001757, -0.348520276299, 0.102569227429, -0.023820253367, 0.194242907683, -0.194973841963, -0.00354279848888, 0.715171987578, 0.679803198566, 0.30085527398, 0.290398225819, -0.481339584921, 0.273581567191, -0.798021118384, -0.364181329645, -0.631718355895, 0.568550659826, -0.492243523831, 0.0678832140837, -1.29021370112, -0.396961890891, 0.347728139082, -0.247139454296, -0.165073883437, 0.398273503965, 0.813287251616, -0.705059513443, 0.830797861999, -0.229576248952, 0.177138024677], 0.169694760351, 50),\n",
+ " (u'Tennessee', [0.286460719233, -0.391554641774, 0.647611140251, -0.670341290996, 0.111261651571, -0.323445227135, -0.362568050061, 0.556843820772, 0.542593756362, -0.849092718814, 0.639022610897, 0.289608013322, -0.771514626336, -0.225524049129, 0.55654170876, 0.699286372946, -0.0971044516431, -0.856859889972, -0.53385894952, 0.181718585915, -0.635491855087, -0.334708753092, 0.676444860354, 0.798824989025, 0.712555678327, -0.362929653511, 0.347434351706, 0.177418782808, 0.225111014926, 0.178241297519, -0.0229063368163, 0.0404201612098, -1.12689954392, -0.523963295766, 1.0563791555, 0.642674420838, 0.742594634114], 0.159575522026, 50)]"
+ ]
+ },
+ "execution_count": 129,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM mlp_model_group ORDER BY state;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 10. Classification with warm start and grouping\n",
+ "Next use warm_start to improve the models that are already present in the mlp_model_group table. Note again that we cannot use n_tries with warm_start, and the model table must already be present."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 130,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>mlp_classification</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td></td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[('',)]"
+ ]
+ },
+ "execution_count": 130,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT madlib.mlp_classification(\n",
+ " 'iris_data', -- Source table\n",
+ " 'mlp_model_group', -- Destination table\n",
+ " 'attributes', -- Input features\n",
+ " 'class_text', -- Label\n",
+ " ARRAY[5], -- Number of units per layer\n",
+ " 'learning_rate_init=0.003,\n",
+ " n_iterations=450,\n",
+ " tolerance=0', -- Optimizer params\n",
+ " 'tanh', -- Activation function\n",
+ " NULL, -- Default weight (1)\n",
+ " TRUE, -- Warm start\n",
+ " FALSE, -- Not verbose\n",
+ " 'state' -- Grouping column\n",
+ ");"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 11. View the classification model with warm start and grouping\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 132,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "2 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>state</th>\n",
+ " <th>coeff</th>\n",
+ " <th>loss</th>\n",
+ " <th>num_iterations</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>Alaska</td>\n",
+ " <td>[0.372342289818, 0.0954960560128, -0.153810124763, 1.00725097032, 1.13495556432, -0.429533598061, -0.409198527269, 0.478752722727, -0.551271316257, -0.0950828556324, 0.0237081636546, 0.297325734546, -0.307808869913, 0.17901470615, 0.882685356116, 0.67682803433, 0.28787018652, 0.297557142865, -0.499522323052, 0.256295844717, -0.768404886254, -0.266707916954, -0.676330194506, 0.699875226902, -0.36878024444, 0.0723594672762, -1.86161184866, -0.0295697622013, -0.116698438374, -0.188190989233, -0.253894999257, 0.393797250761, 1.38468539912, -1.07245164218, 1.29522443949, -0.288524714015, 0.265959140497]</td>\n",
+ " <td>0.0106816837098</td>\n",
+ " <td>450</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>Tennessee</td>\n",
+ " <td>[0.101408120906, -0.504692476376, 0.754172746712, -0.859366119317, -0.0883419348625, -0.322217245475, -0.488849126018, 0.640836955763, 0.379240951714, -1.01591620805, 0.561633386233, 0.474881549745, -0.820848160685, -0.0367588935663, 0.756738298906, 0.690992107944, -0.0117938780722, -0.911609074051, -0.417367078805, 0.303153350896, -0.502205494032, -0.262138409583, 0.531455906745, 0.931199625293, 0.837077534104, -0.383150148885, 0.722094737958, 0.54110242729, -0.140304542646, 0.107821427424, -0.229319721716, 0.0606406565821, -1.50155993006, -0.887646940248, 1.42179471312, 0.713094290933, 0.949008019012]</td>\n",
+ " <td>0.00853723754598</td>\n",
+ " <td>450</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'Alaska', [0.372342289818, 0.0954960560128, -0.153810124763, 1.00725097032, 1.13495556432, -0.429533598061, -0.409198527269, 0.478752722727, -0.551271316257, -0.0950828556324, 0.0237081636546, 0.297325734546, -0.307808869913, 0.17901470615, 0.882685356116, 0.67682803433, 0.28787018652, 0.297557142865, -0.499522323052, 0.256295844717, -0.768404886254, -0.266707916954, -0.676330194506, 0.699875226902, -0.36878024444, 0.0723594672762, -1.86161184866, -0.0295697622013, -0.116698438374, -0.188190989233, -0.253894999257, 0.393797250761, 1.38468539912, -1.07245164218, 1.29522443949, -0.288524714015, 0.265959140497], 0.0106816837098, 450),\n",
+ " (u'Tennessee', [0.101408120906, -0.504692476376, 0.754172746712, -0.859366119317, -0.0883419348625, -0.322217245475, -0.488849126018, 0.640836955763, 0.379240951714, -1.01591620805, 0.561633386233, 0.474881549745, -0.820848160685, -0.0367588935663, 0.756738298906, 0.690992107944, -0.0117938780722, -0.911609074051, -0.417367078805, 0.303153350896, -0.502205494032, -0.262138409583, 0.531455906745, 0.931199625293, 0.837077534104, -0.383150148885, 0.722094737958, 0.54110242729, -0.140304542646, 0.107821427424, -0.229319721716, 0.0606406565821, -1.50155993006, -0.887646940248, 1.42179471312, 0.713094290933, 0.949008019012], 0.00853723754598, 450)]"
+ ]
+ },
+ "execution_count": 132,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM mlp_model_group ORDER BY state;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 12. Create input data for regression\n",
+ "\n",
+ "This dataset contains housing prices."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 133,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "Done.\n",
+ "20 rows affected.\n",
+ "20 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>x</th>\n",
+ " <th>zipcode</th>\n",
+ " <th>y</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>[Decimal('1'), Decimal('0.00632'), Decimal('18.00'), Decimal('2.310'), Decimal('0'), Decimal('0.5380'), Decimal('6.5750'), Decimal('65.20'), Decimal('4.0900'), Decimal('1'), Decimal('296.0'), Decimal('15.30'), Decimal('396.90'), Decimal('4.98')]</td>\n",
+ " <td>94016</td>\n",
+ " <td>24.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>[Decimal('1'), Decimal('0.02731'), Decimal('0.00'), Decimal('7.070'), Decimal('0'), Decimal('0.4690'), Decimal('6.4210'), Decimal('78.90'), Decimal('4.9671'), Decimal('2'), Decimal('242.0'), Decimal('17.80'), Decimal('396.90'), Decimal('9.14')]</td>\n",
+ " <td>94016</td>\n",
+ " <td>21.6</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>[Decimal('1'), Decimal('0.02729'), Decimal('0.00'), Decimal('7.070'), Decimal('0'), Decimal('0.4690'), Decimal('7.1850'), Decimal('61.10'), Decimal('4.9671'), Decimal('2'), Decimal('242.0'), Decimal('17.80'), Decimal('392.83'), Decimal('4.03')]</td>\n",
+ " <td>94016</td>\n",
+ " <td>34.7</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>[Decimal('1'), Decimal('0.03237'), Decimal('0.00'), Decimal('2.180'), Decimal('0'), Decimal('0.4580'), Decimal('6.9980'), Decimal('45.80'), Decimal('6.0622'), Decimal('3'), Decimal('222.0'), Decimal('18.70'), Decimal('394.63'), Decimal('2.94')]</td>\n",
+ " <td>94016</td>\n",
+ " <td>33.4</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>[Decimal('1'), Decimal('0.06905'), Decimal('0.00'), Decimal('2.180'), Decimal('0'), Decimal('0.4580'), Decimal('7.1470'), Decimal('54.20'), Decimal('6.0622'), Decimal('3'), Decimal('222.0'), Decimal('18.70'), Decimal('396.90'), Decimal('5.33')]</td>\n",
+ " <td>94016</td>\n",
+ " <td>36.2</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>[Decimal('1'), Decimal('0.02985'), Decimal('0.00'), Decimal('2.180'), Decimal('0'), Decimal('0.4580'), Decimal('6.4300'), Decimal('58.70'), Decimal('6.0622'), Decimal('3'), Decimal('222.0'), Decimal('18.70'), Decimal('394.12'), Decimal('5.21')]</td>\n",
+ " <td>94016</td>\n",
+ " <td>28.7</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>7</td>\n",
+ " <td>[Decimal('1'), Decimal('0.08829'), Decimal('12.50'), Decimal('7.870'), Decimal('0'), Decimal('0.5240'), Decimal('6.0120'), Decimal('66.60'), Decimal('5.5605'), Decimal('5'), Decimal('311.0'), Decimal('15.20'), Decimal('395.60'), Decimal('12.43')]</td>\n",
+ " <td>94016</td>\n",
+ " <td>22.9</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>8</td>\n",
+ " <td>[Decimal('1'), Decimal('0.14455'), Decimal('12.50'), Decimal('7.870'), Decimal('0'), Decimal('0.5240'), Decimal('6.1720'), Decimal('96.10'), Decimal('5.9505'), Decimal('5'), Decimal('311.0'), Decimal('15.20'), Decimal('396.90'), Decimal('19.15')]</td>\n",
+ " <td>94016</td>\n",
+ " <td>27.1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>9</td>\n",
+ " <td>[Decimal('1'), Decimal('0.21124'), Decimal('12.50'), Decimal('7.870'), Decimal('0'), Decimal('0.5240'), Decimal('5.6310'), Decimal('100.00'), Decimal('6.0821'), Decimal('5'), Decimal('311.0'), Decimal('15.20'), Decimal('386.63'), Decimal('29.93')]</td>\n",
+ " <td>94016</td>\n",
+ " <td>16.5</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>10</td>\n",
+ " <td>[Decimal('1'), Decimal('0.17004'), Decimal('12.50'), Decimal('7.870'), Decimal('0'), Decimal('0.5240'), Decimal('6.0040'), Decimal('85.90'), Decimal('6.5921'), Decimal('5'), Decimal('311.0'), Decimal('15.20'), Decimal('386.71'), Decimal('17.10')]</td>\n",
+ " <td>94016</td>\n",
+ " <td>18.9</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>11</td>\n",
+ " <td>[Decimal('1'), Decimal('0.22489'), Decimal('12.50'), Decimal('7.870'), Decimal('0'), Decimal('0.5240'), Decimal('6.3770'), Decimal('94.30'), Decimal('6.3467'), Decimal('5'), Decimal('311.0'), Decimal('15.20'), Decimal('392.52'), Decimal('20.45')]</td>\n",
+ " <td>94016</td>\n",
+ " <td>15.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>12</td>\n",
+ " <td>[Decimal('1'), Decimal('0.11747'), Decimal('12.50'), Decimal('7.870'), Decimal('0'), Decimal('0.5240'), Decimal('6.0090'), Decimal('82.90'), Decimal('6.2267'), Decimal('5'), Decimal('311.0'), Decimal('15.20'), Decimal('396.90'), Decimal('13.27')]</td>\n",
+ " <td>20001</td>\n",
+ " <td>18.9</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>13</td>\n",
+ " <td>[Decimal('1'), Decimal('0.09378'), Decimal('12.50'), Decimal('7.870'), Decimal('0'), Decimal('0.5240'), Decimal('5.8890'), Decimal('39.00'), Decimal('5.4509'), Decimal('5'), Decimal('311.0'), Decimal('15.20'), Decimal('390.50'), Decimal('15.71')]</td>\n",
+ " <td>20001</td>\n",
+ " <td>21.7</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>14</td>\n",
+ " <td>[Decimal('1'), Decimal('0.62976'), Decimal('0.00'), Decimal('8.140'), Decimal('0'), Decimal('0.5380'), Decimal('5.9490'), Decimal('61.80'), Decimal('4.7075'), Decimal('4'), Decimal('307.0'), Decimal('21.00'), Decimal('396.90'), Decimal('8.26')]</td>\n",
+ " <td>20001</td>\n",
+ " <td>20.4</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>15</td>\n",
+ " <td>[Decimal('1'), Decimal('0.63796'), Decimal('0.00'), Decimal('8.140'), Decimal('0'), Decimal('0.5380'), Decimal('6.0960'), Decimal('84.50'), Decimal('4.4619'), Decimal('4'), Decimal('307.0'), Decimal('21.00'), Decimal('380.02'), Decimal('10.26')]</td>\n",
+ " <td>20001</td>\n",
+ " <td>18.2</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>16</td>\n",
+ " <td>[Decimal('1'), Decimal('0.62739'), Decimal('0.00'), Decimal('8.140'), Decimal('0'), Decimal('0.5380'), Decimal('5.8340'), Decimal('56.50'), Decimal('4.4986'), Decimal('4'), Decimal('307.0'), Decimal('21.00'), Decimal('395.62'), Decimal('8.47')]</td>\n",
+ " <td>20001</td>\n",
+ " <td>19.9</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>17</td>\n",
+ " <td>[Decimal('1'), Decimal('1.05393'), Decimal('0.00'), Decimal('8.140'), Decimal('0'), Decimal('0.5380'), Decimal('5.9350'), Decimal('29.30'), Decimal('4.4986'), Decimal('4'), Decimal('307.0'), Decimal('21.00'), Decimal('386.85'), Decimal('6.58')]</td>\n",
+ " <td>20001</td>\n",
+ " <td>23.1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>18</td>\n",
+ " <td>[Decimal('1'), Decimal('0.78420'), Decimal('0.00'), Decimal('8.140'), Decimal('0'), Decimal('0.5380'), Decimal('5.9900'), Decimal('81.70'), Decimal('4.2579'), Decimal('4'), Decimal('307.0'), Decimal('21.00'), Decimal('386.75'), Decimal('14.67')]</td>\n",
+ " <td>20001</td>\n",
+ " <td>17.5</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>19</td>\n",
+ " <td>[Decimal('1'), Decimal('0.80271'), Decimal('0.00'), Decimal('8.140'), Decimal('0'), Decimal('0.5380'), Decimal('5.4560'), Decimal('36.60'), Decimal('3.7965'), Decimal('4'), Decimal('307.0'), Decimal('21.00'), Decimal('288.99'), Decimal('11.69')]</td>\n",
+ " <td>20001</td>\n",
+ " <td>20.2</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>20</td>\n",
+ " <td>[Decimal('1'), Decimal('0.72580'), Decimal('0.00'), Decimal('8.140'), Decimal('0'), Decimal('0.5380'), Decimal('5.7270'), Decimal('69.50'), Decimal('3.7965'), Decimal('4'), Decimal('307.0'), Decimal('21.00'), Decimal('390.95'), Decimal('11.28')]</td>\n",
+ " <td>20001</td>\n",
+ " <td>18.2</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, [Decimal('1'), Decimal('0.00632'), Decimal('18.00'), Decimal('2.310'), Decimal('0'), Decimal('0.5380'), Decimal('6.5750'), Decimal('65.20'), Decimal('4.0900'), Decimal('1'), Decimal('296.0'), Decimal('15.30'), Decimal('396.90'), Decimal('4.98')], 94016, 24.0),\n",
+ " (2, [Decimal('1'), Decimal('0.02731'), Decimal('0.00'), Decimal('7.070'), Decimal('0'), Decimal('0.4690'), Decimal('6.4210'), Decimal('78.90'), Decimal('4.9671'), Decimal('2'), Decimal('242.0'), Decimal('17.80'), Decimal('396.90'), Decimal('9.14')], 94016, 21.6),\n",
+ " (3, [Decimal('1'), Decimal('0.02729'), Decimal('0.00'), Decimal('7.070'), Decimal('0'), Decimal('0.4690'), Decimal('7.1850'), Decimal('61.10'), Decimal('4.9671'), Decimal('2'), Decimal('242.0'), Decimal('17.80'), Decimal('392.83'), Decimal('4.03')], 94016, 34.7),\n",
+ " (4, [Decimal('1'), Decimal('0.03237'), Decimal('0.00'), Decimal('2.180'), Decimal('0'), Decimal('0.4580'), Decimal('6.9980'), Decimal('45.80'), Decimal('6.0622'), Decimal('3'), Decimal('222.0'), Decimal('18.70'), Decimal('394.63'), Decimal('2.94')], 94016, 33.4),\n",
+ " (5, [Decimal('1'), Decimal('0.06905'), Decimal('0.00'), Decimal('2.180'), Decimal('0'), Decimal('0.4580'), Decimal('7.1470'), Decimal('54.20'), Decimal('6.0622'), Decimal('3'), Decimal('222.0'), Decimal('18.70'), Decimal('396.90'), Decimal('5.33')], 94016, 36.2),\n",
+ " (6, [Decimal('1'), Decimal('0.02985'), Decimal('0.00'), Decimal('2.180'), Decimal('0'), Decimal('0.4580'), Decimal('6.4300'), Decimal('58.70'), Decimal('6.0622'), Decimal('3'), Decimal('222.0'), Decimal('18.70'), Decimal('394.12'), Decimal('5.21')], 94016, 28.7),\n",
+ " (7, [Decimal('1'), Decimal('0.08829'), Decimal('12.50'), Decimal('7.870'), Decimal('0'), Decimal('0.5240'), Decimal('6.0120'), Decimal('66.60'), Decimal('5.5605'), Decimal('5'), Decimal('311.0'), Decimal('15.20'), Decimal('395.60'), Decimal('12.43')], 94016, 22.9),\n",
+ " (8, [Decimal('1'), Decimal('0.14455'), Decimal('12.50'), Decimal('7.870'), Decimal('0'), Decimal('0.5240'), Decimal('6.1720'), Decimal('96.10'), Decimal('5.9505'), Decimal('5'), Decimal('311.0'), Decimal('15.20'), Decimal('396.90'), Decimal('19.15')], 94016, 27.1),\n",
+ " (9, [Decimal('1'), Decimal('0.21124'), Decimal('12.50'), Decimal('7.870'), Decimal('0'), Decimal('0.5240'), Decimal('5.6310'), Decimal('100.00'), Decimal('6.0821'), Decimal('5'), Decimal('311.0'), Decimal('15.20'), Decimal('386.63'), Decimal('29.93')], 94016, 16.5),\n",
+ " (10, [Decimal('1'), Decimal('0.17004'), Decimal('12.50'), Decimal('7.870'), Decimal('0'), Decimal('0.5240'), Decimal('6.0040'), Decimal('85.90'), Decimal('6.5921'), Decimal('5'), Decimal('311.0'), Decimal('15.20'), Decimal('386.71'), Decimal('17.10')], 94016, 18.9),\n",
+ " (11, [Decimal('1'), Decimal('0.22489'), Decimal('12.50'), Decimal('7.870'), Decimal('0'), Decimal('0.5240'), Decimal('6.3770'), Decimal('94.30'), Decimal('6.3467'), Decimal('5'), Decimal('311.0'), Decimal('15.20'), Decimal('392.52'), Decimal('20.45')], 94016, 15.0),\n",
+ " (12, [Decimal('1'), Decimal('0.11747'), Decimal('12.50'), Decimal('7.870'), Decimal('0'), Decimal('0.5240'), Decimal('6.0090'), Decimal('82.90'), Decimal('6.2267'), Decimal('5'), Decimal('311.0'), Decimal('15.20'), Decimal('396.90'), Decimal('13.27')], 20001, 18.9),\n",
+ " (13, [Decimal('1'), Decimal('0.09378'), Decimal('12.50'), Decimal('7.870'), Decimal('0'), Decimal('0.5240'), Decimal('5.8890'), Decimal('39.00'), Decimal('5.4509'), Decimal('5'), Decimal('311.0'), Decimal('15.20'), Decimal('390.50'), Decimal('15.71')], 20001, 21.7),\n",
+ " (14, [Decimal('1'), Decimal('0.62976'), Decimal('0.00'), Decimal('8.140'), Decimal('0'), Decimal('0.5380'), Decimal('5.9490'), Decimal('61.80'), Decimal('4.7075'), Decimal('4'), Decimal('307.0'), Decimal('21.00'), Decimal('396.90'), Decimal('8.26')], 20001, 20.4),\n",
+ " (15, [Decimal('1'), Decimal('0.63796'), Decimal('0.00'), Decimal('8.140'), Decimal('0'), Decimal('0.5380'), Decimal('6.0960'), Decimal('84.50'), Decimal('4.4619'), Decimal('4'), Decimal('307.0'), Decimal('21.00'), Decimal('380.02'), Decimal('10.26')], 20001, 18.2),\n",
+ " (16, [Decimal('1'), Decimal('0.62739'), Decimal('0.00'), Decimal('8.140'), Decimal('0'), Decimal('0.5380'), Decimal('5.8340'), Decimal('56.50'), Decimal('4.4986'), Decimal('4'), Decimal('307.0'), Decimal('21.00'), Decimal('395.62'), Decimal('8.47')], 20001, 19.9),\n",
+ " (17, [Decimal('1'), Decimal('1.05393'), Decimal('0.00'), Decimal('8.140'), Decimal('0'), Decimal('0.5380'), Decimal('5.9350'), Decimal('29.30'), Decimal('4.4986'), Decimal('4'), Decimal('307.0'), Decimal('21.00'), Decimal('386.85'), Decimal('6.58')], 20001, 23.1),\n",
+ " (18, [Decimal('1'), Decimal('0.78420'), Decimal('0.00'), Decimal('8.140'), Decimal('0'), Decimal('0.5380'), Decimal('5.9900'), Decimal('81.70'), Decimal('4.2579'), Decimal('4'), Decimal('307.0'), Decimal('21.00'), Decimal('386.75'), Decimal('14.67')], 20001, 17.5),\n",
+ " (19, [Decimal('1'), Decimal('0.80271'), Decimal('0.00'), Decimal('8.140'), Decimal('0'), Decimal('0.5380'), Decimal('5.4560'), Decimal('36.60'), Decimal('3.7965'), Decimal('4'), Decimal('307.0'), Decimal('21.00'), Decimal('288.99'), Decimal('11.69')], 20001, 20.2),\n",
+ " (20, [Decimal('1'), Decimal('0.72580'), Decimal('0.00'), Decimal('8.140'), Decimal('0'), Decimal('0.5380'), Decimal('5.7270'), Decimal('69.50'), Decimal('3.7965'), Decimal('4'), Decimal('307.0'), Decimal('21.00'), Decimal('390.95'), Decimal('11.28')], 20001, 18.2)]"
+ ]
+ },
+ "execution_count": 133,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS lin_housing;\n",
+ "\n",
+ "CREATE TABLE lin_housing (id serial, \n",
+ " x numeric[], \n",
+ " zipcode int, \n",
+ " y float8);\n",
+ "\n",
+ "INSERT INTO lin_housing(id, x, zipcode, y) VALUES\n",
+ "(1,ARRAY[1,0.00632,18.00,2.310,0,0.5380,6.5750,65.20,4.0900,1,296.0,15.30,396.90,4.98],94016,24.00),\n",
+ "(2,ARRAY[1,0.02731,0.00,7.070,0,0.4690,6.4210,78.90,4.9671,2,242.0,17.80,396.90,9.14],94016,21.60),\n",
+ "(3,ARRAY[1,0.02729,0.00,7.070,0,0.4690,7.1850,61.10,4.9671,2,242.0,17.80,392.83,4.03],94016,34.70),\n",
+ "(4,ARRAY[1,0.03237,0.00,2.180,0,0.4580,6.9980,45.80,6.0622,3,222.0,18.70,394.63,2.94],94016,33.40),\n",
+ "(5,ARRAY[1,0.06905,0.00,2.180,0,0.4580,7.1470,54.20,6.0622,3,222.0,18.70,396.90,5.33],94016,36.20),\n",
+ "(6,ARRAY[1,0.02985,0.00,2.180,0,0.4580,6.4300,58.70,6.0622,3,222.0,18.70,394.12,5.21],94016,28.70),\n",
+ "(7,ARRAY[1,0.08829,12.50,7.870,0,0.5240,6.0120,66.60,5.5605,5,311.0,15.20,395.60,12.43],94016,22.90),\n",
+ "(8,ARRAY[1,0.14455,12.50,7.870,0,0.5240,6.1720,96.10,5.9505,5,311.0,15.20,396.90,19.15],94016,27.10),\n",
+ "(9,ARRAY[1,0.21124,12.50,7.870,0,0.5240,5.6310,100.00,6.0821,5,311.0,15.20,386.63,29.93],94016,16.50),\n",
+ "(10,ARRAY[1,0.17004,12.50,7.870,0,0.5240,6.0040,85.90,6.5921,5,311.0,15.20,386.71,17.10],94016,18.90),\n",
+ "(11,ARRAY[1,0.22489,12.50,7.870,0,0.5240,6.3770,94.30,6.3467,5,311.0,15.20,392.52,20.45],94016,15.00),\n",
+ "(12,ARRAY[1,0.11747,12.50,7.870,0,0.5240,6.0090,82.90,6.2267,5,311.0,15.20,396.90,13.27],20001,18.90),\n",
+ "(13,ARRAY[1,0.09378,12.50,7.870,0,0.5240,5.8890,39.00,5.4509,5,311.0,15.20,390.50,15.71],20001,21.70),\n",
+ "(14,ARRAY[1,0.62976,0.00,8.140,0,0.5380,5.9490,61.80,4.7075,4,307.0,21.00,396.90,8.26],20001,20.40),\n",
+ "(15,ARRAY[1,0.63796,0.00,8.140,0,0.5380,6.0960,84.50,4.4619,4,307.0,21.00,380.02,10.26],20001,18.20),\n",
+ "(16,ARRAY[1,0.62739,0.00,8.140,0,0.5380,5.8340,56.50,4.4986,4,307.0,21.00,395.62,8.47],20001,19.90),\n",
+ "(17,ARRAY[1,1.05393,0.00,8.140,0,0.5380,5.9350,29.30,4.4986,4,307.0,21.00,386.85,6.58],20001, 23.10),\n",
+ "(18,ARRAY[1,0.78420,0.00,8.140,0,0.5380,5.9900,81.70,4.2579,4,307.0,21.00,386.75,14.67],20001,17.50),\n",
+ "(19,ARRAY[1,0.80271,0.00,8.140,0,0.5380,5.4560,36.60,3.7965,4,307.0,21.00,288.99,11.69],20001,20.20),\n",
+ "(20,ARRAY[1,0.72580,0.00,8.140,0,0.5380,5.7270,69.50,3.7965,4,307.0,21.00,390.95,11.28],20001,18.20);\n",
+ "\n",
+ "SELECT * FROM lin_housing ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 13. Regression model\n",
+ "Now train a regression model using a multilayer perceptron with 2 hidden layers of 25 nodes each."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 134,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>mlp_regression</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td></td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[('',)]"
+ ]
+ },
+ "execution_count": 134,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS mlp_regress, mlp_regress_summary, mlp_regress_standardization;\n",
+ "\n",
+ "SELECT setseed(0);\n",
+ "SELECT madlib.mlp_regression(\n",
+ " 'lin_housing', -- Source table\n",
+ " 'mlp_regress', -- Desination table\n",
+ " 'x', -- Input features\n",
+ " 'y', -- Dependent variable\n",
+ " ARRAY[25,25], -- Number of units per layer\n",
+ " 'learning_rate_init=0.001,\n",
+ " n_iterations=500,\n",
+ " lambda=0.001,\n",
+ " tolerance=0', -- Optimizer params\n",
+ " 'relu', -- Activation function\n",
+ " NULL, -- Default weight (1)\n",
+ " FALSE, -- No warm start\n",
+ " FALSE -- Not verbose\n",
+ ");"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 14. View the regression model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 135,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>coeff</th>\n",
+ " <th>loss</th>\n",
+ " <th>num_iterations</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[0.752109182538, -0.0824025290857, 0.0410059928624, 0.337021944674, 0.65416507111, -0.235970675026, 0.105710990669, 0.0477997938709, -0.25104991458, -0.603068658656, 0.383645101517, 0.401628129242, -0.162802028669, -0.156557103874, 0.2105522042, 0.44411313857, 0.105882403567, 0.267663469827, -0.399241363581, -0.0170604141314, -0.377382699093, -0.299853370639, 0.126338917041, 0.210676270596, -0.070565829362, -0.0757525460553, -0.401460560938, -0.210582542703, 0.383367516914, -0.268075915715, -0.173442916036, 0.264575649181, 0.184456779405, -0.0097165532605, 0.00997431792943, 0.0189488858382, 0.116511649746, 0.558767817585, -0.351001044668, 0.604559944202, 0.224654308969, 0.329122775384, -0.136962649643, 0.35692433053, -0.408651008128, -0.0575033890757, 0.24008692791, 0.375288250556, -0.382268762394, 0.285071651184, 0.0202815938214, -0.365421119429, -0.0645604701344, 0.0781523727731, 0.323576095594, -0.131978158405, -0.393724869374, -0.332825704434, -0.044181338530
4, -0.417956144869, -0.256460714895, 0.367189143298, 0.511310537138, 0.214672345403, -0.226875489378, 0.0310210509308, -0.118906658933, 0.344708505926, -0.17563889882, 0.336038890156, -0.0667462014994, -0.405936668753, 0.0138491914875, 0.144754149015, 0.616079043975, 0.130704353611, -0.168294467682, 0.392677221941, -0.00288123069978, -0.0508927001029, 0.146569613764, -0.245656287204, -0.00855988100946, 0.231879433703, 0.214937925868, -0.237298057184, -0.221856092555, 0.419855473628, -0.184932398926, 0.509549986398, 0.848364676338, 0.0691571059329, -0.12501098348, 0.354573894926, -0.0312632006993, 0.330781584014, -0.0945575781122, 0.599881646941, 0.477930952035, 0.0368644507092, 0.291402370717, -0.178220047967, 0.25696983585, 0.423955762121, -0.224823810899, 0.117759914943, 0.110071199153, 0.165406320093, 0.000169648304621, -0.0994666021946, 0.22313907609, -0.212569531367, -0.00804181774423, -0.161692425466, -0.0795345385668, -0.414853143099, -0.0319878410506, -0.0228943999305, -0.40
3365320244, 0.827452938969, -0.241468133377, -0.291736333618, 0.128741060377, 0.147398430131, 0.420266409293, 0.339390474256, 0.197863253743, -0.0954143221801, 0.162946507755, -0.105789077643, -0.151555115552, -0.171680803545, 0.148136907649, -0.327476021688, -0.218371335448, 0.479974277928, -0.292204793018, 0.0691508765986, -0.136512537546, 0.34812612476, -0.331952205898, 0.494087559139, -0.449934439832, 0.0275333229132, -0.655728766517, 0.00147701174048, 0.41330862736, 0.0922963507921, 0.00128558582822, 0.352445255694, 0.266983248469, 0.202645340868, -0.466875319026, -0.274237283821, -0.0344874375585, -0.32906363626, -0.390803035776, -0.0975552077039, -0.137380758754, 0.0485297845885, 0.0880149390821, 0.15598584193, -0.381638091268, -0.349829147079, 0.187767379305, -0.19627281968, 0.304243720029, -0.440061372901, 0.513066236833, -0.551600207373, 0.289095512602, -0.28223824147, -0.544441635982, 0.402488722797, -0.0610017651608, 0.050843874836, -0.208241178518, -0.131554252864, 0.32
0678048538, 0.312675207357, -0.0355083214795, 0.0409488582943, 0.0599442092144, -0.102774575851, 0.119165691458, -0.312361157743, -0.0604515759764, 0.328759826876, -0.164492431141, 0.381209091231, -0.0199720553027, 0.207546700513, 0.219121807697, 0.130512901404, -0.421054928057, 0.331651621998, 0.259992989232, 0.503678285778, 0.043267841401, 0.0763523948028, 0.374054908823, 0.0306548439025, 0.837393801255, 0.337209566495, 0.245463036569, -0.0771582704414, -0.467210091513, 0.475084836639, 0.323805117209, -0.383025045521, 0.149104049736, -0.196576189906, 0.0857511302967, -0.226165994054, 0.0930391937861, -0.142880412787, -0.198743603174, -0.00155007277353, -0.406210296892, 0.227628316285, -0.185312440189, 0.0599849059358, 0.00114737047939, 0.358812682523, -0.234918996681, 0.0341316200714, 0.171073526446, -0.265833332355, -0.0338383472716, 0.100681791486, 0.346657350338, -0.00280320436295, -0.143089617661, 0.267059348013, -0.0749074502189, -0.35274511192, -0.0881180093039, 0.1036167568
18, 0.243472935978, -0.212184518815, 0.371656813353, -0.0133022062202, -0.169948754086, -0.050169477766, 0.28854477509, -0.247670240606, 0.206518527569, -0.407263320198, -0.0875444603835, -0.219289007812, 0.29772747341, 0.249765603598, -0.193444306365, -0.3756050192, 0.00933288367133, 0.25338572147, 0.180015228375, 0.0481910372225, 0.00625538487102, -0.331214327151, 0.328397733498, 0.0693285673286, 0.463588209653, -0.421832831361, 0.0945031623682, -0.281134339656, -0.288763777993, 0.470294399364, 0.187407065939, -0.520244205225, -0.330706311039, -0.107979087849, 0.00581580308799, 0.0933149530593, 0.114432436575, -0.164795756725, 0.195250625489, -0.173799869779, -0.126505013472, -0.252795746611, 0.374565684598, -0.401033212312, 0.225103791284, -0.0979930446003, -0.271302810145, -0.0687136432519, 0.357828941367, 0.287478107344, 0.183861254443, 0.146214251331, -0.295584422439, 0.139370507824, -0.0552549373904, 0.140441522791, 0.201746415561, 0.305001452414, 0.26991025696, -0.2077220879
83, 0.230781894054, -0.0486218777638, -0.0412348938202, 0.315928652965, 0.125159289433, 0.392733141413, 0.0942907960465, 0.0594509510464, -0.229245398136, -0.200056312406, 0.422105787063, -0.219990997434, 0.135655804075, 0.354436596476, -0.116903307569, -0.252831437806, -0.236734192445, 0.493232572173, 0.177534921412, -0.678853712627, 0.226202104373, -0.0323052978471, 0.263812119755, 0.089582259122, -0.0985711472508, -0.209050983422, -0.309590352687, -0.101211948619, -0.296854486931, -0.311745535732, -0.163607048654, 0.161819431155, 0.452881128009, 0.0952589740169, 0.0403205648779, 0.342412677553, -0.00831468239068, -0.128755225228, -0.1026409879, 0.263234751392, 0.220815182949, -0.418091962045, -0.108964160739, -0.311328084993, -0.418564046376, 0.149056423585, -0.15204357297, -0.104682971719, -0.131552924302, -0.330594897393, -0.0659597104145, 0.386126711024, 0.0652016163329, -0.156974345125, -0.144792035304, 0.249947670412, 0.0257226218968, -0.224522181941, 0.330236685238, 0.11136
616445, 0.0739317678304, -0.365834573271, -0.0324767794578, -0.0575793089191, 0.496246255328, -0.215710272757, -0.227194701361, -0.135744441802, 0.1728123765, 0.141816439082, 0.240667199282, 0.00858662933253, 0.213427135043, -0.330636893147, 0.0777718524069, -0.559156785984, 0.498265960413, 0.209778446912, -0.145947336288, -0.266426465146, 0.252851873862, 0.151522256361, -0.309242860686, -0.0350308626896, 0.331248490806, 0.1419392122, -0.199459616973, -0.0213700977558, 0.248837176779, -0.286063678025, -0.276146230114, -0.0814984324179, -0.137366990033, 0.106359985039, 0.208982442892, -0.257427152576, -0.309611555998, -0.310751098667, -0.0311767558708, 0.190492805248, 0.129142573427, -0.0391727039717, -0.261204971454, 0.249837580776, 0.200897595242, 0.0383361036709, -0.0387032891193, -0.0918487256207, -0.270056132045, 0.172558054517, -0.0132385418327, -0.286557173647, 0.0929875665356, 0.162012729233, 0.165406489192, 0.121008860403, -0.156947924955, 0.284377314576, -0.243265896911, -0
.280683609746, 0.416737029901, -0.193692116418, 0.0152023086065, -0.093935202539, 0.254722884645, -0.0642145257547, -0.0616945522594, 0.216955507309, -0.162788593689, 0.254986678588, 0.152350058428, -0.318056067645, -0.209719314718, 0.194242622371, 0.0258810460687, 0.185412661006, -0.0699401585259, -0.242404586191, -0.21440719561, 0.114801408822, -0.269942771176, 0.171018724298, 0.203206518851, 0.174799707147, -0.193630287448, -0.0696319228885, -0.107780391631, -0.0282335659516, -0.193910667074, -0.0486396776352, 0.0557440046157, -0.206565305877, 0.0862357293264, -0.339741514586, -0.127191631972, 0.124359382067, -0.0141058150601, 0.232805292572, -0.263514971502, 0.150806386841, -0.0346701759476, 0.249497446079, 0.265546301338, 0.126291122764, 0.148543207094, -0.0286692743067, 0.15146961004, -0.237376816328, 0.0687592938435, 0.365660129865, 0.146854462476, 0.111312253346, 0.106389494534, 0.0860261132407, -0.0801031524271, 0.2215740019, 0.229513514159, 0.184094389495, -0.141639048475,
-0.352187892212, -0.370947206587, 0.100207140957, -0.233303738937, 0.0659470948013, 0.131842857247, 0.0412011108345, -0.346271914648, -0.345004644313, -0.156477501045, -0.169236295728, 0.104396250782, 0.222890330655, -0.232839410752, -0.126741678466, 0.117716627674, 0.256808516208, 0.0988927700332, -0.136817119135, 0.246878526721, -0.2195475631, -0.245902066799, -0.0146809845532, 0.214495231609, 0.110456804381, 0.267937678488, -0.21706991956, -0.0729932248967, 0.163226543705, 0.270886498349, 0.0303297141028, 0.0740627204247, 0.301075069033, -0.260391810593, 0.225448230163, 0.0471766186593, 0.202609637525, -0.000474654895856, 0.0525110487299, 0.277428468551, 0.218112090594, 0.237248217946, 0.115696905674, 0.119145238901, 0.314854060624, 0.307766306929, -0.0864852288825, 0.249819293104, 0.0406396004587, 0.0956411376162, 0.174556825299, 0.201179881168, 0.16889299831, -0.234380497652, 0.0788832379041, 0.00509369647572, -0.232378757706, 0.210631652523, 0.249053248105, 0.269640790643, 0.
120385609726, -0.0379448372466, 0.0816159884196, 0.0665162437355, 0.105234826008, 0.0331455168967, -0.221739006927, 0.0508328910784, -0.327836017178, 0.130925662083, 0.0114803329716, 0.237225031827, 0.00105047005538, -0.269552990759, 0.0140746082482, 0.0191748435626, -0.30790101868, 0.223496785302, -0.0763824761994, 0.101637411513, -0.0869473040314, -0.276541934073, -0.0762085209744, -0.176417496427, -0.0742782136004, -0.318442635474, 0.181171223461, -0.0407299126913, 0.248864877534, 0.0470731556909, -0.00915274483404, -0.0298517111479, 0.356246941633, -0.304436149304, -0.304780538682, 0.0922429092507, 0.0498552363138, -0.203631678932, -0.206737281264, 0.0738809607608, 0.245094848993, 0.148612974535, -0.0450895361044, -0.0995692135602, 0.23371747268, 0.325057385642, 0.688029772857, 0.498211944692, 0.282447596624, -0.424060265979, 0.095109903181, -0.433955815134, -0.179350146186, 0.525563825366, -0.292548307338, 0.28734109201, 0.547111790455, -0.151637518173, 0.194954334713, 0.296578
592487, 0.467200508955, 0.398406502323, -0.178084080797, 0.361043817455, 0.603349882859, -0.0894855661077, 0.0341132537929, 0.0813043219617, 0.132749164368, 0.261846225082, -0.318470140541, -0.222911236183, 0.240433056414, 0.292373570593, 0.399675478014, -0.403598203854, 0.151345249017, 0.109539102567, -0.355671229949, 0.128303977246, -0.402272396514, 0.0994929690911, 0.194111113676, 0.215027274636, 0.273526596185, -0.230757707891, 0.275866768244, 0.141494595571, -0.150065463478, -0.204086647493, 0.157805556529, -0.344281519938, 0.425336554718, 0.0942775584056, 0.29589323168, 0.44415618877, 0.240682516494, -0.380045847031, -0.107042132178, 0.011227869917, 0.144291159855, 0.0726944292228, 0.277782049318, -0.274223450028, -0.180806550311, 0.150283518394, -0.10010379353, 0.28607701417, 0.180986326602, -0.0197764911826, -0.207646444397, 0.278304557477, -0.323854804091, 0.194252713399, 0.0464626106596, 0.159973018484, 0.16774315265, -0.228900851828, -0.151722137851, -0.218081801389, -0.2
24863288321, 0.0163125453526, -0.238671218526, -0.248615197194, -0.0973904680693, 0.407435102394, -0.335944866491, -0.343263585841, -0.261848152847, -0.149338441767, -0.00814680669417, 0.259445469049, -0.368529153813, 0.0787084504542, -0.239950636357, -0.0144118469096, 0.062427517873, -0.0433511479091, -0.153298723346, -0.171289182491, 0.280616884851, 0.12618429598, -0.220657987191, -0.320605892733, 0.00807802207382, 0.250748438906, 0.234964228413, -0.0782506268437, 0.304492136627, -0.431849369134, -0.0265485445918, -0.217743928218, 0.10055054797, 0.1229711217, -0.121221290895, 0.268278951424, 0.0916211010013, -0.129156283075, -0.33116372839, 0.180985043729, 0.160054208431, 0.131688013546, 0.125354030563, 0.254899559451, -0.164120767568, 0.128291812513, -0.138698298294, 0.194596652004, -0.238860708843, 0.0380178899546, -0.315336382899, -0.271256164553, -0.239873872705, -0.176788093091, -0.251522752554, 0.0383030805781, -0.187553800654, 0.323175268881, 0.359247751487, -0.425963475773
, 0.328243158581, -0.109234678515, -0.220835740328, 0.154468930097, -0.344663447627, 0.0587600299868, 0.0489638466795, 0.225910394992, 0.381801018542, 0.0532836237719, 0.166686728127, 0.205352310597, -0.185271381378, -0.122305494262, 0.204072487562, 0.0253275618271, -0.318368496799, 0.305852917252, -0.0785795132244, -0.132177599249, -0.0223956984967, -0.163778215677, -0.169101075056, 0.128809661107, 0.0633987982098, -0.0479849392255, -0.181109043085, 0.131688752752, -0.135023155226, -0.247046468675, 0.178534189455, -0.167388108544, 0.115145386166, -0.325216612746, 0.236212801864, 0.232417265253, 0.0275544887584, -0.146142886173, 0.092729951582, 0.0261231482138, 0.189902724504, -0.176079531118, -0.160724460124, -0.287686513579, 0.00686818723107, 0.233107367595, 0.0653536619344, 0.0193265084735, 0.247091855313, 0.341688206798, 0.160194596684, -0.146211187577, -0.0952567766309, -0.365147895332, -0.367467123704, -0.329717548045, -0.0437938540316, -0.142335090754, 0.157764133898, 0.22495
5390564, 0.0204180463078, -0.0156459435779, -0.282572516478, -0.202027800222, -0.0136348703444, 0.305992096924, -0.303347842165, -0.289871604408, -0.169550796146, 0.154117753589, 0.0924538930779, -0.26450552206, 0.300265330006, 0.216765054547, -0.209280298389, -0.00186568214219, 0.181566238741, 0.185709585016, -0.33846860054, 0.0461766277893, 0.156783719634, 0.16938865959, 0.288475748493, -0.274261972183, 0.214575787046, 0.308679474779, -0.275685814424, -0.151737938083, -0.173838335325, 0.219918242517, -0.278211607193, 0.180636284551, -0.148096426835, -0.219930916877, 0.304871225267, 0.181504574575, -0.27226963089, 0.336461176652, 0.23620590145, -0.112188864248, 0.143862685961, -0.0257748685333, -0.0402822352288, 0.107704153034, -0.119800002273, 0.0692525917571, -0.251428924813, -0.287161897936, -0.0820397350264, -0.24985383117, 0.122040141968, -0.26536711973, 0.268599372811, 0.057325393046, -0.198505816142, 0.124212723206, 0.022444656401, -0.126355137902, 0.307666369659, 0.18945176
4301, -0.259494799334, -0.31968072217, 0.00795670677648, -0.0702930322583, -0.198409405243, -0.033565517624, -0.139009556785, -0.068780961667, -0.065143175056, -0.445839054973, 0.103211129046, -0.376957497667, -0.108534112303, -0.299969258418, 0.137017641376, 0.201240261999, 0.142048120523, 0.350365820349, 0.298654421098, -0.303893952532, -0.302419412963, 0.0154535549696, -0.227188700672, 0.362204759035, -0.272842415841, -0.0823877508299, 0.133683951892, 0.120891443405, 0.205429634656, 0.0577294305128, -0.10340890905, 0.171571449423, 0.531371883887, 0.566553808629, -0.0520780503133, -0.0787628177761, 0.00340433472768, -0.286500355447, -0.402525295392, 0.488865014742, -0.300403773014, 0.200403968008, 0.588210667321, 0.355749565106, 0.487066679508, 0.264312221547, 0.409127729653, 0.167116753031, 0.0968962336003, 0.23274696174, 0.403827993036, 0.172081801878, -0.172811905026, 0.224453246399, 0.206396444251, -0.0353081725339, -0.343618661148, -0.383318563674, 0.678522844038, 0.515604185
25, 0.304806792601, -0.540454220138, 0.211092277939, -0.37290009409, -0.424084622064, 0.558288983844, -0.551626421418, 0.0108675062321, 0.111220143202, -0.102986852498, 0.134262060987, 0.227250439934, 0.646380169069, -0.0413053530353, 0.019261677, 0.412406597516, 0.663487857047, -0.214333225115, 0.00941033931974, 0.0907604927236, 0.227487539853, 0.491812135301, -0.289395739038, -0.0989568817795, -0.0262702600697, 0.114319043143, 0.20218270762, 0.272918422787, -0.334187836668, 0.130656540733, 0.113742555096, 0.248545131737, 0.198373705871, -0.148636489676, -0.398846723198, 0.142164037048, -0.368336027287, -0.222540537536, -0.10247249525, -0.0480856573101, -0.0961086669627, -0.00954688712445, -0.284862864839, -0.120094349984, 0.1485254925, 0.0377023761729, -0.338321329148, 0.209238823745, 0.195455149231, -0.0344374503483, -0.0980497286311, -0.0728752887693, -0.160014974494, 0.0655260486291, -0.300091123327, 0.270579804162, -0.0520005735765, 0.252068025653, 0.120294068234, -0.027113165
6795, -0.0195424190368, -0.143025885522, -0.193283148435, -0.175744610701, 0.049721115222, -0.138231334164, -0.259957215742, 0.171879987304, -0.0239295347774, -0.00973926408874, -0.211894114588, 0.247278301157, -0.253814002921, -0.14818291327, -0.217343410028, -0.342578856498, 0.159236637749, -0.191596828412, 0.144625068113, 0.0672091074798, 0.0887828441897, -0.253359727958, -0.355869668291, 0.217899398614, 0.142573640195, -0.277351762767, 0.167348886105, -0.230258858193, 0.294488687145, 0.00417458414963, 0.0692115922053, -0.0748945531361, 0.182125172326, 0.224351298742, 0.0461249134393, -0.099153957096, -0.240098006521, 0.218772649181, -0.278552555453, 0.0500467682231, -0.142268076649, -0.13388419
<TRUNCATED>
[6/7] madlib-site git commit: add new workbooks for 1dot13
Posted by fm...@apache.org.
http://git-wip-us.apache.org/repos/asf/madlib-site/blob/95826612/community-artifacts/Elastic-net-v2.ipynb
----------------------------------------------------------------------
diff --git a/community-artifacts/Elastic-net-v2.ipynb b/community-artifacts/Elastic-net-v2.ipynb
new file mode 100644
index 0000000..b6082f0
--- /dev/null
+++ b/community-artifacts/Elastic-net-v2.ipynb
@@ -0,0 +1,2078 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Elastic net (MADlib v1.10+)\n",
+ "Demonstrates elastic net, including these updates:\n",
+ "- in MADlib 1.10: grouping and cross validation which were introduced \n",
+ "- in MADlib 1.13: report negative root mean squared error instead of the negative mean squared error"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/config.py:13: ShimWarning: The `IPython.config` package has been deprecated. You should import from traitlets.config instead.\n",
+ " \"You should import from traitlets.config instead.\", ShimWarning)\n",
+ "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/utils/traitlets.py:5: UserWarning: IPython.utils.traitlets has moved to a top-level traitlets package.\n",
+ " warn(\"IPython.utils.traitlets has moved to a top-level traitlets package.\")\n"
+ ]
+ }
+ ],
+ "source": [
+ "%load_ext sql"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "u'Connected: gpdbchina@madlib'"
+ ]
+ },
+ "execution_count": 2,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Greenplum 4.3.10.0\n",
+ "%sql postgresql://gpdbchina@10.194.10.68:61000/madlib\n",
+ " \n",
+ "# PostgreSQL local\n",
+ "#%sql postgresql://fmcquillan@localhost:5432/madlib\n",
+ "\n",
+ "# Greenplum 4.2.3.0\n",
+ "#%sql postgresql://gpdbchina@10.194.10.68:55000/madlib"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>version</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>MADlib version: 1.13-dev, git revision: rel/v1.12-42-gedc93f5, cmake configuration time: Fri Dec 8 18:28:18 UTC 2017, build type: Release, build system: Linux-2.6.18-238.27.1.el5.hotfix.bz516490, C compiler: gcc 4.4.0, C++ compiler: g++ 4.4.0</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'MADlib version: 1.13-dev, git revision: rel/v1.12-42-gedc93f5, cmake configuration time: Fri Dec 8 18:28:18 UTC 2017, build type: Release, build system: Linux-2.6.18-238.27.1.el5.hotfix.bz516490, C compiler: gcc 4.4.0, C++ compiler: g++ 4.4.0',)]"
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%sql select madlib.version();\n",
+ "#%sql select version();"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## 1. Create data set\n",
+ "House prices and characteristics."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "Done.\n",
+ "27 rows affected.\n",
+ "27 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>tax</th>\n",
+ " <th>bedroom</th>\n",
+ " <th>bath</th>\n",
+ " <th>price</th>\n",
+ " <th>size</th>\n",
+ " <th>lot</th>\n",
+ " <th>zipcode</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>590</td>\n",
+ " <td>2</td>\n",
+ " <td>1.0</td>\n",
+ " <td>50000</td>\n",
+ " <td>770</td>\n",
+ " <td>22100</td>\n",
+ " <td>94301</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>1050</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>85000</td>\n",
+ " <td>1410</td>\n",
+ " <td>12000</td>\n",
+ " <td>94301</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>20</td>\n",
+ " <td>3</td>\n",
+ " <td>1.0</td>\n",
+ " <td>22500</td>\n",
+ " <td>1060</td>\n",
+ " <td>3500</td>\n",
+ " <td>94301</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>870</td>\n",
+ " <td>2</td>\n",
+ " <td>2.0</td>\n",
+ " <td>90000</td>\n",
+ " <td>1300</td>\n",
+ " <td>17500</td>\n",
+ " <td>94301</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>1320</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>133000</td>\n",
+ " <td>1500</td>\n",
+ " <td>30000</td>\n",
+ " <td>94301</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>1350</td>\n",
+ " <td>2</td>\n",
+ " <td>1.0</td>\n",
+ " <td>90500</td>\n",
+ " <td>820</td>\n",
+ " <td>25700</td>\n",
+ " <td>94301</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>7</td>\n",
+ " <td>2790</td>\n",
+ " <td>3</td>\n",
+ " <td>2.5</td>\n",
+ " <td>260000</td>\n",
+ " <td>2130</td>\n",
+ " <td>25000</td>\n",
+ " <td>94301</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>8</td>\n",
+ " <td>680</td>\n",
+ " <td>2</td>\n",
+ " <td>1.0</td>\n",
+ " <td>142500</td>\n",
+ " <td>1170</td>\n",
+ " <td>22000</td>\n",
+ " <td>94301</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>9</td>\n",
+ " <td>1840</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>160000</td>\n",
+ " <td>1500</td>\n",
+ " <td>19000</td>\n",
+ " <td>94301</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>10</td>\n",
+ " <td>3680</td>\n",
+ " <td>4</td>\n",
+ " <td>2.0</td>\n",
+ " <td>240000</td>\n",
+ " <td>2790</td>\n",
+ " <td>20000</td>\n",
+ " <td>94301</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>11</td>\n",
+ " <td>1660</td>\n",
+ " <td>3</td>\n",
+ " <td>1.0</td>\n",
+ " <td>87000</td>\n",
+ " <td>1030</td>\n",
+ " <td>17500</td>\n",
+ " <td>94301</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>12</td>\n",
+ " <td>1620</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>118600</td>\n",
+ " <td>1250</td>\n",
+ " <td>20000</td>\n",
+ " <td>94301</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>13</td>\n",
+ " <td>3100</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>140000</td>\n",
+ " <td>1760</td>\n",
+ " <td>38000</td>\n",
+ " <td>94301</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>14</td>\n",
+ " <td>2070</td>\n",
+ " <td>2</td>\n",
+ " <td>3.0</td>\n",
+ " <td>148000</td>\n",
+ " <td>1550</td>\n",
+ " <td>14000</td>\n",
+ " <td>94301</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>15</td>\n",
+ " <td>650</td>\n",
+ " <td>3</td>\n",
+ " <td>1.5</td>\n",
+ " <td>65000</td>\n",
+ " <td>1450</td>\n",
+ " <td>12000</td>\n",
+ " <td>94301</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>16</td>\n",
+ " <td>770</td>\n",
+ " <td>2</td>\n",
+ " <td>2.0</td>\n",
+ " <td>91000</td>\n",
+ " <td>1300</td>\n",
+ " <td>17500</td>\n",
+ " <td>76010</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>17</td>\n",
+ " <td>1220</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>132300</td>\n",
+ " <td>1500</td>\n",
+ " <td>30000</td>\n",
+ " <td>76010</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>18</td>\n",
+ " <td>1150</td>\n",
+ " <td>2</td>\n",
+ " <td>1.0</td>\n",
+ " <td>91100</td>\n",
+ " <td>820</td>\n",
+ " <td>25700</td>\n",
+ " <td>76010</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>19</td>\n",
+ " <td>2690</td>\n",
+ " <td>3</td>\n",
+ " <td>2.5</td>\n",
+ " <td>260011</td>\n",
+ " <td>2130</td>\n",
+ " <td>25000</td>\n",
+ " <td>76010</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>20</td>\n",
+ " <td>780</td>\n",
+ " <td>2</td>\n",
+ " <td>1.0</td>\n",
+ " <td>141800</td>\n",
+ " <td>1170</td>\n",
+ " <td>22000</td>\n",
+ " <td>76010</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>21</td>\n",
+ " <td>1910</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>160900</td>\n",
+ " <td>1500</td>\n",
+ " <td>19000</td>\n",
+ " <td>76010</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>22</td>\n",
+ " <td>3600</td>\n",
+ " <td>4</td>\n",
+ " <td>2.0</td>\n",
+ " <td>239000</td>\n",
+ " <td>2790</td>\n",
+ " <td>20000</td>\n",
+ " <td>76010</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>23</td>\n",
+ " <td>1600</td>\n",
+ " <td>3</td>\n",
+ " <td>1.0</td>\n",
+ " <td>81010</td>\n",
+ " <td>1030</td>\n",
+ " <td>17500</td>\n",
+ " <td>76010</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>24</td>\n",
+ " <td>1590</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>117910</td>\n",
+ " <td>1250</td>\n",
+ " <td>20000</td>\n",
+ " <td>76010</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>25</td>\n",
+ " <td>3200</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>141100</td>\n",
+ " <td>1760</td>\n",
+ " <td>38000</td>\n",
+ " <td>76010</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>26</td>\n",
+ " <td>2270</td>\n",
+ " <td>2</td>\n",
+ " <td>3.0</td>\n",
+ " <td>148011</td>\n",
+ " <td>1550</td>\n",
+ " <td>14000</td>\n",
+ " <td>76010</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>27</td>\n",
+ " <td>750</td>\n",
+ " <td>3</td>\n",
+ " <td>1.5</td>\n",
+ " <td>66000</td>\n",
+ " <td>1450</td>\n",
+ " <td>12000</td>\n",
+ " <td>76010</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, 590, 2, 1.0, 50000, 770, 22100, 94301),\n",
+ " (2, 1050, 3, 2.0, 85000, 1410, 12000, 94301),\n",
+ " (3, 20, 3, 1.0, 22500, 1060, 3500, 94301),\n",
+ " (4, 870, 2, 2.0, 90000, 1300, 17500, 94301),\n",
+ " (5, 1320, 3, 2.0, 133000, 1500, 30000, 94301),\n",
+ " (6, 1350, 2, 1.0, 90500, 820, 25700, 94301),\n",
+ " (7, 2790, 3, 2.5, 260000, 2130, 25000, 94301),\n",
+ " (8, 680, 2, 1.0, 142500, 1170, 22000, 94301),\n",
+ " (9, 1840, 3, 2.0, 160000, 1500, 19000, 94301),\n",
+ " (10, 3680, 4, 2.0, 240000, 2790, 20000, 94301),\n",
+ " (11, 1660, 3, 1.0, 87000, 1030, 17500, 94301),\n",
+ " (12, 1620, 3, 2.0, 118600, 1250, 20000, 94301),\n",
+ " (13, 3100, 3, 2.0, 140000, 1760, 38000, 94301),\n",
+ " (14, 2070, 2, 3.0, 148000, 1550, 14000, 94301),\n",
+ " (15, 650, 3, 1.5, 65000, 1450, 12000, 94301),\n",
+ " (16, 770, 2, 2.0, 91000, 1300, 17500, 76010),\n",
+ " (17, 1220, 3, 2.0, 132300, 1500, 30000, 76010),\n",
+ " (18, 1150, 2, 1.0, 91100, 820, 25700, 76010),\n",
+ " (19, 2690, 3, 2.5, 260011, 2130, 25000, 76010),\n",
+ " (20, 780, 2, 1.0, 141800, 1170, 22000, 76010),\n",
+ " (21, 1910, 3, 2.0, 160900, 1500, 19000, 76010),\n",
+ " (22, 3600, 4, 2.0, 239000, 2790, 20000, 76010),\n",
+ " (23, 1600, 3, 1.0, 81010, 1030, 17500, 76010),\n",
+ " (24, 1590, 3, 2.0, 117910, 1250, 20000, 76010),\n",
+ " (25, 3200, 3, 2.0, 141100, 1760, 38000, 76010),\n",
+ " (26, 2270, 2, 3.0, 148011, 1550, 14000, 76010),\n",
+ " (27, 750, 3, 1.5, 66000, 1450, 12000, 76010)]"
+ ]
+ },
+ "execution_count": 4,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql \n",
+ "DROP TABLE IF EXISTS houses;\n",
+ "\n",
+ "CREATE TABLE houses ( id INT,\n",
+ " tax INT,\n",
+ " bedroom INT,\n",
+ " bath FLOAT,\n",
+ " price INT,\n",
+ " size INT,\n",
+ " lot INT,\n",
+ " zipcode INT);\n",
+ "\n",
+ "INSERT INTO houses (id, tax, bedroom, bath, price, size, lot, zipcode) VALUES\n",
+ "(1 , 590 , 2 , 1 , 50000 , 770 , 22100 , 94301),\n",
+ "(2 , 1050 , 3 , 2 , 85000 , 1410 , 12000 , 94301),\n",
+ "(3 , 20 , 3 , 1 , 22500 , 1060 , 3500 , 94301),\n",
+ "(4 , 870 , 2 , 2 , 90000 , 1300 , 17500 , 94301),\n",
+ "(5 , 1320 , 3 , 2 , 133000 , 1500 , 30000 , 94301),\n",
+ "(6 , 1350 , 2 , 1 , 90500 , 820 , 25700 , 94301),\n",
+ "(7 , 2790 , 3 , 2.5 , 260000 , 2130 , 25000 , 94301),\n",
+ "(8 , 680 , 2 , 1 , 142500 , 1170 , 22000 , 94301),\n",
+ "(9 , 1840 , 3 , 2 , 160000 , 1500 , 19000 , 94301),\n",
+ "(10 , 3680 , 4 , 2 , 240000 , 2790 , 20000 , 94301),\n",
+ "(11 , 1660 , 3 , 1 , 87000 , 1030 , 17500 , 94301),\n",
+ "(12 , 1620 , 3 , 2 , 118600 , 1250 , 20000 , 94301),\n",
+ "(13 , 3100 , 3 , 2 , 140000 , 1760 , 38000 , 94301),\n",
+ "(14 , 2070 , 2 , 3 , 148000 , 1550 , 14000 , 94301),\n",
+ "(15 , 650 , 3 , 1.5 , 65000 , 1450 , 12000 , 94301),\n",
+ "(16 , 770 , 2 , 2 , 91000 , 1300 , 17500 , 76010),\n",
+ "(17 , 1220 , 3 , 2 , 132300 , 1500 , 30000 , 76010),\n",
+ "(18 , 1150 , 2 , 1 , 91100 , 820 , 25700 , 76010),\n",
+ "(19 , 2690 , 3 , 2.5 , 260011 , 2130 , 25000 , 76010),\n",
+ "(20 , 780 , 2 , 1 , 141800 , 1170 , 22000 , 76010),\n",
+ "(21 , 1910 , 3 , 2 , 160900 , 1500 , 19000 , 76010),\n",
+ "(22 , 3600 , 4 , 2 , 239000 , 2790 , 20000 , 76010),\n",
+ "(23 , 1600 , 3 , 1 , 81010 , 1030 , 17500 , 76010),\n",
+ "(24 , 1590 , 3 , 2 , 117910 , 1250 , 20000 , 76010),\n",
+ "(25 , 3200 , 3 , 2 , 141100 , 1760 , 38000 , 76010),\n",
+ "(26 , 2270 , 2 , 3 , 148011 , 1550 , 14000 , 76010),\n",
+ "(27 , 750 , 3 , 1.5 , 66000 , 1450 , 12000 , 76010);\n",
+ "\n",
+ "SELECT * FROM houses ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## 2. Train the model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>family</th>\n",
+ " <th>features</th>\n",
+ " <th>features_selected</th>\n",
+ " <th>coef_nonzero</th>\n",
+ " <th>coef_all</th>\n",
+ " <th>intercept</th>\n",
+ " <th>log_likelihood</th>\n",
+ " <th>standardize</th>\n",
+ " <th>iteration_run</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>gaussian</td>\n",
+ " <td>[u'tax', u'bath', u'size']</td>\n",
+ " <td>[u'tax', u'bath', u'size']</td>\n",
+ " <td>[22.7851131403, 10707.9523048, 54.7961553151]</td>\n",
+ " <td>[22.7851131403, 10707.9523048, 54.7961553151]</td>\n",
+ " <td>-7798.8034742</td>\n",
+ " <td>-512248641.969</td>\n",
+ " <td>True</td>\n",
+ " <td>457</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'gaussian', [u'tax', u'bath', u'size'], [u'tax', u'bath', u'size'], [22.7851131403, 10707.9523048, 54.7961553151], [22.7851131403, 10707.9523048, 54.7961553151], -7798.8034742, -512248641.969, True, 457)]"
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS houses_en, houses_en_summary;\n",
+ "SELECT madlib.elastic_net_train( 'houses', -- Source table\n",
+ " 'houses_en', -- Result table\n",
+ " 'price', -- Dependent variable\n",
+ " 'array[tax, bath, size]', -- Independent variable\n",
+ " 'gaussian', -- Regression family\n",
+ " 0.5, -- Alpha value\n",
+ " 0.1, -- Lambda value\n",
+ " TRUE, -- Standardize\n",
+ " NULL, -- Grouping column(s)\n",
+ " 'fista', -- Optimizer\n",
+ " '', -- Optimizer parameters\n",
+ " NULL, -- Excluded columns\n",
+ " 10000, -- Maximum iterations\n",
+ " 1e-6 -- Tolerance value\n",
+ " );\n",
+ "SELECT * FROM houses_en;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 3. Prediction\n",
+ "Evaluate residuals."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "27 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>price</th>\n",
+ " <th>predict</th>\n",
+ " <th>residual</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>50000</td>\n",
+ " <td>58545.405176</td>\n",
+ " <td>-8545.405176</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>85000</td>\n",
+ " <td>114804.048927</td>\n",
+ " <td>-29804.048927</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>22500</td>\n",
+ " <td>61448.7757274</td>\n",
+ " <td>-38948.7757274</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>90000</td>\n",
+ " <td>104675.151477</td>\n",
+ " <td>-14675.1514771</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>133000</td>\n",
+ " <td>125887.683453</td>\n",
+ " <td>7112.31654675</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>90500</td>\n",
+ " <td>78601.8989284</td>\n",
+ " <td>11898.1010716</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>7</td>\n",
+ " <td>260000</td>\n",
+ " <td>199257.35377</td>\n",
+ " <td>60742.6462296</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>8</td>\n",
+ " <td>142500</td>\n",
+ " <td>82514.5274847</td>\n",
+ " <td>59985.4725153</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>9</td>\n",
+ " <td>160000</td>\n",
+ " <td>137735.942286</td>\n",
+ " <td>22264.0577138</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>10</td>\n",
+ " <td>240000</td>\n",
+ " <td>250347.590821</td>\n",
+ " <td>-10347.5908208</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>11</td>\n",
+ " <td>87000</td>\n",
+ " <td>97172.4766181</td>\n",
+ " <td>-10172.4766181</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>12</td>\n",
+ " <td>118600</td>\n",
+ " <td>119024.178567</td>\n",
+ " <td>-424.178566561</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>13</td>\n",
+ " <td>140000</td>\n",
+ " <td>180692.185225</td>\n",
+ " <td>-40692.1852249</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>14</td>\n",
+ " <td>148000</td>\n",
+ " <td>156424.278379</td>\n",
+ " <td>-8424.27837903</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>15</td>\n",
+ " <td>65000</td>\n",
+ " <td>102527.873731</td>\n",
+ " <td>-37527.8737311</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>16</td>\n",
+ " <td>91000</td>\n",
+ " <td>102396.640163</td>\n",
+ " <td>-11396.6401631</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>17</td>\n",
+ " <td>132300</td>\n",
+ " <td>123609.172139</td>\n",
+ " <td>8690.82786078</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>18</td>\n",
+ " <td>91100</td>\n",
+ " <td>74044.8763003</td>\n",
+ " <td>17055.1236997</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>19</td>\n",
+ " <td>260011</td>\n",
+ " <td>196978.842456</td>\n",
+ " <td>63032.1575436</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>20</td>\n",
+ " <td>141800</td>\n",
+ " <td>84793.0387987</td>\n",
+ " <td>57006.9612013</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>21</td>\n",
+ " <td>160900</td>\n",
+ " <td>139330.900206</td>\n",
+ " <td>21569.099794</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>22</td>\n",
+ " <td>239000</td>\n",
+ " <td>248524.78177</td>\n",
+ " <td>-9524.78176961</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>23</td>\n",
+ " <td>81010</td>\n",
+ " <td>95805.3698296</td>\n",
+ " <td>-14795.3698296</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>24</td>\n",
+ " <td>117910</td>\n",
+ " <td>118340.625172</td>\n",
+ " <td>-430.625172352</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>25</td>\n",
+ " <td>141100</td>\n",
+ " <td>182970.696539</td>\n",
+ " <td>-41870.6965389</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>26</td>\n",
+ " <td>148011</td>\n",
+ " <td>160981.301007</td>\n",
+ " <td>-12970.3010071</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>27</td>\n",
+ " <td>66000</td>\n",
+ " <td>104806.385045</td>\n",
+ " <td>-38806.3850451</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, 50000, 58545.405176004, -8545.405176004),\n",
+ " (2, 85000, 114804.048927006, -29804.048927006),\n",
+ " (3, 22500, 61448.775727412, -38948.775727412),\n",
+ " (4, 90000, 104675.151477091, -14675.151477091),\n",
+ " (5, 133000, 125887.683453246, 7112.31654675399),\n",
+ " (6, 90500, 78601.898928387, 11898.101071613),\n",
+ " (7, 260000, 199257.3537704, 60742.6462296),\n",
+ " (8, 142500, 82514.527484671, 59985.472515329),\n",
+ " (9, 160000, 137735.942286202, 22264.057713798),\n",
+ " (10, 240000, 250347.590820833, -10347.590820833),\n",
+ " (11, 87000, 97172.476618051, -10172.476618051),\n",
+ " (12, 118600, 119024.178566561, -424.178566560993),\n",
+ " (13, 140000, 180692.185224906, -40692.185224906),\n",
+ " (14, 148000, 156424.278379026, -8424.27837902599),\n",
+ " (15, 65000, 102527.87373109, -37527.87373109),\n",
+ " (16, 91000, 102396.640163061, -11396.640163061),\n",
+ " (17, 132300, 123609.172139216, 8690.827860784),\n",
+ " (18, 91100, 74044.876300327, 17055.123699673),\n",
+ " (19, 260011, 196978.84245637, 63032.15754363),\n",
+ " (20, 141800, 84793.038798701, 57006.961201299),\n",
+ " (21, 160900, 139330.900206023, 21569.099793977),\n",
+ " (22, 239000, 248524.781769609, -9524.78176960899),\n",
+ " (23, 81010, 95805.369829633, -14795.369829633),\n",
+ " (24, 117910, 118340.625172352, -430.625172351996),\n",
+ " (25, 141100, 182970.696538936, -41870.696538936),\n",
+ " (26, 148011, 160981.301007086, -12970.301007086),\n",
+ " (27, 66000, 104806.38504512, -38806.38504512)]"
+ ]
+ },
+ "execution_count": 7,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT id, price, predict, price - predict AS residual\n",
+ "FROM (\n",
+ " SELECT\n",
+ " houses.*,\n",
+ " madlib.elastic_net_gaussian_predict(\n",
+ " m.coef_all, -- Coefficients\n",
+ " m.intercept, -- Intercept\n",
+ " ARRAY[tax,bath,size] -- Features (corresponding to coefficients)\n",
+ " ) AS predict\n",
+ " FROM houses, houses_en m) s\n",
+ "ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 4. Grouping \n",
+ "Group on zip code."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "2 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>zipcode</th>\n",
+ " <th>family</th>\n",
+ " <th>features</th>\n",
+ " <th>features_selected</th>\n",
+ " <th>coef_nonzero</th>\n",
+ " <th>coef_all</th>\n",
+ " <th>intercept</th>\n",
+ " <th>log_likelihood</th>\n",
+ " <th>standardize</th>\n",
+ " <th>iteration_run</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>76010</td>\n",
+ " <td>gaussian</td>\n",
+ " <td>[u'tax', u'bath', u'size']</td>\n",
+ " <td>[u'tax', u'bath', u'size']</td>\n",
+ " <td>[14.9802020928, 9133.17041265, 62.8225614522]</td>\n",
+ " <td>[14.9802020928, 9133.17041265, 62.8225614522]</td>\n",
+ " <td>14.7294468096</td>\n",
+ " <td>-525667117.987</td>\n",
+ " <td>True</td>\n",
+ " <td>10000</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>94301</td>\n",
+ " <td>gaussian</td>\n",
+ " <td>[u'tax', u'bath', u'size']</td>\n",
+ " <td>[u'tax', u'bath', u'size']</td>\n",
+ " <td>[27.6945649037, 11509.010807, 49.0945476263]</td>\n",
+ " <td>[27.6945649037, 11509.010807, 49.0945476263]</td>\n",
+ " <td>-11145.5017384</td>\n",
+ " <td>-520358795.785</td>\n",
+ " <td>True</td>\n",
+ " <td>10000</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(76010, u'gaussian', [u'tax', u'bath', u'size'], [u'tax', u'bath', u'size'], [14.9802020928, 9133.17041265, 62.8225614522], [14.9802020928, 9133.17041265, 62.8225614522], 14.7294468096, -525667117.987, True, 10000),\n",
+ " (94301, u'gaussian', [u'tax', u'bath', u'size'], [u'tax', u'bath', u'size'], [27.6945649037, 11509.010807, 49.0945476263], [27.6945649037, 11509.010807, 49.0945476263], -11145.5017384, -520358795.785, True, 10000)]"
+ ]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS houses_en1, houses_en1_summary;\n",
+ "SELECT madlib.elastic_net_train( 'houses', -- Source table\n",
+ " 'houses_en1', -- Result table\n",
+ " 'price', -- Dependent variable\n",
+ " 'array[tax, bath, size]', -- Independent variable\n",
+ " 'gaussian', -- Regression family\n",
+ " 0.5, -- Alpha value\n",
+ " 0.1, -- Lambda value\n",
+ " TRUE, -- Standardize\n",
+ " 'zipcode', -- Grouping column(s)\n",
+ " 'fista', -- Optimizer\n",
+ " '', -- Optimizer parameters\n",
+ " NULL, -- Excluded columns\n",
+ " 10000, -- Maximum iterations\n",
+ " 1e-6 -- Tolerance value\n",
+ " );\n",
+ "SELECT * FROM houses_en1;"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "metadata": {},
+ "source": [
+ "Prediction function"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n",
+ "27 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>price</th>\n",
+ " <th>prediction</th>\n",
+ " <th>residual</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>50000</td>\n",
+ " <td>54506.104034</td>\n",
+ " <td>-4506.10403403</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>85000</td>\n",
+ " <td>110175.125178</td>\n",
+ " <td>-25175.1251776</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>22500</td>\n",
+ " <td>52957.6208506</td>\n",
+ " <td>-30457.6208506</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>90000</td>\n",
+ " <td>99789.703256</td>\n",
+ " <td>-9789.70325601</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>133000</td>\n",
+ " <td>122071.166988</td>\n",
+ " <td>10928.8330121</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>90500</td>\n",
+ " <td>78008.7007422</td>\n",
+ " <td>12491.2992578</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>7</td>\n",
+ " <td>260000</td>\n",
+ " <td>199466.247804</td>\n",
+ " <td>60533.7521956</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>8</td>\n",
+ " <td>142500</td>\n",
+ " <td>76636.4339259</td>\n",
+ " <td>65863.5660741</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>9</td>\n",
+ " <td>160000</td>\n",
+ " <td>136472.340738</td>\n",
+ " <td>23527.6592621</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>10</td>\n",
+ " <td>240000</td>\n",
+ " <td>250762.306599</td>\n",
+ " <td>-10762.3065986</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>11</td>\n",
+ " <td>87000</td>\n",
+ " <td>96903.8708638</td>\n",
+ " <td>-9903.87086383</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>12</td>\n",
+ " <td>118600</td>\n",
+ " <td>118105.899552</td>\n",
+ " <td>494.100447531</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>13</td>\n",
+ " <td>140000</td>\n",
+ " <td>184132.074899</td>\n",
+ " <td>-44132.0748994</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>14</td>\n",
+ " <td>148000</td>\n",
+ " <td>156805.828854</td>\n",
+ " <td>-8805.82885402</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>15</td>\n",
+ " <td>65000</td>\n",
+ " <td>95306.5757176</td>\n",
+ " <td>-30306.5757176</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>16</td>\n",
+ " <td>91000</td>\n",
+ " <td>111485.155771</td>\n",
+ " <td>-20485.1557714</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>17</td>\n",
+ " <td>132300</td>\n",
+ " <td>130790.759004</td>\n",
+ " <td>1509.24099637</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>18</td>\n",
+ " <td>91100</td>\n",
+ " <td>77889.632657</td>\n",
+ " <td>13210.367343</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>19</td>\n",
+ " <td>260011</td>\n",
+ " <td>196956.455001</td>\n",
+ " <td>63054.5449987</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>20</td>\n",
+ " <td>141800</td>\n",
+ " <td>94334.8543909</td>\n",
+ " <td>47465.1456091</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>21</td>\n",
+ " <td>160900</td>\n",
+ " <td>141127.098448</td>\n",
+ " <td>19772.9015523</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>22</td>\n",
+ " <td>239000</td>\n",
+ " <td>247484.744258</td>\n",
+ " <td>-8484.74425783</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>23</td>\n",
+ " <td>81010</td>\n",
+ " <td>97823.4615037</td>\n",
+ " <td>-16813.4615037</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>24</td>\n",
+ " <td>117910</td>\n",
+ " <td>120627.793415</td>\n",
+ " <td>-2717.79341491</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>25</td>\n",
+ " <td>141100</td>\n",
+ " <td>176785.425125</td>\n",
+ " <td>-35685.4251249</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>26</td>\n",
+ " <td>148011</td>\n",
+ " <td>158794.269686</td>\n",
+ " <td>-10783.2696863</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>27</td>\n",
+ " <td>66000</td>\n",
+ " <td>116042.350741</td>\n",
+ " <td>-50042.3507411</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, 50000, 54506.104034034, -4506.104034034),\n",
+ " (2, 85000, 110175.125177568, -25175.125177568),\n",
+ " (3, 22500, 52957.620850552, -30457.620850552),\n",
+ " (4, 90000, 99789.703256009, -9789.703256009),\n",
+ " (5, 133000, 122071.166987934, 10928.833012066),\n",
+ " (6, 90500, 78008.700742161, 12491.299257839),\n",
+ " (7, 260000, 199466.247804442, 60533.752195558),\n",
+ " (8, 142500, 76636.433925887, 65863.566074113),\n",
+ " (9, 160000, 136472.340737858, 23527.659262142),\n",
+ " (10, 240000, 250762.306598593, -10762.306598593),\n",
+ " (11, 87000, 96903.870863831, -9903.87086383101),\n",
+ " (12, 118600, 118105.899552469, 494.100447531004),\n",
+ " (13, 140000, 184132.074899358, -44132.074899358),\n",
+ " (14, 148000, 156805.828854024, -8805.828854024),\n",
+ " (15, 65000, 95306.57571764, -30306.57571764),\n",
+ " (16, 91000, 111485.155771426, -20485.1557714256),\n",
+ " (17, 132300, 130790.759003626, 1509.2409963744),\n",
+ " (18, 91100, 77889.6326569836, 13210.3673430164),\n",
+ " (19, 260011, 196956.455001253, 63054.5449987474),\n",
+ " (20, 141800, 94334.8543909176, 47465.1456090824),\n",
+ " (21, 160900, 141127.098447658, 19772.9015523424),\n",
+ " (22, 239000, 247484.744257828, -8484.74425782761),\n",
+ " (23, 81010, 97823.4615037056, -16813.4615037056),\n",
+ " (24, 117910, 120627.793414912, -2717.7934149116),\n",
+ " (25, 141100, 176785.425124942, -35685.4251249416),\n",
+ " (26, 148011, 158794.269686326, -10783.2696863256),\n",
+ " (27, 66000, 116042.350741075, -50042.3507410746)]"
+ ]
+ },
+ "execution_count": 9,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT madlib.elastic_net_predict(\n",
+ " 'houses_en1', -- Model table\n",
+ " 'houses', -- New source data table\n",
+ " 'id', -- Unique ID associated with each row\n",
+ " 'houses_en1_prediction' -- Table to store prediction result\n",
+ " );\n",
+ "\n",
+ "SELECT houses.id,\n",
+ " houses.price,\n",
+ " houses_en1_prediction.prediction,\n",
+ " houses.price - houses_en1_prediction.prediction AS residual\n",
+ "FROM houses_en1_prediction, houses\n",
+ "WHERE houses.id = houses_en1_prediction.id ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## 5. When coef_nonzero is different from coef_all\n",
+ "Train"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>family</th>\n",
+ " <th>features</th>\n",
+ " <th>features_selected</th>\n",
+ " <th>coef_nonzero</th>\n",
+ " <th>coef_all</th>\n",
+ " <th>intercept</th>\n",
+ " <th>log_likelihood</th>\n",
+ " <th>standardize</th>\n",
+ " <th>iteration_run</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>gaussian</td>\n",
+ " <td>[u'tax', u'bath', u'size']</td>\n",
+ " <td>[u'tax', u'size']</td>\n",
+ " <td>[6.94383308191, 29.7206857861]</td>\n",
+ " <td>[6.94383308191, 0.0, 29.7206857861]</td>\n",
+ " <td>74441.4573381</td>\n",
+ " <td>-1635348584.1</td>\n",
+ " <td>True</td>\n",
+ " <td>173</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'gaussian', [u'tax', u'bath', u'size'], [u'tax', u'size'], [6.94383308191, 29.7206857861], [6.94383308191, 0.0, 29.7206857861], 74441.4573381, -1635348584.1, True, 173)]"
+ ]
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS houses_en2, houses_en2_summary;\n",
+ "SELECT madlib.elastic_net_train( 'houses', -- Source table\n",
+ " 'houses_en2', -- Result table\n",
+ " 'price', -- Dependent variable\n",
+ " 'array[tax, bath, size]', -- Independent variable\n",
+ " 'gaussian', -- Regression family\n",
+ " 1, -- Alpha value\n",
+ " 30000, -- Lambda value\n",
+ " TRUE, -- Standardize\n",
+ " NULL, -- Grouping column(s)\n",
+ " 'fista', -- Optimizer\n",
+ " '', -- Optimizer parameters\n",
+ " NULL, -- Excluded columns\n",
+ " 10000, -- Maximum iterations\n",
+ " 1e-6 -- Tolerance value\n",
+ " );\n",
+ "SELECT * FROM houses_en2;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Prediction function with coef_all to evaluate residuals."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "27 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>price</th>\n",
+ " <th>predict</th>\n",
+ " <th>residual</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>50000</td>\n",
+ " <td>101423.246912</td>\n",
+ " <td>-51423.2469117</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>85000</td>\n",
+ " <td>123638.649033</td>\n",
+ " <td>-38638.6490325</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>22500</td>\n",
+ " <td>106084.260933</td>\n",
+ " <td>-83584.260933</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>90000</td>\n",
+ " <td>119119.483641</td>\n",
+ " <td>-29119.4836413</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>133000</td>\n",
+ " <td>128188.345685</td>\n",
+ " <td>4811.65431463</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>90500</td>\n",
+ " <td>108186.594343</td>\n",
+ " <td>-17686.5943433</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>7</td>\n",
+ " <td>260000</td>\n",
+ " <td>157119.812361</td>\n",
+ " <td>102880.187639</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>8</td>\n",
+ " <td>142500</td>\n",
+ " <td>113936.466204</td>\n",
+ " <td>28563.5337965</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>9</td>\n",
+ " <td>160000</td>\n",
+ " <td>131799.138888</td>\n",
+ " <td>28200.861112</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>10</td>\n",
+ " <td>240000</td>\n",
+ " <td>182915.476423</td>\n",
+ " <td>57084.5235773</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>11</td>\n",
+ " <td>87000</td>\n",
+ " <td>116580.526614</td>\n",
+ " <td>-29580.5266138</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>12</td>\n",
+ " <td>118600</td>\n",
+ " <td>122841.324163</td>\n",
+ " <td>-4241.32416342</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>13</td>\n",
+ " <td>140000</td>\n",
+ " <td>148275.746876</td>\n",
+ " <td>-8275.74687556</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>14</td>\n",
+ " <td>148000</td>\n",
+ " <td>134882.254786</td>\n",
+ " <td>13117.7452139</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>15</td>\n",
+ " <td>65000</td>\n",
+ " <td>122049.943231</td>\n",
+ " <td>-57049.9432312</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>16</td>\n",
+ " <td>91000</td>\n",
+ " <td>118425.100333</td>\n",
+ " <td>-27425.1003331</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>17</td>\n",
+ " <td>132300</td>\n",
+ " <td>127493.962377</td>\n",
+ " <td>4806.03762282</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>18</td>\n",
+ " <td>91100</td>\n",
+ " <td>106797.827727</td>\n",
+ " <td>-15697.8277269</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>19</td>\n",
+ " <td>260011</td>\n",
+ " <td>156425.429053</td>\n",
+ " <td>103585.570947</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>20</td>\n",
+ " <td>141800</td>\n",
+ " <td>114630.849512</td>\n",
+ " <td>27169.1504883</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>21</td>\n",
+ " <td>160900</td>\n",
+ " <td>132285.207204</td>\n",
+ " <td>28614.7927963</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>22</td>\n",
+ " <td>239000</td>\n",
+ " <td>182359.969776</td>\n",
+ " <td>56640.0302238</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>23</td>\n",
+ " <td>81010</td>\n",
+ " <td>116163.896629</td>\n",
+ " <td>-35153.8966288</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>24</td>\n",
+ " <td>117910</td>\n",
+ " <td>122633.009171</td>\n",
+ " <td>-4723.00917096</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>25</td>\n",
+ " <td>141100</td>\n",
+ " <td>148970.130184</td>\n",
+ " <td>-7870.13018375</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>26</td>\n",
+ " <td>148011</td>\n",
+ " <td>136271.021402</td>\n",
+ " <td>11739.9785975</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>27</td>\n",
+ " <td>66000</td>\n",
+ " <td>122744.326539</td>\n",
+ " <td>-56744.3265394</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, 50000, 101423.246911724, -51423.2469117239),\n",
+ " (2, 85000, 123638.649032506, -38638.6490325065),\n",
+ " (3, 22500, 106084.260933004, -83584.2609330042),\n",
+ " (4, 90000, 119119.483641292, -29119.4836412917),\n",
+ " (5, 133000, 128188.345685371, 4811.6543146288),\n",
+ " (6, 90500, 108186.59434328, -17686.5943432805),\n",
+ " (7, 260000, 157119.812361022, 102880.187638978),\n",
+ " (8, 142500, 113936.466203536, 28563.5337964642),\n",
+ " (9, 160000, 131799.138887964, 28200.8611120356),\n",
+ " (10, 240000, 182915.476422748, 57084.5235772522),\n",
+ " (11, 87000, 116580.526613754, -29580.5266137536),\n",
+ " (12, 118600, 122841.324163419, -4241.32416341919),\n",
+ " (13, 140000, 148275.746875557, -8275.746875557),\n",
+ " (14, 148000, 134882.254786109, 13117.7452138913),\n",
+ " (15, 65000, 122049.943231186, -57049.9432311865),\n",
+ " (16, 91000, 118425.100333101, -27425.1003331007),\n",
+ " (17, 132300, 127493.96237718, 4806.03762281981),\n",
+ " (18, 91100, 106797.827726898, -15697.8277268985),\n",
+ " (19, 260011, 156425.429052831, 103585.570947169),\n",
+ " (20, 141800, 114630.849511727, 27169.1504882732),\n",
+ " (21, 160900, 132285.207203698, 28614.7927963019),\n",
+ " (22, 239000, 182359.969776195, 56640.030223805),\n",
+ " (23, 81010, 116163.896628839, -35153.896628839),\n",
+ " (24, 117910, 122633.009170962, -4723.00917096189),\n",
+ " (25, 141100, 148970.130183748, -7870.130183748),\n",
+ " (26, 148011, 136271.021402491, 11739.9785975093),\n",
+ " (27, 66000, 122744.326539377, -56744.3265393775)]"
+ ]
+ },
+ "execution_count": 11,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT id, price, predict, price - predict AS residual\n",
+ "FROM (\n",
+ " SELECT\n",
+ " houses.*,\n",
+ " madlib.elastic_net_gaussian_predict(\n",
+ " m.coef_all, -- All coefficients\n",
+ " m.intercept, -- Intercept\n",
+ " ARRAY[tax,bath,size] -- All features\n",
+ " ) AS predict\n",
+ " FROM houses, houses_en2 m) s\n",
+ "ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We can speed up the prediction function with coef_nonzero to evaluate residuals. This requires the user to examine the feature_selected column in the result table to construct the correct set of independent variables to provide to the prediction function."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "27 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>price</th>\n",
+ " <th>predict</th>\n",
+ " <th>residual</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>50000</td>\n",
+ " <td>101423.246912</td>\n",
+ " <td>-51423.2469117</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>85000</td>\n",
+ " <td>123638.649033</td>\n",
+ " <td>-38638.6490325</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>22500</td>\n",
+ " <td>106084.260933</td>\n",
+ " <td>-83584.260933</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>90000</td>\n",
+ " <td>119119.483641</td>\n",
+ " <td>-29119.4836413</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>133000</td>\n",
+ " <td>128188.345685</td>\n",
+ " <td>4811.65431463</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>90500</td>\n",
+ " <td>108186.594343</td>\n",
+ " <td>-17686.5943433</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>7</td>\n",
+ " <td>260000</td>\n",
+ " <td>157119.812361</td>\n",
+ " <td>102880.187639</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>8</td>\n",
+ " <td>142500</td>\n",
+ " <td>113936.466204</td>\n",
+ " <td>28563.5337965</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>9</td>\n",
+ " <td>160000</td>\n",
+ " <td>131799.138888</td>\n",
+ " <td>28200.861112</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>10</td>\n",
+ " <td>240000</td>\n",
+ " <td>182915.476423</td>\n",
+ " <td>57084.5235773</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>11</td>\n",
+ " <td>87000</td>\n",
+ " <td>116580.526614</td>\n",
+ " <td>-29580.5266138</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>12</td>\n",
+ " <td>118600</td>\n",
+ " <td>122841.324163</td>\n",
+ " <td>-4241.32416342</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>13</td>\n",
+ " <td>140000</td>\n",
+ " <td>148275.746876</td>\n",
+ " <td>-8275.74687556</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>14</td>\n",
+ " <td>148000</td>\n",
+ " <td>134882.254786</td>\n",
+ " <td>13117.7452139</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>15</td>\n",
+ " <td>65000</td>\n",
+ " <td>122049.943231</td>\n",
+ " <td>-57049.9432312</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>16</td>\n",
+ " <td>91000</td>\n",
+ " <td>118425.100333</td>\n",
+ " <td>-27425.1003331</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>17</td>\n",
+ " <td>132300</td>\n",
+ " <td>127493.962377</td>\n",
+ " <td>4806.03762282</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>18</td>\n",
+ " <td>91100</td>\n",
+ " <td>106797.827727</td>\n",
+ " <td>-15697.8277269</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>19</td>\n",
+ " <td>260011</td>\n",
+ " <td>156425.429053</td>\n",
+ " <td>103585.570947</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>20</td>\n",
+ " <td>141800</td>\n",
+ " <td>114630.849512</td>\n",
+ " <td>27169.1504883</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>21</td>\n",
+ " <td>160900</td>\n",
+ " <td>132285.207204</td>\n",
+ " <td>28614.7927963</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>22</td>\n",
+ " <td>239000</td>\n",
+ " <td>182359.969776</td>\n",
+ " <td>56640.0302238</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>23</td>\n",
+ " <td>81010</td>\n",
+ " <td>116163.896629</td>\n",
+ " <td>-35153.8966288</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>24</td>\n",
+ " <td>117910</td>\n",
+ " <td>122633.009171</td>\n",
+ " <td>-4723.00917096</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>25</td>\n",
+ " <td>141100</td>\n",
+ " <td>148970.130184</td>\n",
+ " <td>-7870.13018375</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>26</td>\n",
+ " <td>148011</td>\n",
+ " <td>136271.021402</td>\n",
+ " <td>11739.9785975</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>27</td>\n",
+ " <td>66000</td>\n",
+ " <td>122744.326539</td>\n",
+ " <td>-56744.3265394</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, 50000, 101423.246911724, -51423.2469117239),\n",
+ " (2, 85000, 123638.649032506, -38638.6490325065),\n",
+ " (3, 22500, 106084.260933004, -83584.2609330042),\n",
+ " (4, 90000, 119119.483641292, -29119.4836412917),\n",
+ " (5, 133000, 128188.345685371, 4811.6543146288),\n",
+ " (6, 90500, 108186.59434328, -17686.5943432805),\n",
+ " (7, 260000, 157119.812361022, 102880.187638978),\n",
+ " (8, 142500, 113936.466203536, 28563.5337964642),\n",
+ " (9, 160000, 131799.138887964, 28200.8611120356),\n",
+ " (10, 240000, 182915.476422748, 57084.5235772522),\n",
+ " (11, 87000, 116580.526613754, -29580.5266137536),\n",
+ " (12, 118600, 122841.324163419, -4241.32416341919),\n",
+ " (13, 140000, 148275.746875557, -8275.746875557),\n",
+ " (14, 148000, 134882.254786109, 13117.7452138913),\n",
+ " (15, 65000, 122049.943231186, -57049.9432311865),\n",
+ " (16, 91000, 118425.100333101, -27425.1003331007),\n",
+ " (17, 132300, 127493.96237718, 4806.03762281981),\n",
+ " (18, 91100, 106797.827726898, -15697.8277268985),\n",
+ " (19, 260011, 156425.429052831, 103585.570947169),\n",
+ " (20, 141800, 114630.849511727, 27169.1504882732),\n",
+ " (21, 160900, 132285.207203698, 28614.7927963019),\n",
+ " (22, 239000, 182359.969776195, 56640.030223805),\n",
+ " (23, 81010, 116163.896628839, -35153.896628839),\n",
+ " (24, 117910, 122633.009170962, -4723.00917096189),\n",
+ " (25, 141100, 148970.130183748, -7870.130183748),\n",
+ " (26, 148011, 136271.021402491, 11739.9785975093),\n",
+ " (27, 66000, 122744.326539377, -56744.3265393775)]"
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT id, price, predict, price - predict AS residual\n",
+ "FROM (\n",
+ " SELECT\n",
+ " houses.*,\n",
+ " madlib.elastic_net_gaussian_predict(\n",
+ " m.coef_nonzero, -- Non-zero coefficients\n",
+ " m.intercept, -- Intercept\n",
+ " ARRAY[tax,size] -- Features corresponding to non-zero coefficients\n",
+ " ) AS predict\n",
+ " FROM houses, houses_en2 m) s\n",
+ "ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## 6. Cross validation\n",
+ "Here we use 3-fold cross validation with 3 automatically generated\n",
+ "lambda values and 3 specified alpha values. (This can take some time\n",
+ "to run since elastic net is effectively being called 27 times.)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>family</th>\n",
+ " <th>features</th>\n",
+ " <th>features_selected</th>\n",
+ " <th>coef_nonzero</th>\n",
+ " <th>coef_all</th>\n",
+ " <th>intercept</th>\n",
+ " <th>log_likelihood</th>\n",
+ " <th>standardize</th>\n",
+ " <th>iteration_run</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>gaussian</td>\n",
+ " <td>[u'tax', u'bath', u'size']</td>\n",
+ " <td>[u'tax', u'bath', u'size']</td>\n",
+ " <td>[22.886204694, 9352.92944389, 58.1284827702]</td>\n",
+ " <td>[22.886204694, 9352.92944389, 58.1284827702]</td>\n",
+ " <td>-10462.3782585</td>\n",
+ " <td>-484874347.1</td>\n",
+ " <td>True</td>\n",
+ " <td>227</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'gaussian', [u'tax', u'bath', u'size'], [u'tax', u'bath', u'size'], [22.886204694, 9352.92944389, 58.1284827702], [22.886204694, 9352.92944389, 58.1284827702], -10462.3782585, -484874347.1, True, 227)]"
+ ]
+ },
+ "execution_count": 13,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS houses_en3, houses_en3_summary, houses_en3_cv;\n",
+ "SELECT madlib.elastic_net_train( 'houses', -- Source table\n",
+ " 'houses_en3', -- Result table\n",
+ " 'price', -- Dependent variable\n",
+ " 'array[tax, bath, size]', -- Independent variable\n",
+ " 'gaussian', -- Regression family\n",
+ " 0.5, -- Alpha value\n",
+ " 0.1, -- Lambda value\n",
+ " TRUE, -- Standardize\n",
+ " NULL, -- Grouping column(s)\n",
+ " 'fista', -- Optimizer\n",
+ " $$ n_folds = 3, -- Optimizer parameters\n",
+ " validation_result=houses_en3_cv,\n",
+ " n_lambdas = 3, \n",
+ " alpha = {0, 0.1, 1}\n",
+ " $$, \n",
+ " NULL, -- Excluded columns\n",
+ " 10000, -- Maximum iterations\n",
+ " 1e-6 -- Tolerance value\n",
+ " );\n",
+ "SELECT * FROM houses_en3;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Details of the cross validation:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "9 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>alpha</th>\n",
+ " <th>lambda_value</th>\n",
+ " <th>mean_neg_loss</th>\n",
+ " <th>std_neg_loss</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>0.0</td>\n",
+ " <td>100000.0</td>\n",
+ " <td>-6302794708170000000000000000000000000000000000000000000</td>\n",
+ " <td>5737622334690000000000000000000000000000000000000000000</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>0.1</td>\n",
+ " <td>100000.0</td>\n",
+ " <td>-139743178256000000000000000000000000000000000000000000</td>\n",
+ " <td>61732605975500000000000000000000000000000000000000000</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1.0</td>\n",
+ " <td>100000.0</td>\n",
+ " <td>-59675.5495462</td>\n",
+ " <td>22077.3570139</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>0.0</td>\n",
+ " <td>100.0</td>\n",
+ " <td>-58713.1759154</td>\n",
+ " <td>21837.2924888</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>0.1</td>\n",
+ " <td>100.0</td>\n",
+ " <td>-58609.7360066</td>\n",
+ " <td>21811.0504158</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1.0</td>\n",
+ " <td>100.0</td>\n",
+ " <td>-34745.228161</td>\n",
+ " <td>3567.21352473</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>0.0</td>\n",
+ " <td>0.1</td>\n",
+ " <td>-34795.1479263</td>\n",
+ " <td>3737.96400823</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>0.1</td>\n",
+ " <td>0.1</td>\n",
+ " <td>-34784.3501787</td>\n",
+ " <td>3710.61952339</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1.0</td>\n",
+ " <td>0.1</td>\n",
+ " <td>-34747.5234117</td>\n",
+ " <td>3602.44041867</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(Decimal('0.0'), Decimal('100000.0'), Decimal('-6302794708170000000000000000000000000000000000000000000'), Decimal('5737622334690000000000000000000000000000000000000000000')),\n",
+ " (Decimal('0.1'), Decimal('100000.0'), Decimal('-139743178256000000000000000000000000000000000000000000'), Decimal('61732605975500000000000000000000000000000000000000000')),\n",
+ " (Decimal('1.0'), Decimal('100000.0'), Decimal('-59675.5495462'), Decimal('22077.3570139')),\n",
+ " (Decimal('0.0'), Decimal('100.0'), Decimal('-58713.1759154'), Decimal('21837.2924888')),\n",
+ " (Decimal('0.1'), Decimal('100.0'), Decimal('-58609.7360066'), Decimal('21811.0504158')),\n",
+ " (Decimal('1.0'), Decimal('100.0'), Decimal('-34745.228161'), Decimal('3567.21352473')),\n",
+ " (Decimal('0.0'), Decimal('0.1'), Decimal('-34795.1479263'), Decimal('3737.96400823')),\n",
+ " (Decimal('0.1'), Decimal('0.1'), Decimal('-34784.3501787'), Decimal('3710.61952339')),\n",
+ " (Decimal('1.0'), Decimal('0.1'), Decimal('-34747.5234117'), Decimal('3602.44041867'))]"
+ ]
+ },
+ "execution_count": 14,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM houses_en3_cv ORDER BY lambda_value DESC, alpha ASC;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "collapsed": true
+ },
+ "source": [
+ "# 6a. Cross validation\n",
+ "Here we use 3-fold cross validation with 3 automatically generated lambda values and 1 alpha value (i.e., 9 times)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>family</th>\n",
+ " <th>features</th>\n",
+ " <th>features_selected</th>\n",
+ " <th>coef_nonzero</th>\n",
+ " <th>coef_all</th>\n",
+ " <th>intercept</th>\n",
+ " <th>log_likelihood</th>\n",
+ " <th>standardize</th>\n",
+ " <th>iteration_run</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>gaussian</td>\n",
+ " <td>[u'tax', u'bath', u'size']</td>\n",
+ " <td>[u'tax', u'bath', u'size']</td>\n",
+ " <td>[22.7850091737, 10707.9354212, 54.7963640251]</td>\n",
+ " <td>[22.7850091737, 10707.9354212, 54.7963640251]</td>\n",
+ " <td>-7798.90806453</td>\n",
+ " <td>-512248641.972</td>\n",
+ " <td>True</td>\n",
+ " <td>10000</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'gaussian', [u'tax', u'bath', u'size'], [u'tax', u'bath', u'size'], [22.7850091737, 10707.9354212, 54.7963640251], [22.7850091737, 10707.9354212, 54.7963640251], -7798.90806453, -512248641.972, True, 10000)]"
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS houses_en3, houses_en3_summary, houses_en3_cv;\n",
+ "SELECT madlib.elastic_net_train( 'houses', -- Source table\n",
+ " 'houses_en3', -- Result table\n",
+ " 'price', -- Dependent variable\n",
+ " 'array[tax, bath, size]', -- Independent variable\n",
+ " 'gaussian', -- Regression family\n",
+ " 0.5, -- Alpha value\n",
+ " 0.1, -- Lambda value\n",
+ " TRUE, -- Standardize\n",
+ " NULL, -- Grouping column(s)\n",
+ " 'fista', -- Optimizer\n",
+ " $$ n_folds = 3, -- Optimizer parameters\n",
+ " validation_result=houses_en3_cv,\n",
+ " n_lambdas = 3\n",
+ " $$, \n",
+ " NULL, -- Excluded columns\n",
+ " 10000, -- Maximum iterations\n",
+ " 1e-6 -- Tolerance value\n",
+ " );\n",
+ "SELECT * FROM houses_en3;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Details of the cross validation:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "3 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>lambda_value</th>\n",
+ " <th>mean_neg_loss</th>\n",
+ " <th>std_neg_loss</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>100000.0</td>\n",
+ " <td>-255543791799000000000000000000000000000000000</td>\n",
+ " <td>442158712729000000000000000000000000000000000</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>100.0</td>\n",
+ " <td>-59332.2198813</td>\n",
+ " <td>8220.8755071</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>0.1</td>\n",
+ " <td>-51938.9613421</td>\n",
+ " <td>28946.523247</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(Decimal('100000.0'), Decimal('-255543791799000000000000000000000000000000000'), Decimal('442158712729000000000000000000000000000000000')),\n",
+ " (Decimal('100.0'), Decimal('-59332.2198813'), Decimal('8220.8755071')),\n",
+ " (Decimal('0.1'), Decimal('-51938.9613421'), Decimal('28946.523247'))]"
+ ]
+ },
+ "execution_count": 16,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM houses_en3_cv ORDER BY lambda_value DESC;"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 2",
+ "language": "python",
+ "name": "python2"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.12"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
[3/7] madlib-site git commit: add new workbooks for 1dot13
Posted by fm...@apache.org.
http://git-wip-us.apache.org/repos/asf/madlib-site/blob/95826612/community-artifacts/kNN-v2.ipynb
----------------------------------------------------------------------
diff --git a/community-artifacts/kNN-v2.ipynb b/community-artifacts/kNN-v2.ipynb
new file mode 100644
index 0000000..5b74e48
--- /dev/null
+++ b/community-artifacts/kNN-v2.ipynb
@@ -0,0 +1,751 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# k-Nearest Neighbors\n",
+ "Finds k nearest data points to a given data point and outputs majority vote value of output classes in case of classification, and average value of target values in case of regression. KNN was first added in MADlib 1.10 and the interface was updated in 1.13."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/config.py:13: ShimWarning: The `IPython.config` package has been deprecated. You should import from traitlets.config instead.\n",
+ " \"You should import from traitlets.config instead.\", ShimWarning)\n",
+ "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/utils/traitlets.py:5: UserWarning: IPython.utils.traitlets has moved to a top-level traitlets package.\n",
+ " warn(\"IPython.utils.traitlets has moved to a top-level traitlets package.\")\n"
+ ]
+ }
+ ],
+ "source": [
+ "%load_ext sql"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "u'Connected: gpdbchina@madlib'"
+ ]
+ },
+ "execution_count": 2,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Greenplum 4.3.10.0\n",
+ "%sql postgresql://gpdbchina@10.194.10.68:61000/madlib\n",
+ " \n",
+ "# PostgreSQL local\n",
+ "#%sql postgresql://fmcquillan@localhost:5432/madlib\n",
+ "\n",
+ "# Greenplum 4.2.3.0\n",
+ "#%sql postgresql://gpdbchina@10.194.10.68:55000/madlib"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>version</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>MADlib version: 1.13-dev, git revision: rel/v1.12-41-g4aa0732, cmake configuration time: Tue Dec 5 20:44:49 UTC 2017, build type: Release, build system: Linux-2.6.18-238.27.1.el5.hotfix.bz516490, C compiler: gcc 4.4.0, C++ compiler: g++ 4.4.0</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'MADlib version: 1.13-dev, git revision: rel/v1.12-41-g4aa0732, cmake configuration time: Tue Dec 5 20:44:49 UTC 2017, build type: Release, build system: Linux-2.6.18-238.27.1.el5.hotfix.bz516490, C compiler: gcc 4.4.0, C++ compiler: g++ 4.4.0',)]"
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%sql select madlib.version();\n",
+ "#%sql select version();"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 1. Load data for classification"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "Done.\n",
+ "9 rows affected.\n",
+ "9 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>data</th>\n",
+ " <th>label</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>[1, 1]</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>[2, 2]</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>[3, 3]</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>[4, 4]</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>[4, 5]</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>[20, 50]</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>7</td>\n",
+ " <td>[10, 31]</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>8</td>\n",
+ " <td>[81, 13]</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>9</td>\n",
+ " <td>[1, 111]</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, [1, 1], 1),\n",
+ " (2, [2, 2], 1),\n",
+ " (3, [3, 3], 1),\n",
+ " (4, [4, 4], 1),\n",
+ " (5, [4, 5], 1),\n",
+ " (6, [20, 50], 0),\n",
+ " (7, [10, 31], 0),\n",
+ " (8, [81, 13], 0),\n",
+ " (9, [1, 111], 0)]"
+ ]
+ },
+ "execution_count": 4,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql \n",
+ "DROP TABLE IF EXISTS knn_train_data;\n",
+ "\n",
+ "CREATE TABLE knn_train_data (\n",
+ " id integer, \n",
+ " data integer[], \n",
+ " label integer -- Integer label means for classification\n",
+ " );\n",
+ "\n",
+ "INSERT INTO knn_train_data VALUES\n",
+ "(1, '{1,1}', 1),\n",
+ "(2, '{2,2}', 1),\n",
+ "(3, '{3,3}', 1),\n",
+ "(4, '{4,4}', 1),\n",
+ "(5, '{4,5}', 1),\n",
+ "(6, '{20,50}', 0),\n",
+ "(7, '{10,31}', 0),\n",
+ "(8, '{81,13}', 0),\n",
+ "(9, '{1,111}', 0);\n",
+ "\n",
+ "SELECT * FROM knn_train_data ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 2. Load data for regression"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "Done.\n",
+ "9 rows affected.\n",
+ "9 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>data</th>\n",
+ " <th>label</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>[1, 1]</td>\n",
+ " <td>1.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>[2, 2]</td>\n",
+ " <td>1.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>[3, 3]</td>\n",
+ " <td>1.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>[4, 4]</td>\n",
+ " <td>1.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>[4, 5]</td>\n",
+ " <td>1.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>[20, 50]</td>\n",
+ " <td>0.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>7</td>\n",
+ " <td>[10, 31]</td>\n",
+ " <td>0.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>8</td>\n",
+ " <td>[81, 13]</td>\n",
+ " <td>0.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>9</td>\n",
+ " <td>[1, 111]</td>\n",
+ " <td>0.0</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, [1, 1], 1.0),\n",
+ " (2, [2, 2], 1.0),\n",
+ " (3, [3, 3], 1.0),\n",
+ " (4, [4, 4], 1.0),\n",
+ " (5, [4, 5], 1.0),\n",
+ " (6, [20, 50], 0.0),\n",
+ " (7, [10, 31], 0.0),\n",
+ " (8, [81, 13], 0.0),\n",
+ " (9, [1, 111], 0.0)]"
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS knn_train_data_reg;\n",
+ "\n",
+ "CREATE TABLE knn_train_data_reg (\n",
+ " id integer, \n",
+ " data integer[], \n",
+ " label float -- Float label means for regression\n",
+ " );\n",
+ "\n",
+ "INSERT INTO knn_train_data_reg VALUES\n",
+ "(1, '{1,1}', 1.0),\n",
+ "(2, '{2,2}', 1.0),\n",
+ "(3, '{3,3}', 1.0),\n",
+ "(4, '{4,4}', 1.0),\n",
+ "(5, '{4,5}', 1.0),\n",
+ "(6, '{20,50}', 0.0),\n",
+ "(7, '{10,31}', 0.0),\n",
+ "(8, '{81,13}', 0.0),\n",
+ "(9, '{1,111}', 0.0);\n",
+ "\n",
+ "SELECT * FROM knn_train_data_reg ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 3. Load testing data"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "Done.\n",
+ "6 rows affected.\n",
+ "6 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>data</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>[2, 1]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>[2, 6]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>[15, 40]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>[12, 1]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>[2, 90]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>[50, 45]</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, [2, 1]),\n",
+ " (2, [2, 6]),\n",
+ " (3, [15, 40]),\n",
+ " (4, [12, 1]),\n",
+ " (5, [2, 90]),\n",
+ " (6, [50, 45])]"
+ ]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql \n",
+ "DROP TABLE IF EXISTS knn_test_data;\n",
+ "\n",
+ "CREATE TABLE knn_test_data (\n",
+ " id integer, \n",
+ " data integer[]\n",
+ " );\n",
+ "\n",
+ "INSERT INTO knn_test_data VALUES\n",
+ "(1, '{2,1}'),\n",
+ "(2, '{2,6}'),\n",
+ "(3, '{15,40}'),\n",
+ "(4, '{12,1}'),\n",
+ "(5, '{2,90}'),\n",
+ "(6, '{50,45}');\n",
+ "\n",
+ "SELECT * from knn_test_data ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 4. Run KNN for classification\n",
+ "Note that the nearest neighbors are sorted from closest to furthest from the corresponding test point."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "6 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>data</th>\n",
+ " <th>prediction</th>\n",
+ " <th>k_nearest_neighbours</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>[2, 1]</td>\n",
+ " <td>1.0</td>\n",
+ " <td>[1, 2, 3]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>[2, 6]</td>\n",
+ " <td>1.0</td>\n",
+ " <td>[5, 4, 3]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>[15, 40]</td>\n",
+ " <td>0.0</td>\n",
+ " <td>[7, 6, 5]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>[12, 1]</td>\n",
+ " <td>1.0</td>\n",
+ " <td>[4, 5, 3]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>[2, 90]</td>\n",
+ " <td>0.0</td>\n",
+ " <td>[9, 6, 7]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>[50, 45]</td>\n",
+ " <td>0.0</td>\n",
+ " <td>[6, 7, 8]</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, [2, 1], 1.0, [1, 2, 3]),\n",
+ " (2, [2, 6], 1.0, [5, 4, 3]),\n",
+ " (3, [15, 40], 0.0, [7, 6, 5]),\n",
+ " (4, [12, 1], 1.0, [4, 5, 3]),\n",
+ " (5, [2, 90], 0.0, [9, 6, 7]),\n",
+ " (6, [50, 45], 0.0, [6, 7, 8])]"
+ ]
+ },
+ "execution_count": 7,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS knn_result_classification;\n",
+ "\n",
+ "SELECT * FROM madlib.knn(\n",
+ " 'knn_train_data', -- Table of training data\n",
+ " 'data', -- Col name of training data\n",
+ " 'id', -- Col name of id in train data\n",
+ " 'label', -- Training labels\n",
+ " 'knn_test_data', -- Table of test data\n",
+ " 'data', -- Col name of test data\n",
+ " 'id', -- Col name of id in test data\n",
+ " 'knn_result_classification', -- Output table\n",
+ " 3, -- Number of nearest neighbors\n",
+ " True, -- True to list nearest-neighbors by id\n",
+ " 'madlib.squared_dist_norm2' -- Distance function\n",
+ " );\n",
+ "\n",
+ "SELECT * from knn_result_classification ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 5. Run KNN for regression"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "6 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>data</th>\n",
+ " <th>prediction</th>\n",
+ " <th>k_nearest_neighbours</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>[2, 1]</td>\n",
+ " <td>1.0</td>\n",
+ " <td>[1, 2, 3]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>[2, 6]</td>\n",
+ " <td>1.0</td>\n",
+ " <td>[5, 4, 3]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>[15, 40]</td>\n",
+ " <td>0.333333333333</td>\n",
+ " <td>[7, 6, 5]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>[12, 1]</td>\n",
+ " <td>1.0</td>\n",
+ " <td>[4, 5, 3]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>[2, 90]</td>\n",
+ " <td>0.0</td>\n",
+ " <td>[9, 6, 7]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>[50, 45]</td>\n",
+ " <td>0.0</td>\n",
+ " <td>[6, 7, 8]</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, [2, 1], 1.0, [1, 2, 3]),\n",
+ " (2, [2, 6], 1.0, [5, 4, 3]),\n",
+ " (3, [15, 40], 0.333333333333333, [7, 6, 5]),\n",
+ " (4, [12, 1], 1.0, [4, 5, 3]),\n",
+ " (5, [2, 90], 0.0, [9, 6, 7]),\n",
+ " (6, [50, 45], 0.0, [6, 7, 8])]"
+ ]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS knn_result_regression;\n",
+ "\n",
+ "SELECT * FROM madlib.knn(\n",
+ " 'knn_train_data_reg', -- Table of training data\n",
+ " 'data', -- Col name of training data\n",
+ " 'id', -- Col Name of id in train data\n",
+ " 'label', -- Training labels\n",
+ " 'knn_test_data', -- Table of test data\n",
+ " 'data', -- Col name of test data\n",
+ " 'id', -- Col name of id in test data\n",
+ " 'knn_result_regression', -- Output table\n",
+ " 3, -- Number of nearest neighbors\n",
+ " True, -- True to list nearest-neighbors by id\n",
+ " 'madlib.dist_norm2' -- Distance function\n",
+ " );\n",
+ "\n",
+ "SELECT * FROM knn_result_regression ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 6. List nearest neighbors only\n",
+ "(without doing classification or regression). Note that the nearest neighbors are sorted from closest to furthest from the corresponding test point."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "6 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>data</th>\n",
+ " <th>k_nearest_neighbours</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>[2, 1]</td>\n",
+ " <td>[2, 1, 3]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>[2, 6]</td>\n",
+ " <td>[5, 4, 3]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>[15, 40]</td>\n",
+ " <td>[7, 6, 5]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>[12, 1]</td>\n",
+ " <td>[4, 5, 3]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>[2, 90]</td>\n",
+ " <td>[9, 6, 7]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>[50, 45]</td>\n",
+ " <td>[6, 7, 8]</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, [2, 1], [2, 1, 3]),\n",
+ " (2, [2, 6], [5, 4, 3]),\n",
+ " (3, [15, 40], [7, 6, 5]),\n",
+ " (4, [12, 1], [4, 5, 3]),\n",
+ " (5, [2, 90], [9, 6, 7]),\n",
+ " (6, [50, 45], [6, 7, 8])]"
+ ]
+ },
+ "execution_count": 9,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS knn_result_list_neighbors;\n",
+ "\n",
+ "SELECT * FROM madlib.knn(\n",
+ " 'knn_train_data_reg', -- Table of training data\n",
+ " 'data', -- Col name of training data\n",
+ " 'id', -- Col Name of id in train data\n",
+ " NULL, -- NULL training labels means just list neighbors\n",
+ " 'knn_test_data', -- Table of test data\n",
+ " 'data', -- Col name of test data\n",
+ " 'id', -- Col name of id in test data\n",
+ " 'knn_result_list_neighbors', -- Output table\n",
+ " 3 -- Number of nearest neighbors\n",
+ " );\n",
+ "\n",
+ "SELECT * FROM knn_result_list_neighbors ORDER BY id;"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 2",
+ "language": "python",
+ "name": "python2"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.12"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
[7/7] madlib-site git commit: add new workbooks for 1dot13
Posted by fm...@apache.org.
add new workbooks for 1dot13
Project: http://git-wip-us.apache.org/repos/asf/madlib-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/madlib-site/commit/95826612
Tree: http://git-wip-us.apache.org/repos/asf/madlib-site/tree/95826612
Diff: http://git-wip-us.apache.org/repos/asf/madlib-site/diff/95826612
Branch: refs/heads/asf-site
Commit: 95826612dd1cc6c52d8977c0582028c636013cb5
Parents: 4ddf175
Author: Frank McQuillan <fm...@pivotal.io>
Authored: Fri Dec 8 12:51:41 2017 -0800
Committer: Frank McQuillan <fm...@pivotal.io>
Committed: Fri Dec 8 12:51:41 2017 -0800
----------------------------------------------------------------------
community-artifacts/Elastic-net-v1.ipynb | 1960 ---------
community-artifacts/Elastic-net-v2.ipynb | 2078 ++++++++++
community-artifacts/HITS-v1.ipynb | 1240 ++++++
community-artifacts/Linear-regression-v1.ipynb | 1188 ++++++
.../Logistic-regression-v1.ipynb | 892 +++++
community-artifacts/Summary-v1.ipynb | 1026 +++++
community-artifacts/kNN-v1.ipynb | 497 ---
community-artifacts/kNN-v2.ipynb | 751 ++++
community-artifacts/mlp-v1.ipynb | 1525 -------
community-artifacts/mlp-v2.ipynb | 3755 ++++++++++++++++++
10 files changed, 10930 insertions(+), 3982 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/madlib-site/blob/95826612/community-artifacts/Elastic-net-v1.ipynb
----------------------------------------------------------------------
diff --git a/community-artifacts/Elastic-net-v1.ipynb b/community-artifacts/Elastic-net-v1.ipynb
deleted file mode 100644
index d7e5202..0000000
--- a/community-artifacts/Elastic-net-v1.ipynb
+++ /dev/null
@@ -1,1960 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Elastic net (MADlib v1.10+)\n",
- "Demonstrates elastic net, including grouping and cross validation, which were introduced in MADlib v1.10"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/config.py:13: ShimWarning: The `IPython.config` package has been deprecated. You should import from traitlets.config instead.\n",
- " \"You should import from traitlets.config instead.\", ShimWarning)\n",
- "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/utils/traitlets.py:5: UserWarning: IPython.utils.traitlets has moved to a top-level traitlets package.\n",
- " warn(\"IPython.utils.traitlets has moved to a top-level traitlets package.\")\n"
- ]
- }
- ],
- "source": [
- "%load_ext sql"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "data": {
- "text/plain": [
- "u'Connected: gpdbchina@madlib'"
- ]
- },
- "execution_count": 2,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%sql postgresql://gpdbchina@10.194.10.68:55000/madlib\n",
- "#%sql postgresql://fmcquillan@localhost:5432/madlib\n",
- "#%sql postgresql://gpadmin@54.197.30.46:10432/gpadmin"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>version</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>MADlib version: 1.10.0-dev, git revision: rel/v1.9.1-12-g6939fd6, cmake configuration time: Thu Dec 22 00:35:17 UTC 2016, build type: Release, build system: Linux-2.6.18-238.27.1.el5.hotfix.bz516490, C compiler: gcc 4.4.0, C++ compiler: g++ 4.4.0</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'MADlib version: 1.10.0-dev, git revision: rel/v1.9.1-12-g6939fd6, cmake configuration time: Thu Dec 22 00:35:17 UTC 2016, build type: Release, build system: Linux-2.6.18-238.27.1.el5.hotfix.bz516490, C compiler: gcc 4.4.0, C++ compiler: g++ 4.4.0',)]"
- ]
- },
- "execution_count": 3,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%sql select madlib.version();\n",
- "#%sql select version();"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## 1. Create data set\n",
- "House prices and characteristics."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "Done.\n",
- "27 rows affected.\n",
- "27 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>id</th>\n",
- " <th>tax</th>\n",
- " <th>bedroom</th>\n",
- " <th>bath</th>\n",
- " <th>price</th>\n",
- " <th>size</th>\n",
- " <th>lot</th>\n",
- " <th>zipcode</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1</td>\n",
- " <td>590</td>\n",
- " <td>2</td>\n",
- " <td>1.0</td>\n",
- " <td>50000</td>\n",
- " <td>770</td>\n",
- " <td>22100</td>\n",
- " <td>94301</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>2</td>\n",
- " <td>1050</td>\n",
- " <td>3</td>\n",
- " <td>2.0</td>\n",
- " <td>85000</td>\n",
- " <td>1410</td>\n",
- " <td>12000</td>\n",
- " <td>94301</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>3</td>\n",
- " <td>20</td>\n",
- " <td>3</td>\n",
- " <td>1.0</td>\n",
- " <td>22500</td>\n",
- " <td>1060</td>\n",
- " <td>3500</td>\n",
- " <td>94301</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>4</td>\n",
- " <td>870</td>\n",
- " <td>2</td>\n",
- " <td>2.0</td>\n",
- " <td>90000</td>\n",
- " <td>1300</td>\n",
- " <td>17500</td>\n",
- " <td>94301</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>5</td>\n",
- " <td>1320</td>\n",
- " <td>3</td>\n",
- " <td>2.0</td>\n",
- " <td>133000</td>\n",
- " <td>1500</td>\n",
- " <td>30000</td>\n",
- " <td>94301</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>6</td>\n",
- " <td>1350</td>\n",
- " <td>2</td>\n",
- " <td>1.0</td>\n",
- " <td>90500</td>\n",
- " <td>820</td>\n",
- " <td>25700</td>\n",
- " <td>94301</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>7</td>\n",
- " <td>2790</td>\n",
- " <td>3</td>\n",
- " <td>2.5</td>\n",
- " <td>260000</td>\n",
- " <td>2130</td>\n",
- " <td>25000</td>\n",
- " <td>94301</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>8</td>\n",
- " <td>680</td>\n",
- " <td>2</td>\n",
- " <td>1.0</td>\n",
- " <td>142500</td>\n",
- " <td>1170</td>\n",
- " <td>22000</td>\n",
- " <td>94301</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>9</td>\n",
- " <td>1840</td>\n",
- " <td>3</td>\n",
- " <td>2.0</td>\n",
- " <td>160000</td>\n",
- " <td>1500</td>\n",
- " <td>19000</td>\n",
- " <td>94301</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>10</td>\n",
- " <td>3680</td>\n",
- " <td>4</td>\n",
- " <td>2.0</td>\n",
- " <td>240000</td>\n",
- " <td>2790</td>\n",
- " <td>20000</td>\n",
- " <td>94301</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>11</td>\n",
- " <td>1660</td>\n",
- " <td>3</td>\n",
- " <td>1.0</td>\n",
- " <td>87000</td>\n",
- " <td>1030</td>\n",
- " <td>17500</td>\n",
- " <td>94301</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>12</td>\n",
- " <td>1620</td>\n",
- " <td>3</td>\n",
- " <td>2.0</td>\n",
- " <td>118600</td>\n",
- " <td>1250</td>\n",
- " <td>20000</td>\n",
- " <td>94301</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>13</td>\n",
- " <td>3100</td>\n",
- " <td>3</td>\n",
- " <td>2.0</td>\n",
- " <td>140000</td>\n",
- " <td>1760</td>\n",
- " <td>38000</td>\n",
- " <td>94301</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>14</td>\n",
- " <td>2070</td>\n",
- " <td>2</td>\n",
- " <td>3.0</td>\n",
- " <td>148000</td>\n",
- " <td>1550</td>\n",
- " <td>14000</td>\n",
- " <td>94301</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>15</td>\n",
- " <td>650</td>\n",
- " <td>3</td>\n",
- " <td>1.5</td>\n",
- " <td>65000</td>\n",
- " <td>1450</td>\n",
- " <td>12000</td>\n",
- " <td>94301</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>16</td>\n",
- " <td>770</td>\n",
- " <td>2</td>\n",
- " <td>2.0</td>\n",
- " <td>91000</td>\n",
- " <td>1300</td>\n",
- " <td>17500</td>\n",
- " <td>76010</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>17</td>\n",
- " <td>1220</td>\n",
- " <td>3</td>\n",
- " <td>2.0</td>\n",
- " <td>132300</td>\n",
- " <td>1500</td>\n",
- " <td>30000</td>\n",
- " <td>76010</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>18</td>\n",
- " <td>1150</td>\n",
- " <td>2</td>\n",
- " <td>1.0</td>\n",
- " <td>91100</td>\n",
- " <td>820</td>\n",
- " <td>25700</td>\n",
- " <td>76010</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>19</td>\n",
- " <td>2690</td>\n",
- " <td>3</td>\n",
- " <td>2.5</td>\n",
- " <td>260011</td>\n",
- " <td>2130</td>\n",
- " <td>25000</td>\n",
- " <td>76010</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>20</td>\n",
- " <td>780</td>\n",
- " <td>2</td>\n",
- " <td>1.0</td>\n",
- " <td>141800</td>\n",
- " <td>1170</td>\n",
- " <td>22000</td>\n",
- " <td>76010</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>21</td>\n",
- " <td>1910</td>\n",
- " <td>3</td>\n",
- " <td>2.0</td>\n",
- " <td>160900</td>\n",
- " <td>1500</td>\n",
- " <td>19000</td>\n",
- " <td>76010</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>22</td>\n",
- " <td>3600</td>\n",
- " <td>4</td>\n",
- " <td>2.0</td>\n",
- " <td>239000</td>\n",
- " <td>2790</td>\n",
- " <td>20000</td>\n",
- " <td>76010</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>23</td>\n",
- " <td>1600</td>\n",
- " <td>3</td>\n",
- " <td>1.0</td>\n",
- " <td>81010</td>\n",
- " <td>1030</td>\n",
- " <td>17500</td>\n",
- " <td>76010</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>24</td>\n",
- " <td>1590</td>\n",
- " <td>3</td>\n",
- " <td>2.0</td>\n",
- " <td>117910</td>\n",
- " <td>1250</td>\n",
- " <td>20000</td>\n",
- " <td>76010</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>25</td>\n",
- " <td>3200</td>\n",
- " <td>3</td>\n",
- " <td>2.0</td>\n",
- " <td>141100</td>\n",
- " <td>1760</td>\n",
- " <td>38000</td>\n",
- " <td>76010</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>26</td>\n",
- " <td>2270</td>\n",
- " <td>2</td>\n",
- " <td>3.0</td>\n",
- " <td>148011</td>\n",
- " <td>1550</td>\n",
- " <td>14000</td>\n",
- " <td>76010</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>27</td>\n",
- " <td>750</td>\n",
- " <td>3</td>\n",
- " <td>1.5</td>\n",
- " <td>66000</td>\n",
- " <td>1450</td>\n",
- " <td>12000</td>\n",
- " <td>76010</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(1, 590, 2, 1.0, 50000, 770, 22100, 94301),\n",
- " (2, 1050, 3, 2.0, 85000, 1410, 12000, 94301),\n",
- " (3, 20, 3, 1.0, 22500, 1060, 3500, 94301),\n",
- " (4, 870, 2, 2.0, 90000, 1300, 17500, 94301),\n",
- " (5, 1320, 3, 2.0, 133000, 1500, 30000, 94301),\n",
- " (6, 1350, 2, 1.0, 90500, 820, 25700, 94301),\n",
- " (7, 2790, 3, 2.5, 260000, 2130, 25000, 94301),\n",
- " (8, 680, 2, 1.0, 142500, 1170, 22000, 94301),\n",
- " (9, 1840, 3, 2.0, 160000, 1500, 19000, 94301),\n",
- " (10, 3680, 4, 2.0, 240000, 2790, 20000, 94301),\n",
- " (11, 1660, 3, 1.0, 87000, 1030, 17500, 94301),\n",
- " (12, 1620, 3, 2.0, 118600, 1250, 20000, 94301),\n",
- " (13, 3100, 3, 2.0, 140000, 1760, 38000, 94301),\n",
- " (14, 2070, 2, 3.0, 148000, 1550, 14000, 94301),\n",
- " (15, 650, 3, 1.5, 65000, 1450, 12000, 94301),\n",
- " (16, 770, 2, 2.0, 91000, 1300, 17500, 76010),\n",
- " (17, 1220, 3, 2.0, 132300, 1500, 30000, 76010),\n",
- " (18, 1150, 2, 1.0, 91100, 820, 25700, 76010),\n",
- " (19, 2690, 3, 2.5, 260011, 2130, 25000, 76010),\n",
- " (20, 780, 2, 1.0, 141800, 1170, 22000, 76010),\n",
- " (21, 1910, 3, 2.0, 160900, 1500, 19000, 76010),\n",
- " (22, 3600, 4, 2.0, 239000, 2790, 20000, 76010),\n",
- " (23, 1600, 3, 1.0, 81010, 1030, 17500, 76010),\n",
- " (24, 1590, 3, 2.0, 117910, 1250, 20000, 76010),\n",
- " (25, 3200, 3, 2.0, 141100, 1760, 38000, 76010),\n",
- " (26, 2270, 2, 3.0, 148011, 1550, 14000, 76010),\n",
- " (27, 750, 3, 1.5, 66000, 1450, 12000, 76010)]"
- ]
- },
- "execution_count": 4,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql \n",
- "DROP TABLE IF EXISTS houses;\n",
- "\n",
- "CREATE TABLE houses ( id INT,\n",
- " tax INT,\n",
- " bedroom INT,\n",
- " bath FLOAT,\n",
- " price INT,\n",
- " size INT,\n",
- " lot INT,\n",
- " zipcode INT);\n",
- "\n",
- "INSERT INTO houses (id, tax, bedroom, bath, price, size, lot, zipcode) VALUES\n",
- "(1 , 590 , 2 , 1 , 50000 , 770 , 22100 , 94301),\n",
- "(2 , 1050 , 3 , 2 , 85000 , 1410 , 12000 , 94301),\n",
- "(3 , 20 , 3 , 1 , 22500 , 1060 , 3500 , 94301),\n",
- "(4 , 870 , 2 , 2 , 90000 , 1300 , 17500 , 94301),\n",
- "(5 , 1320 , 3 , 2 , 133000 , 1500 , 30000 , 94301),\n",
- "(6 , 1350 , 2 , 1 , 90500 , 820 , 25700 , 94301),\n",
- "(7 , 2790 , 3 , 2.5 , 260000 , 2130 , 25000 , 94301),\n",
- "(8 , 680 , 2 , 1 , 142500 , 1170 , 22000 , 94301),\n",
- "(9 , 1840 , 3 , 2 , 160000 , 1500 , 19000 , 94301),\n",
- "(10 , 3680 , 4 , 2 , 240000 , 2790 , 20000 , 94301),\n",
- "(11 , 1660 , 3 , 1 , 87000 , 1030 , 17500 , 94301),\n",
- "(12 , 1620 , 3 , 2 , 118600 , 1250 , 20000 , 94301),\n",
- "(13 , 3100 , 3 , 2 , 140000 , 1760 , 38000 , 94301),\n",
- "(14 , 2070 , 2 , 3 , 148000 , 1550 , 14000 , 94301),\n",
- "(15 , 650 , 3 , 1.5 , 65000 , 1450 , 12000 , 94301),\n",
- "(16 , 770 , 2 , 2 , 91000 , 1300 , 17500 , 76010),\n",
- "(17 , 1220 , 3 , 2 , 132300 , 1500 , 30000 , 76010),\n",
- "(18 , 1150 , 2 , 1 , 91100 , 820 , 25700 , 76010),\n",
- "(19 , 2690 , 3 , 2.5 , 260011 , 2130 , 25000 , 76010),\n",
- "(20 , 780 , 2 , 1 , 141800 , 1170 , 22000 , 76010),\n",
- "(21 , 1910 , 3 , 2 , 160900 , 1500 , 19000 , 76010),\n",
- "(22 , 3600 , 4 , 2 , 239000 , 2790 , 20000 , 76010),\n",
- "(23 , 1600 , 3 , 1 , 81010 , 1030 , 17500 , 76010),\n",
- "(24 , 1590 , 3 , 2 , 117910 , 1250 , 20000 , 76010),\n",
- "(25 , 3200 , 3 , 2 , 141100 , 1760 , 38000 , 76010),\n",
- "(26 , 2270 , 2 , 3 , 148011 , 1550 , 14000 , 76010),\n",
- "(27 , 750 , 3 , 1.5 , 66000 , 1450 , 12000 , 76010);\n",
- "\n",
- "SELECT * FROM houses ORDER BY id;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## 2. Train the model"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 16,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>family</th>\n",
- " <th>features</th>\n",
- " <th>features_selected</th>\n",
- " <th>coef_nonzero</th>\n",
- " <th>coef_all</th>\n",
- " <th>intercept</th>\n",
- " <th>log_likelihood</th>\n",
- " <th>standardize</th>\n",
- " <th>iteration_run</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>gaussian</td>\n",
- " <td>[u'tax', u'bath', u'size']</td>\n",
- " <td>[u'tax', u'bath', u'size']</td>\n",
- " <td>[22.785125212, 10707.9542786, 54.796130396]</td>\n",
- " <td>[22.785125212, 10707.9542786, 54.796130396]</td>\n",
- " <td>-7798.79034492</td>\n",
- " <td>-512248641.969</td>\n",
- " <td>True</td>\n",
- " <td>375</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'gaussian', [u'tax', u'bath', u'size'], [u'tax', u'bath', u'size'], [22.785125212, 10707.9542786, 54.796130396], [22.785125212, 10707.9542786, 54.796130396], -7798.79034492, -512248641.969, True, 375)]"
- ]
- },
- "execution_count": 16,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS houses_en, houses_en_summary;\n",
- "SELECT madlib.elastic_net_train( 'houses', -- Source table\n",
- " 'houses_en', -- Result table\n",
- " 'price', -- Dependent variable\n",
- " 'array[tax, bath, size]', -- Independent variable\n",
- " 'gaussian', -- Regression family\n",
- " 0.5, -- Alpha value\n",
- " 0.1, -- Lambda value\n",
- " TRUE, -- Standardize\n",
- " NULL, -- Grouping column(s)\n",
- " 'fista', -- Optimizer\n",
- " '', -- Optimizer parameters\n",
- " NULL, -- Excluded columns\n",
- " 10000, -- Maximum iterations\n",
- " 1e-6 -- Tolerance value\n",
- " );\n",
- "SELECT * FROM houses_en;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# 3. Prediction\n",
- "Evaluate residuals."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 17,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "27 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>id</th>\n",
- " <th>price</th>\n",
- " <th>predict</th>\n",
- " <th>residual</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1</td>\n",
- " <td>50000</td>\n",
- " <td>58545.4082137</td>\n",
- " <td>-8545.40821368</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>2</td>\n",
- " <td>85000</td>\n",
- " <td>114804.043543</td>\n",
- " <td>-29804.0435432</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>3</td>\n",
- " <td>22500</td>\n",
- " <td>61448.7646577</td>\n",
- " <td>-38948.7646577</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>4</td>\n",
- " <td>90000</td>\n",
- " <td>104675.146662</td>\n",
- " <td>-14675.1466615</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>5</td>\n",
- " <td>133000</td>\n",
- " <td>125887.679086</td>\n",
- " <td>7112.32091388</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>6</td>\n",
- " <td>90500</td>\n",
- " <td>78601.9098946</td>\n",
- " <td>11898.0901054</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>7</td>\n",
- " <td>260000</td>\n",
- " <td>199257.352437</td>\n",
- " <td>60742.6475635</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>8</td>\n",
- " <td>142500</td>\n",
- " <td>82514.5216412</td>\n",
- " <td>59985.4783588</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>9</td>\n",
- " <td>160000</td>\n",
- " <td>137735.944196</td>\n",
- " <td>22264.0558036</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>10</td>\n",
- " <td>240000</td>\n",
- " <td>250347.582797</td>\n",
- " <td>-10347.5827973</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>11</td>\n",
- " <td>87000</td>\n",
- " <td>97172.4860935</td>\n",
- " <td>-10172.4860935</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>12</td>\n",
- " <td>118600</td>\n",
- " <td>119024.184051</td>\n",
- " <td>-424.18405072</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>13</td>\n",
- " <td>140000</td>\n",
- " <td>180692.195866</td>\n",
- " <td>-40692.1958664</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>14</td>\n",
- " <td>148000</td>\n",
- " <td>156424.283794</td>\n",
- " <td>-8424.28379352</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>15</td>\n",
- " <td>65000</td>\n",
- " <td>102527.861535</td>\n",
- " <td>-37527.861535</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>16</td>\n",
- " <td>91000</td>\n",
- " <td>102396.63414</td>\n",
- " <td>-11396.6341403</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>17</td>\n",
- " <td>132300</td>\n",
- " <td>123609.166565</td>\n",
- " <td>8690.83343508</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>18</td>\n",
- " <td>91100</td>\n",
- " <td>74044.8848522</td>\n",
- " <td>17055.1151478</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>19</td>\n",
- " <td>260011</td>\n",
- " <td>196978.839915</td>\n",
- " <td>63032.1600847</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>20</td>\n",
- " <td>141800</td>\n",
- " <td>84793.0341624</td>\n",
- " <td>57006.9658376</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>21</td>\n",
- " <td>160900</td>\n",
- " <td>139330.902961</td>\n",
- " <td>21569.0970388</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>22</td>\n",
- " <td>239000</td>\n",
- " <td>248524.77278</td>\n",
- " <td>-9524.77278032</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>23</td>\n",
- " <td>81010</td>\n",
- " <td>95805.3785808</td>\n",
- " <td>-14795.3785808</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>24</td>\n",
- " <td>117910</td>\n",
- " <td>118340.630294</td>\n",
- " <td>-430.63029436</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>25</td>\n",
- " <td>141100</td>\n",
- " <td>182970.708388</td>\n",
- " <td>-41870.7083876</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>26</td>\n",
- " <td>148011</td>\n",
- " <td>160981.308836</td>\n",
- " <td>-12970.3088359</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>27</td>\n",
- " <td>66000</td>\n",
- " <td>104806.374056</td>\n",
- " <td>-38806.3740562</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(1, 50000, 58545.40821368, -8545.40821368),\n",
- " (2, 85000, 114804.04354324, -29804.04354324),\n",
- " (3, 22500, 61448.76465768, -38948.76465768),\n",
- " (4, 90000, 104675.14666152, -14675.14666152),\n",
- " (5, 133000, 125887.67908612, 7112.32091388),\n",
- " (6, 90500, 78601.9098946, 11898.0901054),\n",
- " (7, 260000, 199257.35243654, 60742.64756346),\n",
- " (8, 142500, 82514.52164116, 59985.47835884),\n",
- " (9, 160000, 137735.94419636, 22264.05580364),\n",
- " (10, 240000, 250347.58279728, -10347.58279728),\n",
- " (11, 87000, 97172.48609348, -10172.48609348),\n",
- " (12, 118600, 119024.18405072, -424.18405072001),\n",
- " (13, 140000, 180692.19586644, -40692.19586644),\n",
- " (14, 148000, 156424.28379352, -8424.28379352001),\n",
- " (15, 65000, 102527.86153498, -37527.86153498),\n",
- " (16, 91000, 102396.63414032, -11396.63414032),\n",
- " (17, 132300, 123609.16656492, 8690.83343508),\n",
- " (18, 91100, 74044.8848522, 17055.1151478),\n",
- " (19, 260011, 196978.83991534, 63032.16008466),\n",
- " (20, 141800, 84793.03416236, 57006.96583764),\n",
- " (21, 160900, 139330.9029612, 21569.0970388),\n",
- " (22, 239000, 248524.77278032, -9524.77278032),\n",
- " (23, 81010, 95805.37858076, -14795.37858076),\n",
- " (24, 117910, 118340.63029436, -430.630294360002),\n",
- " (25, 141100, 182970.70838764, -41870.70838764),\n",
- " (26, 148011, 160981.30883592, -12970.30883592),\n",
- " (27, 66000, 104806.37405618, -38806.37405618)]"
- ]
- },
- "execution_count": 17,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT id, price, predict, price - predict AS residual\n",
- "FROM (\n",
- " SELECT\n",
- " houses.*,\n",
- " madlib.elastic_net_gaussian_predict(\n",
- " m.coef_all, -- Coefficients\n",
- " m.intercept, -- Intercept\n",
- " ARRAY[tax,bath,size] -- Features (corresponding to coefficients)\n",
- " ) AS predict\n",
- " FROM houses, houses_en m) s\n",
- "ORDER BY id;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# 4. Grouping \n",
- "Group on zip code."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 18,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "2 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>zipcode</th>\n",
- " <th>family</th>\n",
- " <th>features</th>\n",
- " <th>features_selected</th>\n",
- " <th>coef_nonzero</th>\n",
- " <th>coef_all</th>\n",
- " <th>intercept</th>\n",
- " <th>log_likelihood</th>\n",
- " <th>standardize</th>\n",
- " <th>iteration_run</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>76010</td>\n",
- " <td>gaussian</td>\n",
- " <td>[u'tax', u'bath', u'size']</td>\n",
- " <td>[u'tax', u'bath', u'size']</td>\n",
- " <td>[15.6325563808, 10166.6562969, 57.8690658309]</td>\n",
- " <td>[15.6325563808, 10166.6562969, 57.8690658309]</td>\n",
- " <td>513.875727396</td>\n",
- " <td>-538806627.977</td>\n",
- " <td>True</td>\n",
- " <td>10000</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>94301</td>\n",
- " <td>gaussian</td>\n",
- " <td>[u'tax', u'bath', u'size']</td>\n",
- " <td>[u'tax', u'bath', u'size']</td>\n",
- " <td>[27.0542060555, 12351.5246493, 47.5833358009]</td>\n",
- " <td>[27.0542060555, 12351.5246493, 47.5833358009]</td>\n",
- " <td>-7191.20234448</td>\n",
- " <td>-519199958.999</td>\n",
- " <td>True</td>\n",
- " <td>10000</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(76010, u'gaussian', [u'tax', u'bath', u'size'], [u'tax', u'bath', u'size'], [15.6325563808, 10166.6562969, 57.8690658309], [15.6325563808, 10166.6562969, 57.8690658309], 513.875727396, -538806627.977, True, 10000),\n",
- " (94301, u'gaussian', [u'tax', u'bath', u'size'], [u'tax', u'bath', u'size'], [27.0542060555, 12351.5246493, 47.5833358009], [27.0542060555, 12351.5246493, 47.5833358009], -7191.20234448, -519199958.999, True, 10000)]"
- ]
- },
- "execution_count": 18,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS houses_en1, houses_en1_summary;\n",
- "SELECT madlib.elastic_net_train( 'houses', -- Source table\n",
- " 'houses_en1', -- Result table\n",
- " 'price', -- Dependent variable\n",
- " 'array[tax, bath, size]', -- Independent variable\n",
- " 'gaussian', -- Regression family\n",
- " 0.5, -- Alpha value\n",
- " 0.1, -- Lambda value\n",
- " TRUE, -- Standardize\n",
- " 'zipcode', -- Grouping column(s)\n",
- " 'fista', -- Optimizer\n",
- " '', -- Optimizer parameters\n",
- " NULL, -- Excluded columns\n",
- " 10000, -- Maximum iterations\n",
- " 1e-6 -- Tolerance value\n",
- " );\n",
- "SELECT * FROM houses_en1;"
- ]
- },
- {
- "cell_type": "raw",
- "metadata": {},
- "source": [
- "Prediction function"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 19,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n",
- "27 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>id</th>\n",
- " <th>price</th>\n",
- " <th>prediction</th>\n",
- " <th>residual</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1</td>\n",
- " <td>50000</td>\n",
- " <td>57761.4724443</td>\n",
- " <td>-7761.47244426</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>2</td>\n",
- " <td>85000</td>\n",
- " <td>113011.266792</td>\n",
- " <td>-28011.2667917</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>3</td>\n",
- " <td>22500</td>\n",
- " <td>56139.7423749</td>\n",
- " <td>-33639.7423749</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>4</td>\n",
- " <td>90000</td>\n",
- " <td>102907.342764</td>\n",
- " <td>-12907.3427636</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>5</td>\n",
- " <td>133000</td>\n",
- " <td>124598.402649</td>\n",
- " <td>8401.59735127</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>6</td>\n",
- " <td>90500</td>\n",
- " <td>80701.8358365</td>\n",
- " <td>9798.16416352</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>7</td>\n",
- " <td>260000</td>\n",
- " <td>200521.34943</td>\n",
- " <td>59478.6505705</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>8</td>\n",
- " <td>142500</td>\n",
- " <td>79229.6853096</td>\n",
- " <td>63270.3146904</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>9</td>\n",
- " <td>160000</td>\n",
- " <td>138666.589798</td>\n",
- " <td>21333.4102024</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>10</td>\n",
- " <td>240000</td>\n",
- " <td>249828.832123</td>\n",
- " <td>-9828.83212287</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>11</td>\n",
- " <td>87000</td>\n",
- " <td>99081.1402319</td>\n",
- " <td>-12081.1402319</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>12</td>\n",
- " <td>118600</td>\n",
- " <td>120818.830515</td>\n",
- " <td>-2218.83051515</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>13</td>\n",
- " <td>140000</td>\n",
- " <td>185126.556736</td>\n",
- " <td>-45126.5567358</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>14</td>\n",
- " <td>148000</td>\n",
- " <td>159619.74863</td>\n",
- " <td>-11619.7486297</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>15</td>\n",
- " <td>65000</td>\n",
- " <td>97917.1554769</td>\n",
- " <td>-32917.1554768</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>16</td>\n",
- " <td>91000</td>\n",
- " <td>108114.042315</td>\n",
- " <td>-17114.0423146</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>17</td>\n",
- " <td>132300</td>\n",
- " <td>126722.505852</td>\n",
- " <td>5577.49414788</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>18</td>\n",
- " <td>91100</td>\n",
- " <td>76110.6058436</td>\n",
- " <td>14989.3941564</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>19</td>\n",
- " <td>260011</td>\n",
- " <td>191243.203354</td>\n",
- " <td>68767.7966462</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>20</td>\n",
- " <td>141800</td>\n",
- " <td>90580.7330235</td>\n",
- " <td>51219.2669765</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>21</td>\n",
- " <td>160900</td>\n",
- " <td>137508.969755</td>\n",
- " <td>23391.0302451</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>22</td>\n",
- " <td>239000</td>\n",
- " <td>238579.08496</td>\n",
- " <td>420.915039713</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>23</td>\n",
- " <td>81010</td>\n",
- " <td>95297.7600394</td>\n",
- " <td>-14287.7600394</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>24</td>\n",
- " <td>117910</td>\n",
- " <td>118039.285255</td>\n",
- " <td>-129.285255293</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>25</td>\n",
- " <td>141100</td>\n",
- " <td>172720.924602</td>\n",
- " <td>-31620.9246021</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>26</td>\n",
- " <td>148011</td>\n",
- " <td>156196.79964</td>\n",
- " <td>-8185.79964041</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>27</td>\n",
- " <td>66000</td>\n",
- " <td>111398.422913</td>\n",
- " <td>-45398.4229132</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(1, 50000, 57761.472444258, -7761.472444258),\n",
- " (2, 85000, 113011.266791664, -28011.266791664),\n",
- " (3, 22500, 56139.742374884, -33639.742374884),\n",
- " (4, 90000, 102907.342763575, -12907.342763575),\n",
- " (5, 133000, 124598.40264873, 8401.59735127),\n",
- " (6, 90500, 80701.835836483, 9798.16416351699),\n",
- " (7, 260000, 200521.349429532, 59478.650570468),\n",
- " (8, 142500, 79229.685309613, 63270.314690387),\n",
- " (9, 160000, 138666.58979759, 21333.41020241),\n",
- " (10, 240000, 249828.832122871, -9828.83212287101),\n",
- " (11, 87000, 99081.140231877, -12081.140231877),\n",
- " (12, 118600, 120818.830515155, -2218.830515155),\n",
- " (13, 140000, 185126.556735754, -45126.556735754),\n",
- " (14, 148000, 159619.7486297, -11619.7486297),\n",
- " (15, 65000, 97917.15547685, -32917.15547685),\n",
- " (16, 91000, 108114.042314582, -17114.042314582),\n",
- " (17, 132300, 126722.505852122, 5577.49414787799),\n",
- " (18, 91100, 76110.605843554, 14989.394156446),\n",
- " (19, 260011, 191243.203353815, 68767.796646185),\n",
- " (20, 141800, 90580.733023473, 51219.266976527),\n",
- " (21, 160900, 137508.969754874, 23391.030245126),\n",
- " (22, 239000, 238579.084960287, 420.915039713),\n",
- " (23, 81010, 95297.760039403, -14287.760039403),\n",
- " (24, 117910, 118039.285255293, -129.285255292998),\n",
- " (25, 141100, 172720.92460214, -31620.92460214),\n",
- " (26, 148011, 156196.799640407, -8185.799640407),\n",
- " (27, 66000, 111398.422913151, -45398.422913151)]"
- ]
- },
- "execution_count": 19,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT madlib.elastic_net_predict(\n",
- " 'houses_en1', -- Model table\n",
- " 'houses', -- New source data table\n",
- " 'id', -- Unique ID associated with each row\n",
- " 'houses_en1_prediction' -- Table to store prediction result\n",
- " );\n",
- "\n",
- "SELECT houses.id,\n",
- " houses.price,\n",
- " houses_en1_prediction.prediction,\n",
- " houses.price - houses_en1_prediction.prediction AS residual\n",
- "FROM houses_en1_prediction, houses\n",
- "WHERE houses.id = houses_en1_prediction.id ORDER BY id;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## 5. When coef_nonzero is different from coef_all\n",
- "Train"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 20,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>family</th>\n",
- " <th>features</th>\n",
- " <th>features_selected</th>\n",
- " <th>coef_nonzero</th>\n",
- " <th>coef_all</th>\n",
- " <th>intercept</th>\n",
- " <th>log_likelihood</th>\n",
- " <th>standardize</th>\n",
- " <th>iteration_run</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>gaussian</td>\n",
- " <td>[u'tax', u'bath', u'size']</td>\n",
- " <td>[u'tax', u'size']</td>\n",
- " <td>[6.94744249834, 29.7137297658]</td>\n",
- " <td>[6.94744249834, 0.0, 29.7137297658]</td>\n",
- " <td>74445.7039382</td>\n",
- " <td>-1635348585.07</td>\n",
- " <td>True</td>\n",
- " <td>151</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'gaussian', [u'tax', u'bath', u'size'], [u'tax', u'size'], [6.94744249834, 29.7137297658], [6.94744249834, 0.0, 29.7137297658], 74445.7039382, -1635348585.07, True, 151)]"
- ]
- },
- "execution_count": 20,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS houses_en2, houses_en2_summary;\n",
- "SELECT madlib.elastic_net_train( 'houses', -- Source table\n",
- " 'houses_en2', -- Result table\n",
- " 'price', -- Dependent variable\n",
- " 'array[tax, bath, size]', -- Independent variable\n",
- " 'gaussian', -- Regression family\n",
- " 1, -- Alpha value\n",
- " 30000, -- Lambda value\n",
- " TRUE, -- Standardize\n",
- " NULL, -- Grouping column(s)\n",
- " 'fista', -- Optimizer\n",
- " '', -- Optimizer parameters\n",
- " NULL, -- Excluded columns\n",
- " 10000, -- Maximum iterations\n",
- " 1e-6 -- Tolerance value\n",
- " );\n",
- "SELECT * FROM houses_en2;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Prediction function with coef_all to evaluate residuals."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 21,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "27 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>id</th>\n",
- " <th>price</th>\n",
- " <th>predict</th>\n",
- " <th>residual</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1</td>\n",
- " <td>50000</td>\n",
- " <td>101424.266932</td>\n",
- " <td>-51424.2669319</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>2</td>\n",
- " <td>85000</td>\n",
- " <td>123636.877531</td>\n",
- " <td>-38636.8775312</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>3</td>\n",
- " <td>22500</td>\n",
- " <td>106081.20634</td>\n",
- " <td>-83581.2063399</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>4</td>\n",
- " <td>90000</td>\n",
- " <td>119117.827607</td>\n",
- " <td>-29117.8276073</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>5</td>\n",
- " <td>133000</td>\n",
- " <td>128186.922685</td>\n",
- " <td>4813.07731529</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>6</td>\n",
- " <td>90500</td>\n",
- " <td>108190.009719</td>\n",
- " <td>-17690.0097189</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>7</td>\n",
- " <td>260000</td>\n",
- " <td>157119.31291</td>\n",
- " <td>102880.68709</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>8</td>\n",
- " <td>142500</td>\n",
- " <td>113935.028663</td>\n",
- " <td>28564.9713369</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>9</td>\n",
- " <td>160000</td>\n",
- " <td>131799.592784</td>\n",
- " <td>28200.4072162</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>10</td>\n",
- " <td>240000</td>\n",
- " <td>182913.598379</td>\n",
- " <td>57086.4016213</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>11</td>\n",
- " <td>87000</td>\n",
- " <td>116583.600144</td>\n",
- " <td>-29583.6001442</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>12</td>\n",
- " <td>118600</td>\n",
- " <td>122842.722993</td>\n",
- " <td>-4242.72299276</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>13</td>\n",
- " <td>140000</td>\n",
- " <td>148278.940071</td>\n",
- " <td>-8278.94007086</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>14</td>\n",
- " <td>148000</td>\n",
- " <td>134883.191047</td>\n",
- " <td>13116.8089532</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>15</td>\n",
- " <td>65000</td>\n",
- " <td>122046.449723</td>\n",
- " <td>-57046.4497225</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>16</td>\n",
- " <td>91000</td>\n",
- " <td>118423.083357</td>\n",
- " <td>-27423.0833575</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>17</td>\n",
- " <td>132300</td>\n",
- " <td>127492.178435</td>\n",
- " <td>4807.82156513</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>18</td>\n",
- " <td>91100</td>\n",
- " <td>106800.521219</td>\n",
- " <td>-15700.5212192</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>19</td>\n",
- " <td>260011</td>\n",
- " <td>156424.56866</td>\n",
- " <td>103586.43134</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>20</td>\n",
- " <td>141800</td>\n",
- " <td>114629.772913</td>\n",
- " <td>27170.2270871</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>21</td>\n",
- " <td>160900</td>\n",
- " <td>132285.913759</td>\n",
- " <td>28614.0862413</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>22</td>\n",
- " <td>239000</td>\n",
- " <td>182357.802979</td>\n",
- " <td>56642.1970212</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>23</td>\n",
- " <td>81010</td>\n",
- " <td>116166.753594</td>\n",
- " <td>-35156.7535943</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>24</td>\n",
- " <td>117910</td>\n",
- " <td>122634.299718</td>\n",
- " <td>-4724.29971781</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>25</td>\n",
- " <td>141100</td>\n",
- " <td>148973.684321</td>\n",
- " <td>-7873.6843207</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>26</td>\n",
- " <td>148011</td>\n",
- " <td>136272.679546</td>\n",
- " <td>11738.3204536</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>27</td>\n",
- " <td>66000</td>\n",
- " <td>122741.193972</td>\n",
- " <td>-56741.1939724</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(1, 50000, 101424.266931887, -51424.2669318866),\n",
- " (2, 85000, 123636.877531235, -38636.877531235),\n",
- " (3, 22500, 106081.206339915, -83581.2063399148),\n",
- " (4, 90000, 119117.827607296, -29117.8276072958),\n",
- " (5, 133000, 128186.922684709, 4813.0773152912),\n",
- " (6, 90500, 108190.009718915, -17690.009718915),\n",
- " (7, 260000, 157119.312909723, 102880.687090277),\n",
- " (8, 142500, 113935.028663057, 28564.9713369428),\n",
- " (9, 160000, 131799.592783846, 28200.4072161544),\n",
- " (10, 240000, 182913.598378673, 57086.4016213268),\n",
- " (11, 87000, 116583.600144218, -29583.6001442184),\n",
- " (12, 118600, 122842.722992761, -4242.7229927608),\n",
- " (13, 140000, 148278.940070862, -8278.94007086198),\n",
- " (14, 148000, 134883.191046754, 13116.8089532462),\n",
- " (15, 65000, 122046.449722531, -57046.449722531),\n",
- " (16, 91000, 118423.083357462, -27423.0833574618),\n",
- " (17, 132300, 127492.178434875, 4807.82156512521),\n",
- " (18, 91100, 106800.521219247, -15700.521219247),\n",
- " (19, 260011, 156424.568659889, 103586.431340111),\n",
- " (20, 141800, 114629.772912891, 27170.2270871088),\n",
- " (21, 160900, 132285.913758729, 28614.0862412706),\n",
- " (22, 239000, 182357.802978806, 56642.197021194),\n",
- " (23, 81010, 116166.753594318, -35156.753594318),\n",
- " (24, 117910, 122634.299717811, -4724.29971781059),\n",
- " (25, 141100, 148973.684320696, -7873.68432069599),\n",
- " (26, 148011, 136272.679546422, 11738.3204535782),\n",
- " (27, 66000, 122741.193972365, -56741.193972365)]"
- ]
- },
- "execution_count": 21,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT id, price, predict, price - predict AS residual\n",
- "FROM (\n",
- " SELECT\n",
- " houses.*,\n",
- " madlib.elastic_net_gaussian_predict(\n",
- " m.coef_all, -- All coefficients\n",
- " m.intercept, -- Intercept\n",
- " ARRAY[tax,bath,size] -- All features\n",
- " ) AS predict\n",
- " FROM houses, houses_en2 m) s\n",
- "ORDER BY id;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "We can speed up the prediction function with coef_nonzero to evaluate residuals. This requires the user to examine the feature_selected column in the result table to construct the correct set of independent variables to provide to the prediction function."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 22,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "27 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>id</th>\n",
- " <th>price</th>\n",
- " <th>predict</th>\n",
- " <th>residual</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1</td>\n",
- " <td>50000</td>\n",
- " <td>101424.266932</td>\n",
- " <td>-51424.2669319</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>2</td>\n",
- " <td>85000</td>\n",
- " <td>123636.877531</td>\n",
- " <td>-38636.8775312</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>3</td>\n",
- " <td>22500</td>\n",
- " <td>106081.20634</td>\n",
- " <td>-83581.2063399</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>4</td>\n",
- " <td>90000</td>\n",
- " <td>119117.827607</td>\n",
- " <td>-29117.8276073</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>5</td>\n",
- " <td>133000</td>\n",
- " <td>128186.922685</td>\n",
- " <td>4813.07731529</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>6</td>\n",
- " <td>90500</td>\n",
- " <td>108190.009719</td>\n",
- " <td>-17690.0097189</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>7</td>\n",
- " <td>260000</td>\n",
- " <td>157119.31291</td>\n",
- " <td>102880.68709</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>8</td>\n",
- " <td>142500</td>\n",
- " <td>113935.028663</td>\n",
- " <td>28564.9713369</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>9</td>\n",
- " <td>160000</td>\n",
- " <td>131799.592784</td>\n",
- " <td>28200.4072162</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>10</td>\n",
- " <td>240000</td>\n",
- " <td>182913.598379</td>\n",
- " <td>57086.4016213</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>11</td>\n",
- " <td>87000</td>\n",
- " <td>116583.600144</td>\n",
- " <td>-29583.6001442</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>12</td>\n",
- " <td>118600</td>\n",
- " <td>122842.722993</td>\n",
- " <td>-4242.72299276</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>13</td>\n",
- " <td>140000</td>\n",
- " <td>148278.940071</td>\n",
- " <td>-8278.94007086</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>14</td>\n",
- " <td>148000</td>\n",
- " <td>134883.191047</td>\n",
- " <td>13116.8089532</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>15</td>\n",
- " <td>65000</td>\n",
- " <td>122046.449723</td>\n",
- " <td>-57046.4497225</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>16</td>\n",
- " <td>91000</td>\n",
- " <td>118423.083357</td>\n",
- " <td>-27423.0833575</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>17</td>\n",
- " <td>132300</td>\n",
- " <td>127492.178435</td>\n",
- " <td>4807.82156513</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>18</td>\n",
- " <td>91100</td>\n",
- " <td>106800.521219</td>\n",
- " <td>-15700.5212192</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>19</td>\n",
- " <td>260011</td>\n",
- " <td>156424.56866</td>\n",
- " <td>103586.43134</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>20</td>\n",
- " <td>141800</td>\n",
- " <td>114629.772913</td>\n",
- " <td>27170.2270871</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>21</td>\n",
- " <td>160900</td>\n",
- " <td>132285.913759</td>\n",
- " <td>28614.0862413</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>22</td>\n",
- " <td>239000</td>\n",
- " <td>182357.802979</td>\n",
- " <td>56642.1970212</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>23</td>\n",
- " <td>81010</td>\n",
- " <td>116166.753594</td>\n",
- " <td>-35156.7535943</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>24</td>\n",
- " <td>117910</td>\n",
- " <td>122634.299718</td>\n",
- " <td>-4724.29971781</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>25</td>\n",
- " <td>141100</td>\n",
- " <td>148973.684321</td>\n",
- " <td>-7873.6843207</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>26</td>\n",
- " <td>148011</td>\n",
- " <td>136272.679546</td>\n",
- " <td>11738.3204536</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>27</td>\n",
- " <td>66000</td>\n",
- " <td>122741.193972</td>\n",
- " <td>-56741.1939724</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(1, 50000, 101424.266931887, -51424.2669318866),\n",
- " (2, 85000, 123636.877531235, -38636.877531235),\n",
- " (3, 22500, 106081.206339915, -83581.2063399148),\n",
- " (4, 90000, 119117.827607296, -29117.8276072958),\n",
- " (5, 133000, 128186.922684709, 4813.0773152912),\n",
- " (6, 90500, 108190.009718915, -17690.009718915),\n",
- " (7, 260000, 157119.312909723, 102880.687090277),\n",
- " (8, 142500, 113935.028663057, 28564.9713369428),\n",
- " (9, 160000, 131799.592783846, 28200.4072161544),\n",
- " (10, 240000, 182913.598378673, 57086.4016213268),\n",
- " (11, 87000, 116583.600144218, -29583.6001442184),\n",
- " (12, 118600, 122842.722992761, -4242.7229927608),\n",
- " (13, 140000, 148278.940070862, -8278.94007086198),\n",
- " (14, 148000, 134883.191046754, 13116.8089532462),\n",
- " (15, 65000, 122046.449722531, -57046.449722531),\n",
- " (16, 91000, 118423.083357462, -27423.0833574618),\n",
- " (17, 132300, 127492.178434875, 4807.82156512521),\n",
- " (18, 91100, 106800.521219247, -15700.521219247),\n",
- " (19, 260011, 156424.568659889, 103586.431340111),\n",
- " (20, 141800, 114629.772912891, 27170.2270871088),\n",
- " (21, 160900, 132285.913758729, 28614.0862412706),\n",
- " (22, 239000, 182357.802978806, 56642.197021194),\n",
- " (23, 81010, 116166.753594318, -35156.753594318),\n",
- " (24, 117910, 122634.299717811, -4724.29971781059),\n",
- " (25, 141100, 148973.684320696, -7873.68432069599),\n",
- " (26, 148011, 136272.679546422, 11738.3204535782),\n",
- " (27, 66000, 122741.193972365, -56741.193972365)]"
- ]
- },
- "execution_count": 22,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT id, price, predict, price - predict AS residual\n",
- "FROM (\n",
- " SELECT\n",
- " houses.*,\n",
- " madlib.elastic_net_gaussian_predict(\n",
- " m.coef_nonzero, -- Non-zero coefficients\n",
- " m.intercept, -- Intercept\n",
- " ARRAY[tax,size] -- Features corresponding to non-zero coefficients\n",
- " ) AS predict\n",
- " FROM houses, houses_en2 m) s\n",
- "ORDER BY id;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## 6. Cross validation\n",
- "Here we use 3-fold cross validation with 3 automatically generated\n",
- "lambda values and 3 specified alpha values. (This can take some time\n",
- "to run since elastic net is effectively being called 27 times.)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 9,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>family</th>\n",
- " <th>features</th>\n",
- " <th>features_selected</th>\n",
- " <th>coef_nonzero</th>\n",
- " <th>coef_all</th>\n",
- " <th>intercept</th>\n",
- " <th>log_likelihood</th>\n",
- " <th>standardize</th>\n",
- " <th>iteration_run</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>gaussian</td>\n",
- " <td>[u'tax', u'bath', u'size']</td>\n",
- " <td>[u'tax', u'bath', u'size']</td>\n",
- " <td>[22.4584783679, 11657.0825871, 52.1622899664]</td>\n",
- " <td>[22.4584783679, 11657.0825871, 52.1622899664]</td>\n",
- " <td>-5067.27288499</td>\n",
- " <td>-543193170.15</td>\n",
- " <td>True</td>\n",
- " <td>392</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'gaussian', [u'tax', u'bath', u'size'], [u'tax', u'bath', u'size'], [22.4584783679, 11657.0825871, 52.1622899664], [22.4584783679, 11657.0825871, 52.1622899664], -5067.27288499, -543193170.15, True, 392)]"
- ]
- },
- "execution_count": 9,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS houses_en3, houses_en3_summary, houses_en3_cv;\n",
- "SELECT madlib.elastic_net_train( 'houses', -- Source table\n",
- " 'houses_en3', -- Result table\n",
- " 'price', -- Dependent variable\n",
- " 'array[tax, bath, size]', -- Independent variable\n",
- " 'gaussian', -- Regression family\n",
- " 0.5, -- Alpha value\n",
- " 0.1, -- Lambda value\n",
- " TRUE, -- Standardize\n",
- " NULL, -- Grouping column(s)\n",
- " 'fista', -- Optimizer\n",
- " $$ n_folds = 3, -- Optimizer parameters\n",
- " validation_result=houses_en3_cv,\n",
- " n_lambdas = 3, \n",
- " alpha = {0, 0.1, 1}\n",
- " $$, \n",
- " NULL, -- Excluded columns\n",
- " 10000, -- Maximum iterations\n",
- " 1e-6 -- Tolerance value\n",
- " );\n",
- "SELECT * FROM houses_en3;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Details of the cross validation:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 13,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "9 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>std</th>\n",
- " <th>alpha</th>\n",
- " <th>lambda_value</th>\n",
- " <th>mean</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1.80536123195e+110</td>\n",
- " <td>0.0</td>\n",
- " <td>100000.0</td>\n",
- " <td>-1.41777698585e+110</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1.72846143163e+107</td>\n",
- " <td>0.1</td>\n",
- " <td>100000.0</td>\n",
- " <td>-1.19953054719e+107</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>2485189261.38</td>\n",
- " <td>1.0</td>\n",
- " <td>100000.0</td>\n",
- " <td>-4175743937.91</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>2424765457.66</td>\n",
- " <td>0.0</td>\n",
- " <td>100.0</td>\n",
- " <td>-4054694238.18</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>2418294966.72</td>\n",
- " <td>0.1</td>\n",
- " <td>100.0</td>\n",
- " <td>-4041768667.28</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>483327430.802</td>\n",
- " <td>1.0</td>\n",
- " <td>100.0</td>\n",
- " <td>-1458791218.11</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>426795110.876</td>\n",
- " <td>0.0</td>\n",
- " <td>0.1</td>\n",
- " <td>-1442293698.38</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>429680202.16</td>\n",
- " <td>0.1</td>\n",
- " <td>0.1</td>\n",
- " <td>-1442705511.6</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>485107796.02</td>\n",
- " <td>1.0</td>\n",
- " <td>0.1</td>\n",
- " <td>-1459206061.39</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(1.80536123195e+110, 0.0, 100000.0, -1.41777698585e+110),\n",
- " (1.72846143163e+107, 0.1, 100000.0, -1.19953054719e+107),\n",
- " (2485189261.38, 1.0, 100000.0, -4175743937.91),\n",
- " (2424765457.66, 0.0, 100.0, -4054694238.18),\n",
- " (2418294966.72, 0.1, 100.0, -4041768667.28),\n",
- " (483327430.802, 1.0, 100.0, -1458791218.11),\n",
- " (426795110.876, 0.0, 0.1, -1442293698.38),\n",
- " (429680202.16, 0.1, 0.1, -1442705511.6),\n",
- " (485107796.02, 1.0, 0.1, -1459206061.39)]"
- ]
- },
- "execution_count": 13,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT * FROM houses_en3_cv ORDER BY lambda_value DESC, alpha ASC;"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": true
- },
- "outputs": [],
- "source": []
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 2",
- "language": "python",
- "name": "python2"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 2
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython2",
- "version": "2.7.12"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 0
-}
[5/7] madlib-site git commit: add new workbooks for 1dot13
Posted by fm...@apache.org.
http://git-wip-us.apache.org/repos/asf/madlib-site/blob/95826612/community-artifacts/HITS-v1.ipynb
----------------------------------------------------------------------
diff --git a/community-artifacts/HITS-v1.ipynb b/community-artifacts/HITS-v1.ipynb
new file mode 100644
index 0000000..40fef57
--- /dev/null
+++ b/community-artifacts/HITS-v1.ipynb
@@ -0,0 +1,1240 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# HITS (Hyperlink-Induced Topic Search) \n",
+ "Outputs the authority score and hub score of every vertex, where authority estimates the value of the content of the page and hub estimates the value of its links to other pages. Added in MADlib 1.13."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/config.py:13: ShimWarning: The `IPython.config` package has been deprecated. You should import from traitlets.config instead.\n",
+ " \"You should import from traitlets.config instead.\", ShimWarning)\n",
+ "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/utils/traitlets.py:5: UserWarning: IPython.utils.traitlets has moved to a top-level traitlets package.\n",
+ " warn(\"IPython.utils.traitlets has moved to a top-level traitlets package.\")\n"
+ ]
+ }
+ ],
+ "source": [
+ "%load_ext sql"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "u'Connected: gpdbchina@madlib'"
+ ]
+ },
+ "execution_count": 2,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Greenplum 4.3.10.0\n",
+ "%sql postgresql://gpdbchina@10.194.10.68:61000/madlib\n",
+ " \n",
+ "# PostgreSQL local\n",
+ "#%sql postgresql://fmcquillan@localhost:5432/madlib"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>version</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>MADlib version: 1.13-dev, git revision: rel/v1.12-31-g4b7d9cc, cmake configuration time: Tue Nov 21 22:31:28 UTC 2017, build type: Release, build system: Linux-2.6.18-238.27.1.el5.hotfix.bz516490, C compiler: gcc 4.4.0, C++ compiler: g++ 4.4.0</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'MADlib version: 1.13-dev, git revision: rel/v1.12-31-g4b7d9cc, cmake configuration time: Tue Nov 21 22:31:28 UTC 2017, build type: Release, build system: Linux-2.6.18-238.27.1.el5.hotfix.bz516490, C compiler: gcc 4.4.0, C++ compiler: g++ 4.4.0',)]"
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%sql select madlib.version();\n",
+ "#%sql select version();"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 1. Create vertex and edge tables"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "Done.\n",
+ "Done.\n",
+ "7 rows affected.\n",
+ "12 rows affected.\n",
+ "12 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>src</th>\n",
+ " <th>dest</th>\n",
+ " <th>user_id</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>0</td>\n",
+ " <td>1</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>0</td>\n",
+ " <td>2</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>0</td>\n",
+ " <td>4</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>2</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>3</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>3</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>5</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>6</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>0</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>0</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>6</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>3</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(0, 1, 1),\n",
+ " (0, 2, 1),\n",
+ " (0, 4, 1),\n",
+ " (1, 2, 1),\n",
+ " (1, 3, 1),\n",
+ " (2, 3, 1),\n",
+ " (2, 5, 1),\n",
+ " (2, 6, 1),\n",
+ " (3, 0, 1),\n",
+ " (4, 0, 1),\n",
+ " (5, 6, 1),\n",
+ " (6, 3, 1)]"
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS vertex, edge;\n",
+ "\n",
+ "CREATE TABLE vertex(\n",
+ " id INTEGER\n",
+ " );\n",
+ "\n",
+ "CREATE TABLE edge(\n",
+ " src INTEGER,\n",
+ " dest INTEGER,\n",
+ " user_id INTEGER\n",
+ " );\n",
+ "\n",
+ "INSERT INTO vertex VALUES\n",
+ "(0),\n",
+ "(1),\n",
+ "(2),\n",
+ "(3),\n",
+ "(4),\n",
+ "(5),\n",
+ "(6);\n",
+ "\n",
+ "INSERT INTO edge VALUES\n",
+ "(0, 1, 1),\n",
+ "(0, 2, 1),\n",
+ "(0, 4, 1),\n",
+ "(1, 2, 1),\n",
+ "(1, 3, 1),\n",
+ "(2, 3, 1),\n",
+ "(2, 5, 1),\n",
+ "(2, 6, 1),\n",
+ "(3, 0, 1),\n",
+ "(4, 0, 1),\n",
+ "(5, 6, 1),\n",
+ "(6, 3, 1);\n",
+ "\n",
+ "SELECT * from edge ORDER BY src, dest;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 2. Compute the HITS scores"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "7 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>authority</th>\n",
+ " <th>hub</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>0</td>\n",
+ " <td>8.43871829093e-07</td>\n",
+ " <td>0.338306115083</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>0.158459587238</td>\n",
+ " <td>0.527865350448</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>0.40562796969</td>\n",
+ " <td>0.675800764728</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>0.721775835522</td>\n",
+ " <td>3.95111934817e-07</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>0.158459587238</td>\n",
+ " <td>3.95111934817e-07</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>0.316385413093</td>\n",
+ " <td>0.189719957843</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>0.405199928761</td>\n",
+ " <td>0.337944978189</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(0, 8.4387182909347e-07, 0.338306115082665),\n",
+ " (1, 0.158459587238244, 0.527865350448059),\n",
+ " (2, 0.405627969689677, 0.675800764727558),\n",
+ " (3, 0.721775835521825, 3.95111934817447e-07),\n",
+ " (4, 0.158459587238244, 3.95111934817447e-07),\n",
+ " (5, 0.316385413093048, 0.189719957843216),\n",
+ " (6, 0.405199928761102, 0.337944978189241)]"
+ ]
+ },
+ "execution_count": 4,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS hits_out, hits_out_summary;\n",
+ "\n",
+ "SELECT madlib.hits(\n",
+ " 'vertex', -- Vertex table\n",
+ " 'id', -- Vertex id column\n",
+ " 'edge', -- Edge table\n",
+ " 'src=src, dest=dest', -- Comma delimited string of edge arguments\n",
+ " 'hits_out'); -- Output table of HITS\n",
+ "\n",
+ "SELECT * FROM hits_out ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Look at the summary table:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>__iterations__</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>17</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(17,)]"
+ ]
+ },
+ "execution_count": 9,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM hits_out_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 3. Different number of iterations\n",
+ "Results in different hub and authority scores."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "7 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>authority</th>\n",
+ " <th>hub</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>0</td>\n",
+ " <td>0.0865332738778</td>\n",
+ " <td>0.375721659592</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>0.18388320699</td>\n",
+ " <td>0.533118571043</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>0.432666369389</td>\n",
+ " <td>0.654974244425</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>0.703082850257</td>\n",
+ " <td>0.0406185577938</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>0.18388320699</td>\n",
+ " <td>0.0406185577938</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>0.302866458572</td>\n",
+ " <td>0.182783510072</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>0.38939973245</td>\n",
+ " <td>0.330025782074</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(0, 0.0865332738777835, 0.375721659592363),\n",
+ " (1, 0.18388320699029, 0.533118571043218),\n",
+ " (2, 0.432666369388918, 0.654974244424525),\n",
+ " (3, 0.703082850256991, 0.040618557793769),\n",
+ " (4, 0.18388320699029, 0.040618557793769),\n",
+ " (5, 0.302866458572242, 0.182783510071961),\n",
+ " (6, 0.389399732450026, 0.330025782074373)]"
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS hits_out, hits_out_summary;\n",
+ "\n",
+ "SELECT madlib.hits(\n",
+ " 'vertex', -- Vertex table\n",
+ " 'id', -- Vertex id column\n",
+ " 'edge', -- Edge table\n",
+ " 'src=src, dest=dest', -- Comma delimited string of edge arguments\n",
+ " 'hits_out', -- Output table of HITS\n",
+ " 3); -- Max iteration\n",
+ "\n",
+ "SELECT * FROM hits_out ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Look at the summary table: "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>__iterations__</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(3,)]"
+ ]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM hits_out_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 4. Different theshold\n",
+ "Running HITS with a low threshold of 0.00001 results in more iterations for convergence"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "7 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>authority</th>\n",
+ " <th>hub</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>0</td>\n",
+ " <td>0.194028500029</td>\n",
+ " <td>0.39062401003</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>0.194028500029</td>\n",
+ " <td>0.528491307688</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>0.436564125065</td>\n",
+ " <td>0.643380722403</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>0.679099750102</td>\n",
+ " <td>0.0919115317719</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>0.194028500029</td>\n",
+ " <td>0.0919115317719</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>0.291042750044</td>\n",
+ " <td>0.183823063544</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>0.388057000058</td>\n",
+ " <td>0.321690361202</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(0, 0.194028500029066, 0.390624010030425),\n",
+ " (1, 0.194028500029066, 0.528491307688221),\n",
+ " (2, 0.436564125065399, 0.643380722403052),\n",
+ " (3, 0.679099750101732, 0.0919115317718646),\n",
+ " (4, 0.194028500029066, 0.0919115317718646),\n",
+ " (5, 0.2910427500436, 0.183823063543729),\n",
+ " (6, 0.388057000058133, 0.321690361201526)]"
+ ]
+ },
+ "execution_count": 16,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS hits_out, hits_out_summary;\n",
+ "\n",
+ "SELECT madlib.hits(\n",
+ " 'vertex', -- Vertex table\n",
+ " 'id', -- Vertex id column\n",
+ " 'edge', -- Edge table\n",
+ " 'src=src, dest=dest', -- Comma delimited string of edge arguments\n",
+ " 'hits_out', -- Output table of HITS\n",
+ " NULL, -- Default max_iter\n",
+ " 0.5); -- Threshold\n",
+ "\n",
+ "SELECT * FROM hits_out ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Look at the summary table:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>__iterations__</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(2,)]"
+ ]
+ },
+ "execution_count": 17,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM hits_out_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 5. Different number of iterations and threshold"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "7 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>authority</th>\n",
+ " <th>hub</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>0</td>\n",
+ " <td>7.11260011826e-08</td>\n",
+ " <td>0.33810307986</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>0.158326035588</td>\n",
+ " <td>0.527815233931</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>0.40546145318</td>\n",
+ " <td>0.675913495026</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>0.72183534323</td>\n",
+ " <td>3.33021322089e-08</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>0.158326035588</td>\n",
+ " <td>3.33021322089e-08</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>0.316459563894</td>\n",
+ " <td>0.189770119974</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>0.405307074424</td>\n",
+ " <td>0.337972831786</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(0, 7.11260011825861e-08, 0.33810307986005),\n",
+ " (1, 0.158326035587958, 0.527815233930963),\n",
+ " (2, 0.405461453180491, 0.675913495026452),\n",
+ " (3, 0.721835343230399, 3.3302132208914e-08),\n",
+ " (4, 0.158326035587958, 3.3302132208914e-08),\n",
+ " (5, 0.316459563893809, 0.189770119973925),\n",
+ " (6, 0.405307074424261, 0.337972831786458)]"
+ ]
+ },
+ "execution_count": 14,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS hits_out, hits_out_summary;\n",
+ "\n",
+ "SELECT madlib.hits(\n",
+ " 'vertex', -- Vertex table\n",
+ " 'id', -- Vertex id column\n",
+ " 'edge', -- Edge table\n",
+ " 'src=src, dest=dest', -- Comma delimited string of edge arguments\n",
+ " 'hits_out', -- Output table\n",
+ " 20, -- Default max_iter\n",
+ " 0.00001); -- Threshold\n",
+ "\n",
+ "SELECT * FROM hits_out ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Look at the summary table. The algorithm stopped at 20 iterations even though the convergence for threshold of 0.00001 is at 25 iterations. This is because max_iter was set to 20."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>__iterations__</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>20</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(20,)]"
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM hits_out_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 6. Grouping\n",
+ "Running HITS with grouping column and default values for max_iter and threshold. Add more rows to the edge table to create different graphs based on the user_id column."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "10 rows affected.\n",
+ "Done.\n",
+ "1 rows affected.\n",
+ "14 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>user_id</th>\n",
+ " <th>id</th>\n",
+ " <th>authority</th>\n",
+ " <th>hub</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>0</td>\n",
+ " <td>8.43871829093e-07</td>\n",
+ " <td>0.338306115083</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>1</td>\n",
+ " <td>0.158459587238</td>\n",
+ " <td>0.527865350448</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>2</td>\n",
+ " <td>0.40562796969</td>\n",
+ " <td>0.675800764728</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>3</td>\n",
+ " <td>0.721775835522</td>\n",
+ " <td>3.95111934817e-07</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>4</td>\n",
+ " <td>0.158459587238</td>\n",
+ " <td>3.95111934817e-07</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>5</td>\n",
+ " <td>0.316385413093</td>\n",
+ " <td>0.189719957843</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>6</td>\n",
+ " <td>0.405199928761</td>\n",
+ " <td>0.337944978189</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>0</td>\n",
+ " <td>1.60841750445e-05</td>\n",
+ " <td>0.632262085114</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>1</td>\n",
+ " <td>0.316079985713</td>\n",
+ " <td>0.6325293909</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>2</td>\n",
+ " <td>0.632364174872</td>\n",
+ " <td>0.31634729748</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>3</td>\n",
+ " <td>0.632694582988</td>\n",
+ " <td>8.04208767443e-06</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>4</td>\n",
+ " <td>0.316079985713</td>\n",
+ " <td>8.04208767443e-06</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>5</td>\n",
+ " <td>0.0</td>\n",
+ " <td>1.22712519446e-10</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>6</td>\n",
+ " <td>2.45425034248e-10</td>\n",
+ " <td>0.31634729748</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, 0, 8.4387182909347e-07, 0.338306115082665),\n",
+ " (1, 1, 0.158459587238244, 0.527865350448059),\n",
+ " (1, 2, 0.405627969689677, 0.675800764727558),\n",
+ " (1, 3, 0.721775835521825, 3.95111934817447e-07),\n",
+ " (1, 4, 0.158459587238244, 3.95111934817447e-07),\n",
+ " (1, 5, 0.316385413093048, 0.189719957843216),\n",
+ " (1, 6, 0.405199928761102, 0.337944978189241),\n",
+ " (2, 0, 1.60841750444904e-05, 0.632262085114062),\n",
+ " (2, 1, 0.316079985713431, 0.632529390899584),\n",
+ " (2, 2, 0.632364174872359, 0.316347297480213),\n",
+ " (2, 3, 0.632694582987791, 8.04208767442759e-06),\n",
+ " (2, 4, 0.316079985713431, 8.04208767442759e-06),\n",
+ " (2, 5, 0.0, 1.22712519446222e-10),\n",
+ " (2, 6, 2.45425034248205e-10, 0.316347297480213)]"
+ ]
+ },
+ "execution_count": 16,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "INSERT INTO edge VALUES\n",
+ "(0, 1, 2),\n",
+ "(0, 2, 2),\n",
+ "(0, 4, 2),\n",
+ "(1, 2, 2),\n",
+ "(1, 3, 2),\n",
+ "(2, 3, 2),\n",
+ "(3, 0, 2),\n",
+ "(4, 0, 2),\n",
+ "(5, 6, 2),\n",
+ "(6, 3, 2);\n",
+ "\n",
+ "DROP TABLE IF EXISTS hits_out, hits_out_summary;\n",
+ "\n",
+ "SELECT madlib.hits(\n",
+ " 'vertex', -- Vertex table\n",
+ " 'id', -- Vertex id column\n",
+ " 'edge', -- Edge table\n",
+ " 'src=src, dest=dest', -- Comma delimited string of edge arguments\n",
+ " 'hits_out', -- Output table\n",
+ " NULL, -- Default max_iter\n",
+ " NULL, -- Threshold\n",
+ " 'user_id'); -- Grouping column\n",
+ "\n",
+ "SELECT * FROM hits_out ORDER BY user_id, id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 7. Other\n",
+ "Let's check against the output from p. 8 of http://www.cis.hut.fi/Opinnot/T-61.6020/2008/pagerank_hits.pdf"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "Done.\n",
+ "Done.\n",
+ "4 rows affected.\n",
+ "6 rows affected.\n",
+ "6 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>src</th>\n",
+ " <th>dest</th>\n",
+ " <th>user_id</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>0</td>\n",
+ " <td>1</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>0</td>\n",
+ " <td>2</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>0</td>\n",
+ " <td>3</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>2</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>3</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>1</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(0, 1, 1), (0, 2, 1), (0, 3, 1), (1, 2, 1), (1, 3, 1), (2, 1, 1)]"
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS vertex, edge;\n",
+ "\n",
+ "CREATE TABLE vertex(\n",
+ " id INTEGER\n",
+ " );\n",
+ "\n",
+ "CREATE TABLE edge(\n",
+ " src INTEGER,\n",
+ " dest INTEGER,\n",
+ " user_id INTEGER\n",
+ " );\n",
+ "\n",
+ "INSERT INTO vertex VALUES\n",
+ "(0),\n",
+ "(1),\n",
+ "(2),\n",
+ "(3);\n",
+ "\n",
+ "INSERT INTO edge VALUES\n",
+ "(0, 1, 1),\n",
+ "(0, 2, 1),\n",
+ "(0, 3, 1),\n",
+ "(1, 2, 1),\n",
+ "(1, 3, 1),\n",
+ "(2, 1, 1);\n",
+ "\n",
+ "SELECT * from edge ORDER BY src, dest;"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "2 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>user_id</th>\n",
+ " <th>__iterations__</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>17</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>16</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, 17), (2, 16)]"
+ ]
+ },
+ "execution_count": 18,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM hits_out_summary order by user_id;"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "4 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>authority</th>\n",
+ " <th>hub</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>0</td>\n",
+ " <td>0.0</td>\n",
+ " <td>0.788680749581</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>0.459746429928</td>\n",
+ " <td>0.577334927798</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>0.627946343316</td>\n",
+ " <td>0.211345821783</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>0.627946343316</td>\n",
+ " <td>0.0</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(0, 0.0, 0.788680749580922),\n",
+ " (1, 0.459746429927966, 0.577334927797799),\n",
+ " (2, 0.627946343316246, 0.211345821783123),\n",
+ " (3, 0.627946343316246, 0.0)]"
+ ]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS hits_out, hits_out_summary;\n",
+ "\n",
+ "SELECT madlib.hits(\n",
+ " 'vertex', -- Vertex table\n",
+ " 'id', -- Vertex id column\n",
+ " 'edge', -- Edge table\n",
+ " 'src=src, dest=dest', -- Comma delimited string of edge arguments\n",
+ " 'hits_out', -- Output table of HITS\n",
+ " 100); -- Max iteration\n",
+ "\n",
+ "SELECT * FROM hits_out ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Yes ^^^ matches the results from the reference."
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 2",
+ "language": "python",
+ "name": "python2"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.12"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
http://git-wip-us.apache.org/repos/asf/madlib-site/blob/95826612/community-artifacts/Linear-regression-v1.ipynb
----------------------------------------------------------------------
diff --git a/community-artifacts/Linear-regression-v1.ipynb b/community-artifacts/Linear-regression-v1.ipynb
new file mode 100644
index 0000000..4ae89cf
--- /dev/null
+++ b/community-artifacts/Linear-regression-v1.ipynb
@@ -0,0 +1,1188 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Linear regression"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/config.py:13: ShimWarning: The `IPython.config` package has been deprecated. You should import from traitlets.config instead.\n",
+ " \"You should import from traitlets.config instead.\", ShimWarning)\n",
+ "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/utils/traitlets.py:5: UserWarning: IPython.utils.traitlets has moved to a top-level traitlets package.\n",
+ " warn(\"IPython.utils.traitlets has moved to a top-level traitlets package.\")\n"
+ ]
+ }
+ ],
+ "source": [
+ "%load_ext sql"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "u'Connected: gpdbchina@madlib'"
+ ]
+ },
+ "execution_count": 2,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Greenplum 4.3.10.0\n",
+ "%sql postgresql://gpdbchina@10.194.10.68:61000/madlib\n",
+ " \n",
+ "# PostgreSQL local\n",
+ "#%sql postgresql://fmcquillan@localhost:5432/madlib"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>version</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>MADlib version: 1.13-dev, git revision: rel/v1.12-31-g4b7d9cc, cmake configuration time: Tue Nov 21 22:31:28 UTC 2017, build type: Release, build system: Linux-2.6.18-238.27.1.el5.hotfix.bz516490, C compiler: gcc 4.4.0, C++ compiler: g++ 4.4.0</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'MADlib version: 1.13-dev, git revision: rel/v1.12-31-g4b7d9cc, cmake configuration time: Tue Nov 21 22:31:28 UTC 2017, build type: Release, build system: Linux-2.6.18-238.27.1.el5.hotfix.bz516490, C compiler: gcc 4.4.0, C++ compiler: g++ 4.4.0',)]"
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%sql select madlib.version();"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 1. Load test data"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "Done.\n",
+ "15 rows affected.\n",
+ "15 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>tax</th>\n",
+ " <th>bedroom</th>\n",
+ " <th>bath</th>\n",
+ " <th>price</th>\n",
+ " <th>size</th>\n",
+ " <th>lot</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>590</td>\n",
+ " <td>2</td>\n",
+ " <td>1.0</td>\n",
+ " <td>50000</td>\n",
+ " <td>770</td>\n",
+ " <td>22100</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>1050</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>85000</td>\n",
+ " <td>1410</td>\n",
+ " <td>12000</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>20</td>\n",
+ " <td>3</td>\n",
+ " <td>1.0</td>\n",
+ " <td>22500</td>\n",
+ " <td>1060</td>\n",
+ " <td>3500</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>870</td>\n",
+ " <td>2</td>\n",
+ " <td>2.0</td>\n",
+ " <td>90000</td>\n",
+ " <td>1300</td>\n",
+ " <td>17500</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>1320</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>133000</td>\n",
+ " <td>1500</td>\n",
+ " <td>30000</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>1350</td>\n",
+ " <td>2</td>\n",
+ " <td>1.0</td>\n",
+ " <td>90500</td>\n",
+ " <td>820</td>\n",
+ " <td>25700</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>7</td>\n",
+ " <td>2790</td>\n",
+ " <td>3</td>\n",
+ " <td>2.5</td>\n",
+ " <td>260000</td>\n",
+ " <td>2130</td>\n",
+ " <td>25000</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>8</td>\n",
+ " <td>680</td>\n",
+ " <td>2</td>\n",
+ " <td>1.0</td>\n",
+ " <td>142500</td>\n",
+ " <td>1170</td>\n",
+ " <td>22000</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>9</td>\n",
+ " <td>1840</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>160000</td>\n",
+ " <td>1500</td>\n",
+ " <td>19000</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>10</td>\n",
+ " <td>3680</td>\n",
+ " <td>4</td>\n",
+ " <td>2.0</td>\n",
+ " <td>240000</td>\n",
+ " <td>2790</td>\n",
+ " <td>20000</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>11</td>\n",
+ " <td>1660</td>\n",
+ " <td>3</td>\n",
+ " <td>1.0</td>\n",
+ " <td>87000</td>\n",
+ " <td>1030</td>\n",
+ " <td>17500</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>12</td>\n",
+ " <td>1620</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>118600</td>\n",
+ " <td>1250</td>\n",
+ " <td>20000</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>13</td>\n",
+ " <td>3100</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>140000</td>\n",
+ " <td>1760</td>\n",
+ " <td>38000</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>14</td>\n",
+ " <td>2070</td>\n",
+ " <td>2</td>\n",
+ " <td>3.0</td>\n",
+ " <td>148000</td>\n",
+ " <td>1550</td>\n",
+ " <td>14000</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>15</td>\n",
+ " <td>650</td>\n",
+ " <td>3</td>\n",
+ " <td>1.5</td>\n",
+ " <td>65000</td>\n",
+ " <td>1450</td>\n",
+ " <td>12000</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, 590, 2, 1.0, 50000, 770, 22100),\n",
+ " (2, 1050, 3, 2.0, 85000, 1410, 12000),\n",
+ " (3, 20, 3, 1.0, 22500, 1060, 3500),\n",
+ " (4, 870, 2, 2.0, 90000, 1300, 17500),\n",
+ " (5, 1320, 3, 2.0, 133000, 1500, 30000),\n",
+ " (6, 1350, 2, 1.0, 90500, 820, 25700),\n",
+ " (7, 2790, 3, 2.5, 260000, 2130, 25000),\n",
+ " (8, 680, 2, 1.0, 142500, 1170, 22000),\n",
+ " (9, 1840, 3, 2.0, 160000, 1500, 19000),\n",
+ " (10, 3680, 4, 2.0, 240000, 2790, 20000),\n",
+ " (11, 1660, 3, 1.0, 87000, 1030, 17500),\n",
+ " (12, 1620, 3, 2.0, 118600, 1250, 20000),\n",
+ " (13, 3100, 3, 2.0, 140000, 1760, 38000),\n",
+ " (14, 2070, 2, 3.0, 148000, 1550, 14000),\n",
+ " (15, 650, 3, 1.5, 65000, 1450, 12000)]"
+ ]
+ },
+ "execution_count": 17,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql \n",
+ "DROP TABLE IF EXISTS houses;\n",
+ "\n",
+ "CREATE TABLE houses (id INT, tax INT, bedroom INT, bath FLOAT, price INT,\n",
+ " size INT, lot INT);\n",
+ "\n",
+ "INSERT INTO houses VALUES \n",
+ " (1 , 590 , 2 , 1 , 50000 , 770 , 22100),\n",
+ " (2 , 1050 , 3 , 2 , 85000 , 1410 , 12000),\n",
+ " (3 , 20 , 3 , 1 , 22500 , 1060 , 3500),\n",
+ " (4 , 870 , 2 , 2 , 90000 , 1300 , 17500),\n",
+ " (5 , 1320 , 3 , 2 , 133000 , 1500 , 30000),\n",
+ " (6 , 1350 , 2 , 1 , 90500 , 820 , 25700),\n",
+ " (7 , 2790 , 3 , 2.5 , 260000 , 2130 , 25000),\n",
+ " (8 , 680 , 2 , 1 , 142500 , 1170 , 22000),\n",
+ " (9 , 1840 , 3 , 2 , 160000 , 1500 , 19000),\n",
+ " (10 , 3680 , 4 , 2 , 240000 , 2790 , 20000),\n",
+ " (11 , 1660 , 3 , 1 , 87000 , 1030 , 17500),\n",
+ " (12 , 1620 , 3 , 2 , 118600 , 1250 , 20000),\n",
+ " (13 , 3100 , 3 , 2 , 140000 , 1760 , 38000),\n",
+ " (14 , 2070 , 2 , 3 , 148000 , 1550 , 14000),\n",
+ " (15 , 650 , 3 , 1.5 , 65000 , 1450 , 12000);\n",
+ " \n",
+ "SELECT * FROM houses ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 2. Train regression model\n",
+ "First, we generate a single regression for all data."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>method</th>\n",
+ " <th>source_table</th>\n",
+ " <th>out_table</th>\n",
+ " <th>dependent_varname</th>\n",
+ " <th>independent_varname</th>\n",
+ " <th>num_rows_processed</th>\n",
+ " <th>num_missing_rows_skipped</th>\n",
+ " <th>grouping_col</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>linregr</td>\n",
+ " <td>houses</td>\n",
+ " <td>houses_linregr</td>\n",
+ " <td>price</td>\n",
+ " <td>ARRAY[1, tax, bath, size]</td>\n",
+ " <td>15</td>\n",
+ " <td>0</td>\n",
+ " <td>None</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'linregr', u'houses', u'houses_linregr', u'price', u'ARRAY[1, tax, bath, size]', 15, 0, None)]"
+ ]
+ },
+ "execution_count": 18,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS houses_linregr, houses_linregr_summary;\n",
+ "\n",
+ "SELECT madlib.linregr_train( 'houses',\n",
+ " 'houses_linregr',\n",
+ " 'price',\n",
+ " 'ARRAY[1, tax, bath, size]'\n",
+ " );\n",
+ "\n",
+ "SELECT * FROM houses_linregr_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 3. Train regression model with grouping\n",
+ "Next we generate three output models, one for each value of \"bedroom\"."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>method</th>\n",
+ " <th>source_table</th>\n",
+ " <th>out_table</th>\n",
+ " <th>dependent_varname</th>\n",
+ " <th>independent_varname</th>\n",
+ " <th>num_rows_processed</th>\n",
+ " <th>num_missing_rows_skipped</th>\n",
+ " <th>grouping_col</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>linregr</td>\n",
+ " <td>houses</td>\n",
+ " <td>houses_linregr_bedroom</td>\n",
+ " <td>price</td>\n",
+ " <td>ARRAY[1, tax, bath, size]</td>\n",
+ " <td>15</td>\n",
+ " <td>0</td>\n",
+ " <td>bedroom</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'linregr', u'houses', u'houses_linregr_bedroom', u'price', u'ARRAY[1, tax, bath, size]', 15, 0, u'bedroom')]"
+ ]
+ },
+ "execution_count": 19,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql \n",
+ "DROP TABLE IF EXISTS houses_linregr_bedroom, houses_linregr_bedroom_summary;\n",
+ "\n",
+ "SELECT madlib.linregr_train( 'houses',\n",
+ " 'houses_linregr_bedroom',\n",
+ " 'price',\n",
+ " 'ARRAY[1, tax, bath, size]',\n",
+ " 'bedroom'\n",
+ " );\n",
+ "\n",
+ "SELECT * FROM houses_linregr_bedroom_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 4. Review model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>coef</th>\n",
+ " <th>r2</th>\n",
+ " <th>std_err</th>\n",
+ " <th>t_stats</th>\n",
+ " <th>p_values</th>\n",
+ " <th>condition_no</th>\n",
+ " <th>num_rows_processed</th>\n",
+ " <th>num_missing_rows_skipped</th>\n",
+ " <th>variance_covariance</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[-12849.4168959872, 28.9613922651775, 10181.6290712649, 50.516894915353]</td>\n",
+ " <td>0.768577580597</td>\n",
+ " <td>[33453.0344331377, 15.8992104963991, 19437.7710925915, 32.9280231740856]</td>\n",
+ " <td>[-0.384103179688204, 1.82156166004197, 0.523806408809164, 1.53416118083608]</td>\n",
+ " <td>[0.708223134615411, 0.0958005827189554, 0.610804093526515, 0.153235085548177]</td>\n",
+ " <td>9002.5045707</td>\n",
+ " <td>15</td>\n",
+ " <td>0</td>\n",
+ " <td>[[1119105512.7847, 217782.067878005, -283344228.394538, -616679.693190829], [217782.067878005, 252.784894408806, -46373.1796964037, -369.864520095145], [-283344228.394538, -46373.1796964038, 377826945.047986, -209088.217319699], [-616679.693190829, -369.864520095145, -209088.217319699, 1084.25471015312]]</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[([-12849.4168959872, 28.9613922651775, 10181.6290712649, 50.516894915353], 0.768577580597462, [33453.0344331377, 15.8992104963991, 19437.7710925915, 32.9280231740856], [-0.384103179688204, 1.82156166004197, 0.523806408809164, 1.53416118083608], [0.708223134615411, 0.0958005827189554, 0.610804093526515, 0.153235085548177], 9002.50457069859, 15L, 0L, [[1119105512.7847, 217782.067878005, -283344228.394538, -616679.693190829], [217782.067878005, 252.784894408806, -46373.1796964037, -369.864520095145], [-283344228.394538, -46373.1796964038, 377826945.047986, -209088.217319699], [-616679.693190829, -369.864520095145, -209088.217319699, 1084.25471015312]])]"
+ ]
+ },
+ "execution_count": 20,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM houses_linregr;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Alternatively you can unnest the results for easier reading of output."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "4 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>attribute</th>\n",
+ " <th>coefficient</th>\n",
+ " <th>standard_error</th>\n",
+ " <th>t_stat</th>\n",
+ " <th>pvalue</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>intercept</td>\n",
+ " <td>-12849.416896</td>\n",
+ " <td>33453.0344331</td>\n",
+ " <td>-0.384103179688</td>\n",
+ " <td>0.708223134615</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>tax</td>\n",
+ " <td>28.9613922652</td>\n",
+ " <td>15.8992104964</td>\n",
+ " <td>1.82156166004</td>\n",
+ " <td>0.095800582719</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>bath</td>\n",
+ " <td>10181.6290713</td>\n",
+ " <td>19437.7710926</td>\n",
+ " <td>0.523806408809</td>\n",
+ " <td>0.610804093527</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>size</td>\n",
+ " <td>50.5168949154</td>\n",
+ " <td>32.9280231741</td>\n",
+ " <td>1.53416118084</td>\n",
+ " <td>0.153235085548</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'intercept', -12849.4168959872, 33453.0344331377, -0.384103179688204, 0.708223134615411),\n",
+ " (u'tax', 28.9613922651775, 15.8992104963991, 1.82156166004197, 0.0958005827189554),\n",
+ " (u'bath', 10181.6290712649, 19437.7710925915, 0.523806408809164, 0.610804093526515),\n",
+ " (u'size', 50.516894915353, 32.9280231740856, 1.53416118083608, 0.153235085548177)]"
+ ]
+ },
+ "execution_count": 25,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT unnest(ARRAY['intercept','tax','bath','size']) as attribute,\n",
+ " unnest(coef) as coefficient,\n",
+ " unnest(std_err) as standard_error,\n",
+ " unnest(t_stats) as t_stat,\n",
+ " unnest(p_values) as pvalue\n",
+ "FROM houses_linregr;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "collapsed": true
+ },
+ "source": [
+ "# 5. Review model with grouping"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "3 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>bedroom</th>\n",
+ " <th>coef</th>\n",
+ " <th>r2</th>\n",
+ " <th>std_err</th>\n",
+ " <th>t_stats</th>\n",
+ " <th>p_values</th>\n",
+ " <th>condition_no</th>\n",
+ " <th>num_rows_processed</th>\n",
+ " <th>num_missing_rows_skipped</th>\n",
+ " <th>variance_covariance</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>[-84242.0345406619, 55.4430144648689, -78966.9753675336, 225.611910021196]</td>\n",
+ " <td>0.968809546465</td>\n",
+ " <td>[35018.9991666351, 19.5731125321026, 23036.8071292953, 49.0448678149637]</td>\n",
+ " <td>[-2.40560942760823, 2.83261103076655, -3.42786111479457, 4.60011251069905]</td>\n",
+ " <td>[0.250804617665626, 0.21605133377637, 0.180704400437667, 0.136272031474349]</td>\n",
+ " <td>10086.1048725</td>\n",
+ " <td>5</td>\n",
+ " <td>0</td>\n",
+ " <td>[[1226330302.63279, -300921.595597853, 551696673.399772, -1544160.63236657], [-300921.595597853, 383.106734194352, -304863.397298569, 323.251642470093], [551696673.399772, -304863.397298569, 530694482.712349, -946345.586402425], [-1544160.63236657, 323.251642470093, -946345.586402425, 2405.39905898726]]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>[-88155.8292501588, 27.1966436294421, 41404.0293363614, 62.6375210753234]</td>\n",
+ " <td>0.841699901311</td>\n",
+ " <td>[57867.9999702679, 17.8272309154706, 43643.1321511158, 70.8506824864022]</td>\n",
+ " <td>[-1.52339512848988, 1.52556747362489, 0.948695185143874, 0.884077878675974]</td>\n",
+ " <td>[0.188161432894911, 0.187636685729916, 0.38634003237497, 0.417132778705835]</td>\n",
+ " <td>11722.6225642</td>\n",
+ " <td>9</td>\n",
+ " <td>0</td>\n",
+ " <td>[[3348705420.55893, 433697.545104307, -70253017.4577515, -2593488.13800241], [433697.545104307, 317.810162113512, -90019.0797451144, -529.274668274391], [-70253017.4577515, -90019.0797451146, 1904722983.95976, -2183233.19448568], [-2593488.13800241, -529.27466827439, -2183233.19448568, 5019.81920878897]]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>[0.0112536020318379, 41.4132554771633, 0.0225072040636757, 31.3975496688276]</td>\n",
+ " <td>1.0</td>\n",
+ " <td>[0.0, 0.0, 0.0, 0.0]</td>\n",
+ " <td>[inf, inf, inf, inf]</td>\n",
+ " <td>None</td>\n",
+ " <td>inf</td>\n",
+ " <td>1</td>\n",
+ " <td>0</td>\n",
+ " <td>[[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]]</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(2, [-84242.0345406619, 55.4430144648689, -78966.9753675336, 225.611910021196], 0.968809546465205, [35018.9991666351, 19.5731125321026, 23036.8071292953, 49.0448678149637], [-2.40560942760823, 2.83261103076655, -3.42786111479457, 4.60011251069905], [0.250804617665626, 0.21605133377637, 0.180704400437667, 0.136272031474349], 10086.1048725104, 5L, 0L, [[1226330302.63279, -300921.595597853, 551696673.399772, -1544160.63236657], [-300921.595597853, 383.106734194352, -304863.397298569, 323.251642470093], [551696673.399772, -304863.397298569, 530694482.712349, -946345.586402425], [-1544160.63236657, 323.251642470093, -946345.586402425, 2405.39905898726]]),\n",
+ " (3, [-88155.8292501588, 27.1966436294421, 41404.0293363614, 62.6375210753234], 0.841699901311221, [57867.9999702679, 17.8272309154706, 43643.1321511158, 70.8506824864022], [-1.52339512848988, 1.52556747362489, 0.948695185143874, 0.884077878675974], [0.188161432894911, 0.187636685729916, 0.38634003237497, 0.417132778705835], 11722.6225642065, 9L, 0L, [[3348705420.55893, 433697.545104307, -70253017.4577515, -2593488.13800241], [433697.545104307, 317.810162113512, -90019.0797451144, -529.274668274391], [-70253017.4577515, -90019.0797451146, 1904722983.95976, -2183233.19448568], [-2593488.13800241, -529.27466827439, -2183233.19448568, 5019.81920878897]]),\n",
+ " (4, [0.0112536020318379, 41.4132554771633, 0.0225072040636757, 31.3975496688276], 1.0, [0.0, 0.0, 0.0, 0.0], [inf, inf, inf, inf], None, inf, 1L, 0L, [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]])]"
+ ]
+ },
+ "execution_count": 22,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM houses_linregr_bedroom ORDER BY bedroom;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 6. Prediction\n",
+ "Compare predicted price with actual, in the case where not grouping is used. That is, there is only one regression model for all data. (This example uses the original data table to perform the prediction. Typically a different test dataset with the same features as the original training dataset would be used for prediction.)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "15 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>tax</th>\n",
+ " <th>bedroom</th>\n",
+ " <th>bath</th>\n",
+ " <th>price</th>\n",
+ " <th>size</th>\n",
+ " <th>lot</th>\n",
+ " <th>predict</th>\n",
+ " <th>residual</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>590</td>\n",
+ " <td>2</td>\n",
+ " <td>1.0</td>\n",
+ " <td>50000</td>\n",
+ " <td>770</td>\n",
+ " <td>22100</td>\n",
+ " <td>53317.4426966</td>\n",
+ " <td>-3317.44269655</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>1050</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>85000</td>\n",
+ " <td>1410</td>\n",
+ " <td>12000</td>\n",
+ " <td>109152.124956</td>\n",
+ " <td>-24152.1249556</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>20</td>\n",
+ " <td>3</td>\n",
+ " <td>1.0</td>\n",
+ " <td>22500</td>\n",
+ " <td>1060</td>\n",
+ " <td>3500</td>\n",
+ " <td>51459.3486309</td>\n",
+ " <td>-28959.3486309</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>870</td>\n",
+ " <td>2</td>\n",
+ " <td>2.0</td>\n",
+ " <td>90000</td>\n",
+ " <td>1300</td>\n",
+ " <td>17500</td>\n",
+ " <td>98382.2159072</td>\n",
+ " <td>-8382.21590721</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>1320</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>133000</td>\n",
+ " <td>1500</td>\n",
+ " <td>30000</td>\n",
+ " <td>121518.22141</td>\n",
+ " <td>11481.7785904</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>1350</td>\n",
+ " <td>2</td>\n",
+ " <td>1.0</td>\n",
+ " <td>90500</td>\n",
+ " <td>820</td>\n",
+ " <td>25700</td>\n",
+ " <td>77853.9455639</td>\n",
+ " <td>12646.0544361</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>7</td>\n",
+ " <td>2790</td>\n",
+ " <td>3</td>\n",
+ " <td>2.5</td>\n",
+ " <td>260000</td>\n",
+ " <td>2130</td>\n",
+ " <td>25000</td>\n",
+ " <td>201007.926372</td>\n",
+ " <td>58992.0736283</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>8</td>\n",
+ " <td>680</td>\n",
+ " <td>2</td>\n",
+ " <td>1.0</td>\n",
+ " <td>142500</td>\n",
+ " <td>1170</td>\n",
+ " <td>22000</td>\n",
+ " <td>76130.7259666</td>\n",
+ " <td>66369.2740334</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>9</td>\n",
+ " <td>1840</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>160000</td>\n",
+ " <td>1500</td>\n",
+ " <td>19000</td>\n",
+ " <td>136578.145387</td>\n",
+ " <td>23421.8546125</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>10</td>\n",
+ " <td>3680</td>\n",
+ " <td>4</td>\n",
+ " <td>2.0</td>\n",
+ " <td>240000</td>\n",
+ " <td>2790</td>\n",
+ " <td>20000</td>\n",
+ " <td>255033.901596</td>\n",
+ " <td>-15033.9015962</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>11</td>\n",
+ " <td>1660</td>\n",
+ " <td>3</td>\n",
+ " <td>1.0</td>\n",
+ " <td>87000</td>\n",
+ " <td>1030</td>\n",
+ " <td>17500</td>\n",
+ " <td>97440.5250983</td>\n",
+ " <td>-10440.5250983</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>12</td>\n",
+ " <td>1620</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>118600</td>\n",
+ " <td>1250</td>\n",
+ " <td>20000</td>\n",
+ " <td>117577.41536</td>\n",
+ " <td>1022.58463968</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>13</td>\n",
+ " <td>3100</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>140000</td>\n",
+ " <td>1760</td>\n",
+ " <td>38000</td>\n",
+ " <td>186203.89232</td>\n",
+ " <td>-46203.8923196</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>14</td>\n",
+ " <td>2070</td>\n",
+ " <td>2</td>\n",
+ " <td>3.0</td>\n",
+ " <td>148000</td>\n",
+ " <td>1550</td>\n",
+ " <td>14000</td>\n",
+ " <td>155946.739426</td>\n",
+ " <td>-7946.73942552</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>15</td>\n",
+ " <td>650</td>\n",
+ " <td>3</td>\n",
+ " <td>1.5</td>\n",
+ " <td>65000</td>\n",
+ " <td>1450</td>\n",
+ " <td>12000</td>\n",
+ " <td>94497.4293105</td>\n",
+ " <td>-29497.4293105</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, 590, 2, 1.0, 50000, 770, 22100, 53317.4426965543, -3317.44269655428),\n",
+ " (2, 1050, 3, 2.0, 85000, 1410, 12000, 109152.124955627, -24152.1249556268),\n",
+ " (3, 20, 3, 1.0, 22500, 1060, 3500, 51459.3486308555, -28959.3486308555),\n",
+ " (4, 870, 2, 2.0, 90000, 1300, 17500, 98382.215907206, -8382.21590720599),\n",
+ " (5, 1320, 3, 2.0, 133000, 1500, 30000, 121518.221409606, 11481.7785903935),\n",
+ " (6, 1350, 2, 1.0, 90500, 820, 25700, 77853.9455638568, 12646.0544361432),\n",
+ " (7, 2790, 3, 2.5, 260000, 2130, 25000, 201007.926371722, 58992.0736282778),\n",
+ " (8, 680, 2, 1.0, 142500, 1170, 22000, 76130.7259665615, 66369.2740334385),\n",
+ " (9, 1840, 3, 2.0, 160000, 1500, 19000, 136578.145387499, 23421.8546125013),\n",
+ " (10, 3680, 4, 2.0, 240000, 2790, 20000, 255033.901596231, -15033.9015962306),\n",
+ " (11, 1660, 3, 1.0, 87000, 1030, 17500, 97440.5250982859, -10440.5250982859),\n",
+ " (12, 1620, 3, 2.0, 118600, 1250, 20000, 117577.415360321, 1022.58463967856),\n",
+ " (13, 3100, 3, 2.0, 140000, 1760, 38000, 186203.892319614, -46203.8923196141),\n",
+ " (14, 2070, 2, 3.0, 148000, 1550, 14000, 155946.739425522, -7946.73942552213),\n",
+ " (15, 650, 3, 1.5, 65000, 1450, 12000, 94497.4293105374, -29497.4293105374)]"
+ ]
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT houses.*,\n",
+ " madlib.linregr_predict( m.coef,\n",
+ " ARRAY[1,tax,bath,size]\n",
+ " ) as predict,\n",
+ " price -\n",
+ " madlib.linregr_predict( m.coef,\n",
+ " ARRAY[1,tax,bath,size] \n",
+ " ) as residual\n",
+ "FROM houses, houses_linregr m ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 7. Prediction with grouping\n",
+ "Compare predicted price with actual with grouping. It means a different model is used depending on the number of bedrooms."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 49,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "15 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>tax</th>\n",
+ " <th>bedroom</th>\n",
+ " <th>bath</th>\n",
+ " <th>price</th>\n",
+ " <th>size</th>\n",
+ " <th>lot</th>\n",
+ " <th>predict</th>\n",
+ " <th>residual</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>590</td>\n",
+ " <td>2</td>\n",
+ " <td>1.0</td>\n",
+ " <td>50000</td>\n",
+ " <td>770</td>\n",
+ " <td>22100</td>\n",
+ " <td>43223.5393424</td>\n",
+ " <td>6776.4606576</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>1050</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>85000</td>\n",
+ " <td>1410</td>\n",
+ " <td>12000</td>\n",
+ " <td>111527.60995</td>\n",
+ " <td>-26527.6099497</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>20</td>\n",
+ " <td>3</td>\n",
+ " <td>1.0</td>\n",
+ " <td>22500</td>\n",
+ " <td>1060</td>\n",
+ " <td>3500</td>\n",
+ " <td>20187.9052986</td>\n",
+ " <td>2312.09470137</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>870</td>\n",
+ " <td>2</td>\n",
+ " <td>2.0</td>\n",
+ " <td>90000</td>\n",
+ " <td>1300</td>\n",
+ " <td>17500</td>\n",
+ " <td>99354.9203363</td>\n",
+ " <td>-9354.92033626</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>1320</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>133000</td>\n",
+ " <td>1500</td>\n",
+ " <td>30000</td>\n",
+ " <td>124508.080626</td>\n",
+ " <td>8491.91937359</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>1350</td>\n",
+ " <td>2</td>\n",
+ " <td>1.0</td>\n",
+ " <td>90500</td>\n",
+ " <td>820</td>\n",
+ " <td>25700</td>\n",
+ " <td>96640.8258368</td>\n",
+ " <td>-6140.82583676</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>7</td>\n",
+ " <td>2790</td>\n",
+ " <td>3</td>\n",
+ " <td>2.5</td>\n",
+ " <td>260000</td>\n",
+ " <td>2130</td>\n",
+ " <td>25000</td>\n",
+ " <td>224650.799707</td>\n",
+ " <td>35349.2002927</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>8</td>\n",
+ " <td>680</td>\n",
+ " <td>2</td>\n",
+ " <td>1.0</td>\n",
+ " <td>142500</td>\n",
+ " <td>1170</td>\n",
+ " <td>22000</td>\n",
+ " <td>138458.174653</td>\n",
+ " <td>4041.82534729</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>9</td>\n",
+ " <td>1840</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>160000</td>\n",
+ " <td>1500</td>\n",
+ " <td>19000</td>\n",
+ " <td>138650.335314</td>\n",
+ " <td>21349.6646863</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>10</td>\n",
+ " <td>3680</td>\n",
+ " <td>4</td>\n",
+ " <td>2.0</td>\n",
+ " <td>240000</td>\n",
+ " <td>2790</td>\n",
+ " <td>20000</td>\n",
+ " <td>240000.0</td>\n",
+ " <td>0.0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>11</td>\n",
+ " <td>1660</td>\n",
+ " <td>3</td>\n",
+ " <td>1.0</td>\n",
+ " <td>87000</td>\n",
+ " <td>1030</td>\n",
+ " <td>17500</td>\n",
+ " <td>62911.2752187</td>\n",
+ " <td>24088.7247813</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>12</td>\n",
+ " <td>1620</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>118600</td>\n",
+ " <td>1250</td>\n",
+ " <td>20000</td>\n",
+ " <td>117007.693446</td>\n",
+ " <td>1592.30655359</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>13</td>\n",
+ " <td>3100</td>\n",
+ " <td>3</td>\n",
+ " <td>2.0</td>\n",
+ " <td>140000</td>\n",
+ " <td>1760</td>\n",
+ " <td>38000</td>\n",
+ " <td>189203.861766</td>\n",
+ " <td>-49203.8617664</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>14</td>\n",
+ " <td>2070</td>\n",
+ " <td>2</td>\n",
+ " <td>3.0</td>\n",
+ " <td>148000</td>\n",
+ " <td>1550</td>\n",
+ " <td>14000</td>\n",
+ " <td>143322.539832</td>\n",
+ " <td>4677.46016813</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>15</td>\n",
+ " <td>650</td>\n",
+ " <td>3</td>\n",
+ " <td>1.5</td>\n",
+ " <td>65000</td>\n",
+ " <td>1450</td>\n",
+ " <td>12000</td>\n",
+ " <td>82452.4386727</td>\n",
+ " <td>-17452.4386727</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, 590, 2, 1.0, 50000, 770, 22100, 43223.5393423978, 6776.46065760222),\n",
+ " (2, 1050, 3, 2.0, 85000, 1410, 12000, 111527.609949684, -26527.609949684),\n",
+ " (3, 20, 3, 1.0, 22500, 1060, 3500, 20187.9052986341, 2312.09470136587),\n",
+ " (4, 870, 2, 2.0, 90000, 1300, 17500, 99354.9203362612, -9354.92033626116),\n",
+ " (5, 1320, 3, 2.0, 133000, 1500, 30000, 124508.080626412, 8491.91937358756),\n",
+ " (6, 1350, 2, 1.0, 90500, 820, 25700, 96640.8258367579, -6140.8258367579),\n",
+ " (7, 2790, 3, 2.5, 260000, 2130, 25000, 224650.799707327, 35349.2002926733),\n",
+ " (8, 680, 2, 1.0, 142500, 1170, 22000, 138458.174652714, 4041.82534728572),\n",
+ " (9, 1840, 3, 2.0, 160000, 1500, 19000, 138650.335313722, 21349.6646862777),\n",
+ " (10, 3680, 4, 2.0, 240000, 2790, 20000, 240000.0, 0.0),\n",
+ " (11, 1660, 3, 1.0, 87000, 1030, 17500, 62911.2752186594, 24088.7247813406),\n",
+ " (12, 1620, 3, 2.0, 118600, 1250, 20000, 117007.693446414, 1592.30655358579),\n",
+ " (13, 3100, 3, 2.0, 140000, 1760, 38000, 189203.861766403, -49203.8617664034),\n",
+ " (14, 2070, 2, 3.0, 148000, 1550, 14000, 143322.539831869, 4677.46016813093),\n",
+ " (15, 650, 3, 1.5, 65000, 1450, 12000, 82452.4386727394, -17452.4386727394)]"
+ ]
+ },
+ "execution_count": 49,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT houses.*,\n",
+ " madlib.linregr_predict( m.coef,\n",
+ " ARRAY[1,tax,bath,size]\n",
+ " ) as predict,\n",
+ " price -\n",
+ " madlib.linregr_predict( m.coef,\n",
+ " ARRAY[1,tax,bath,size] \n",
+ " ) as residual\n",
+ "FROM houses, houses_linregr_bedroom m\n",
+ "WHERE houses.bedroom = m.bedroom\n",
+ "ORDER BY id;"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 2",
+ "language": "python",
+ "name": "python2"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.12"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
[4/7] madlib-site git commit: add new workbooks for 1dot13
Posted by fm...@apache.org.
http://git-wip-us.apache.org/repos/asf/madlib-site/blob/95826612/community-artifacts/Logistic-regression-v1.ipynb
----------------------------------------------------------------------
diff --git a/community-artifacts/Logistic-regression-v1.ipynb b/community-artifacts/Logistic-regression-v1.ipynb
new file mode 100644
index 0000000..226049d
--- /dev/null
+++ b/community-artifacts/Logistic-regression-v1.ipynb
@@ -0,0 +1,892 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Logistic regression"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/config.py:13: ShimWarning: The `IPython.config` package has been deprecated. You should import from traitlets.config instead.\n",
+ " \"You should import from traitlets.config instead.\", ShimWarning)\n",
+ "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/utils/traitlets.py:5: UserWarning: IPython.utils.traitlets has moved to a top-level traitlets package.\n",
+ " warn(\"IPython.utils.traitlets has moved to a top-level traitlets package.\")\n"
+ ]
+ }
+ ],
+ "source": [
+ "%load_ext sql"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "u'Connected: fmcquillan@madlib'"
+ ]
+ },
+ "execution_count": 2,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Greenplum 4.3.10.0\n",
+ "# %sql postgresql://gpdbchina@10.194.10.68:61000/madlib\n",
+ " \n",
+ "# PostgreSQL local\n",
+ "%sql postgresql://fmcquillan@localhost:5432/madlib"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>version</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>MADlib version: 1.12, git revision: unknown, cmake configuration time: Wed Aug 23 23:07:18 UTC 2017, build type: Release, build system: Darwin-16.7.0, C compiler: Clang, C++ compiler: Clang</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'MADlib version: 1.12, git revision: unknown, cmake configuration time: Wed Aug 23 23:07:18 UTC 2017, build type: Release, build system: Darwin-16.7.0, C compiler: Clang, C++ compiler: Clang',)]"
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%sql select madlib.version();\n",
+ "#%sql select version();"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 1. Load data\n",
+ "This data set is related to predicting a second heart attack given treatment and health factors."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 30,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "Done.\n",
+ "20 rows affected.\n",
+ "20 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>second_attack</th>\n",
+ " <th>treatment</th>\n",
+ " <th>trait_anxiety</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>1</td>\n",
+ " <td>1</td>\n",
+ " <td>70</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>1</td>\n",
+ " <td>1</td>\n",
+ " <td>80</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>1</td>\n",
+ " <td>1</td>\n",
+ " <td>50</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>1</td>\n",
+ " <td>0</td>\n",
+ " <td>60</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>1</td>\n",
+ " <td>0</td>\n",
+ " <td>40</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>1</td>\n",
+ " <td>0</td>\n",
+ " <td>65</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>7</td>\n",
+ " <td>1</td>\n",
+ " <td>0</td>\n",
+ " <td>75</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>8</td>\n",
+ " <td>1</td>\n",
+ " <td>0</td>\n",
+ " <td>80</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>9</td>\n",
+ " <td>1</td>\n",
+ " <td>0</td>\n",
+ " <td>70</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>10</td>\n",
+ " <td>1</td>\n",
+ " <td>0</td>\n",
+ " <td>60</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>11</td>\n",
+ " <td>0</td>\n",
+ " <td>1</td>\n",
+ " <td>65</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>12</td>\n",
+ " <td>0</td>\n",
+ " <td>1</td>\n",
+ " <td>50</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>13</td>\n",
+ " <td>0</td>\n",
+ " <td>1</td>\n",
+ " <td>45</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>14</td>\n",
+ " <td>0</td>\n",
+ " <td>1</td>\n",
+ " <td>35</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>15</td>\n",
+ " <td>0</td>\n",
+ " <td>1</td>\n",
+ " <td>40</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>16</td>\n",
+ " <td>0</td>\n",
+ " <td>1</td>\n",
+ " <td>50</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>17</td>\n",
+ " <td>0</td>\n",
+ " <td>0</td>\n",
+ " <td>55</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>18</td>\n",
+ " <td>0</td>\n",
+ " <td>0</td>\n",
+ " <td>45</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>19</td>\n",
+ " <td>0</td>\n",
+ " <td>0</td>\n",
+ " <td>50</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>20</td>\n",
+ " <td>0</td>\n",
+ " <td>0</td>\n",
+ " <td>60</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, 1, 1, 70),\n",
+ " (2, 1, 1, 80),\n",
+ " (3, 1, 1, 50),\n",
+ " (4, 1, 0, 60),\n",
+ " (5, 1, 0, 40),\n",
+ " (6, 1, 0, 65),\n",
+ " (7, 1, 0, 75),\n",
+ " (8, 1, 0, 80),\n",
+ " (9, 1, 0, 70),\n",
+ " (10, 1, 0, 60),\n",
+ " (11, 0, 1, 65),\n",
+ " (12, 0, 1, 50),\n",
+ " (13, 0, 1, 45),\n",
+ " (14, 0, 1, 35),\n",
+ " (15, 0, 1, 40),\n",
+ " (16, 0, 1, 50),\n",
+ " (17, 0, 0, 55),\n",
+ " (18, 0, 0, 45),\n",
+ " (19, 0, 0, 50),\n",
+ " (20, 0, 0, 60)]"
+ ]
+ },
+ "execution_count": 30,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS patients;\n",
+ "\n",
+ "CREATE TABLE patients( id INTEGER NOT NULL,\n",
+ " second_attack INTEGER,\n",
+ " treatment INTEGER,\n",
+ " trait_anxiety INTEGER);\n",
+ " \n",
+ "INSERT INTO patients VALUES \n",
+ "(1, 1, 1, 70),\n",
+ "(2, 1, 1, 80),\n",
+ "(3, 1, 1, 50),\n",
+ "(4, 1, 0, 60),\n",
+ "(5, 1, 0, 40),\n",
+ "(6, 1, 0, 65),\n",
+ "(7, 1, 0, 75),\n",
+ "(8, 1, 0, 80),\n",
+ "(9, 1, 0, 70),\n",
+ "(10, 1, 0, 60),\n",
+ "(11, 0, 1, 65),\n",
+ "(12, 0, 1, 50),\n",
+ "(13, 0, 1, 45),\n",
+ "(14, 0, 1, 35),\n",
+ "(15, 0, 1, 40),\n",
+ "(16, 0, 1, 50),\n",
+ "(17, 0, 0, 55),\n",
+ "(18, 0, 0, 45),\n",
+ "(19, 0, 0, 50),\n",
+ "(20, 0, 0, 60);\n",
+ "\n",
+ "SELECT * FROM patients ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 2. Train regression model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 31,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>method</th>\n",
+ " <th>source_table</th>\n",
+ " <th>out_table</th>\n",
+ " <th>dependent_varname</th>\n",
+ " <th>independent_varname</th>\n",
+ " <th>optimizer_params</th>\n",
+ " <th>num_all_groups</th>\n",
+ " <th>num_failed_groups</th>\n",
+ " <th>num_rows_processed</th>\n",
+ " <th>num_missing_rows_skipped</th>\n",
+ " <th>grouping_col</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>logregr</td>\n",
+ " <td>patients</td>\n",
+ " <td>patients_logregr</td>\n",
+ " <td>second_attack</td>\n",
+ " <td>ARRAY[1, treatment, trait_anxiety]</td>\n",
+ " <td>optimizer=irls, max_iter=20, tolerance=0.0001</td>\n",
+ " <td>1</td>\n",
+ " <td>0</td>\n",
+ " <td>20</td>\n",
+ " <td>0</td>\n",
+ " <td>None</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'logregr', u'patients', u'patients_logregr', u'second_attack', u'ARRAY[1, treatment, trait_anxiety]', u'optimizer=irls, max_iter=20, tolerance=0.0001', 1, 0, 20, 0, None)]"
+ ]
+ },
+ "execution_count": 31,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS patients_logregr, patients_logregr_summary;\n",
+ "\n",
+ "SELECT madlib.logregr_train( 'patients', -- Source table\n",
+ " 'patients_logregr', -- Output table\n",
+ " 'second_attack', -- Dependent variable\n",
+ " 'ARRAY[1, treatment, trait_anxiety]', -- Feature vector\n",
+ " NULL, -- Grouping\n",
+ " 20, -- Max iterations\n",
+ " 'irls' -- Optimizer to use\n",
+ " );\n",
+ "\n",
+ "SELECT * FROM patients_logregr_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 3. View regression results"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>coef</th>\n",
+ " <th>log_likelihood</th>\n",
+ " <th>std_err</th>\n",
+ " <th>z_stats</th>\n",
+ " <th>p_values</th>\n",
+ " <th>odds_ratios</th>\n",
+ " <th>condition_no</th>\n",
+ " <th>num_rows_processed</th>\n",
+ " <th>num_missing_rows_skipped</th>\n",
+ " <th>num_iterations</th>\n",
+ " <th>variance_covariance</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>[-6.36346994178192, -1.02410605239327, 0.119044916668607]</td>\n",
+ " <td>-9.41018298389</td>\n",
+ " <td>[3.21389766375099, 1.17107844860319, 0.0549790458269317]</td>\n",
+ " <td>[-1.97998524145757, -0.874498248699539, 2.16527796868916]</td>\n",
+ " <td>[0.0477051870698145, 0.381846973530455, 0.0303664045046183]</td>\n",
+ " <td>[0.00172337630923221, 0.359117354054956, 1.12642051220895]</td>\n",
+ " <td>326.081922792</td>\n",
+ " <td>20</td>\n",
+ " <td>0</td>\n",
+ " <td>5</td>\n",
+ " <td>[[10.329138193064, -0.474304665195738, -0.171995901260057], [-0.474304665195738, 1.37142473278286, -0.00119520703381591], [-0.171995901260057, -0.00119520703381591, 0.00302269548003986]]</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[([-6.36346994178192, -1.02410605239327, 0.119044916668607], -9.41018298388876, [3.21389766375099, 1.17107844860319, 0.0549790458269317], [-1.97998524145757, -0.874498248699539, 2.16527796868916], [0.0477051870698145, 0.381846973530455, 0.0303664045046183], [0.00172337630923221, 0.359117354054956, 1.12642051220895], 326.081922791575, 20L, 0L, 5, [[10.329138193064, -0.474304665195738, -0.171995901260057], [-0.474304665195738, 1.37142473278286, -0.00119520703381591], [-0.171995901260057, -0.00119520703381591, 0.00302269548003986]])]"
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * from patients_logregr;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 4. Output formatting\n",
+ "Alternatively, unnest the arrays in the results for easier reading of output:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "3 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>attribute</th>\n",
+ " <th>coefficient</th>\n",
+ " <th>standard_error</th>\n",
+ " <th>z_stat</th>\n",
+ " <th>pvalue</th>\n",
+ " <th>odds_ratio</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>intercept</td>\n",
+ " <td>-6.36346994178</td>\n",
+ " <td>3.21389766375</td>\n",
+ " <td>-1.97998524146</td>\n",
+ " <td>0.0477051870698</td>\n",
+ " <td>0.00172337630923</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>treatment</td>\n",
+ " <td>-1.02410605239</td>\n",
+ " <td>1.1710784486</td>\n",
+ " <td>-0.8744982487</td>\n",
+ " <td>0.38184697353</td>\n",
+ " <td>0.359117354055</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>trait_anxiety</td>\n",
+ " <td>0.119044916669</td>\n",
+ " <td>0.0549790458269</td>\n",
+ " <td>2.16527796869</td>\n",
+ " <td>0.0303664045046</td>\n",
+ " <td>1.12642051221</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'intercept', -6.36346994178192, 3.21389766375099, -1.97998524145757, 0.0477051870698145, 0.00172337630923221),\n",
+ " (u'treatment', -1.02410605239327, 1.17107844860319, -0.874498248699539, 0.381846973530455, 0.359117354054956),\n",
+ " (u'trait_anxiety', 0.119044916668607, 0.0549790458269317, 2.16527796868916, 0.0303664045046183, 1.12642051220895)]"
+ ]
+ },
+ "execution_count": 13,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT unnest(array['intercept', 'treatment', 'trait_anxiety']) as attribute,\n",
+ " unnest(coef) as coefficient,\n",
+ " unnest(std_err) as standard_error,\n",
+ " unnest(z_stats) as z_stat,\n",
+ " unnest(p_values) as pvalue,\n",
+ " unnest(odds_ratios) as odds_ratio\n",
+ " FROM patients_logregr;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "scrolled": true
+ },
+ "source": [
+ "# 5. Predict outcome\n",
+ "This example uses the original data table to perform the prediction. Typically a different test dataset with the same features as the original training dataset would be used for prediction."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 32,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "20 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>logregr_predict</th>\n",
+ " <th>second_attack</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>False</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>False</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>7</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>8</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>9</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>10</td>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>11</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>12</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>13</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>14</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>15</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>16</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>17</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>18</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>19</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>20</td>\n",
+ " <td>True</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, True, True),\n",
+ " (2, True, True),\n",
+ " (3, False, True),\n",
+ " (4, True, True),\n",
+ " (5, False, True),\n",
+ " (6, True, True),\n",
+ " (7, True, True),\n",
+ " (8, True, True),\n",
+ " (9, True, True),\n",
+ " (10, True, True),\n",
+ " (11, True, False),\n",
+ " (12, False, False),\n",
+ " (13, False, False),\n",
+ " (14, False, False),\n",
+ " (15, False, False),\n",
+ " (16, False, False),\n",
+ " (17, True, False),\n",
+ " (18, False, False),\n",
+ " (19, False, False),\n",
+ " (20, True, False)]"
+ ]
+ },
+ "execution_count": 32,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "-- Display prediction value along with the original value\n",
+ "SELECT p.id, madlib.logregr_predict(coef, ARRAY[1, treatment, trait_anxiety]),\n",
+ " p.second_attack::BOOLEAN\n",
+ "FROM patients p, patients_logregr m\n",
+ "ORDER BY p.id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 6. Predict probability\n",
+ "Predicting the probability of the dependent variable being TRUE."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 33,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "20 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>logregr_predict_prob</th>\n",
+ " <th>second_attack</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>0.720223028942</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>0.894354902502</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>0.192269541755</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>0.685513072239</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>0.167747881509</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>0.798098108915</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>7</td>\n",
+ " <td>0.928568075753</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>8</td>\n",
+ " <td>0.959305763694</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>9</td>\n",
+ " <td>0.877576117431</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>10</td>\n",
+ " <td>0.685513072239</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>11</td>\n",
+ " <td>0.586700895943</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>12</td>\n",
+ " <td>0.192269541755</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>13</td>\n",
+ " <td>0.116032010633</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>14</td>\n",
+ " <td>0.0383829143135</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>15</td>\n",
+ " <td>0.0674976224148</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>16</td>\n",
+ " <td>0.192269541755</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>17</td>\n",
+ " <td>0.545870774303</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>18</td>\n",
+ " <td>0.267675422387</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>19</td>\n",
+ " <td>0.398618639285</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>20</td>\n",
+ " <td>0.685513072239</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, 0.720223028941527, True),\n",
+ " (2, 0.894354902502048, True),\n",
+ " (3, 0.192269541755171, True),\n",
+ " (4, 0.685513072239347, True),\n",
+ " (5, 0.167747881508857, True),\n",
+ " (6, 0.79809810891514, True),\n",
+ " (7, 0.928568075752503, True),\n",
+ " (8, 0.959305763693571, True),\n",
+ " (9, 0.877576117431452, True),\n",
+ " (10, 0.685513072239347, True),\n",
+ " (11, 0.586700895943317, False),\n",
+ " (12, 0.192269541755171, False),\n",
+ " (13, 0.116032010632994, False),\n",
+ " (14, 0.0383829143134982, False),\n",
+ " (15, 0.0674976224147597, False),\n",
+ " (16, 0.192269541755171, False),\n",
+ " (17, 0.545870774302621, False),\n",
+ " (18, 0.267675422387132, False),\n",
+ " (19, 0.398618639285111, False),\n",
+ " (20, 0.685513072239347, False)]"
+ ]
+ },
+ "execution_count": 33,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "-- Display prediction value along with the original value\n",
+ "SELECT p.id, madlib.logregr_predict_prob(coef, ARRAY[1, treatment, trait_anxiety]),\n",
+ " p.second_attack::BOOLEAN\n",
+ "FROM patients p, patients_logregr m\n",
+ "ORDER BY p.id;"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 2",
+ "language": "python",
+ "name": "python2"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.12"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
http://git-wip-us.apache.org/repos/asf/madlib-site/blob/95826612/community-artifacts/Summary-v1.ipynb
----------------------------------------------------------------------
diff --git a/community-artifacts/Summary-v1.ipynb b/community-artifacts/Summary-v1.ipynb
new file mode 100644
index 0000000..57c3611
--- /dev/null
+++ b/community-artifacts/Summary-v1.ipynb
@@ -0,0 +1,1026 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "The sql extension is already loaded. To reload it, use:\n",
+ " %reload_ext sql\n"
+ ]
+ }
+ ],
+ "source": [
+ "%load_ext sql"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "u'Connected: fmcquillan@madlib'"
+ ]
+ },
+ "execution_count": 14,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Greenplum 4.3.10.0\n",
+ "# %sql postgresql://gpdbchina@10.194.10.68:61000/madlib\n",
+ " \n",
+ "# PostgreSQL local\n",
+ "%sql postgresql://fmcquillan@localhost:5432/madlib"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>version</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>MADlib version: 1.12, git revision: unknown, cmake configuration time: Wed Aug 23 23:07:18 UTC 2017, build type: Release, build system: Darwin-16.7.0, C compiler: Clang, C++ compiler: Clang</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'MADlib version: 1.12, git revision: unknown, cmake configuration time: Wed Aug 23 23:07:18 UTC 2017, build type: Release, build system: Darwin-16.7.0, C compiler: Clang, C++ compiler: Clang',)]"
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%sql select madlib.version();\n",
+ "#%sql select version();"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 1. On-line help"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>summary</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td><br> 'summary' is a generic function used to produce summary statistics<br> of any data table. The function invokes particular 'methods' from<br> the MADlib library to provide an overview of the data.<br> -------<br> For an overview on usage, run:<br> SELECT madlib.summary('usage');<br> -------<br> For an example, run:<br> SELECT madlib.summary('example')<br> </td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u\"\\n 'summary' is a generic function used to produce summary statistics\\n of any data table. The function invokes particular 'methods' from\\n the MADlib library to provide an overview of the data.\\n -------\\n For an overview on usage, run:\\n SELECT madlib.summary('usage');\\n -------\\n For an example, run:\\n SELECT madlib.summary('example')\\n \",)]"
+ ]
+ },
+ "execution_count": 16,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%sql SELECT * FROM madlib.summary();"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 2. Load data\n",
+ "Using part of the popular iris data set from https://archive.ics.uci.edu/ml/datasets/iris"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 51,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "Done.\n",
+ "30 rows affected.\n",
+ "30 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>id</th>\n",
+ " <th>sepal_length</th>\n",
+ " <th>sepal_width</th>\n",
+ " <th>petal_length</th>\n",
+ " <th>petal_width</th>\n",
+ " <th>class_name</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>5.1</td>\n",
+ " <td>3.5</td>\n",
+ " <td>1.4</td>\n",
+ " <td>0.2</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>4.9</td>\n",
+ " <td>3.0</td>\n",
+ " <td>1.4</td>\n",
+ " <td>0.2</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>4.7</td>\n",
+ " <td>3.2</td>\n",
+ " <td>1.3</td>\n",
+ " <td>0.2</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>4.6</td>\n",
+ " <td>3.1</td>\n",
+ " <td>1.5</td>\n",
+ " <td>0.2</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>5</td>\n",
+ " <td>5.0</td>\n",
+ " <td>3.6</td>\n",
+ " <td>1.4</td>\n",
+ " <td>0.2</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>6</td>\n",
+ " <td>5.4</td>\n",
+ " <td>3.9</td>\n",
+ " <td>1.7</td>\n",
+ " <td>0.4</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>7</td>\n",
+ " <td>4.6</td>\n",
+ " <td>3.4</td>\n",
+ " <td>1.4</td>\n",
+ " <td>0.3</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>8</td>\n",
+ " <td>5.0</td>\n",
+ " <td>3.4</td>\n",
+ " <td>1.5</td>\n",
+ " <td>0.2</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>9</td>\n",
+ " <td>4.4</td>\n",
+ " <td>2.9</td>\n",
+ " <td>1.4</td>\n",
+ " <td>0.2</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>10</td>\n",
+ " <td>4.9</td>\n",
+ " <td>3.1</td>\n",
+ " <td>1.5</td>\n",
+ " <td>0.1</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>11</td>\n",
+ " <td>7.0</td>\n",
+ " <td>3.2</td>\n",
+ " <td>4.7</td>\n",
+ " <td>1.4</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>12</td>\n",
+ " <td>6.4</td>\n",
+ " <td>3.2</td>\n",
+ " <td>4.5</td>\n",
+ " <td>1.5</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>13</td>\n",
+ " <td>6.9</td>\n",
+ " <td>3.1</td>\n",
+ " <td>4.9</td>\n",
+ " <td>1.5</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>14</td>\n",
+ " <td>5.5</td>\n",
+ " <td>2.3</td>\n",
+ " <td>4.0</td>\n",
+ " <td>1.3</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>15</td>\n",
+ " <td>6.5</td>\n",
+ " <td>2.8</td>\n",
+ " <td>4.6</td>\n",
+ " <td>1.5</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>16</td>\n",
+ " <td>5.7</td>\n",
+ " <td>2.8</td>\n",
+ " <td>4.5</td>\n",
+ " <td>1.3</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>17</td>\n",
+ " <td>6.3</td>\n",
+ " <td>3.3</td>\n",
+ " <td>4.7</td>\n",
+ " <td>1.6</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>18</td>\n",
+ " <td>4.9</td>\n",
+ " <td>2.4</td>\n",
+ " <td>3.3</td>\n",
+ " <td>1.0</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>19</td>\n",
+ " <td>6.6</td>\n",
+ " <td>2.9</td>\n",
+ " <td>4.6</td>\n",
+ " <td>1.3</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>20</td>\n",
+ " <td>5.2</td>\n",
+ " <td>2.7</td>\n",
+ " <td>3.9</td>\n",
+ " <td>1.4</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>21</td>\n",
+ " <td>6.3</td>\n",
+ " <td>3.3</td>\n",
+ " <td>6.0</td>\n",
+ " <td>2.5</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>22</td>\n",
+ " <td>5.8</td>\n",
+ " <td>2.7</td>\n",
+ " <td>5.1</td>\n",
+ " <td>1.9</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>23</td>\n",
+ " <td>7.1</td>\n",
+ " <td>3.0</td>\n",
+ " <td>5.9</td>\n",
+ " <td>2.1</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>24</td>\n",
+ " <td>6.3</td>\n",
+ " <td>2.9</td>\n",
+ " <td>5.6</td>\n",
+ " <td>1.8</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>25</td>\n",
+ " <td>6.5</td>\n",
+ " <td>3.0</td>\n",
+ " <td>5.8</td>\n",
+ " <td>2.2</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>26</td>\n",
+ " <td>7.6</td>\n",
+ " <td>3.0</td>\n",
+ " <td>6.6</td>\n",
+ " <td>2.1</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>27</td>\n",
+ " <td>4.9</td>\n",
+ " <td>2.5</td>\n",
+ " <td>4.5</td>\n",
+ " <td>1.7</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>28</td>\n",
+ " <td>7.3</td>\n",
+ " <td>2.9</td>\n",
+ " <td>6.3</td>\n",
+ " <td>1.8</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>29</td>\n",
+ " <td>6.7</td>\n",
+ " <td>2.5</td>\n",
+ " <td>5.8</td>\n",
+ " <td>1.8</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>30</td>\n",
+ " <td>7.2</td>\n",
+ " <td>3.6</td>\n",
+ " <td>6.1</td>\n",
+ " <td>2.5</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, 5.1, 3.5, 1.4, 0.2, u'Iris-setosa'),\n",
+ " (2, 4.9, 3.0, 1.4, 0.2, u'Iris-setosa'),\n",
+ " (3, 4.7, 3.2, 1.3, 0.2, u'Iris-setosa'),\n",
+ " (4, 4.6, 3.1, 1.5, 0.2, u'Iris-setosa'),\n",
+ " (5, 5.0, 3.6, 1.4, 0.2, u'Iris-setosa'),\n",
+ " (6, 5.4, 3.9, 1.7, 0.4, u'Iris-setosa'),\n",
+ " (7, 4.6, 3.4, 1.4, 0.3, u'Iris-setosa'),\n",
+ " (8, 5.0, 3.4, 1.5, 0.2, u'Iris-setosa'),\n",
+ " (9, 4.4, 2.9, 1.4, 0.2, u'Iris-setosa'),\n",
+ " (10, 4.9, 3.1, 1.5, 0.1, u'Iris-setosa'),\n",
+ " (11, 7.0, 3.2, 4.7, 1.4, u'Iris-versicolor'),\n",
+ " (12, 6.4, 3.2, 4.5, 1.5, u'Iris-versicolor'),\n",
+ " (13, 6.9, 3.1, 4.9, 1.5, u'Iris-versicolor'),\n",
+ " (14, 5.5, 2.3, 4.0, 1.3, u'Iris-versicolor'),\n",
+ " (15, 6.5, 2.8, 4.6, 1.5, u'Iris-versicolor'),\n",
+ " (16, 5.7, 2.8, 4.5, 1.3, u'Iris-versicolor'),\n",
+ " (17, 6.3, 3.3, 4.7, 1.6, u'Iris-versicolor'),\n",
+ " (18, 4.9, 2.4, 3.3, 1.0, u'Iris-versicolor'),\n",
+ " (19, 6.6, 2.9, 4.6, 1.3, u'Iris-versicolor'),\n",
+ " (20, 5.2, 2.7, 3.9, 1.4, u'Iris-versicolor'),\n",
+ " (21, 6.3, 3.3, 6.0, 2.5, u'Iris-virginica'),\n",
+ " (22, 5.8, 2.7, 5.1, 1.9, u'Iris-virginica'),\n",
+ " (23, 7.1, 3.0, 5.9, 2.1, u'Iris-virginica'),\n",
+ " (24, 6.3, 2.9, 5.6, 1.8, u'Iris-virginica'),\n",
+ " (25, 6.5, 3.0, 5.8, 2.2, u'Iris-virginica'),\n",
+ " (26, 7.6, 3.0, 6.6, 2.1, u'Iris-virginica'),\n",
+ " (27, 4.9, 2.5, 4.5, 1.7, u'Iris-virginica'),\n",
+ " (28, 7.3, 2.9, 6.3, 1.8, u'Iris-virginica'),\n",
+ " (29, 6.7, 2.5, 5.8, 1.8, u'Iris-virginica'),\n",
+ " (30, 7.2, 3.6, 6.1, 2.5, u'Iris-virginica')]"
+ ]
+ },
+ "execution_count": 51,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS iris;\n",
+ "\n",
+ "CREATE TABLE iris (id INT, sepal_length FLOAT, sepal_width FLOAT,\n",
+ " petal_length FLOAT, petal_width FLOAT, \n",
+ " class_name text);\n",
+ " \n",
+ "INSERT INTO iris VALUES \n",
+ "(1,5.1,3.5,1.4,0.2,'Iris-setosa'),\n",
+ "(2,4.9,3.0,1.4,0.2,'Iris-setosa'),\n",
+ "(3,4.7,3.2,1.3,0.2,'Iris-setosa'),\n",
+ "(4,4.6,3.1,1.5,0.2,'Iris-setosa'),\n",
+ "(5,5.0,3.6,1.4,0.2,'Iris-setosa'),\n",
+ "(6,5.4,3.9,1.7,0.4,'Iris-setosa'),\n",
+ "(7,4.6,3.4,1.4,0.3,'Iris-setosa'),\n",
+ "(8,5.0,3.4,1.5,0.2,'Iris-setosa'),\n",
+ "(9,4.4,2.9,1.4,0.2,'Iris-setosa'),\n",
+ "(10,4.9,3.1,1.5,0.1,'Iris-setosa'),\n",
+ "(11,7.0,3.2,4.7,1.4,'Iris-versicolor'),\n",
+ "(12,6.4,3.2,4.5,1.5,'Iris-versicolor'),\n",
+ "(13,6.9,3.1,4.9,1.5,'Iris-versicolor'),\n",
+ "(14,5.5,2.3,4.0,1.3,'Iris-versicolor'),\n",
+ "(15,6.5,2.8,4.6,1.5,'Iris-versicolor'),\n",
+ "(16,5.7,2.8,4.5,1.3,'Iris-versicolor'),\n",
+ "(17,6.3,3.3,4.7,1.6,'Iris-versicolor'),\n",
+ "(18,4.9,2.4,3.3,1.0,'Iris-versicolor'),\n",
+ "(19,6.6,2.9,4.6,1.3,'Iris-versicolor'),\n",
+ "(20,5.2,2.7,3.9,1.4,'Iris-versicolor'),\n",
+ "(21,6.3,3.3,6.0,2.5,'Iris-virginica'),\n",
+ "(22,5.8,2.7,5.1,1.9,'Iris-virginica'),\n",
+ "(23,7.1,3.0,5.9,2.1,'Iris-virginica'),\n",
+ "(24,6.3,2.9,5.6,1.8,'Iris-virginica'),\n",
+ "(25,6.5,3.0,5.8,2.2,'Iris-virginica'),\n",
+ "(26,7.6,3.0,6.6,2.1,'Iris-virginica'),\n",
+ "(27,4.9,2.5,4.5,1.7,'Iris-virginica'),\n",
+ "(28,7.3,2.9,6.3,1.8,'Iris-virginica'),\n",
+ "(29,6.7,2.5,5.8,1.8,'Iris-virginica'),\n",
+ "(30,7.2,3.6,6.1,2.5,'Iris-virginica');\n",
+ "\n",
+ "SELECT * FROM iris ORDER BY id;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 3. Get summary data \n",
+ "Basic summary data using all defaults."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 52,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>output_table</th>\n",
+ " <th>row_count</th>\n",
+ " <th>duration</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>iris_summary</td>\n",
+ " <td>6</td>\n",
+ " <td>0.0161118507385</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'iris_summary', 6, 0.0161118507385254)]"
+ ]
+ },
+ "execution_count": 52,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS iris_summary;\n",
+ "\n",
+ "SELECT * FROM madlib.summary( 'iris', -- Source table\n",
+ " 'iris_summary' -- Output table\n",
+ " );"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 53,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "2 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>group_by</th>\n",
+ " <th>group_by_value</th>\n",
+ " <th>target_column</th>\n",
+ " <th>column_number</th>\n",
+ " <th>data_type</th>\n",
+ " <th>row_count</th>\n",
+ " <th>distinct_values</th>\n",
+ " <th>missing_values</th>\n",
+ " <th>blank_values</th>\n",
+ " <th>fraction_missing</th>\n",
+ " <th>fraction_blank</th>\n",
+ " <th>mean</th>\n",
+ " <th>variance</th>\n",
+ " <th>min</th>\n",
+ " <th>max</th>\n",
+ " <th>quantile_array</th>\n",
+ " <th>most_frequent_values</th>\n",
+ " <th>mfv_frequencies</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " <td>sepal_length</td>\n",
+ " <td>2</td>\n",
+ " <td>float8</td>\n",
+ " <td>30</td>\n",
+ " <td>22</td>\n",
+ " <td>0</td>\n",
+ " <td>None</td>\n",
+ " <td>0.0</td>\n",
+ " <td>None</td>\n",
+ " <td>5.84333333333</td>\n",
+ " <td>0.929436781609</td>\n",
+ " <td>4.4</td>\n",
+ " <td>7.6</td>\n",
+ " <td>[5.057, 6.414]</td>\n",
+ " <td>[u'4.9', u'6.3', u'5']</td>\n",
+ " <td>[4L, 3L, 2L]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " <td>sepal_width</td>\n",
+ " <td>3</td>\n",
+ " <td>float8</td>\n",
+ " <td>30</td>\n",
+ " <td>14</td>\n",
+ " <td>0</td>\n",
+ " <td>None</td>\n",
+ " <td>0.0</td>\n",
+ " <td>None</td>\n",
+ " <td>3.04</td>\n",
+ " <td>0.139034482759</td>\n",
+ " <td>2.3</td>\n",
+ " <td>3.9</td>\n",
+ " <td>[2.9, 3.2]</td>\n",
+ " <td>[u'3', u'2.9', u'3.2']</td>\n",
+ " <td>[4L, 4L, 3L]</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(None, None, u'sepal_length', 2, u'float8', 30L, 22L, 0L, None, 0.0, None, 5.84333333333333, 0.9294367816092, 4.4, 7.6, [5.057, 6.414], [u'4.9', u'6.3', u'5'], [4L, 3L, 2L]),\n",
+ " (None, None, u'sepal_width', 3, u'float8', 30L, 14L, 0L, None, 0.0, None, 3.04, 0.13903448275862, 2.3, 3.9, [2.9, 3.2], [u'3', u'2.9', u'3.2'], [4L, 4L, 3L])]"
+ ]
+ },
+ "execution_count": 53,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT * FROM iris_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 4. Grouping"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 45,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "8 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>group_by</th>\n",
+ " <th>group_by_value</th>\n",
+ " <th>target_column</th>\n",
+ " <th>column_number</th>\n",
+ " <th>data_type</th>\n",
+ " <th>row_count</th>\n",
+ " <th>distinct_values</th>\n",
+ " <th>missing_values</th>\n",
+ " <th>blank_values</th>\n",
+ " <th>fraction_missing</th>\n",
+ " <th>fraction_blank</th>\n",
+ " <th>mean</th>\n",
+ " <th>variance</th>\n",
+ " <th>min</th>\n",
+ " <th>max</th>\n",
+ " <th>first_quartile</th>\n",
+ " <th>median</th>\n",
+ " <th>third_quartile</th>\n",
+ " <th>most_frequent_values</th>\n",
+ " <th>mfv_frequencies</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>class_name</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " <td>sepal_length</td>\n",
+ " <td>2</td>\n",
+ " <td>float8</td>\n",
+ " <td>10</td>\n",
+ " <td>7</td>\n",
+ " <td>0</td>\n",
+ " <td>None</td>\n",
+ " <td>0.0</td>\n",
+ " <td>None</td>\n",
+ " <td>4.86</td>\n",
+ " <td>0.0848888888889</td>\n",
+ " <td>4.4</td>\n",
+ " <td>5.4</td>\n",
+ " <td>4.625</td>\n",
+ " <td>4.9</td>\n",
+ " <td>5.0</td>\n",
+ " <td>[u'4.6', u'4.9', u'5', u'5.1', u'4.4', u'5.4', u'4.7']</td>\n",
+ " <td>[2L, 2L, 2L, 1L, 1L, 1L, 1L]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>class_name</td>\n",
+ " <td>Iris-setosa</td>\n",
+ " <td>sepal_width</td>\n",
+ " <td>3</td>\n",
+ " <td>float8</td>\n",
+ " <td>10</td>\n",
+ " <td>8</td>\n",
+ " <td>0</td>\n",
+ " <td>None</td>\n",
+ " <td>0.0</td>\n",
+ " <td>None</td>\n",
+ " <td>3.31</td>\n",
+ " <td>0.0943333333333</td>\n",
+ " <td>2.9</td>\n",
+ " <td>3.9</td>\n",
+ " <td>3.1</td>\n",
+ " <td>3.3</td>\n",
+ " <td>3.475</td>\n",
+ " <td>[u'3.4', u'3.1', u'3.2', u'3.6', u'3.9', u'3.5', u'2.9', u'3']</td>\n",
+ " <td>[2L, 2L, 1L, 1L, 1L, 1L, 1L, 1L]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>class_name</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " <td>sepal_length</td>\n",
+ " <td>2</td>\n",
+ " <td>float8</td>\n",
+ " <td>10</td>\n",
+ " <td>10</td>\n",
+ " <td>0</td>\n",
+ " <td>None</td>\n",
+ " <td>0.0</td>\n",
+ " <td>None</td>\n",
+ " <td>6.1</td>\n",
+ " <td>0.528888888889</td>\n",
+ " <td>4.9</td>\n",
+ " <td>7.0</td>\n",
+ " <td>5.55</td>\n",
+ " <td>6.35</td>\n",
+ " <td>6.575</td>\n",
+ " <td>[u'7', u'6.4', u'6.9', u'5.5', u'6.5', u'5.7', u'6.3', u'4.9', u'6.6', u'5.2']</td>\n",
+ " <td>[1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>class_name</td>\n",
+ " <td>Iris-versicolor</td>\n",
+ " <td>sepal_width</td>\n",
+ " <td>3</td>\n",
+ " <td>float8</td>\n",
+ " <td>10</td>\n",
+ " <td>8</td>\n",
+ " <td>0</td>\n",
+ " <td>None</td>\n",
+ " <td>0.0</td>\n",
+ " <td>None</td>\n",
+ " <td>2.87</td>\n",
+ " <td>0.115666666667</td>\n",
+ " <td>2.3</td>\n",
+ " <td>3.3</td>\n",
+ " <td>2.725</td>\n",
+ " <td>2.85</td>\n",
+ " <td>3.175</td>\n",
+ " <td>[u'2.8', u'3.2', u'2.3', u'3.3', u'2.4', u'2.9', u'2.7', u'3.1']</td>\n",
+ " <td>[2L, 2L, 1L, 1L, 1L, 1L, 1L, 1L]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>class_name</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " <td>sepal_length</td>\n",
+ " <td>2</td>\n",
+ " <td>float8</td>\n",
+ " <td>10</td>\n",
+ " <td>9</td>\n",
+ " <td>0</td>\n",
+ " <td>None</td>\n",
+ " <td>0.0</td>\n",
+ " <td>None</td>\n",
+ " <td>6.57</td>\n",
+ " <td>0.646777777778</td>\n",
+ " <td>4.9</td>\n",
+ " <td>7.6</td>\n",
+ " <td>6.3</td>\n",
+ " <td>6.6</td>\n",
+ " <td>7.175</td>\n",
+ " <td>[u'6.3', u'5.8', u'7.1', u'6.5', u'7.6', u'4.9', u'7.3', u'6.7', u'7.2']</td>\n",
+ " <td>[2L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>class_name</td>\n",
+ " <td>Iris-virginica</td>\n",
+ " <td>sepal_width</td>\n",
+ " <td>3</td>\n",
+ " <td>float8</td>\n",
+ " <td>10</td>\n",
+ " <td>6</td>\n",
+ " <td>0</td>\n",
+ " <td>None</td>\n",
+ " <td>0.0</td>\n",
+ " <td>None</td>\n",
+ " <td>2.94</td>\n",
+ " <td>0.113777777778</td>\n",
+ " <td>2.5</td>\n",
+ " <td>3.6</td>\n",
+ " <td>2.75</td>\n",
+ " <td>2.95</td>\n",
+ " <td>3.0</td>\n",
+ " <td>[u'3', u'2.9', u'2.5', u'3.3', u'2.7', u'3.6']</td>\n",
+ " <td>[3L, 2L, 2L, 1L, 1L, 1L]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " <td>sepal_length</td>\n",
+ " <td>2</td>\n",
+ " <td>float8</td>\n",
+ " <td>30</td>\n",
+ " <td>22</td>\n",
+ " <td>0</td>\n",
+ " <td>None</td>\n",
+ " <td>0.0</td>\n",
+ " <td>None</td>\n",
+ " <td>5.84333333333</td>\n",
+ " <td>0.929436781609</td>\n",
+ " <td>4.4</td>\n",
+ " <td>7.6</td>\n",
+ " <td>4.925</td>\n",
+ " <td>5.75</td>\n",
+ " <td>6.575</td>\n",
+ " <td>[u'4.9', u'6.3', u'6.5', u'4.6', u'5', u'6.9', u'5.4', u'4.4', u'7', u'6.4']</td>\n",
+ " <td>[4L, 3L, 2L, 2L, 2L, 1L, 1L, 1L, 1L, 1L]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " <td>sepal_width</td>\n",
+ " <td>3</td>\n",
+ " <td>float8</td>\n",
+ " <td>30</td>\n",
+ " <td>14</td>\n",
+ " <td>0</td>\n",
+ " <td>None</td>\n",
+ " <td>0.0</td>\n",
+ " <td>None</td>\n",
+ " <td>3.04</td>\n",
+ " <td>0.139034482759</td>\n",
+ " <td>2.3</td>\n",
+ " <td>3.9</td>\n",
+ " <td>2.825</td>\n",
+ " <td>3.0</td>\n",
+ " <td>3.275</td>\n",
+ " <td>[u'2.9', u'3', u'3.1', u'3.2', u'3.6', u'3.3', u'2.8', u'2.7', u'2.5', u'3.4']</td>\n",
+ " <td>[4L, 4L, 3L, 3L, 2L, 2L, 2L, 2L, 2L, 2L]</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'class_name', u'Iris-setosa', u'sepal_length', 2, u'float8', 10L, 7L, 0L, None, 0.0, None, 4.86, 0.0848888888888976, 4.4, 5.4, 4.625, 4.9, 5.0, [u'4.6', u'4.9', u'5', u'5.1', u'4.4', u'5.4', u'4.7'], [2L, 2L, 2L, 1L, 1L, 1L, 1L]),\n",
+ " (u'class_name', u'Iris-setosa', u'sepal_width', 3, u'float8', 10L, 8L, 0L, None, 0.0, None, 3.31, 0.094333333333336, 2.9, 3.9, 3.1, 3.3, 3.475, [u'3.4', u'3.1', u'3.2', u'3.6', u'3.9', u'3.5', u'2.9', u'3'], [2L, 2L, 1L, 1L, 1L, 1L, 1L, 1L]),\n",
+ " (u'class_name', u'Iris-versicolor', u'sepal_length', 2, u'float8', 10L, 10L, 0L, None, 0.0, None, 6.1, 0.528888888888893, 4.9, 7.0, 5.55, 6.35, 6.575, [u'7', u'6.4', u'6.9', u'5.5', u'6.5', u'5.7', u'6.3', u'4.9', u'6.6', u'5.2'], [1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L]),\n",
+ " (u'class_name', u'Iris-versicolor', u'sepal_width', 3, u'float8', 10L, 8L, 0L, None, 0.0, None, 2.87, 0.115666666666666, 2.3, 3.3, 2.725, 2.85, 3.175, [u'2.8', u'3.2', u'2.3', u'3.3', u'2.4', u'2.9', u'2.7', u'3.1'], [2L, 2L, 1L, 1L, 1L, 1L, 1L, 1L]),\n",
+ " (u'class_name', u'Iris-virginica', u'sepal_length', 2, u'float8', 10L, 9L, 0L, None, 0.0, None, 6.57, 0.646777777777778, 4.9, 7.6, 6.3, 6.6, 7.175, [u'6.3', u'5.8', u'7.1', u'6.5', u'7.6', u'4.9', u'7.3', u'6.7', u'7.2'], [2L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L]),\n",
+ " (u'class_name', u'Iris-virginica', u'sepal_width', 3, u'float8', 10L, 6L, 0L, None, 0.0, None, 2.94, 0.11377777777778, 2.5, 3.6, 2.75, 2.95, 3.0, [u'3', u'2.9', u'2.5', u'3.3', u'2.7', u'3.6'], [3L, 2L, 2L, 1L, 1L, 1L]),\n",
+ " (None, None, u'sepal_length', 2, u'float8', 30L, 22L, 0L, None, 0.0, None, 5.84333333333333, 0.9294367816092, 4.4, 7.6, 4.925, 5.75, 6.575, [u'4.9', u'6.3', u'6.5', u'4.6', u'5', u'6.9', u'5.4', u'4.4', u'7', u'6.4'], [4L, 3L, 2L, 2L, 2L, 1L, 1L, 1L, 1L, 1L]),\n",
+ " (None, None, u'sepal_width', 3, u'float8', 30L, 14L, 0L, None, 0.0, None, 3.04, 0.13903448275862, 2.3, 3.9, 2.825, 3.0, 3.275, [u'2.9', u'3', u'3.1', u'3.2', u'3.6', u'3.3', u'2.8', u'2.7', u'2.5', u'3.4'], [4L, 4L, 3L, 3L, 2L, 2L, 2L, 2L, 2L, 2L])]"
+ ]
+ },
+ "execution_count": 45,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS iris_summary;\n",
+ "\n",
+ "SELECT * FROM madlib.summary( 'iris', -- Source table\n",
+ " 'iris_summary', -- Output table\n",
+ " 'sepal_length, sepal_width', -- Columns to summarize\n",
+ " 'class_name' -- Grouping column\n",
+ " );\n",
+ "\n",
+ "SELECT * FROM iris_summary;"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 5. Other parameters"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 50,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "1 rows affected.\n",
+ "2 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>group_by</th>\n",
+ " <th>group_by_value</th>\n",
+ " <th>target_column</th>\n",
+ " <th>column_number</th>\n",
+ " <th>data_type</th>\n",
+ " <th>row_count</th>\n",
+ " <th>distinct_values</th>\n",
+ " <th>missing_values</th>\n",
+ " <th>blank_values</th>\n",
+ " <th>fraction_missing</th>\n",
+ " <th>fraction_blank</th>\n",
+ " <th>mean</th>\n",
+ " <th>variance</th>\n",
+ " <th>min</th>\n",
+ " <th>max</th>\n",
+ " <th>quantile_array</th>\n",
+ " <th>most_frequent_values</th>\n",
+ " <th>mfv_frequencies</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " <td>sepal_length</td>\n",
+ " <td>2</td>\n",
+ " <td>float8</td>\n",
+ " <td>30</td>\n",
+ " <td>22</td>\n",
+ " <td>0</td>\n",
+ " <td>None</td>\n",
+ " <td>0.0</td>\n",
+ " <td>None</td>\n",
+ " <td>5.84333333333</td>\n",
+ " <td>0.929436781609</td>\n",
+ " <td>4.4</td>\n",
+ " <td>7.6</td>\n",
+ " <td>[5.057, 6.414]</td>\n",
+ " <td>[u'4.9', u'6.3', u'5']</td>\n",
+ " <td>[4L, 3L, 2L]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " <td>sepal_width</td>\n",
+ " <td>3</td>\n",
+ " <td>float8</td>\n",
+ " <td>30</td>\n",
+ " <td>14</td>\n",
+ " <td>0</td>\n",
+ " <td>None</td>\n",
+ " <td>0.0</td>\n",
+ " <td>None</td>\n",
+ " <td>3.04</td>\n",
+ " <td>0.139034482759</td>\n",
+ " <td>2.3</td>\n",
+ " <td>3.9</td>\n",
+ " <td>[2.9, 3.2]</td>\n",
+ " <td>[u'3', u'2.9', u'3.2']</td>\n",
+ " <td>[4L, 4L, 3L]</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(None, None, u'sepal_length', 2, u'float8', 30L, 22L, 0L, None, 0.0, None, 5.84333333333333, 0.9294367816092, 4.4, 7.6, [5.057, 6.414], [u'4.9', u'6.3', u'5'], [4L, 3L, 2L]),\n",
+ " (None, None, u'sepal_width', 3, u'float8', 30L, 14L, 0L, None, 0.0, None, 3.04, 0.13903448275862, 2.3, 3.9, [2.9, 3.2], [u'3', u'2.9', u'3.2'], [4L, 4L, 3L])]"
+ ]
+ },
+ "execution_count": 50,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "DROP TABLE IF EXISTS iris_summary;\n",
+ "\n",
+ "SELECT * FROM madlib.summary( 'iris', -- Source table\n",
+ " 'iris_summary', -- Output table\n",
+ " 'sepal_length, sepal_width', -- Columns to summarize\n",
+ " NULL, -- No grouping\n",
+ " TRUE, -- Get distinct values\n",
+ " FALSE, -- Dont get quartiles\n",
+ " ARRAY[0.33, 0.66], -- Get ntiles\n",
+ " 3, -- Number of MFV to compute\n",
+ " FALSE -- Get exact values\n",
+ " );\n",
+ "\n",
+ "SELECT * FROM iris_summary;"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 2",
+ "language": "python",
+ "name": "python2"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.12"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
http://git-wip-us.apache.org/repos/asf/madlib-site/blob/95826612/community-artifacts/kNN-v1.ipynb
----------------------------------------------------------------------
diff --git a/community-artifacts/kNN-v1.ipynb b/community-artifacts/kNN-v1.ipynb
deleted file mode 100644
index cca3251..0000000
--- a/community-artifacts/kNN-v1.ipynb
+++ /dev/null
@@ -1,497 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# k-Nearest Neighbors (MADlib v1.10+)\n",
- "Finds k nearest data points to a given data point and outputs majority vote value of output classes in case of classification, and average value of target values in case of regression."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 8,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "The sql extension is already loaded. To reload it, use:\n",
- " %reload_ext sql\n"
- ]
- }
- ],
- "source": [
- "%load_ext sql"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 9,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "data": {
- "text/plain": [
- "u'Connected: gpdbchina@madlib'"
- ]
- },
- "execution_count": 9,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%sql postgresql://gpdbchina@10.194.10.68:55000/madlib\n",
- "#%sql postgresql://fmcquillan@localhost:5432/madlib\n",
- "#%sql postgresql://gpadmin@54.197.30.46:10432/gpadmin"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 18,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>version</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>MADlib version: 1.10.0-dev, git revision: rel/v1.9.1-47-g2d5a5ed, cmake configuration time: Tue Feb 7 19:45:19 UTC 2017, build type: Release, build system: Linux-2.6.18-238.27.1.el5.hotfix.bz516490, C compiler: gcc 4.4.0, C++ compiler: g++ 4.4.0</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'MADlib version: 1.10.0-dev, git revision: rel/v1.9.1-47-g2d5a5ed, cmake configuration time: Tue Feb 7 19:45:19 UTC 2017, build type: Release, build system: Linux-2.6.18-238.27.1.el5.hotfix.bz516490, C compiler: gcc 4.4.0, C++ compiler: g++ 4.4.0',)]"
- ]
- },
- "execution_count": 18,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%sql select madlib.version();\n",
- "#%sql select version();"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# 1. Misc testing"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 19,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "Done.\n",
- "9 rows affected.\n",
- "9 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>id</th>\n",
- " <th>data</th>\n",
- " <th>label</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1</td>\n",
- " <td>[1, 1]</td>\n",
- " <td>1.0</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>2</td>\n",
- " <td>[2, 2]</td>\n",
- " <td>1.0</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>3</td>\n",
- " <td>[3, 3]</td>\n",
- " <td>1.0</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>4</td>\n",
- " <td>[4, 4]</td>\n",
- " <td>1.0</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>5</td>\n",
- " <td>[4, 5]</td>\n",
- " <td>1.0</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>6</td>\n",
- " <td>[20, 50]</td>\n",
- " <td>0.0</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>7</td>\n",
- " <td>[10, 31]</td>\n",
- " <td>0.0</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>8</td>\n",
- " <td>[81, 13]</td>\n",
- " <td>0.0</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>9</td>\n",
- " <td>[1, 111]</td>\n",
- " <td>0.0</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(1, [1, 1], 1.0),\n",
- " (2, [2, 2], 1.0),\n",
- " (3, [3, 3], 1.0),\n",
- " (4, [4, 4], 1.0),\n",
- " (5, [4, 5], 1.0),\n",
- " (6, [20, 50], 0.0),\n",
- " (7, [10, 31], 0.0),\n",
- " (8, [81, 13], 0.0),\n",
- " (9, [1, 111], 0.0)]"
- ]
- },
- "execution_count": 19,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql \n",
- "DROP TABLE IF EXISTS knn_train_data;\n",
- "CREATE TABLE knn_train_data (\n",
- " id integer, \n",
- " data integer[], \n",
- " label float\n",
- " );\n",
- "\n",
- "INSERT INTO knn_train_data VALUES\n",
- "(1, '{1,1}', 1.0),\n",
- "(2, '{2,2}', 1.0),\n",
- "(3, '{3,3}', 1.0),\n",
- "(4, '{4,4}', 1.0),\n",
- "(5, '{4,5}', 1.0),\n",
- "(6, '{20,50}', 0.0),\n",
- "(7, '{10,31}', 0.0),\n",
- "(8, '{81,13}', 0.0),\n",
- "(9, '{1,111}', 0.0);\n",
- "\n",
- "SELECT * from knn_train_data ORDER BY id;"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 23,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "Done.\n",
- "6 rows affected.\n",
- "6 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>id</th>\n",
- " <th>data</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1</td>\n",
- " <td>[2, 1]</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>2</td>\n",
- " <td>[2, 6]</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>3</td>\n",
- " <td>[15, 40]</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>4</td>\n",
- " <td>[12, 1]</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>5</td>\n",
- " <td>[2, 90]</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>6</td>\n",
- " <td>[50, 45]</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(1, [2, 1]),\n",
- " (2, [2, 6]),\n",
- " (3, [15, 40]),\n",
- " (4, [12, 1]),\n",
- " (5, [2, 90]),\n",
- " (6, [50, 45])]"
- ]
- },
- "execution_count": 23,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql \n",
- "DROP TABLE IF EXISTS knn_test_data;\n",
- "CREATE TABLE knn_test_data (\n",
- " id integer, \n",
- " data integer[]\n",
- " );\n",
- "\n",
- "INSERT INTO knn_test_data VALUES\n",
- "(1, '{2,1}'),\n",
- "(2, '{2,6}'),\n",
- "(3, '{15,40}'),\n",
- "(4, '{12,1}'),\n",
- "(5, '{2,90}'),\n",
- "(6, '{50,45}');\n",
- "\n",
- "SELECT * from knn_test_data ORDER BY id;"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 24,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "6 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>id</th>\n",
- " <th>data</th>\n",
- " <th>prediction</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1</td>\n",
- " <td>[2, 1]</td>\n",
- " <td>1.0</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>2</td>\n",
- " <td>[2, 6]</td>\n",
- " <td>1.0</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>3</td>\n",
- " <td>[15, 40]</td>\n",
- " <td>0.0</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>4</td>\n",
- " <td>[12, 1]</td>\n",
- " <td>1.0</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>5</td>\n",
- " <td>[2, 90]</td>\n",
- " <td>0.0</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>6</td>\n",
- " <td>[50, 45]</td>\n",
- " <td>0.0</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(1, [2, 1], 1.0),\n",
- " (2, [2, 6], 1.0),\n",
- " (3, [15, 40], 0.0),\n",
- " (4, [12, 1], 1.0),\n",
- " (5, [2, 90], 0.0),\n",
- " (6, [50, 45], 0.0)]"
- ]
- },
- "execution_count": 24,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS madlib_knn_result_classification;\n",
- "SELECT * FROM madlib.knn( \n",
- " 'knn_train_data', -- Table of training data\n",
- " 'data', -- Col name of training data\n",
- " 'label', -- Training labels\n",
- " 'knn_test_data', -- Table of test data\n",
- " 'data', -- Col name of test data\n",
- " 'id', -- Col name of id in test data \n",
- " 'madlib_knn_result_classification', -- Output table\n",
- " 'c', -- Classification\n",
- " 3 -- Number of nearest neighbours\n",
- " );\n",
- "SELECT * from madlib_knn_result_classification ORDER BY id;"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 25,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "6 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>id</th>\n",
- " <th>data</th>\n",
- " <th>prediction</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1</td>\n",
- " <td>[2, 1]</td>\n",
- " <td>1.0</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>2</td>\n",
- " <td>[2, 6]</td>\n",
- " <td>1.0</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>3</td>\n",
- " <td>[15, 40]</td>\n",
- " <td>0.333333333333</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>4</td>\n",
- " <td>[12, 1]</td>\n",
- " <td>1.0</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>5</td>\n",
- " <td>[2, 90]</td>\n",
- " <td>0.0</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>6</td>\n",
- " <td>[50, 45]</td>\n",
- " <td>0.0</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(1, [2, 1], 1.0),\n",
- " (2, [2, 6], 1.0),\n",
- " (3, [15, 40], 0.333333333333333),\n",
- " (4, [12, 1], 1.0),\n",
- " (5, [2, 90], 0.0),\n",
- " (6, [50, 45], 0.0)]"
- ]
- },
- "execution_count": 25,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS madlib_knn_result_regression;\n",
- "SELECT * FROM madlib.knn( \n",
- " 'knn_train_data', -- Table of training data\n",
- " 'data', -- Col name of training data\n",
- " 'label', -- Training labels\n",
- " 'knn_test_data', -- Table of test data\n",
- " 'data', -- Col name of test data\n",
- " 'id', -- Col name of id in test data \n",
- " 'madlib_knn_result_regression', -- Output table\n",
- " 'r', -- Regressions\n",
- " 3 -- Number of nearest neighbours\n",
- " );\n",
- "SELECT * from madlib_knn_result_regression ORDER BY id;"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 2",
- "language": "python",
- "name": "python2"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 2
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython2",
- "version": "2.7.12"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 0
-}
[2/7] madlib-site git commit: add new workbooks for 1dot13
Posted by fm...@apache.org.
http://git-wip-us.apache.org/repos/asf/madlib-site/blob/95826612/community-artifacts/mlp-v1.ipynb
----------------------------------------------------------------------
diff --git a/community-artifacts/mlp-v1.ipynb b/community-artifacts/mlp-v1.ipynb
deleted file mode 100644
index eaba7fb..0000000
--- a/community-artifacts/mlp-v1.ipynb
+++ /dev/null
@@ -1,1525 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Multilayer Perceptron"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {
- "scrolled": true
- },
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/config.py:13: ShimWarning: The `IPython.config` package has been deprecated. You should import from traitlets.config instead.\n",
- " \"You should import from traitlets.config instead.\", ShimWarning)\n",
- "/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/utils/traitlets.py:5: UserWarning: IPython.utils.traitlets has moved to a top-level traitlets package.\n",
- " warn(\"IPython.utils.traitlets has moved to a top-level traitlets package.\")\n"
- ]
- }
- ],
- "source": [
- "%load_ext sql"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "u'Connected: gpdbchina@madlib'"
- ]
- },
- "execution_count": 2,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "# Greenplum 4.3.10.0\n",
- "%sql postgresql://gpdbchina@10.194.10.68:61000/madlib\n",
- " \n",
- "# PostgreSQL local\n",
- "#%sql postgresql://fmcquillan@localhost:5432/madlib\n",
- "\n",
- "# Greenplum 4.2.3.0\n",
- "#%sql postgresql://gpdbchina@10.194.10.68:55000/madlib"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>version</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>MADlib version: 1.12-dev, git revision: rel/v1.11-48-gff1b0f8, cmake configuration time: Tue Aug 15 00:36:04 UTC 2017, build type: Release, build system: Linux-2.6.18-238.27.1.el5.hotfix.bz516490, C compiler: gcc 4.4.0, C++ compiler: g++ 4.4.0</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(u'MADlib version: 1.12-dev, git revision: rel/v1.11-48-gff1b0f8, cmake configuration time: Tue Aug 15 00:36:04 UTC 2017, build type: Release, build system: Linux-2.6.18-238.27.1.el5.hotfix.bz516490, C compiler: gcc 4.4.0, C++ compiler: g++ 4.4.0',)]"
- ]
- },
- "execution_count": 3,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%sql select madlib.version();\n",
- "#%sql select version();"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# 1. Create input table for classification"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "Done.\n",
- "20 rows affected.\n",
- "20 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>id</th>\n",
- " <th>attributes</th>\n",
- " <th>class_text</th>\n",
- " <th>class</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1</td>\n",
- " <td>[Decimal('5.1'), Decimal('3.5'), Decimal('1.4'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>2</td>\n",
- " <td>[Decimal('4.9'), Decimal('3.0'), Decimal('1.4'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>3</td>\n",
- " <td>[Decimal('4.7'), Decimal('3.2'), Decimal('1.3'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>4</td>\n",
- " <td>[Decimal('4.6'), Decimal('3.1'), Decimal('1.5'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>5</td>\n",
- " <td>[Decimal('5.0'), Decimal('3.6'), Decimal('1.4'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>6</td>\n",
- " <td>[Decimal('5.4'), Decimal('3.9'), Decimal('1.7'), Decimal('0.4')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>7</td>\n",
- " <td>[Decimal('4.6'), Decimal('3.4'), Decimal('1.4'), Decimal('0.3')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>8</td>\n",
- " <td>[Decimal('5.0'), Decimal('3.4'), Decimal('1.5'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>9</td>\n",
- " <td>[Decimal('4.4'), Decimal('2.9'), Decimal('1.4'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>10</td>\n",
- " <td>[Decimal('4.9'), Decimal('3.1'), Decimal('1.5'), Decimal('0.1')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>11</td>\n",
- " <td>[Decimal('7.0'), Decimal('3.2'), Decimal('4.7'), Decimal('1.4')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>2</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>12</td>\n",
- " <td>[Decimal('6.4'), Decimal('3.2'), Decimal('4.5'), Decimal('1.5')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>2</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>13</td>\n",
- " <td>[Decimal('6.9'), Decimal('3.1'), Decimal('4.9'), Decimal('1.5')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>2</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>14</td>\n",
- " <td>[Decimal('5.5'), Decimal('2.3'), Decimal('4.0'), Decimal('1.3')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>2</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>15</td>\n",
- " <td>[Decimal('6.5'), Decimal('2.8'), Decimal('4.6'), Decimal('1.5')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>2</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>16</td>\n",
- " <td>[Decimal('5.7'), Decimal('2.8'), Decimal('4.5'), Decimal('1.3')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>2</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>17</td>\n",
- " <td>[Decimal('6.3'), Decimal('3.3'), Decimal('4.7'), Decimal('1.6')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>2</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>18</td>\n",
- " <td>[Decimal('4.9'), Decimal('2.4'), Decimal('3.3'), Decimal('1.0')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>2</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>19</td>\n",
- " <td>[Decimal('6.6'), Decimal('2.9'), Decimal('4.6'), Decimal('1.3')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>2</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>20</td>\n",
- " <td>[Decimal('5.2'), Decimal('2.7'), Decimal('3.9'), Decimal('1.4')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>2</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(1, [Decimal('5.1'), Decimal('3.5'), Decimal('1.4'), Decimal('0.2')], u'Iris-setosa', 1),\n",
- " (2, [Decimal('4.9'), Decimal('3.0'), Decimal('1.4'), Decimal('0.2')], u'Iris-setosa', 1),\n",
- " (3, [Decimal('4.7'), Decimal('3.2'), Decimal('1.3'), Decimal('0.2')], u'Iris-setosa', 1),\n",
- " (4, [Decimal('4.6'), Decimal('3.1'), Decimal('1.5'), Decimal('0.2')], u'Iris-setosa', 1),\n",
- " (5, [Decimal('5.0'), Decimal('3.6'), Decimal('1.4'), Decimal('0.2')], u'Iris-setosa', 1),\n",
- " (6, [Decimal('5.4'), Decimal('3.9'), Decimal('1.7'), Decimal('0.4')], u'Iris-setosa', 1),\n",
- " (7, [Decimal('4.6'), Decimal('3.4'), Decimal('1.4'), Decimal('0.3')], u'Iris-setosa', 1),\n",
- " (8, [Decimal('5.0'), Decimal('3.4'), Decimal('1.5'), Decimal('0.2')], u'Iris-setosa', 1),\n",
- " (9, [Decimal('4.4'), Decimal('2.9'), Decimal('1.4'), Decimal('0.2')], u'Iris-setosa', 1),\n",
- " (10, [Decimal('4.9'), Decimal('3.1'), Decimal('1.5'), Decimal('0.1')], u'Iris-setosa', 1),\n",
- " (11, [Decimal('7.0'), Decimal('3.2'), Decimal('4.7'), Decimal('1.4')], u'Iris-versicolor', 2),\n",
- " (12, [Decimal('6.4'), Decimal('3.2'), Decimal('4.5'), Decimal('1.5')], u'Iris-versicolor', 2),\n",
- " (13, [Decimal('6.9'), Decimal('3.1'), Decimal('4.9'), Decimal('1.5')], u'Iris-versicolor', 2),\n",
- " (14, [Decimal('5.5'), Decimal('2.3'), Decimal('4.0'), Decimal('1.3')], u'Iris-versicolor', 2),\n",
- " (15, [Decimal('6.5'), Decimal('2.8'), Decimal('4.6'), Decimal('1.5')], u'Iris-versicolor', 2),\n",
- " (16, [Decimal('5.7'), Decimal('2.8'), Decimal('4.5'), Decimal('1.3')], u'Iris-versicolor', 2),\n",
- " (17, [Decimal('6.3'), Decimal('3.3'), Decimal('4.7'), Decimal('1.6')], u'Iris-versicolor', 2),\n",
- " (18, [Decimal('4.9'), Decimal('2.4'), Decimal('3.3'), Decimal('1.0')], u'Iris-versicolor', 2),\n",
- " (19, [Decimal('6.6'), Decimal('2.9'), Decimal('4.6'), Decimal('1.3')], u'Iris-versicolor', 2),\n",
- " (20, [Decimal('5.2'), Decimal('2.7'), Decimal('3.9'), Decimal('1.4')], u'Iris-versicolor', 2)]"
- ]
- },
- "execution_count": 4,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql \n",
- "DROP TABLE IF EXISTS iris_data;\n",
- "\n",
- "CREATE TABLE iris_data(\n",
- " id integer,\n",
- " attributes numeric[],\n",
- " class_text varchar,\n",
- " class integer\n",
- ");\n",
- "\n",
- "INSERT INTO iris_data VALUES\n",
- "(1,ARRAY[5.1,3.5,1.4,0.2],'Iris-setosa',1),\n",
- "(2,ARRAY[4.9,3.0,1.4,0.2],'Iris-setosa',1),\n",
- "(3,ARRAY[4.7,3.2,1.3,0.2],'Iris-setosa',1),\n",
- "(4,ARRAY[4.6,3.1,1.5,0.2],'Iris-setosa',1),\n",
- "(5,ARRAY[5.0,3.6,1.4,0.2],'Iris-setosa',1),\n",
- "(6,ARRAY[5.4,3.9,1.7,0.4],'Iris-setosa',1),\n",
- "(7,ARRAY[4.6,3.4,1.4,0.3],'Iris-setosa',1),\n",
- "(8,ARRAY[5.0,3.4,1.5,0.2],'Iris-setosa',1),\n",
- "(9,ARRAY[4.4,2.9,1.4,0.2],'Iris-setosa',1),\n",
- "(10,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa',1),\n",
- "(11,ARRAY[7.0,3.2,4.7,1.4],'Iris-versicolor',2),\n",
- "(12,ARRAY[6.4,3.2,4.5,1.5],'Iris-versicolor',2),\n",
- "(13,ARRAY[6.9,3.1,4.9,1.5],'Iris-versicolor',2),\n",
- "(14,ARRAY[5.5,2.3,4.0,1.3],'Iris-versicolor',2),\n",
- "(15,ARRAY[6.5,2.8,4.6,1.5],'Iris-versicolor',2),\n",
- "(16,ARRAY[5.7,2.8,4.5,1.3],'Iris-versicolor',2),\n",
- "(17,ARRAY[6.3,3.3,4.7,1.6],'Iris-versicolor',2),\n",
- "(18,ARRAY[4.9,2.4,3.3,1.0],'Iris-versicolor',2),\n",
- "(19,ARRAY[6.6,2.9,4.6,1.3],'Iris-versicolor',2),\n",
- "(20,ARRAY[5.2,2.7,3.9,1.4],'Iris-versicolor',2);\n",
- "\n",
- "SELECT * FROM iris_data ORDER BY id;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# 2. Classification\n",
- "\n",
- "Generate a multilayer perceptron with a single hidden layer of 5 units. Use the attributes column as the independent variables, and use the class column as the classification. Set the tolerance to 0 so that 500 iterations will be run. Use a hyperbolic tangent activation function. The model will be written to mlp_model."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>mlp_classification</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td></td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[('',)]"
- ]
- },
- "execution_count": 5,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS mlp_model, mlp_model_summary;\n",
- "\n",
- "-- Set seed so results are reproducible\n",
- "SELECT setseed(0);\n",
- "SELECT madlib.mlp_classification(\n",
- " 'iris_data', -- Source table\n",
- " 'mlp_model', -- Destination table\n",
- " 'attributes', -- Input features\n",
- " 'class_text', -- Label\n",
- " ARRAY[5], -- Number of units per layer\n",
- " 'learning_rate_init=0.003,\n",
- " n_iterations=500,\n",
- " tolerance=0', -- Optimizer params\n",
- " 'tanh', -- Activation function\n",
- " NULL, -- Default weight (1)\n",
- " FALSE, -- No warm start\n",
- " TRUE -- Verbose\n",
- ");"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# 3. View the classification model"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 6,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>coeff</th>\n",
- " <th>loss</th>\n",
- " <th>num_iterations</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[0.134859643584, 0.200285896402, -0.281831690249, 0.755199357168, 0.857042304782, -0.188536767412, -0.291668520498, 0.343686800435, -0.399408727166, -0.179921593947, -0.010662340824, 0.23622232339, -0.257390617236, 0.213182376685, 0.576459373081, 0.306524087619, 0.248260630252, 0.050175145813, -0.101614469022, 0.281200318932, -0.391835525435, -0.0953767781907, -0.384721111012, 0.402854448732, -0.122585128952, 0.110591514785, -1.40623002748, 0.177662074116, -0.247743897977, -0.258200774495, -0.203357963386, 0.122486844237, 1.1677668027, -0.728672776285, 0.837006898515, 0.0198429228676, 0.209390034006]</td>\n",
- " <td>0.0320841110315</td>\n",
- " <td>500</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[([0.134859643584, 0.200285896402, -0.281831690249, 0.755199357168, 0.857042304782, -0.188536767412, -0.291668520498, 0.343686800435, -0.399408727166, -0.179921593947, -0.010662340824, 0.23622232339, -0.257390617236, 0.213182376685, 0.576459373081, 0.306524087619, 0.248260630252, 0.050175145813, -0.101614469022, 0.281200318932, -0.391835525435, -0.0953767781907, -0.384721111012, 0.402854448732, -0.122585128952, 0.110591514785, -1.40623002748, 0.177662074116, -0.247743897977, -0.258200774495, -0.203357963386, 0.122486844237, 1.1677668027, -0.728672776285, 0.837006898515, 0.0198429228676, 0.209390034006], Decimal('0.0320841110315'), 500)]"
- ]
- },
- "execution_count": 6,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT * FROM mlp_model;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# 4. Create input data for regression\n",
- "\n",
- "This dataset contains housing prices."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 7,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "Done.\n",
- "20 rows affected.\n",
- "20 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>id</th>\n",
- " <th>x</th>\n",
- " <th>grp_by_col</th>\n",
- " <th>y</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1</td>\n",
- " <td>[0.00632, 18.0, 2.31, 0.0, 0.538, 6.575, 65.2, 4.09, 1.0, 296.0, 15.3, 396.9, 4.98]</td>\n",
- " <td>1</td>\n",
- " <td>24.0</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>2</td>\n",
- " <td>[0.02731, 0.0, 7.07, 0.0, 0.469, 6.421, 78.9, 4.9671, 2.0, 242.0, 17.8, 396.9, 9.14]</td>\n",
- " <td>1</td>\n",
- " <td>21.6</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>3</td>\n",
- " <td>[0.02729, 0.0, 7.07, 0.0, 0.469, 7.185, 61.1, 4.9671, 2.0, 242.0, 17.8, 392.83, 4.03]</td>\n",
- " <td>1</td>\n",
- " <td>34.7</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>4</td>\n",
- " <td>[0.03237, 0.0, 2.18, 0.0, 0.458, 6.998, 45.8, 6.0622, 3.0, 222.0, 18.7, 394.63, 2.94]</td>\n",
- " <td>1</td>\n",
- " <td>33.4</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>5</td>\n",
- " <td>[0.06905, 0.0, 2.18, 0.0, 0.458, 7.147, 54.2, 6.0622, 3.0, 222.0, 18.7, 396.9, 5.33]</td>\n",
- " <td>1</td>\n",
- " <td>36.2</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>6</td>\n",
- " <td>[0.02985, 0.0, 2.18, 0.0, 0.458, 6.43, 58.7, 6.0622, 3.0, 222.0, 18.7, 394.12, 5.21]</td>\n",
- " <td>1</td>\n",
- " <td>28.7</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>7</td>\n",
- " <td>[0.08829, 12.5, 7.87, 0.0, 0.524, 6.012, 66.6, 5.5605, 5.0, 311.0, 15.2, 395.6, 12.43]</td>\n",
- " <td>1</td>\n",
- " <td>22.9</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>8</td>\n",
- " <td>[0.14455, 12.5, 7.87, 0.0, 0.524, 6.172, 96.1, 5.9505, 5.0, 311.0, 15.2, 396.9, 19.15]</td>\n",
- " <td>1</td>\n",
- " <td>27.1</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>9</td>\n",
- " <td>[0.21124, 12.5, 7.87, 0.0, 0.524, 5.631, 100.0, 6.0821, 5.0, 311.0, 15.2, 386.63, 29.93]</td>\n",
- " <td>1</td>\n",
- " <td>16.5</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>10</td>\n",
- " <td>[0.17004, 12.5, 7.87, 0.0, 0.524, 6.004, 85.9, 6.5921, 5.0, 311.0, 15.2, 386.71, 17.1]</td>\n",
- " <td>1</td>\n",
- " <td>18.9</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>11</td>\n",
- " <td>[0.22489, 12.5, 7.87, 0.0, 0.524, 6.377, 94.3, 6.3467, 5.0, 311.0, 15.2, 392.52, 20.45]</td>\n",
- " <td>1</td>\n",
- " <td>15.0</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>12</td>\n",
- " <td>[0.11747, 12.5, 7.87, 0.0, 0.524, 6.009, 82.9, 6.2267, 5.0, 311.0, 15.2, 396.9, 13.27]</td>\n",
- " <td>1</td>\n",
- " <td>18.9</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>13</td>\n",
- " <td>[0.09378, 12.5, 7.87, 0.0, 0.524, 5.889, 39.0, 5.4509, 5.0, 311.0, 15.2, 390.5, 15.71]</td>\n",
- " <td>1</td>\n",
- " <td>21.7</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>14</td>\n",
- " <td>[0.62976, 0.0, 8.14, 0.0, 0.538, 5.949, 61.8, 4.7075, 4.0, 307.0, 21.0, 396.9, 8.26]</td>\n",
- " <td>1</td>\n",
- " <td>20.4</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>15</td>\n",
- " <td>[0.63796, 0.0, 8.14, 0.0, 0.538, 6.096, 84.5, 4.4619, 4.0, 307.0, 21.0, 380.02, 10.26]</td>\n",
- " <td>1</td>\n",
- " <td>18.2</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>16</td>\n",
- " <td>[0.62739, 0.0, 8.14, 0.0, 0.538, 5.834, 56.5, 4.4986, 4.0, 307.0, 21.0, 395.62, 8.47]</td>\n",
- " <td>1</td>\n",
- " <td>19.9</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>17</td>\n",
- " <td>[1.05393, 0.0, 8.14, 0.0, 0.538, 5.935, 29.3, 4.4986, 4.0, 307.0, 21.0, 386.85, 6.58]</td>\n",
- " <td>1</td>\n",
- " <td>23.1</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>18</td>\n",
- " <td>[0.7842, 0.0, 8.14, 0.0, 0.538, 5.99, 81.7, 4.2579, 4.0, 307.0, 21.0, 386.75, 14.67]</td>\n",
- " <td>1</td>\n",
- " <td>17.5</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>19</td>\n",
- " <td>[0.80271, 0.0, 8.14, 0.0, 0.538, 5.456, 36.6, 3.7965, 4.0, 307.0, 21.0, 288.99, 11.69]</td>\n",
- " <td>1</td>\n",
- " <td>20.2</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>20</td>\n",
- " <td>[0.7258, 0.0, 8.14, 0.0, 0.538, 5.727, 69.5, 3.7965, 4.0, 307.0, 21.0, 390.95, 11.28]</td>\n",
- " <td>1</td>\n",
- " <td>18.2</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(1, [0.00632, 18.0, 2.31, 0.0, 0.538, 6.575, 65.2, 4.09, 1.0, 296.0, 15.3, 396.9, 4.98], 1, 24.0),\n",
- " (2, [0.02731, 0.0, 7.07, 0.0, 0.469, 6.421, 78.9, 4.9671, 2.0, 242.0, 17.8, 396.9, 9.14], 1, 21.6),\n",
- " (3, [0.02729, 0.0, 7.07, 0.0, 0.469, 7.185, 61.1, 4.9671, 2.0, 242.0, 17.8, 392.83, 4.03], 1, 34.7),\n",
- " (4, [0.03237, 0.0, 2.18, 0.0, 0.458, 6.998, 45.8, 6.0622, 3.0, 222.0, 18.7, 394.63, 2.94], 1, 33.4),\n",
- " (5, [0.06905, 0.0, 2.18, 0.0, 0.458, 7.147, 54.2, 6.0622, 3.0, 222.0, 18.7, 396.9, 5.33], 1, 36.2),\n",
- " (6, [0.02985, 0.0, 2.18, 0.0, 0.458, 6.43, 58.7, 6.0622, 3.0, 222.0, 18.7, 394.12, 5.21], 1, 28.7),\n",
- " (7, [0.08829, 12.5, 7.87, 0.0, 0.524, 6.012, 66.6, 5.5605, 5.0, 311.0, 15.2, 395.6, 12.43], 1, 22.9),\n",
- " (8, [0.14455, 12.5, 7.87, 0.0, 0.524, 6.172, 96.1, 5.9505, 5.0, 311.0, 15.2, 396.9, 19.15], 1, 27.1),\n",
- " (9, [0.21124, 12.5, 7.87, 0.0, 0.524, 5.631, 100.0, 6.0821, 5.0, 311.0, 15.2, 386.63, 29.93], 1, 16.5),\n",
- " (10, [0.17004, 12.5, 7.87, 0.0, 0.524, 6.004, 85.9, 6.5921, 5.0, 311.0, 15.2, 386.71, 17.1], 1, 18.9),\n",
- " (11, [0.22489, 12.5, 7.87, 0.0, 0.524, 6.377, 94.3, 6.3467, 5.0, 311.0, 15.2, 392.52, 20.45], 1, 15.0),\n",
- " (12, [0.11747, 12.5, 7.87, 0.0, 0.524, 6.009, 82.9, 6.2267, 5.0, 311.0, 15.2, 396.9, 13.27], 1, 18.9),\n",
- " (13, [0.09378, 12.5, 7.87, 0.0, 0.524, 5.889, 39.0, 5.4509, 5.0, 311.0, 15.2, 390.5, 15.71], 1, 21.7),\n",
- " (14, [0.62976, 0.0, 8.14, 0.0, 0.538, 5.949, 61.8, 4.7075, 4.0, 307.0, 21.0, 396.9, 8.26], 1, 20.4),\n",
- " (15, [0.63796, 0.0, 8.14, 0.0, 0.538, 6.096, 84.5, 4.4619, 4.0, 307.0, 21.0, 380.02, 10.26], 1, 18.2),\n",
- " (16, [0.62739, 0.0, 8.14, 0.0, 0.538, 5.834, 56.5, 4.4986, 4.0, 307.0, 21.0, 395.62, 8.47], 1, 19.9),\n",
- " (17, [1.05393, 0.0, 8.14, 0.0, 0.538, 5.935, 29.3, 4.4986, 4.0, 307.0, 21.0, 386.85, 6.58], 1, 23.1),\n",
- " (18, [0.7842, 0.0, 8.14, 0.0, 0.538, 5.99, 81.7, 4.2579, 4.0, 307.0, 21.0, 386.75, 14.67], 1, 17.5),\n",
- " (19, [0.80271, 0.0, 8.14, 0.0, 0.538, 5.456, 36.6, 3.7965, 4.0, 307.0, 21.0, 288.99, 11.69], 1, 20.2),\n",
- " (20, [0.7258, 0.0, 8.14, 0.0, 0.538, 5.727, 69.5, 3.7965, 4.0, 307.0, 21.0, 390.95, 11.28], 1, 18.2)]"
- ]
- },
- "execution_count": 7,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS lin_housing;\n",
- "\n",
- "CREATE TABLE lin_housing (id serial, \n",
- " x float8[], \n",
- " grp_by_col int, \n",
- " y float8);\n",
- "\n",
- "INSERT INTO lin_housing VALUES\n",
- "(1,ARRAY[0.00632,18.00,2.310,0,0.5380,6.5750,65.20,4.0900,1,296.0,15.30,396.90,4.98],1,24.00),\n",
- "(2,ARRAY[0.02731,0.00,7.070,0,0.4690,6.4210,78.90,4.9671,2,242.0,17.80,396.90,9.14],1,21.60),\n",
- "(3,ARRAY[0.02729,0.00,7.070,0,0.4690,7.1850,61.10,4.9671,2,242.0,17.80,392.83,4.03],1,34.70),\n",
- "(4,ARRAY[0.03237,0.00,2.180,0,0.4580,6.9980,45.80,6.0622,3,222.0,18.70,394.63,2.94],1,33.40),\n",
- "(5,ARRAY[0.06905,0.00,2.180,0,0.4580,7.1470,54.20,6.0622,3,222.0,18.70,396.90,5.33],1,36.20),\n",
- "(6,ARRAY[0.02985,0.00,2.180,0,0.4580,6.4300,58.70,6.0622,3,222.0,18.70,394.12,5.21],1,28.70),\n",
- "(7,ARRAY[0.08829,12.50,7.870,0,0.5240,6.0120,66.60,5.5605,5,311.0,15.20,395.60,12.43],1,22.90),\n",
- "(8,ARRAY[0.14455,12.50,7.870,0,0.5240,6.1720,96.10,5.9505,5,311.0,15.20,396.90,19.15],1,27.10),\n",
- "(9,ARRAY[0.21124,12.50,7.870,0,0.5240,5.6310,100.00,6.0821,5,311.0,15.20,386.63,29.93],1,16.50),\n",
- "(10,ARRAY[0.17004,12.50,7.870,0,0.5240,6.0040,85.90,6.5921,5,311.0,15.20,386.71,17.10],1,18.90),\n",
- "(11,ARRAY[0.22489,12.50,7.870,0,0.5240,6.3770,94.30,6.3467,5,311.0,15.20,392.52,20.45],1,15.00),\n",
- "(12,ARRAY[0.11747,12.50,7.870,0,0.5240,6.0090,82.90,6.2267,5,311.0,15.20,396.90,13.27],1,18.90),\n",
- "(13,ARRAY[0.09378,12.50,7.870,0,0.5240,5.8890,39.00,5.4509,5,311.0,15.20,390.50,15.71],1,21.70),\n",
- "(14,ARRAY[0.62976,0.00,8.140,0,0.5380,5.9490,61.80,4.7075,4,307.0,21.00,396.90,8.26],1,20.40),\n",
- "(15,ARRAY[0.63796,0.00,8.140,0,0.5380,6.0960,84.50,4.4619,4,307.0,21.00,380.02,10.26],1,18.20),\n",
- "(16,ARRAY[0.62739,0.00,8.140,0,0.5380,5.8340,56.50,4.4986,4,307.0,21.00,395.62,8.47],1,19.90),\n",
- "(17,ARRAY[1.05393,0.00,8.140,0,0.5380,5.9350,29.30,4.4986,4,307.0,21.00,386.85,6.58],1, 23.10),\n",
- "(18,ARRAY[0.78420,0.00,8.140,0,0.5380,5.9900,81.70,4.2579,4,307.0,21.00,386.75,14.67],1,17.50),\n",
- "(19,ARRAY[0.80271,0.00,8.140,0,0.5380,5.4560,36.60,3.7965,4,307.0,21.00,288.99,11.69],1,20.20),\n",
- "(20,ARRAY[0.72580,0.00,8.140,0,0.5380,5.7270,69.50,3.7965,4,307.0,21.00,390.95,11.28],1,18.20);\n",
- "\n",
- "SELECT * FROM lin_housing ORDER BY id;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# 5. Regression\n",
- "\n",
- "Now train a regression model using a multilayer perceptron with 2 hidden layers of 25 nodes each."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 8,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "Done.\n",
- "1 rows affected.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>mlp_regression</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td></td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[('',)]"
- ]
- },
- "execution_count": 8,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS mlp_regress;\n",
- "DROP TABLE IF EXISTS mlp_regress_summary;\n",
- "SELECT setseed(0);\n",
- "SELECT madlib.mlp_regression(\n",
- " 'lin_housing', -- Source table\n",
- " 'mlp_regress', -- Desination table\n",
- " 'x', -- Input features\n",
- " 'y', -- Dependent variable\n",
- " ARRAY[25,25], -- Number of units per layer\n",
- " 'learning_rate_init=0.001,\n",
- " n_iterations=500,\n",
- " lambda=0.001,\n",
- " tolerance=0',\n",
- " 'relu',\n",
- " NULL, -- Default weight (1)\n",
- " FALSE, -- No warm start\n",
- " TRUE -- Verbose\n",
- ");"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# 6.0 View the regression model"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 9,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>coeff</th>\n",
- " <th>loss</th>\n",
- " <th>num_iterations</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>[0.122856566817, -0.109511383377, 0.10922147183, 0.142820008065, 0.162756253894, -0.104436945183, 0.112036676092, 0.0467338442275, -0.16453372472, 0.0827992259598, 0.0137404038273, 0.0565353617298, -0.0246679212553, -0.042838429926, 0.381777598727, 0.276073801059, 0.00907788148497, 0.21191870498, -0.141702416989, 0.181936732347, -0.136061155165, -0.721381760449, -0.0560067549982, 0.203545531143, -0.00995912442451, 0.0613143120688, -0.0984596213056, -0.204436524678, 0.470767785122, -0.435311003078, -0.0424606276832, 0.160892203008, 0.0445352965619, -0.0879636030809, 0.154146037853, 0.0288780423027, -0.317282313525, 0.385922416222, -0.0664872317641, 0.146757511446, 0.133876976675, -0.116256543538, -0.0213680210249, 0.20015749221, -0.103457503483, -0.0425956948505, 0.121667463986, 0.204968623833, -0.159914333708, 0.149243857206, 0.0456901480449, -0.136975914168, -0.090899493726, 0.103816779262, 0.157303539757, -0.0813411072267, -0.186218530515, -0.18296662406, -0.01
06273344335, -0.15268221771, -0.103478313334, 0.199984721908, 0.140135789025, 0.143966006971, -0.0970102681415, 0.0261086412469, -0.0330555393771, 0.0995076031524, 0.00130931383947, 0.0778756667869, -0.0527086470232, -0.600133053147, 0.0859133658584, 0.24113974288, 0.170332657512, 0.0676027349796, -0.21148456397, 0.248468295235, -0.436384189411, 0.175863820799, 0.123778486759, -0.282581458533, 0.0191701197018, 0.280887575877, 0.21926286976, -0.24666316945, -0.0262117477515, 0.268417193439, -0.0591642733853, 0.168335924156, 0.567262570173, -0.0244295268223, -0.161746761037, 0.239464436128, 0.0927106954455, 0.108399901337, -0.00684509495191, 0.0942018933997, 0.27681768162, -0.0200546475364, -0.0178865707443, 0.0590165895996, 0.178019822826, 0.270901895834, -0.180136838054, 0.201151867097, -0.269229552029, 0.162901821652, 0.176989876451, -0.0562545583224, 0.302891594425, -0.107643708081, 0.124875778612, -0.0717199493045, -0.256875036503, -0.104361508745, 0.0223166114387, -0.07222004733
6, 0.048066243185, 0.0917036420782, -0.0401521607798, -0.166103131, -0.0598667085507, 0.226655306977, 0.196020672052, -0.0019893331732, 0.298047303143, 0.0610271888562, -0.123756854539, -0.138425394609, -0.0813844369306, -0.211061219263, 0.280266006616, -0.456958897687, 0.0957847778583, 0.10211765487, -0.280257589355, 0.31403698235, -0.0988827746221, -0.315188453188, 0.0323558432208, 0.30938173242, -0.337991301037, -0.0746337727327, -0.128019139853, -0.107135213031, 0.401895277966, -0.0856407552996, 0.338660876519, 0.182709918475, 0.00468663392494, 0.296164404708, -0.149328993889, -0.455192142697, 0.123695517097, -0.101042506058, -0.299448111606, -0.121418328349, -0.015253490999, 0.105945563874, 0.206274179032, 0.0110203190195, -0.0198463256646, -0.122433280535, 0.159641322926, 0.0111190972641, 0.146024395139, -0.3292631924, 0.383620965193, -0.407475374511, 0.124656819426, 0.00814682875215, -0.196043525121, 0.259447828442, -0.0110058837001, -0.0911647683094, -0.495353629738, 0.15832
4486446, 0.284649054294, 0.159015240494, 0.127990452507, -0.0450611236751, 0.100307630204, -0.178426360702, 0.0795949564344, -0.0935834137435, 0.0121807964771, 0.13280000068, -0.0780024566724, 0.268183360375, -0.143863232988, 0.229637060781, 0.143828509532, 0.0467453366612, -0.101445117614, 0.110812332279, 0.274695568803, 0.0886704115438, 0.0920752574129, 0.230045146137, 0.189557642894, 0.175650286388, 0.612810225963, 0.445301543545, -0.115078741068, 0.157970457258, -0.0479338968094, 0.212561173909, 0.180880194985, -0.0870642554799, 0.142339596596, -0.151003784829, 0.0688697053283, -0.128056785862, 0.0793206934645, -0.118269324067, -0.0332473245304, -0.311866015062, -0.117922286012, 0.138892683591, -0.13204744327, 0.0179514948766, -0.019965660049, 0.132229635306, -0.0313989569283, -0.130315567213, 0.0845857500725, -0.159680095558, -0.0112748210112, -0.00992759022784, 0.164452997275, 0.0322838283196, -0.270242562524, 0.162183925006, -0.0574346808757, -0.22116233726, -0.047400722579,
0.00863094243228, 0.211311350784, -0.163996331401, -0.0477683759558, -0.124856946878, -0.00714217581038, -0.266411668298, 0.313991233853, -0.225971650736, 0.161591101292, -0.188041216058, -0.214157824743, 0.118347597707, 0.00614700038529, -0.16123423931, -0.102840533896, -0.102239903301, -0.000257003830486, 0.0511787840226, 0.0912252762164, -0.0751847257127, 0.0846024895256, -0.157788057275, 0.0988942818557, 0.0264955057078, 0.0417793769197, -0.0540072711509, -0.00632082438876, -0.180617287286, -0.0526100222044, 0.118036208602, -0.000661796862675, -0.254004752387, -0.153693011234, -0.171988783244, 0.164053555891, -0.0169405183209, 0.142598359067, -0.085909030833, -0.00680028505823, -0.0510574641112, -0.10346364174, -0.25172618999, 0.457311106971, -0.873025286844, 0.183853172995, -0.124620606266, -0.123278406783, -0.191666220417, 0.159077818728, 0.284815904677, -0.735489991194, 0.351899702528, -0.235562757335, -0.0727504017353, 0.040328813679, -0.125064567205, 0.314741875925, 0.03317
54869107, 0.0884781096995, -0.178033350198, 0.116105414459, -0.0304993356093, -0.27746854463, 0.264004909184, 0.0563022180518, 0.201355642027, -0.0517272948481, 0.0748041423242, -0.0262047282685, -0.092904739373, 0.115647917976, 0.0175972187488, 0.0193950400413, 0.140792231397, -0.00159044852904, -0.00590168585226, -0.139502824903, 0.0410346350032, 0.0413657977828, -0.0633203110196, 0.111349814682, 0.00747228750879, 0.140373814921, 0.102221513338, -0.116981487287, -0.0693222177589, 0.0413462607849, -0.105398620292, -0.143032508267, -0.0178459843317, -0.0731008179726, 0.103677863645, 0.245558735393, 0.0864260843503, 0.180994293221, 0.0917188124414, -0.0121479315063, -0.0382354365149, 0.07547910454, -0.165737544578, 0.0807115289326, -0.134231077856, -0.137895944349, -0.23171311458, -0.163296061142, 0.116367060077, -0.275327082108, 0.000654881341552, -0.101458765282, -0.230568790277, 0.00328713579536, 0.121407554293, 0.270474969032, -0.0332705761859, 0.195253595642, 0.202199456653, 0.0
240241261406, -0.117138257744, 0.262847956454, 0.167488405552, 0.0791663226224, -0.0850169677771, 0.127015057297, 0.0881663075291, 0.157579936939, 0.0790931902296, -0.0350584878084, 0.0687973505113, 0.127800875594, 0.0990169958988, 0.261759911012, -0.0209958907533, 0.178141387084, 0.0802477036446, 0.0375667573926, -0.156527532652, 0.09595744853, 0.144071902168, 0.0272982692838, -0.107756789849, 0.191712610983, 0.131812031368, -0.150147659255, -0.0193064462346, 0.248057351446, 0.132636431219, -0.0656579288641, -0.0102056610199, 0.166573513737, -0.089580189987, -0.13267377489, 0.00765881272357, -0.0294600407492, 0.109945889046, 0.127286865292, -0.103663816189, -0.13701947915, -0.140194114889, -0.00273323080925, 0.193293649137, 0.111701285793, -0.00816508369269, -0.128614542133, 0.0578130660316, 0.0262319837289, 0.00944211414361, 0.0349982062763, -0.0497391325644, -0.0684296167909, 0.133638380272, -0.00513533046615, -0.113005170272, 0.0371435244753, 0.00685550220379, 0.0345504421305, 0
.0291314878, -0.0963381594242, 0.113578331339, -0.148312320923, -0.139277275139, 0.145797861416, -0.115402530254, -0.0141028671065, -0.0964248545817, 0.109089742884, -0.0746598414841, -0.0469864710904, 0.130259197266, -0.0537714811321, 0.110238509759, 0.0570749324965, -0.159819021215, -0.0835962095075, 0.0959104137424, 0.0437050342893, 0.116577502313, -0.0661250670803, -0.0961631710138, -0.103486980341, 0.0392920895058, -0.134266639255, 0.0601833600592, 0.0995456197992, 0.0780624339557, -0.103283278004, -0.0337175173846, -0.0631874494788, -0.021864548369, -0.0910979985263, -0.0393805762934, 0.0124725373216, -0.118269891962, 0.0190946263827, -0.167315972761, -0.041287897759, -0.0402292813632, -0.124074804749, 0.0743012733141, -0.0889323559481, 0.0291352210856, 0.0152283683786, 0.137832612062, 0.0299226271789, 0.0973969148803, 0.0455620046357, -0.0690846277006, 0.0463525595754, -0.170969427729, -0.00530098180334, 0.0860765294519, 0.0200025350096, 0.0408217087641, 0.000674906042282, -0
.0484522805343, -0.063904387141, 0.0680186364286, 0.0956646181429, 0.0211126410967, -0.135956703385, -0.173184730333, -0.153592609448, 0.0515420758792, -0.106504865816, 0.0427090308662, 0.0588571345969, 0.038628020038, -0.167495451323, -0.171995990523, -0.0746036469661, -0.0893353944191, 0.0769671432324, 0.129290424004, -0.115018423904, -0.0608217250046, 0.0576405168275, 0.138811008142, 0.0484048069161, -0.0648970775095, 0.12408920527, -0.116670120942, -0.11461248868, -0.000556289750656, 0.113501171408, 0.073011667822, 0.157085701474, -0.103935971103, -0.0285974926781, 0.0668750198788, 0.495352461847, 0.102622017207, 0.343493376056, 0.356233832875, -0.0474178679926, 0.143445828453, 0.276335059268, 0.396289343536, 0.131457852813, 0.0343149415343, 0.343782464698, 0.327289025174, 0.176853423997, 0.24349898742, 0.149069604566, 0.418153967013, 0.235895507356, 0.033694810931, 0.267228687328, 0.0814095665435, 0.120164220897, 0.462571446373, 0.222535013115, 0.11424010583, -0.0906197141098,
0.129458603213, 0.108931721147, -0.0890778578148, 0.191923971562, 0.17901983388, 0.151099630479, 0.0566764331123, 0.0489030212844, 0.073160549313, 0.085809192495, 0.0485886118323, 0.0215963694964, -0.0834544297183, 0.0327265042333, -0.0931091459474, 0.112069055149, 0.043039352435, 0.130235543317, 0.0299101055543, -0.149377273183, 0.00646975988585, -0.000554735025032, -0.0831731375459, 0.13563300492, -0.0165566474276, 0.0485383902634, 0.00888025767832, -0.139154446785, -0.0353843799165, -0.0871916612905, -0.04686546779, -0.166337955736, 0.0750002808334, 0.0168899088843, 0.104283523831, 0.0187089188179, -0.0251945516132, -0.0131352016429, 0.170830441379, -0.154809267796, -0.155763940099, 0.0443056586881, 0.0274827557274, -0.0977496234606, -0.0970180773028, 0.0350993020096, 0.146745061397, 0.078815475605, -0.0389907438712, -0.0335619958692, 0.118211074784, 0.152649314105, 0.147007119673, 0.103915506114, 0.0890045904999, -0.0362673918145, 0.0113676661504, -0.0975364965898, -0.0103831091
45, 0.179494474761, -0.00826024925394, 0.139510966287, 0.132164679775, -0.133918793256, -0.00635655788897, 0.0767882832999, 0.10726612898, 0.142674161454, -0.0883857724845, 0.0855403411171, 0.165220673799, -0.112104374468, -0.0380395831084, 0.00501932226928, 0.00263269004584, 0.0356991274653, -0.113333317511, -0.0124000028152, -0.0515325939146, 0.373876728674, 0.207596737771, 0.0813373842185, 0.183371106337, 0.165925307432, -0.135548407527, 0.12660809146, 0.0536288727356, 0.147716796788, 0.0568205245395, 0.197828755587, 0.176202897769, -0.101477342881, 0.159869918913, 0.0979205147731, 0.0746576092262, -0.103825525431, 0.00928188760788, -0.0504819433487, 0.191625701797, 0.0852383221501, 0.358258019876, 0.227735982591, 0.176158268625, -0.111316520456, 0.0455282389086, -0.0614696960162, 0.0843890679754, 0.0463263728094, 0.119850225612, -0.11927794611, -0.0713883054891, 0.0924825570733, -0.0624807055815, 0.115190061413, 0.0844764278027, -0.0132342347732, -0.0849093011178, 0.152923763906
, -0.148325245058, 0.105231772707, 0.0283105691055, 0.0720970891556, 0.132436487291, -0.10922576116, -0.0815515390022, -0.114722256636, -0.170915126812, 0.0175996345112, -0.124412352668, -0.13304515302, -0.106477616928, 0.120199855771, -0.174671087798, -0.122672321399, -0.144846427759, -0.0154300950821, 0.0507292688075, 0.126551952929, -0.143439054826, 0.0299006158402, -0.15005716624, -0.0401343045517, -0.00252760290587, -0.0237907281236, -0.0797753025531, -0.0991644050106, 0.134308019727, 0.0200117589182, -0.112084247031, -0.167995452395, -0.0295671374508, 0.107351765084, 0.0632551569846, -0.0582422444596, 0.16366750329, -0.162479674609, 0.00214073446262, -0.103278921502, 0.0558777777097, 0.0723732610543, -0.0613630222551, 0.135128811096, 0.050568989534, -0.0546392828733, -0.153971534091, 0.0924751592423, 0.101706791108, 0.0607050649968, 0.0596870445781, 0.136883549105, -0.0637011610473, 0.0871060796234, -0.0681990890046, 0.104202328594, -0.108179302086, 0.0223725516522, -0.1555482
26486, -0.141998314601, -0.122743124803, -0.0875520919781, -0.125286855988, 0.0307176179015, -0.153042075499, 0.23696229498, 0.177058796447, -0.082108841359, 0.20329677785, 0.00175169627491, -0.0846871909768, 0.099835299916, -0.0481316144897, 0.0661170188435, -0.00346217036623, 0.161053868398, 0.207674939326, 0.0281098351705, 0.0860739426955, 0.105192953584, -0.0285058349374, -0.0580393494223, 0.0714749978916, 0.0512850741134, -0.152503380977, 0.168472501535, 0.0485058104422, -0.0672229715629, 0.0117263745766, -0.0365743846743, -0.0412353731283, 0.0291119464053, 0.0441411422804, -0.0316612818875, -0.087567406123, 0.063411170627, -0.0589250368343, -0.107599666557, 0.081908821462, -0.0935361527796, 0.0577467210631, -0.168351581892, 0.11573638204, 0.115117344323, 0.0244735425375, -0.070952849692, 0.0485741149425, 0.00733481525825, 0.116528669326, -0.090380421955, -0.0854344432891, -0.148897840337, -0.00584468517666, 0.125584338188, 0.0303923394942, 0.00866505023759, 0.107045771132, 0.1
04507667791, 0.0759993673344, -0.0560582027212, -0.0601905601019, -0.162052677836, -0.165142098544, -0.171051382284, -0.00747175762698, -0.0725351668202, 0.0574896106535, 0.0883795833561, -0.00115905040131, -0.0208072413433, -0.146708687264, -0.106512542328, -0.0148684889684, 0.135203007488, -0.159366976488, -0.185727272921, -0.0986049216812, 0.0663781391027, 0.00337193126349, -0.143224040776, 0.151377444068, 0.122900039992, -0.123876153218, -0.0616559523895, 0.07474867887, 0.0759827169541, -0.196835030676, 0.0240963945802, 0.0815042565879, 0.0813777417882, 0.122173760283, -0.149634521852, 0.0950858936023, 0.147034831794, -0.143746034738, -0.0783443886749, -0.100895898859, 0.090389376971, -0.163667864143, 0.0799367976539, -0.080443647121, -0.112871360843, 0.147539108095, 0.0899145483935, -0.161477148966, 0.138928943053, 0.110902779082, -0.0666307526333, 0.0729760796573, 0.0290997225314, -0.0156054718084, 0.111971399191, -0.0448196144982, 0.0547285642038, -0.12753315933, -0.144334282
657, -0.0106939193609, -0.0986907128641, 0.0600266162345, -0.120230639837, 0.16669020756, 0.0439133843093, -0.0724689439423, 0.0629139032688, 0.0504403844796, -0.068271656679, 0.156640645958, 0.0919725761945, -0.127266025876, -0.159289153952, 0.00546200555327, -0.0223831026984, -0.0931915620749, -0.0163207716589, -0.12419155542, 0.116520477065, 0.0153427362007, -0.129060892518, 0.157096091442, -0.129280966926, -0.0137527695116, -0.0171380709271, 0.195707287676, 0.142599751827, 0.072152904103, 0.252011620265, 0.207324038903, -0.125405121498, -0.100013512174, 0.0583275733665, -0.0146841391113, 0.196379045132, -0.0875501697175, 0.078958660421, 0.0740925533203, 0.0902143324496, 0.29527471964, 0.073563228215, -0.0180613309394, 0.115906200451, 0.0786093503948, 1.306422046, 0.137688222641, 0.756774881139, 0.496403150097, 0.0972268426637, -0.1337106213, 0.739166706063, 0.550671589035, 0.441561444758, 0.231081592497, 0.622412508822, 0.644482827381, 0.282956642749, 0.587122887501, 0.276576675
538, 0.65481113446, 0.229873153345, 0.271528882808, 0.506621646929, 0.00565366103103, 0.282388297824, 1.02057546753, 0.217085174554, -0.0379775424244, -0.10514999404, 0.332566472108, 0.332449111037, 0.124367878135, 0.0141927018472, 0.15855771012, -0.038603288579, -0.135492381545, 0.254667963224, -0.0234027088103, 0.0593833730872, -0.0644732316305, -0.00845219720435, 0.028657566833, 0.0936179669519, 0.254740895255, -0.0461609993025, 0.120413358916, 0.138933025039, 0.196582134527, -0.0400133811542, -0.040323701499, 0.0581573904481, 0.23554382053, 0.192417275699, -0.0776195707746, 0.0592937520315, 0.089886264069, 0.0539065543803, 0.09886462098, 0.0667640588722, -0.18084306698, 0.0222659539242, 0.0403979776829, 0.0775484353974, 0.0791672960253, -0.0899907599431, -0.150064269247, 0.124833764386, -0.120770820436, -0.114178974003, -0.0639780687205, -0.0336269431702, -0.0608694034386, 0.0467895651684, -0.141267333139, -0.064375264314, 0.121018439709, 0.0366265122228, -0.154133636165, 0.0859
110581911, 0.0756296178076, -0.0424145003869, -0.0456028602463, -0.0407671725434, -0.0791181207568, 0.0288097129653, -0.151513578519, 0.138406753718, -0.0233697746503, 0.127112027349, 0.0588020231249, -0.015333084669, -0.0097756075345, -0.0715451307032, -0.0949891422502, -0.0875247946453, 0.0222796529221, -0.0696183219033, -0.128883685169, 0.0827929784601, -0.00779955596619, -0.00487182386305, -0.106664395522, 0.123694800488, -0.129353816579, -0.071810234853, -0.110089746206, -0.171366525529, 0.136882534904, -0.111206506502, 0.0982967459967, 0.0298035370415, 0.0446501099772, -0.117237913284, -0.156925357628, 0.13528309057, 0.0847468661361, -0.150103993267, 0.117264719501, -0.114274591469, 0.157554316643, 0.00546812137529, 0.038074635687, -0.0263940535714, 0.101334013433, 0.128183877645, 0.0670206279241, -0.0442752598958, -0.109288889221, 0.111823141465, -0.148855288965, 0.0475096756177, -0.0695066497011, -0.0537726255674, 0.000632083516048, 1.57413626503, 0.55559495427, 0.3146391992
95, 0.0363237197077, 0.0298504461036, -0.224219935066, -0.157329470601, 1.21592745382, 0.234224608374, -0.0740354122049, 0.165430831517, 0.66674980751, -0.12077633358, -0.0337960848896, 0.00708649021609, 0.293365487636, 0.000863544472662, -0.199669889395, -0.210671570015, 0.180015272246, 0.467349281518, 2.58471060843, 0.516690045711, -0.0632937912186, -0.15288034917, -0.0418765986663]</td>\n",
- " <td>1.48283239226</td>\n",
- " <td>500</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[([0.122856566817, -0.109511383377, 0.10922147183, 0.142820008065, 0.162756253894, -0.104436945183, 0.112036676092, 0.0467338442275, -0.16453372472, 0.0827992259598, 0.0137404038273, 0.0565353617298, -0.0246679212553, -0.042838429926, 0.381777598727, 0.276073801059, 0.00907788148497, 0.21191870498, -0.141702416989, 0.181936732347, -0.136061155165, -0.721381760449, -0.0560067549982, 0.203545531143, -0.00995912442451, 0.0613143120688, -0.0984596213056, -0.204436524678, 0.470767785122, -0.435311003078, -0.0424606276832, 0.160892203008, 0.0445352965619, -0.0879636030809, 0.154146037853, 0.0288780423027, -0.317282313525, 0.385922416222, -0.0664872317641, 0.146757511446, 0.133876976675, -0.116256543538, -0.0213680210249, 0.20015749221, -0.103457503483, -0.0425956948505, 0.121667463986, 0.204968623833, -0.159914333708, 0.149243857206, 0.0456901480449, -0.136975914168, -0.090899493726, 0.103816779262, 0.157303539757, -0.0813411072267, -0.186218530515, -0.18296662406, -0.010627334433
5, -0.15268221771, -0.103478313334, 0.199984721908, 0.140135789025, 0.143966006971, -0.0970102681415, 0.0261086412469, -0.0330555393771, 0.0995076031524, 0.00130931383947, 0.0778756667869, -0.0527086470232, -0.600133053147, 0.0859133658584, 0.24113974288, 0.170332657512, 0.0676027349796, -0.21148456397, 0.248468295235, -0.436384189411, 0.175863820799, 0.123778486759, -0.282581458533, 0.0191701197018, 0.280887575877, 0.21926286976, -0.24666316945, -0.0262117477515, 0.268417193439, -0.0591642733853, 0.168335924156, 0.567262570173, -0.0244295268223, -0.161746761037, 0.239464436128, 0.0927106954455, 0.108399901337, -0.00684509495191, 0.0942018933997, 0.27681768162, -0.0200546475364, -0.0178865707443, 0.0590165895996, 0.178019822826, 0.270901895834, -0.180136838054, 0.201151867097, -0.269229552029, 0.162901821652, 0.176989876451, -0.0562545583224, 0.302891594425, -0.107643708081, 0.124875778612, -0.0717199493045, -0.256875036503, -0.104361508745, 0.0223166114387, -0.072220047336, 0.04806
6243185, 0.0917036420782, -0.0401521607798, -0.166103131, -0.0598667085507, 0.226655306977, 0.196020672052, -0.0019893331732, 0.298047303143, 0.0610271888562, -0.123756854539, -0.138425394609, -0.0813844369306, -0.211061219263, 0.280266006616, -0.456958897687, 0.0957847778583, 0.10211765487, -0.280257589355, 0.31403698235, -0.0988827746221, -0.315188453188, 0.0323558432208, 0.30938173242, -0.337991301037, -0.0746337727327, -0.128019139853, -0.107135213031, 0.401895277966, -0.0856407552996, 0.338660876519, 0.182709918475, 0.00468663392494, 0.296164404708, -0.149328993889, -0.455192142697, 0.123695517097, -0.101042506058, -0.299448111606, -0.121418328349, -0.015253490999, 0.105945563874, 0.206274179032, 0.0110203190195, -0.0198463256646, -0.122433280535, 0.159641322926, 0.0111190972641, 0.146024395139, -0.3292631924, 0.383620965193, -0.407475374511, 0.124656819426, 0.00814682875215, -0.196043525121, 0.259447828442, -0.0110058837001, -0.0911647683094, -0.495353629738, 0.158324486446, 0
.284649054294, 0.159015240494, 0.127990452507, -0.0450611236751, 0.100307630204, -0.178426360702, 0.0795949564344, -0.0935834137435, 0.0121807964771, 0.13280000068, -0.0780024566724, 0.268183360375, -0.143863232988, 0.229637060781, 0.143828509532, 0.0467453366612, -0.101445117614, 0.110812332279, 0.274695568803, 0.0886704115438, 0.0920752574129, 0.230045146137, 0.189557642894, 0.175650286388, 0.612810225963, 0.445301543545, -0.115078741068, 0.157970457258, -0.0479338968094, 0.212561173909, 0.180880194985, -0.0870642554799, 0.142339596596, -0.151003784829, 0.0688697053283, -0.128056785862, 0.0793206934645, -0.118269324067, -0.0332473245304, -0.311866015062, -0.117922286012, 0.138892683591, -0.13204744327, 0.0179514948766, -0.019965660049, 0.132229635306, -0.0313989569283, -0.130315567213, 0.0845857500725, -0.159680095558, -0.0112748210112, -0.00992759022784, 0.164452997275, 0.0322838283196, -0.270242562524, 0.162183925006, -0.0574346808757, -0.22116233726, -0.047400722579, 0.00863094
243228, 0.211311350784, -0.163996331401, -0.0477683759558, -0.124856946878, -0.00714217581038, -0.266411668298, 0.313991233853, -0.225971650736, 0.161591101292, -0.188041216058, -0.214157824743, 0.118347597707, 0.00614700038529, -0.16123423931, -0.102840533896, -0.102239903301, -0.000257003830486, 0.0511787840226, 0.0912252762164, -0.0751847257127, 0.0846024895256, -0.157788057275, 0.0988942818557, 0.0264955057078, 0.0417793769197, -0.0540072711509, -0.00632082438876, -0.180617287286, -0.0526100222044, 0.118036208602, -0.000661796862675, -0.254004752387, -0.153693011234, -0.171988783244, 0.164053555891, -0.0169405183209, 0.142598359067, -0.085909030833, -0.00680028505823, -0.0510574641112, -0.10346364174, -0.25172618999, 0.457311106971, -0.873025286844, 0.183853172995, -0.124620606266, -0.123278406783, -0.191666220417, 0.159077818728, 0.284815904677, -0.735489991194, 0.351899702528, -0.235562757335, -0.0727504017353, 0.040328813679, -0.125064567205, 0.314741875925, 0.0331754869107,
0.0884781096995, -0.178033350198, 0.116105414459, -0.0304993356093, -0.27746854463, 0.264004909184, 0.0563022180518, 0.201355642027, -0.0517272948481, 0.0748041423242, -0.0262047282685, -0.092904739373, 0.115647917976, 0.0175972187488, 0.0193950400413, 0.140792231397, -0.00159044852904, -0.00590168585226, -0.139502824903, 0.0410346350032, 0.0413657977828, -0.0633203110196, 0.111349814682, 0.00747228750879, 0.140373814921, 0.102221513338, -0.116981487287, -0.0693222177589, 0.0413462607849, -0.105398620292, -0.143032508267, -0.0178459843317, -0.0731008179726, 0.103677863645, 0.245558735393, 0.0864260843503, 0.180994293221, 0.0917188124414, -0.0121479315063, -0.0382354365149, 0.07547910454, -0.165737544578, 0.0807115289326, -0.134231077856, -0.137895944349, -0.23171311458, -0.163296061142, 0.116367060077, -0.275327082108, 0.000654881341552, -0.101458765282, -0.230568790277, 0.00328713579536, 0.121407554293, 0.270474969032, -0.0332705761859, 0.195253595642, 0.202199456653, 0.02402412614
06, -0.117138257744, 0.262847956454, 0.167488405552, 0.0791663226224, -0.0850169677771, 0.127015057297, 0.0881663075291, 0.157579936939, 0.0790931902296, -0.0350584878084, 0.0687973505113, 0.127800875594, 0.0990169958988, 0.261759911012, -0.0209958907533, 0.178141387084, 0.0802477036446, 0.0375667573926, -0.156527532652, 0.09595744853, 0.144071902168, 0.0272982692838, -0.107756789849, 0.191712610983, 0.131812031368, -0.150147659255, -0.0193064462346, 0.248057351446, 0.132636431219, -0.0656579288641, -0.0102056610199, 0.166573513737, -0.089580189987, -0.13267377489, 0.00765881272357, -0.0294600407492, 0.109945889046, 0.127286865292, -0.103663816189, -0.13701947915, -0.140194114889, -0.00273323080925, 0.193293649137, 0.111701285793, -0.00816508369269, -0.128614542133, 0.0578130660316, 0.0262319837289, 0.00944211414361, 0.0349982062763, -0.0497391325644, -0.0684296167909, 0.133638380272, -0.00513533046615, -0.113005170272, 0.0371435244753, 0.00685550220379, 0.0345504421305, 0.029131487
8, -0.0963381594242, 0.113578331339, -0.148312320923, -0.139277275139, 0.145797861416, -0.115402530254, -0.0141028671065, -0.0964248545817, 0.109089742884, -0.0746598414841, -0.0469864710904, 0.130259197266, -0.0537714811321, 0.110238509759, 0.0570749324965, -0.159819021215, -0.0835962095075, 0.0959104137424, 0.0437050342893, 0.116577502313, -0.0661250670803, -0.0961631710138, -0.103486980341, 0.0392920895058, -0.134266639255, 0.0601833600592, 0.0995456197992, 0.0780624339557, -0.103283278004, -0.0337175173846, -0.0631874494788, -0.021864548369, -0.0910979985263, -0.0393805762934, 0.0124725373216, -0.118269891962, 0.0190946263827, -0.167315972761, -0.041287897759, -0.0402292813632, -0.124074804749, 0.0743012733141, -0.0889323559481, 0.0291352210856, 0.0152283683786, 0.137832612062, 0.0299226271789, 0.0973969148803, 0.0455620046357, -0.0690846277006, 0.0463525595754, -0.170969427729, -0.00530098180334, 0.0860765294519, 0.0200025350096, 0.0408217087641, 0.000674906042282, -0.048452280
5343, -0.063904387141, 0.0680186364286, 0.0956646181429, 0.0211126410967, -0.135956703385, -0.173184730333, -0.153592609448, 0.0515420758792, -0.106504865816, 0.0427090308662, 0.0588571345969, 0.038628020038, -0.167495451323, -0.171995990523, -0.0746036469661, -0.0893353944191, 0.0769671432324, 0.129290424004, -0.115018423904, -0.0608217250046, 0.0576405168275, 0.138811008142, 0.0484048069161, -0.0648970775095, 0.12408920527, -0.116670120942, -0.11461248868, -0.000556289750656, 0.113501171408, 0.073011667822, 0.157085701474, -0.103935971103, -0.0285974926781, 0.0668750198788, 0.495352461847, 0.102622017207, 0.343493376056, 0.356233832875, -0.0474178679926, 0.143445828453, 0.276335059268, 0.396289343536, 0.131457852813, 0.0343149415343, 0.343782464698, 0.327289025174, 0.176853423997, 0.24349898742, 0.149069604566, 0.418153967013, 0.235895507356, 0.033694810931, 0.267228687328, 0.0814095665435, 0.120164220897, 0.462571446373, 0.222535013115, 0.11424010583, -0.0906197141098, 0.12945860
3213, 0.108931721147, -0.0890778578148, 0.191923971562, 0.17901983388, 0.151099630479, 0.0566764331123, 0.0489030212844, 0.073160549313, 0.085809192495, 0.0485886118323, 0.0215963694964, -0.0834544297183, 0.0327265042333, -0.0931091459474, 0.112069055149, 0.043039352435, 0.130235543317, 0.0299101055543, -0.149377273183, 0.00646975988585, -0.000554735025032, -0.0831731375459, 0.13563300492, -0.0165566474276, 0.0485383902634, 0.00888025767832, -0.139154446785, -0.0353843799165, -0.0871916612905, -0.04686546779, -0.166337955736, 0.0750002808334, 0.0168899088843, 0.104283523831, 0.0187089188179, -0.0251945516132, -0.0131352016429, 0.170830441379, -0.154809267796, -0.155763940099, 0.0443056586881, 0.0274827557274, -0.0977496234606, -0.0970180773028, 0.0350993020096, 0.146745061397, 0.078815475605, -0.0389907438712, -0.0335619958692, 0.118211074784, 0.152649314105, 0.147007119673, 0.103915506114, 0.0890045904999, -0.0362673918145, 0.0113676661504, -0.0975364965898, -0.010383109145, 0.1794
94474761, -0.00826024925394, 0.139510966287, 0.132164679775, -0.133918793256, -0.00635655788897, 0.0767882832999, 0.10726612898, 0.142674161454, -0.0883857724845, 0.0855403411171, 0.165220673799, -0.112104374468, -0.0380395831084, 0.00501932226928, 0.00263269004584, 0.0356991274653, -0.113333317511, -0.0124000028152, -0.0515325939146, 0.373876728674, 0.207596737771, 0.0813373842185, 0.183371106337, 0.165925307432, -0.135548407527, 0.12660809146, 0.0536288727356, 0.147716796788, 0.0568205245395, 0.197828755587, 0.176202897769, -0.101477342881, 0.159869918913, 0.0979205147731, 0.0746576092262, -0.103825525431, 0.00928188760788, -0.0504819433487, 0.191625701797, 0.0852383221501, 0.358258019876, 0.227735982591, 0.176158268625, -0.111316520456, 0.0455282389086, -0.0614696960162, 0.0843890679754, 0.0463263728094, 0.119850225612, -0.11927794611, -0.0713883054891, 0.0924825570733, -0.0624807055815, 0.115190061413, 0.0844764278027, -0.0132342347732, -0.0849093011178, 0.152923763906, -0.14832
5245058, 0.105231772707, 0.0283105691055, 0.0720970891556, 0.132436487291, -0.10922576116, -0.0815515390022, -0.114722256636, -0.170915126812, 0.0175996345112, -0.124412352668, -0.13304515302, -0.106477616928, 0.120199855771, -0.174671087798, -0.122672321399, -0.144846427759, -0.0154300950821, 0.0507292688075, 0.126551952929, -0.143439054826, 0.0299006158402, -0.15005716624, -0.0401343045517, -0.00252760290587, -0.0237907281236, -0.0797753025531, -0.0991644050106, 0.134308019727, 0.0200117589182, -0.112084247031, -0.167995452395, -0.0295671374508, 0.107351765084, 0.0632551569846, -0.0582422444596, 0.16366750329, -0.162479674609, 0.00214073446262, -0.103278921502, 0.0558777777097, 0.0723732610543, -0.0613630222551, 0.135128811096, 0.050568989534, -0.0546392828733, -0.153971534091, 0.0924751592423, 0.101706791108, 0.0607050649968, 0.0596870445781, 0.136883549105, -0.0637011610473, 0.0871060796234, -0.0681990890046, 0.104202328594, -0.108179302086, 0.0223725516522, -0.155548226486, -0.
141998314601, -0.122743124803, -0.0875520919781, -0.125286855988, 0.0307176179015, -0.153042075499, 0.23696229498, 0.177058796447, -0.082108841359, 0.20329677785, 0.00175169627491, -0.0846871909768, 0.099835299916, -0.0481316144897, 0.0661170188435, -0.00346217036623, 0.161053868398, 0.207674939326, 0.0281098351705, 0.0860739426955, 0.105192953584, -0.0285058349374, -0.0580393494223, 0.0714749978916, 0.0512850741134, -0.152503380977, 0.168472501535, 0.0485058104422, -0.0672229715629, 0.0117263745766, -0.0365743846743, -0.0412353731283, 0.0291119464053, 0.0441411422804, -0.0316612818875, -0.087567406123, 0.063411170627, -0.0589250368343, -0.107599666557, 0.081908821462, -0.0935361527796, 0.0577467210631, -0.168351581892, 0.11573638204, 0.115117344323, 0.0244735425375, -0.070952849692, 0.0485741149425, 0.00733481525825, 0.116528669326, -0.090380421955, -0.0854344432891, -0.148897840337, -0.00584468517666, 0.125584338188, 0.0303923394942, 0.00866505023759, 0.107045771132, 0.10450766779
1, 0.0759993673344, -0.0560582027212, -0.0601905601019, -0.162052677836, -0.165142098544, -0.171051382284, -0.00747175762698, -0.0725351668202, 0.0574896106535, 0.0883795833561, -0.00115905040131, -0.0208072413433, -0.146708687264, -0.106512542328, -0.0148684889684, 0.135203007488, -0.159366976488, -0.185727272921, -0.0986049216812, 0.0663781391027, 0.00337193126349, -0.143224040776, 0.151377444068, 0.122900039992, -0.123876153218, -0.0616559523895, 0.07474867887, 0.0759827169541, -0.196835030676, 0.0240963945802, 0.0815042565879, 0.0813777417882, 0.122173760283, -0.149634521852, 0.0950858936023, 0.147034831794, -0.143746034738, -0.0783443886749, -0.100895898859, 0.090389376971, -0.163667864143, 0.0799367976539, -0.080443647121, -0.112871360843, 0.147539108095, 0.0899145483935, -0.161477148966, 0.138928943053, 0.110902779082, -0.0666307526333, 0.0729760796573, 0.0290997225314, -0.0156054718084, 0.111971399191, -0.0448196144982, 0.0547285642038, -0.12753315933, -0.144334282657, -0.01
06939193609, -0.0986907128641, 0.0600266162345, -0.120230639837, 0.16669020756, 0.0439133843093, -0.0724689439423, 0.0629139032688, 0.0504403844796, -0.068271656679, 0.156640645958, 0.0919725761945, -0.127266025876, -0.159289153952, 0.00546200555327, -0.0223831026984, -0.0931915620749, -0.0163207716589, -0.12419155542, 0.116520477065, 0.0153427362007, -0.129060892518, 0.157096091442, -0.129280966926, -0.0137527695116, -0.0171380709271, 0.195707287676, 0.142599751827, 0.072152904103, 0.252011620265, 0.207324038903, -0.125405121498, -0.100013512174, 0.0583275733665, -0.0146841391113, 0.196379045132, -0.0875501697175, 0.078958660421, 0.0740925533203, 0.0902143324496, 0.29527471964, 0.073563228215, -0.0180613309394, 0.115906200451, 0.0786093503948, 1.306422046, 0.137688222641, 0.756774881139, 0.496403150097, 0.0972268426637, -0.1337106213, 0.739166706063, 0.550671589035, 0.441561444758, 0.231081592497, 0.622412508822, 0.644482827381, 0.282956642749, 0.587122887501, 0.276576675538, 0.654
81113446, 0.229873153345, 0.271528882808, 0.506621646929, 0.00565366103103, 0.282388297824, 1.02057546753, 0.217085174554, -0.0379775424244, -0.10514999404, 0.332566472108, 0.332449111037, 0.124367878135, 0.0141927018472, 0.15855771012, -0.038603288579, -0.135492381545, 0.254667963224, -0.0234027088103, 0.0593833730872, -0.0644732316305, -0.00845219720435, 0.028657566833, 0.0936179669519, 0.254740895255, -0.0461609993025, 0.120413358916, 0.138933025039, 0.196582134527, -0.0400133811542, -0.040323701499, 0.0581573904481, 0.23554382053, 0.192417275699, -0.0776195707746, 0.0592937520315, 0.089886264069, 0.0539065543803, 0.09886462098, 0.0667640588722, -0.18084306698, 0.0222659539242, 0.0403979776829, 0.0775484353974, 0.0791672960253, -0.0899907599431, -0.150064269247, 0.124833764386, -0.120770820436, -0.114178974003, -0.0639780687205, -0.0336269431702, -0.0608694034386, 0.0467895651684, -0.141267333139, -0.064375264314, 0.121018439709, 0.0366265122228, -0.154133636165, 0.0859110581911,
0.0756296178076, -0.0424145003869, -0.0456028602463, -0.0407671725434, -0.0791181207568, 0.0288097129653, -0.151513578519, 0.138406753718, -0.0233697746503, 0.127112027349, 0.0588020231249, -0.015333084669, -0.0097756075345, -0.0715451307032, -0.0949891422502, -0.0875247946453, 0.0222796529221, -0.0696183219033, -0.128883685169, 0.0827929784601, -0.00779955596619, -0.00487182386305, -0.106664395522, 0.123694800488, -0.129353816579, -0.071810234853, -0.110089746206, -0.171366525529, 0.136882534904, -0.111206506502, 0.0982967459967, 0.0298035370415, 0.0446501099772, -0.117237913284, -0.156925357628, 0.13528309057, 0.0847468661361, -0.150103993267, 0.117264719501, -0.114274591469, 0.157554316643, 0.00546812137529, 0.038074635687, -0.0263940535714, 0.101334013433, 0.128183877645, 0.0670206279241, -0.0442752598958, -0.109288889221, 0.111823141465, -0.148855288965, 0.0475096756177, -0.0695066497011, -0.0537726255674, 0.000632083516048, 1.57413626503, 0.55559495427, 0.314639199295, 0.0363
237197077, 0.0298504461036, -0.224219935066, -0.157329470601, 1.21592745382, 0.234224608374, -0.0740354122049, 0.165430831517, 0.66674980751, -0.12077633358, -0.0337960848896, 0.00708649021609, 0.293365487636, 0.000863544472662, -0.199669889395, -0.210671570015, 0.180015272246, 0.467349281518, 2.58471060843, 0.516690045711, -0.0632937912186, -0.15288034917, -0.0418765986663], Decimal('1.48283239226'), 500)]"
- ]
- },
- "execution_count": 9,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT * FROM mlp_regress;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# 7. Prediction for classification\n",
- "\n",
- "In the following examples we will use the training data set for prediction as well, which is not usual but serves to show the syntax. First we will test the classification example. The prediction is in the the estimated_class_text column with the actual value in the class_text column."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 10,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "20 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>id</th>\n",
- " <th>estimated_class_text</th>\n",
- " <th>attributes</th>\n",
- " <th>class_text</th>\n",
- " <th>class</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>[Decimal('5.1'), Decimal('3.5'), Decimal('1.4'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>2</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>[Decimal('4.9'), Decimal('3.0'), Decimal('1.4'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>3</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>[Decimal('4.7'), Decimal('3.2'), Decimal('1.3'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>4</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>[Decimal('4.6'), Decimal('3.1'), Decimal('1.5'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>5</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>[Decimal('5.0'), Decimal('3.6'), Decimal('1.4'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>6</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>[Decimal('5.4'), Decimal('3.9'), Decimal('1.7'), Decimal('0.4')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>7</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>[Decimal('4.6'), Decimal('3.4'), Decimal('1.4'), Decimal('0.3')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>8</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>[Decimal('5.0'), Decimal('3.4'), Decimal('1.5'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>9</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>[Decimal('4.4'), Decimal('2.9'), Decimal('1.4'), Decimal('0.2')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>10</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>[Decimal('4.9'), Decimal('3.1'), Decimal('1.5'), Decimal('0.1')]</td>\n",
- " <td>Iris-setosa</td>\n",
- " <td>1</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>11</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>[Decimal('7.0'), Decimal('3.2'), Decimal('4.7'), Decimal('1.4')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>2</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>12</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>[Decimal('6.4'), Decimal('3.2'), Decimal('4.5'), Decimal('1.5')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>2</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>13</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>[Decimal('6.9'), Decimal('3.1'), Decimal('4.9'), Decimal('1.5')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>2</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>14</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>[Decimal('5.5'), Decimal('2.3'), Decimal('4.0'), Decimal('1.3')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>2</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>15</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>[Decimal('6.5'), Decimal('2.8'), Decimal('4.6'), Decimal('1.5')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>2</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>16</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>[Decimal('5.7'), Decimal('2.8'), Decimal('4.5'), Decimal('1.3')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>2</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>17</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>[Decimal('6.3'), Decimal('3.3'), Decimal('4.7'), Decimal('1.6')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>2</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>18</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>[Decimal('4.9'), Decimal('2.4'), Decimal('3.3'), Decimal('1.0')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>2</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>19</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>[Decimal('6.6'), Decimal('2.9'), Decimal('4.6'), Decimal('1.3')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>2</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>20</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>[Decimal('5.2'), Decimal('2.7'), Decimal('3.9'), Decimal('1.4')]</td>\n",
- " <td>Iris-versicolor</td>\n",
- " <td>2</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(1, u'Iris-setosa', [Decimal('5.1'), Decimal('3.5'), Decimal('1.4'), Decimal('0.2')], u'Iris-setosa', 1),\n",
- " (2, u'Iris-setosa', [Decimal('4.9'), Decimal('3.0'), Decimal('1.4'), Decimal('0.2')], u'Iris-setosa', 1),\n",
- " (3, u'Iris-setosa', [Decimal('4.7'), Decimal('3.2'), Decimal('1.3'), Decimal('0.2')], u'Iris-setosa', 1),\n",
- " (4, u'Iris-setosa', [Decimal('4.6'), Decimal('3.1'), Decimal('1.5'), Decimal('0.2')], u'Iris-setosa', 1),\n",
- " (5, u'Iris-setosa', [Decimal('5.0'), Decimal('3.6'), Decimal('1.4'), Decimal('0.2')], u'Iris-setosa', 1),\n",
- " (6, u'Iris-setosa', [Decimal('5.4'), Decimal('3.9'), Decimal('1.7'), Decimal('0.4')], u'Iris-setosa', 1),\n",
- " (7, u'Iris-setosa', [Decimal('4.6'), Decimal('3.4'), Decimal('1.4'), Decimal('0.3')], u'Iris-setosa', 1),\n",
- " (8, u'Iris-setosa', [Decimal('5.0'), Decimal('3.4'), Decimal('1.5'), Decimal('0.2')], u'Iris-setosa', 1),\n",
- " (9, u'Iris-setosa', [Decimal('4.4'), Decimal('2.9'), Decimal('1.4'), Decimal('0.2')], u'Iris-setosa', 1),\n",
- " (10, u'Iris-setosa', [Decimal('4.9'), Decimal('3.1'), Decimal('1.5'), Decimal('0.1')], u'Iris-setosa', 1),\n",
- " (11, u'Iris-versicolor', [Decimal('7.0'), Decimal('3.2'), Decimal('4.7'), Decimal('1.4')], u'Iris-versicolor', 2),\n",
- " (12, u'Iris-versicolor', [Decimal('6.4'), Decimal('3.2'), Decimal('4.5'), Decimal('1.5')], u'Iris-versicolor', 2),\n",
- " (13, u'Iris-versicolor', [Decimal('6.9'), Decimal('3.1'), Decimal('4.9'), Decimal('1.5')], u'Iris-versicolor', 2),\n",
- " (14, u'Iris-versicolor', [Decimal('5.5'), Decimal('2.3'), Decimal('4.0'), Decimal('1.3')], u'Iris-versicolor', 2),\n",
- " (15, u'Iris-versicolor', [Decimal('6.5'), Decimal('2.8'), Decimal('4.6'), Decimal('1.5')], u'Iris-versicolor', 2),\n",
- " (16, u'Iris-versicolor', [Decimal('5.7'), Decimal('2.8'), Decimal('4.5'), Decimal('1.3')], u'Iris-versicolor', 2),\n",
- " (17, u'Iris-versicolor', [Decimal('6.3'), Decimal('3.3'), Decimal('4.7'), Decimal('1.6')], u'Iris-versicolor', 2),\n",
- " (18, u'Iris-versicolor', [Decimal('4.9'), Decimal('2.4'), Decimal('3.3'), Decimal('1.0')], u'Iris-versicolor', 2),\n",
- " (19, u'Iris-versicolor', [Decimal('6.6'), Decimal('2.9'), Decimal('4.6'), Decimal('1.3')], u'Iris-versicolor', 2),\n",
- " (20, u'Iris-versicolor', [Decimal('5.2'), Decimal('2.7'), Decimal('3.9'), Decimal('1.4')], u'Iris-versicolor', 2)]"
- ]
- },
- "execution_count": 10,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS mlp_prediction;\n",
- "\n",
- "SELECT madlib.mlp_predict(\n",
- " 'mlp_model', -- Model table\n",
- " 'iris_data', -- Test data table\n",
- " 'id', -- Id column in test table\n",
- " 'mlp_prediction', -- Output table for predictions\n",
- " 'response' -- Output classes, not probabilities\n",
- " );\n",
- "\n",
- "SELECT * FROM mlp_prediction JOIN iris_data USING (id) ORDER BY id;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Count the misclassifications"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 11,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>count</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>0</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(0L,)]"
- ]
- },
- "execution_count": 11,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT COUNT(*) FROM mlp_prediction JOIN iris_data USING (id) \n",
- "WHERE mlp_prediction.estimated_class_text != iris_data.class_text;"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# 8. Prediction for regression"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 12,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Done.\n",
- "1 rows affected.\n",
- "20 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>id</th>\n",
- " <th>x</th>\n",
- " <th>grp_by_col</th>\n",
- " <th>y</th>\n",
- " <th>estimated_y</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1</td>\n",
- " <td>[0.00632, 18.0, 2.31, 0.0, 0.538, 6.575, 65.2, 4.09, 1.0, 296.0, 15.3, 396.9, 4.98]</td>\n",
- " <td>1</td>\n",
- " <td>24.0</td>\n",
- " <td>23.997693578</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>2</td>\n",
- " <td>[0.02731, 0.0, 7.07, 0.0, 0.469, 6.421, 78.9, 4.9671, 2.0, 242.0, 17.8, 396.9, 9.14]</td>\n",
- " <td>1</td>\n",
- " <td>21.6</td>\n",
- " <td>22.0225551504</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>3</td>\n",
- " <td>[0.02729, 0.0, 7.07, 0.0, 0.469, 7.185, 61.1, 4.9671, 2.0, 242.0, 17.8, 392.83, 4.03]</td>\n",
- " <td>1</td>\n",
- " <td>34.7</td>\n",
- " <td>34.3269436787</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>4</td>\n",
- " <td>[0.03237, 0.0, 2.18, 0.0, 0.458, 6.998, 45.8, 6.0622, 3.0, 222.0, 18.7, 394.63, 2.94]</td>\n",
- " <td>1</td>\n",
- " <td>33.4</td>\n",
- " <td>34.7421700033</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>5</td>\n",
- " <td>[0.06905, 0.0, 2.18, 0.0, 0.458, 7.147, 54.2, 6.0622, 3.0, 222.0, 18.7, 396.9, 5.33]</td>\n",
- " <td>1</td>\n",
- " <td>36.2</td>\n",
- " <td>35.1914922401</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>6</td>\n",
- " <td>[0.02985, 0.0, 2.18, 0.0, 0.458, 6.43, 58.7, 6.0622, 3.0, 222.0, 18.7, 394.12, 5.21]</td>\n",
- " <td>1</td>\n",
- " <td>28.7</td>\n",
- " <td>29.5286073544</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>7</td>\n",
- " <td>[0.08829, 12.5, 7.87, 0.0, 0.524, 6.012, 66.6, 5.5605, 5.0, 311.0, 15.2, 395.6, 12.43]</td>\n",
- " <td>1</td>\n",
- " <td>22.9</td>\n",
- " <td>23.2022360304</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>8</td>\n",
- " <td>[0.14455, 12.5, 7.87, 0.0, 0.524, 6.172, 96.1, 5.9505, 5.0, 311.0, 15.2, 396.9, 19.15]</td>\n",
- " <td>1</td>\n",
- " <td>27.1</td>\n",
- " <td>23.364906529</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>9</td>\n",
- " <td>[0.21124, 12.5, 7.87, 0.0, 0.524, 5.631, 100.0, 6.0821, 5.0, 311.0, 15.2, 386.63, 29.93]</td>\n",
- " <td>1</td>\n",
- " <td>16.5</td>\n",
- " <td>17.7779926867</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>10</td>\n",
- " <td>[0.17004, 12.5, 7.87, 0.0, 0.524, 6.004, 85.9, 6.5921, 5.0, 311.0, 15.2, 386.71, 17.1]</td>\n",
- " <td>1</td>\n",
- " <td>18.9</td>\n",
- " <td>13.9266690258</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>11</td>\n",
- " <td>[0.22489, 12.5, 7.87, 0.0, 0.524, 6.377, 94.3, 6.3467, 5.0, 311.0, 15.2, 392.52, 20.45]</td>\n",
- " <td>1</td>\n",
- " <td>15.0</td>\n",
- " <td>18.5049155839</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>12</td>\n",
- " <td>[0.11747, 12.5, 7.87, 0.0, 0.524, 6.009, 82.9, 6.2267, 5.0, 311.0, 15.2, 396.9, 13.27]</td>\n",
- " <td>1</td>\n",
- " <td>18.9</td>\n",
- " <td>18.4287114359</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>13</td>\n",
- " <td>[0.09378, 12.5, 7.87, 0.0, 0.524, 5.889, 39.0, 5.4509, 5.0, 311.0, 15.2, 390.5, 15.71]</td>\n",
- " <td>1</td>\n",
- " <td>21.7</td>\n",
- " <td>22.6228336115</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>14</td>\n",
- " <td>[0.62976, 0.0, 8.14, 0.0, 0.538, 5.949, 61.8, 4.7075, 4.0, 307.0, 21.0, 396.9, 8.26]</td>\n",
- " <td>1</td>\n",
- " <td>20.4</td>\n",
- " <td>20.1083536059</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>15</td>\n",
- " <td>[0.63796, 0.0, 8.14, 0.0, 0.538, 6.096, 84.5, 4.4619, 4.0, 307.0, 21.0, 380.02, 10.26]</td>\n",
- " <td>1</td>\n",
- " <td>18.2</td>\n",
- " <td>18.8935467873</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>16</td>\n",
- " <td>[0.62739, 0.0, 8.14, 0.0, 0.538, 5.834, 56.5, 4.4986, 4.0, 307.0, 21.0, 395.62, 8.47]</td>\n",
- " <td>1</td>\n",
- " <td>19.9</td>\n",
- " <td>19.8383202293</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>17</td>\n",
- " <td>[1.05393, 0.0, 8.14, 0.0, 0.538, 5.935, 29.3, 4.4986, 4.0, 307.0, 21.0, 386.85, 6.58]</td>\n",
- " <td>1</td>\n",
- " <td>23.1</td>\n",
- " <td>23.1604635402</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>18</td>\n",
- " <td>[0.7842, 0.0, 8.14, 0.0, 0.538, 5.99, 81.7, 4.2579, 4.0, 307.0, 21.0, 386.75, 14.67]</td>\n",
- " <td>1</td>\n",
- " <td>17.5</td>\n",
- " <td>16.8540384346</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>19</td>\n",
- " <td>[0.80271, 0.0, 8.14, 0.0, 0.538, 5.456, 36.6, 3.7965, 4.0, 307.0, 21.0, 288.99, 11.69]</td>\n",
- " <td>1</td>\n",
- " <td>20.2</td>\n",
- " <td>20.3628760581</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>20</td>\n",
- " <td>[0.7258, 0.0, 8.14, 0.0, 0.538, 5.727, 69.5, 3.7965, 4.0, 307.0, 21.0, 390.95, 11.28]</td>\n",
- " <td>1</td>\n",
- " <td>18.2</td>\n",
- " <td>18.1198369917</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(1, [0.00632, 18.0, 2.31, 0.0, 0.538, 6.575, 65.2, 4.09, 1.0, 296.0, 15.3, 396.9, 4.98], 1, 24.0, 23.9976935779896),\n",
- " (2, [0.02731, 0.0, 7.07, 0.0, 0.469, 6.421, 78.9, 4.9671, 2.0, 242.0, 17.8, 396.9, 9.14], 1, 21.6, 22.0225551503712),\n",
- " (3, [0.02729, 0.0, 7.07, 0.0, 0.469, 7.185, 61.1, 4.9671, 2.0, 242.0, 17.8, 392.83, 4.03], 1, 34.7, 34.3269436787012),\n",
- " (4, [0.03237, 0.0, 2.18, 0.0, 0.458, 6.998, 45.8, 6.0622, 3.0, 222.0, 18.7, 394.63, 2.94], 1, 33.4, 34.7421700032985),\n",
- " (5, [0.06905, 0.0, 2.18, 0.0, 0.458, 7.147, 54.2, 6.0622, 3.0, 222.0, 18.7, 396.9, 5.33], 1, 36.2, 35.1914922401243),\n",
- " (6, [0.02985, 0.0, 2.18, 0.0, 0.458, 6.43, 58.7, 6.0622, 3.0, 222.0, 18.7, 394.12, 5.21], 1, 28.7, 29.5286073543722),\n",
- " (7, [0.08829, 12.5, 7.87, 0.0, 0.524, 6.012, 66.6, 5.5605, 5.0, 311.0, 15.2, 395.6, 12.43], 1, 22.9, 23.2022360304219),\n",
- " (8, [0.14455, 12.5, 7.87, 0.0, 0.524, 6.172, 96.1, 5.9505, 5.0, 311.0, 15.2, 396.9, 19.15], 1, 27.1, 23.3649065290002),\n",
- " (9, [0.21124, 12.5, 7.87, 0.0, 0.524, 5.631, 100.0, 6.0821, 5.0, 311.0, 15.2, 386.63, 29.93], 1, 16.5, 17.7779926866502),\n",
- " (10, [0.17004, 12.5, 7.87, 0.0, 0.524, 6.004, 85.9, 6.5921, 5.0, 311.0, 15.2, 386.71, 17.1], 1, 18.9, 13.9266690257803),\n",
- " (11, [0.22489, 12.5, 7.87, 0.0, 0.524, 6.377, 94.3, 6.3467, 5.0, 311.0, 15.2, 392.52, 20.45], 1, 15.0, 18.5049155838719),\n",
- " (12, [0.11747, 12.5, 7.87, 0.0, 0.524, 6.009, 82.9, 6.2267, 5.0, 311.0, 15.2, 396.9, 13.27], 1, 18.9, 18.4287114359317),\n",
- " (13, [0.09378, 12.5, 7.87, 0.0, 0.524, 5.889, 39.0, 5.4509, 5.0, 311.0, 15.2, 390.5, 15.71], 1, 21.7, 22.6228336114696),\n",
- " (14, [0.62976, 0.0, 8.14, 0.0, 0.538, 5.949, 61.8, 4.7075, 4.0, 307.0, 21.0, 396.9, 8.26], 1, 20.4, 20.1083536059151),\n",
- " (15, [0.63796, 0.0, 8.14, 0.0, 0.538, 6.096, 84.5, 4.4619, 4.0, 307.0, 21.0, 380.02, 10.26], 1, 18.2, 18.8935467873061),\n",
- " (16, [0.62739, 0.0, 8.14, 0.0, 0.538, 5.834, 56.5, 4.4986, 4.0, 307.0, 21.0, 395.62, 8.47], 1, 19.9, 19.8383202293121),\n",
- " (17, [1.05393, 0.0, 8.14, 0.0, 0.538, 5.935, 29.3, 4.4986, 4.0, 307.0, 21.0, 386.85, 6.58], 1, 23.1, 23.160463540176),\n",
- " (18, [0.7842, 0.0, 8.14, 0.0, 0.538, 5.99, 81.7, 4.2579, 4.0, 307.0, 21.0, 386.75, 14.67], 1, 17.5, 16.8540384345856),\n",
- " (19, [0.80271, 0.0, 8.14, 0.0, 0.538, 5.456, 36.6, 3.7965, 4.0, 307.0, 21.0, 288.99, 11.69], 1, 20.2, 20.3628760580577),\n",
- " (20, [0.7258, 0.0, 8.14, 0.0, 0.538, 5.727, 69.5, 3.7965, 4.0, 307.0, 21.0, 390.95, 11.28], 1, 18.2, 18.1198369917265)]"
- ]
- },
- "execution_count": 12,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "DROP TABLE IF EXISTS mlp_regress_prediction;\n",
- "\n",
- "SELECT madlib.mlp_predict(\n",
- " 'mlp_regress', -- Model table\n",
- " 'lin_housing', -- Test data table\n",
- " 'id', -- Id column in test table\n",
- " 'mlp_regress_prediction', -- Output table for predictions\n",
- " 'response' -- Output values, not probabilities\n",
- " );\n",
- "\n",
- "SELECT * FROM lin_housing JOIN mlp_regress_prediction USING (id) ORDER BY id;"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 15,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "20 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>id</th>\n",
- " <th>x</th>\n",
- " <th>grp_by_col</th>\n",
- " <th>y</th>\n",
- " <th>estimated_y</th>\n",
- " <th>abs_diff</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>1</td>\n",
- " <td>[0.00632, 18.0, 2.31, 0.0, 0.538, 6.575, 65.2, 4.09, 1.0, 296.0, 15.3, 396.9, 4.98]</td>\n",
- " <td>1</td>\n",
- " <td>24.0</td>\n",
- " <td>23.997693578</td>\n",
- " <td>0.00230642201043</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>2</td>\n",
- " <td>[0.02731, 0.0, 7.07, 0.0, 0.469, 6.421, 78.9, 4.9671, 2.0, 242.0, 17.8, 396.9, 9.14]</td>\n",
- " <td>1</td>\n",
- " <td>21.6</td>\n",
- " <td>22.0225551504</td>\n",
- " <td>0.422555150371</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>3</td>\n",
- " <td>[0.02729, 0.0, 7.07, 0.0, 0.469, 7.185, 61.1, 4.9671, 2.0, 242.0, 17.8, 392.83, 4.03]</td>\n",
- " <td>1</td>\n",
- " <td>34.7</td>\n",
- " <td>34.3269436787</td>\n",
- " <td>0.373056321299</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>4</td>\n",
- " <td>[0.03237, 0.0, 2.18, 0.0, 0.458, 6.998, 45.8, 6.0622, 3.0, 222.0, 18.7, 394.63, 2.94]</td>\n",
- " <td>1</td>\n",
- " <td>33.4</td>\n",
- " <td>34.7421700033</td>\n",
- " <td>1.3421700033</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>5</td>\n",
- " <td>[0.06905, 0.0, 2.18, 0.0, 0.458, 7.147, 54.2, 6.0622, 3.0, 222.0, 18.7, 396.9, 5.33]</td>\n",
- " <td>1</td>\n",
- " <td>36.2</td>\n",
- " <td>35.1914922401</td>\n",
- " <td>1.00850775988</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>6</td>\n",
- " <td>[0.02985, 0.0, 2.18, 0.0, 0.458, 6.43, 58.7, 6.0622, 3.0, 222.0, 18.7, 394.12, 5.21]</td>\n",
- " <td>1</td>\n",
- " <td>28.7</td>\n",
- " <td>29.5286073544</td>\n",
- " <td>0.828607354372</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>7</td>\n",
- " <td>[0.08829, 12.5, 7.87, 0.0, 0.524, 6.012, 66.6, 5.5605, 5.0, 311.0, 15.2, 395.6, 12.43]</td>\n",
- " <td>1</td>\n",
- " <td>22.9</td>\n",
- " <td>23.2022360304</td>\n",
- " <td>0.302236030422</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>8</td>\n",
- " <td>[0.14455, 12
<TRUNCATED>