You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by gw...@apache.org on 2017/05/01 18:54:17 UTC
incubator-systemml git commit: [SYSTEMML-1571] New Jupyter Python
Notebook to showcase SystemML 2-layer autoencoder for acoustic signal
modeling
Repository: incubator-systemml
Updated Branches:
refs/heads/master e8fbc7539 -> 5aa6fb75b
[SYSTEMML-1571] New Jupyter Python Notebook to showcase SystemML 2-layer autoencoder for acoustic signal modeling
Closes #478.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/5aa6fb75
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/5aa6fb75
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/5aa6fb75
Branch: refs/heads/master
Commit: 5aa6fb75b066228d723dfc88bc0ae24551812e92
Parents: e8fbc75
Author: fmakari <fa...@gmail.com>
Authored: Mon May 1 11:50:23 2017 -0700
Committer: Glenn Weidner <gw...@us.ibm.com>
Committed: Mon May 1 11:50:24 2017 -0700
----------------------------------------------------------------------
samples/jupyter-notebooks/Autoencoder.ipynb | 188 +++++++++++++++++++++++
1 file changed, 188 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5aa6fb75/samples/jupyter-notebooks/Autoencoder.ipynb
----------------------------------------------------------------------
diff --git a/samples/jupyter-notebooks/Autoencoder.ipynb b/samples/jupyter-notebooks/Autoencoder.ipynb
new file mode 100644
index 0000000..ec9c2aa
--- /dev/null
+++ b/samples/jupyter-notebooks/Autoencoder.ipynb
@@ -0,0 +1,188 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Autoencoder\n",
+ "This notebook demonstrates the invocation of the SystemML autoencoder script, and alternative ways of passing in/out data."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "#!pip install --user systemml>0.13.0\n",
+ "!pip show systemml"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "from systemml import MLContext, dml\n",
+ "ml = MLContext(sc)\n",
+ "print(ml.info())\n",
+ "sc.version"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## SystemML Read/Write data from local file system"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "FsPath = \"/tmp/data/\"\n",
+ "inp = FsPath + \"Input/\"\n",
+ "outp = FsPath + \"Output/\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Generate Data and write out to file."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "X_pd = pd.DataFrame(range(1, 2001,1),dtype=float).values.reshape(100,20)\n",
+ "script =\"\"\"\n",
+ " write(X, $Xfile)\n",
+ "\"\"\"\n",
+ "prog = dml(script).input(X=X_pd).input(**{\"$Xfile\":inp+\"X.csv\"})\n",
+ "ml.execute(prog)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "!ls -l /tmp/data/Input"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "autoencoderURL = \"https://raw.githubusercontent.com/apache/incubator-systemml/master/scripts/staging/autoencoder-2layer.dml\"\n",
+ "rets = (\"iter\", \"num_iters_per_epoch\", \"beg\", \"end\", \"o\")\n",
+ "\n",
+ "prog = dml(autoencoderURL).input(**{\"$X\":inp+\"X.csv\"}) \\\n",
+ " .input(**{\"$H1\":500, \"$H2\":2, \"$BATCH\":36, \"$EPOCH\":5 \\\n",
+ " , \"$W1_out\":outp+\"W1_out\", \"$b1_out\":outp+\"b1_out\" \\\n",
+ " , \"$W2_out\":outp+\"W2_out\", \"$b2_out\":outp+\"b2_out\" \\\n",
+ " , \"$W3_out\":outp+\"W3_out\", \"$b3_out\":outp+\"b3_out\" \\\n",
+ " , \"$W4_out\":outp+\"W4_out\", \"$b4_out\":outp+\"b4_out\" \\\n",
+ " }).output(*rets)\n",
+ "iter, num_iters_per_epoch, beg, end, o = ml.execute(prog).get(*rets)\n",
+ "print (iter, num_iters_per_epoch, beg, end, o)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "!ls -l /tmp/data/Output"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Alternatively to passing in/out file names, use Python variables."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "autoencoderURL = \"https://raw.githubusercontent.com/apache/incubator-systemml/master/scripts/staging/autoencoder-2layer.dml\"\n",
+ "rets = (\"iter\", \"num_iters_per_epoch\", \"beg\", \"end\", \"o\")\n",
+ "rets2 = (\"W1\", \"b1\", \"W2\", \"b2\", \"W3\", \"b3\", \"W4\", \"b4\")\n",
+ "\n",
+ "prog = dml(autoencoderURL).input(X=X_pd) \\\n",
+ " .input(**{ \"$H1\":500, \"$H2\":2, \"$BATCH\":36, \"$EPOCH\":5}) \\\n",
+ " .output(*rets) \\\n",
+ " .output(*rets2)\n",
+ "result = ml.execute(prog)\n",
+ "iter, num_iters_per_epoch, beg, end, o = result.get(*rets)\n",
+ "W1, b1, W2, b2, W3, b3, W4, b4 = result.get(*rets2)\n",
+ "\n",
+ "print (iter, num_iters_per_epoch, beg, end, o)\n",
+ "W1.toDF().head(3)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 2",
+ "language": "python",
+ "name": "python2"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.11"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}