You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by du...@apache.org on 2017/03/10 06:36:55 UTC
[1/2] incubator-systemml git commit: [SYSTEMML-1185] Updating
MachineLearning Notebook
Repository: incubator-systemml
Updated Branches:
refs/heads/master e3a75d141 -> 88ad73939
[SYSTEMML-1185] Updating MachineLearning Notebook
Updating the image sizes back to the full 256x256x3 shape.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/88ad7393
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/88ad7393
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/88ad7393
Branch: refs/heads/master
Commit: 88ad73939cc19278979c93b55ce394557535f788
Parents: be99410
Author: Mike Dusenberry <mw...@us.ibm.com>
Authored: Thu Mar 9 22:34:16 2017 -0800
Committer: Mike Dusenberry <mw...@us.ibm.com>
Committed: Thu Mar 9 22:35:56 2017 -0800
----------------------------------------------------------------------
projects/breast_cancer/MachineLearning.ipynb | 170 +++++++++++++++++-----
1 file changed, 130 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/88ad7393/projects/breast_cancer/MachineLearning.ipynb
----------------------------------------------------------------------
diff --git a/projects/breast_cancer/MachineLearning.ipynb b/projects/breast_cancer/MachineLearning.ipynb
index 9a11450..4999250 100644
--- a/projects/breast_cancer/MachineLearning.ipynb
+++ b/projects/breast_cancer/MachineLearning.ipynb
@@ -2,7 +2,10 @@
"cells": [
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "deletable": true,
+ "editable": true
+ },
"source": [
"# Predicting Breast Cancer Proliferation Scores with Apache Spark and Apache SystemML\n",
"\n",
@@ -12,7 +15,10 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "deletable": true,
+ "editable": true
+ },
"source": [
"# Setup"
]
@@ -21,7 +27,9 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "deletable": true,
+ "editable": true
},
"outputs": [],
"source": [
@@ -44,7 +52,9 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "deletable": true,
+ "editable": true
},
"outputs": [],
"source": [
@@ -53,7 +63,10 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "deletable": true,
+ "editable": true
+ },
"source": [
"# Read in train & val data"
]
@@ -62,13 +75,15 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": true
+ "collapsed": true,
+ "deletable": true,
+ "editable": true
},
"outputs": [],
"source": [
"# Settings\n",
- "size=64\n",
- "grayscale = True\n",
+ "size=256\n",
+ "grayscale = False\n",
"c = 1 if grayscale else 3\n",
"p = 0.01"
]
@@ -77,12 +92,18 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "deletable": true,
+ "editable": true
},
"outputs": [],
"source": [
- "tr_sample_filename = os.path.join(\"data\", \"train_{}_sample_{}{}.parquet\".format(p, size, \"_grayscale\" if grayscale else \"\"))\n",
- "val_sample_filename = os.path.join(\"data\", \"val_{}_sample_{}{}.parquet\".format(p, size, \"_grayscale\" if grayscale else \"\"))\n",
+ "if p < 1:\n",
+ " tr_sample_filename = os.path.join(\"data\", \"train_{}_sample_{}{}.parquet\".format(p, size, \"_grayscale\" if grayscale else \"\"))\n",
+ " val_sample_filename = os.path.join(\"data\", \"val_{}_sample_{}{}.parquet\".format(p, size, \"_grayscale\" if grayscale else \"\"))\n",
+ "else:\n",
+ " tr_filename = \"train_{}{}.parquet\".format(size, \"_grayscale\" if grayscale else \"\")\n",
+ " val_filename = \"val_{}{}.parquet\".format(size, \"_grayscale\" if grayscale else \"\")\n",
"train_df = sqlContext.read.load(tr_sample_filename)\n",
"val_df = sqlContext.read.load(val_sample_filename)\n",
"train_df, val_df"
@@ -92,7 +113,9 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "deletable": true,
+ "editable": true
},
"outputs": [],
"source": [
@@ -105,7 +128,9 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "deletable": true,
+ "editable": true
},
"outputs": [],
"source": [
@@ -116,7 +141,10 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "deletable": true,
+ "editable": true
+ },
"source": [
"# Extract X and Y matrices"
]
@@ -125,7 +153,9 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "deletable": true,
+ "editable": true
},
"outputs": [],
"source": [
@@ -140,7 +170,10 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "deletable": true,
+ "editable": true
+ },
"source": [
"# Convert to SystemML Matrices\n",
"Note: This allows for reuse of the matrices on multiple\n",
@@ -154,7 +187,9 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "deletable": true,
+ "editable": true
},
"outputs": [],
"source": [
@@ -180,7 +215,10 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "deletable": true,
+ "editable": true
+ },
"source": [
"# Trigger Caching (Optional)\n",
"Note: This will take a while and is not necessary, but doing it\n",
@@ -193,7 +231,9 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "deletable": true,
+ "editable": true
},
"outputs": [],
"source": [
@@ -211,7 +251,10 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "deletable": true,
+ "editable": true
+ },
"source": [
"# Save Matrices (Optional)"
]
@@ -220,7 +263,9 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": true
+ "collapsed": true,
+ "deletable": true,
+ "editable": true
},
"outputs": [],
"source": [
@@ -236,21 +281,30 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "deletable": true,
+ "editable": true
+ },
"source": [
"---"
]
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "deletable": true,
+ "editable": true
+ },
"source": [
"# Softmax Classifier"
]
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "deletable": true,
+ "editable": true
+ },
"source": [
"## Sanity Check: Overfit Small Portion"
]
@@ -259,7 +313,9 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "deletable": true,
+ "editable": true
},
"outputs": [],
"source": [
@@ -286,7 +342,10 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "deletable": true,
+ "editable": true
+ },
"source": [
"## Train"
]
@@ -295,7 +354,9 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "deletable": true,
+ "editable": true
},
"outputs": [],
"source": [
@@ -321,7 +382,10 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "deletable": true,
+ "editable": true
+ },
"source": [
"## Eval"
]
@@ -330,7 +394,9 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "deletable": true,
+ "editable": true
},
"outputs": [],
"source": [
@@ -351,7 +417,10 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "deletable": true,
+ "editable": true
+ },
"source": [
"---"
]
@@ -359,7 +428,9 @@
{
"cell_type": "markdown",
"metadata": {
- "collapsed": true
+ "collapsed": true,
+ "deletable": true,
+ "editable": true
},
"source": [
"# LeNet-like ConvNet"
@@ -367,7 +438,10 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "deletable": true,
+ "editable": true
+ },
"source": [
"## Sanity Check: Overfit Small Portion"
]
@@ -376,7 +450,9 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "deletable": true,
+ "editable": true
},
"outputs": [],
"source": [
@@ -407,7 +483,10 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "deletable": true,
+ "editable": true
+ },
"source": [
"## Hyperparameter Search"
]
@@ -416,7 +495,9 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "deletable": true,
+ "editable": true
},
"outputs": [],
"source": [
@@ -461,7 +542,10 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "deletable": true,
+ "editable": true
+ },
"source": [
"## Train"
]
@@ -470,7 +554,9 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "deletable": true,
+ "editable": true
},
"outputs": [],
"source": [
@@ -500,7 +586,10 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "deletable": true,
+ "editable": true
+ },
"source": [
"## Eval"
]
@@ -509,7 +598,9 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "deletable": true,
+ "editable": true
},
"outputs": [],
"source": [
@@ -537,7 +628,6 @@
}
],
"metadata": {
- "anaconda-cloud": {},
"kernelspec": {
"display_name": "Python 3",
"language": "python",
[2/2] incubator-systemml git commit: [SYSTEMML-1185] Updating
Preprocessing Notebook
Posted by du...@apache.org.
[SYSTEMML-1185] Updating Preprocessing Notebook
Adding more aggressive filtering by utilizing optical density values and
effectively skipping the 1024x1024 tiles by generating tiles of the
same size as the final "samples".
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/be994109
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/be994109
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/be994109
Branch: refs/heads/master
Commit: be9941097ef88eae0bb221142fd76ae2231ac954
Parents: e3a75d1
Author: Mike Dusenberry <mw...@us.ibm.com>
Authored: Thu Mar 9 22:33:13 2017 -0800
Committer: Mike Dusenberry <mw...@us.ibm.com>
Committed: Thu Mar 9 22:35:56 2017 -0800
----------------------------------------------------------------------
projects/breast_cancer/Preprocessing.ipynb | 44 ++++++++++++++++++++++---
1 file changed, 40 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/be994109/projects/breast_cancer/Preprocessing.ipynb
----------------------------------------------------------------------
diff --git a/projects/breast_cancer/Preprocessing.ipynb b/projects/breast_cancer/Preprocessing.ipynb
index 2c2cc41..e5690a9 100644
--- a/projects/breast_cancer/Preprocessing.ipynb
+++ b/projects/breast_cancer/Preprocessing.ipynb
@@ -320,6 +320,22 @@
},
"outputs": [],
"source": [
+ "def optical_density(tile):\n",
+ " \"\"\"\n",
+ " Convert a tile to optical density values.\n",
+ " \n",
+ " Args:\n",
+ " tile: A 3D NumPy array of shape (tile_size, tile_size, channels).\n",
+ " \n",
+ " Returns:\n",
+ " A 3D NumPy array of shape (tile_size, tile_size, channels) representing\n",
+ " optical density values.\n",
+ " \"\"\"\n",
+ " tile = tile.astype(np.float64)\n",
+ " #od = -np.log10(tile/255 + 1e-8)\n",
+ " od = -np.log((tile+1)/240)\n",
+ " return od\n",
+ "\n",
"def keep_tile(tile_tuple, tile_size, tissue_threshold):\n",
" \"\"\"\n",
" Determine if a tile should be kept.\n",
@@ -343,6 +359,9 @@
" \"\"\"\n",
" slide_num, tile = tile_tuple\n",
" if tile.shape[0:2] == (tile_size, tile_size):\n",
+ " tile_orig = tile\n",
+ " \n",
+ " # Check 1\n",
" # Convert 3D RGB image to 2D grayscale image, from\n",
" # 0 (dense tissue) to 1 (plain background).\n",
" tile = rgb2gray(tile)\n",
@@ -366,7 +385,22 @@
" tile = binary_fill_holes(tile)\n",
" # Calculate percentage of tissue coverage.\n",
" percentage = tile.mean()\n",
- " return percentage >= tissue_threshold\n",
+ " check1 = percentage >= tissue_threshold\n",
+ " \n",
+ " # Check 2\n",
+ " # Convert to optical density values\n",
+ " tile = optical_density(tile_orig)\n",
+ " # Threshold at beta\n",
+ " beta = 0.15\n",
+ " tile = np.min(tile, axis=2) >= beta\n",
+ " # Apply morphology for same reasons as above.\n",
+ " tile = binary_closing(tile, disk(2))\n",
+ " tile = binary_dilation(tile, disk(2))\n",
+ " tile = binary_fill_holes(tile)\n",
+ " percentage = tile.mean()\n",
+ " check2 = percentage >= tissue_threshold\n",
+ " \n",
+ " return check1 and check2\n",
" else:\n",
" return False"
]
@@ -620,6 +654,7 @@
" tile_indices = (slides.flatMap(\n",
" lambda slide: process_slide(slide, folder, training, tile_size, overlap)))\n",
" tile_indices = tile_indices.repartition(num_partitions)\n",
+ " tile_indices.cache()\n",
" tiles = tile_indices.map(lambda tile_index: process_tile_index(tile_index, folder, training))\n",
" filtered_tiles = tiles.filter(lambda tile: keep_tile(tile, tile_size, tissue_threshold))\n",
" samples = filtered_tiles.flatMap(lambda tile: process_tile(tile, sample_size, grayscale))\n",
@@ -705,7 +740,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": true,
+ "collapsed": false,
"deletable": true,
"editable": true
},
@@ -717,7 +752,7 @@
"\n",
"# Settings\n",
"training = True\n",
- "tile_size = 1024\n",
+ "tile_size = 256\n",
"sample_size = 256\n",
"grayscale = False\n",
"num_partitions = 20000\n",
@@ -799,7 +834,8 @@
"metadata": {
"collapsed": false,
"deletable": true,
- "editable": true
+ "editable": true,
+ "scrolled": false
},
"outputs": [],
"source": [