You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by du...@apache.org on 2017/07/18 00:20:48 UTC
[5/5] systemml git commit: [SYSTEMML-1185][SYSTEMML-1766] Merge
experimental breast cancer updates
[SYSTEMML-1185][SYSTEMML-1766] Merge experimental breast cancer updates
This merges additional code from the experimental breast cancer branch
to the main repo, including Keras experiments, updates to the
preprocessing, shell script helpers, file reorganization, and
documentation improvements.
Closes #573.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/532da1bc
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/532da1bc
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/532da1bc
Branch: refs/heads/master
Commit: 532da1bc51fed65cd6c329b1c99c1926fe4cf2cd
Parents: 62b64b3
Author: Mike Dusenberry <mw...@us.ibm.com>
Authored: Mon Jul 17 17:18:46 2017 -0700
Committer: Mike Dusenberry <mw...@us.ibm.com>
Committed: Mon Jul 17 17:18:46 2017 -0700
----------------------------------------------------------------------
pom.xml | 5 +-
.../MachineLearning-Keras-Eval.ipynb | 859 +++++++++++++++++++
.../MachineLearning-Keras-ResNet50.ipynb | 717 ++++++++++++++++
projects/breast_cancer/MachineLearning.ipynb | 338 +++++---
.../Preprocessing-Save-JPEGs.ipynb | 610 +++++++++++++
projects/breast_cancer/Preprocessing.ipynb | 101 +--
projects/breast_cancer/README.md | 40 +-
projects/breast_cancer/approach.svg | 4 +
projects/breast_cancer/bin/clean_spark.sh | 26 +
projects/breast_cancer/bin/monitor_gpu.sh | 23 +
.../breast_cancer/bin/remove_old_processes.sh | 24 +
projects/breast_cancer/bin/run_tensorboard.sh | 23 +
projects/breast_cancer/breastcancer/convnet.dml | 495 +++++++++++
.../breastcancer/convnet_distrib_sgd.dml | 592 +++++++++++++
.../breast_cancer/breastcancer/input_data.py | 229 +++++
.../breast_cancer/breastcancer/softmax_clf.dml | 207 +++++
projects/breast_cancer/convnet.dml | 495 -----------
projects/breast_cancer/hyperparam_tuning.dml | 2 +-
projects/breast_cancer/nn | 1 -
projects/breast_cancer/preprocess.py | 2 +-
projects/breast_cancer/softmax_clf.dml | 207 -----
21 files changed, 4072 insertions(+), 928 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/532da1bc/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e3bf831..ee29fe2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -303,7 +303,7 @@
<outputDirectory>${basedir}/target/lib/hadoop/bin</outputDirectory>
</configuration>
</execution>
-
+
<execution>
<id>copy-resources-filtered</id>
<phase>compile</phase>
@@ -344,7 +344,7 @@
</execution>
</executions>
</plugin>
-
+
<plugin>
<groupId>com.github.os72</groupId>
<artifactId>protoc-jar-maven-plugin</artifactId>
@@ -870,6 +870,7 @@
<exclude>**/*.keep</exclude>
<exclude>**/target/**</exclude>
<exclude>**/README.md</exclude>
+ <exclude>**/*.svg</exclude>
<!-- Jupyter Notebooks -->
<exclude>**/*.ipynb</exclude>
<!-- Generated antlr files -->
http://git-wip-us.apache.org/repos/asf/systemml/blob/532da1bc/projects/breast_cancer/MachineLearning-Keras-Eval.ipynb
----------------------------------------------------------------------
diff --git a/projects/breast_cancer/MachineLearning-Keras-Eval.ipynb b/projects/breast_cancer/MachineLearning-Keras-Eval.ipynb
new file mode 100644
index 0000000..c9d3e49
--- /dev/null
+++ b/projects/breast_cancer/MachineLearning-Keras-Eval.ipynb
@@ -0,0 +1,859 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Imports"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%load_ext autoreload\n",
+ "%autoreload 2\n",
+ "%matplotlib inline\n",
+ "\n",
+ "import math\n",
+ "import multiprocessing as mp\n",
+ "import os\n",
+ "\n",
+ "import keras\n",
+ "import keras.backend as K\n",
+ "from keras.applications.resnet50 import ResNet50\n",
+ "from keras.callbacks import ModelCheckpoint, TensorBoard\n",
+ "from keras.initializers import VarianceScaling\n",
+ "from keras.layers import Dense, Dropout, Flatten, GlobalAveragePooling2D, Input, Lambda, merge\n",
+ "from keras.models import Model, load_model\n",
+ "from keras.optimizers import SGD\n",
+ "# from keras.preprocessing.image import ImageDataGenerator\n",
+ "from keras.regularizers import l2\n",
+ "from keras.utils import to_categorical\n",
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "from PIL import Image\n",
+ "import tensorflow as tf\n",
+ "\n",
+ "# After move to Keras 2.0 API, need to check if this can still be used.\n",
+ "from preprocessing.image_eval import ImageDataGenerator # multiprocessing ImageDataGenerator\n",
+ "\n",
+ "plt.rcParams['figure.figsize'] = (10, 10)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Settings"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "# NOTE: Need to update the following for each model\n",
+ "# 1. train & val data dirs\n",
+ "# 2. train & val data percentages\n",
+ "# 3. experiment directory\n",
+ "# 4. model file\n",
+ "# 5. preprocessing channel means"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#os.environ['CUDA_VISIBLE_DEVICES'] = \"\"\n",
+ "size = 224\n",
+ "channels = 3\n",
+ "classes = 3\n",
+ "p = 0.01 # 0.01\n",
+ "val_p = 0.01 #0.01\n",
+ "num_gpus = 4\n",
+ "batch_size = 32 * num_gpus # for 2 GPUs, 32/GPU has 1.2x systems speedup over 16/GPU\n",
+ "train_dir = \"train_updated_norm_v3\"\n",
+ "val_dir = \"val_updated_norm_v3\"\n",
+ "run = 13\n",
+ "# exp_dir = \"experiments/keras/resnet50-1%-4-gpu-128-batch-size-updated-norm-v3-data-1%-val-sanity/4\"\n",
+ "experiment_template = \"resnet50-{p}%-{num_gpus}-gpu-{batch_size}-batch-size-{train_dir}-data-{val_p}%-val-sanity/{run}\"\n",
+ "experiment = experiment_template.format(p=int(p*100), val_p=int(val_p*100), num_gpus=num_gpus,\n",
+ " batch_size=batch_size, train_dir=train_dir, run=run)\n",
+ "model_file = \"0.38936_acc_0.27847_loss_model.hdf5\"\n",
+ "exp_dir = os.path.join(\"experiments\", \"keras\", experiment)\n",
+ "# experiment_name = model_file.replace(\"/\", \"_\")[:-5]\n",
+ "print(exp_dir)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "# os.makedirs(os.path.join(\"results\", experiment_name), exist_ok=True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Load model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "model = load_model(os.path.join(exp_dir, model_file))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(model.summary())\n",
+ "print(model.get_layer(\"resnet50\").summary())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Visualize Model\n",
+ "from IPython.display import SVG\n",
+ "from keras.utils.vis_utils import model_to_dot\n",
+ "SVG(model_to_dot(model.get_layer(\"resnet50\")).create(prog='dot', format='svg'))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "# Note: previous `model` is already compiled and ready to go.\n",
+ "# However, it may have been built for multi-GPU training, so it\n",
+ "# would still require multiple parallel inputs at eval time.\n",
+ "# Even worse, the device settings will not be retained, so all\n",
+ "# towers would be run on one device. To fix this, we can extract\n",
+ "# a single tower, rewrap in a multi-GPU block, and recompile.\n",
+ "\n",
+ "# Extract single tower\n",
+ "resnet50 = model.get_layer(\"resnet50\")\n",
+ "#model.save(\"resnet50-100%-4-gpu-128-batch-size-updated-norm-v3-data-1%-val-dropout_0_1.56-19_NO_GPU_TOWERS.hdf5\")\n",
+ "\n",
+ "# Multi-GPU exploitation via a linear combination of GPU loss functions.\n",
+ "ins = []\n",
+ "outs = []\n",
+ "for i in range(num_gpus):\n",
+ " with tf.device(\"/gpu:{}\".format(i)):\n",
+ " x = Input(shape=(size,size,channels)) # split of batch\n",
+ " out = resnet50(x) # run split on shared model\n",
+ " ins.append(x)\n",
+ " outs.append(out)\n",
+ "model = Model(inputs=ins, outputs=outs) # multi-GPU, data-parallel model\n",
+ "\n",
+ "# Compile model.\n",
+ "metrics = ['accuracy']\n",
+ "model.compile(optimizer=\"sgd\", loss=\"categorical_crossentropy\",\n",
+ " loss_weights=[1/num_gpus]*num_gpus, metrics=metrics)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# # Explore model\n",
+ "for x in model.inputs + model.outputs + model.metrics_tensors + model.targets:\n",
+ " print(x.name, x.device) # check that tensor devices exploit multi-GPU\n",
+ "\n",
+ "# print(model.summary())\n",
+ "\n",
+ "# print(resnet50.summary())"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Create train & val data generators"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train_save_dir = \"images/{stage}/{p}\".format(stage=train_dir, p=p)\n",
+ "val_save_dir = \"images/{stage}/{p}\".format(stage=val_dir, p=val_p)\n",
+ "print(train_save_dir, val_save_dir)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "def preprocess_input(x):\n",
+ " \"\"\"\n",
+ " Preprocesses a tensor encoding a batch of images.\n",
+ "\n",
+ " Adapted from keras/applications/imagenet_utils.py\n",
+ "\n",
+ " # Arguments\n",
+ " x: input Numpy tensor, 4D of shape (N, H, W, C).\n",
+ " # Returns\n",
+ " Preprocessed tensor.\n",
+ " \"\"\"\n",
+ " # Zero-center by subtracting mean pixel value per channel\n",
+ " # based on means from a 50%, evenly-distributed sample.\n",
+ " # Means: updated-data norm v3, norm, no-norm original\n",
+ " x[:, :, :, 0] -= 183.36777842 #189.54944625 #194.27633667\n",
+ " x[:, :, :, 1] -= 138.81743141 #152.73427159 #145.3067627\n",
+ " x[:, :, :, 2] -= 166.07406199 #176.89543273 #181.27861023 \n",
+ " x = x[:, :, :, ::-1] # 'RGB'->'BGR'\n",
+ " return x\n",
+ "\n",
+ "# Multi-GPU exploitation\n",
+ "def split(x, num_splits):\n",
+ " \"\"\"Split batch into K equal-sized batches.\"\"\"\n",
+ " # Split tensors evenly, even if it means throwing away a few examples.\n",
+ " samples = math.floor(len(x) / num_splits)\n",
+ " x_splits = [arr[:samples] for arr in np.array_split(x, num_splits)]\n",
+ " return x_splits\n",
+ "\n",
+ "def gen_preprocessed_batch(batch_generator, num_gpus):\n",
+ " \"\"\"Yield preprocessed batches of x,y data.\"\"\"\n",
+ "# for xs, ys in batch_generator:\n",
+ "# yield split(preprocess_input(xs), num_gpus), split(ys, num_gpus)\n",
+ "# yield split(xs, num_gpus), split(ys, num_gpus) for tf aug experiments\n",
+ " for xs, ys, filenames in batch_generator:\n",
+ " yield split(preprocess_input(xs), num_gpus), split(ys, num_gpus), split(filenames, num_gpus)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Create train & val image generators\n",
+ "try:\n",
+ " # For interactive work, kill any existing pool.\n",
+ " pool.terminate()\n",
+ "except:\n",
+ " pass\n",
+ "pool = mp.Pool(processes=8)\n",
+ "train_datagen = ImageDataGenerator(pool=pool) #, horizontal_flip=True, vertical_flip=True,\n",
+ "# rotation_range=180, shear_range=0.1, fill_mode='reflect')\n",
+ "val_datagen = ImageDataGenerator(pool=pool)\n",
+ "#train_datagen = ImageDataGenerator()\n",
+ "#val_datagen = ImageDataGenerator()\n",
+ "train_generator_orig = train_datagen.flow_from_directory(train_save_dir, batch_size=batch_size, target_size=(size, size))\n",
+ "val_generator_orig = val_datagen.flow_from_directory(val_save_dir, batch_size=batch_size, target_size=(size, size))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "# Create train & val preprocessed generators\n",
+ "train_generator = gen_preprocessed_batch(train_generator_orig, num_gpus)\n",
+ "val_generator = gen_preprocessed_batch(val_generator_orig, num_gpus)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Get number of samples"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "# Number of examples.\n",
+ "tc = train_generator_orig.nb_sample\n",
+ "vc = val_generator_orig.nb_sample\n",
+ "#tc = train_generator_orig.samples\n",
+ "#vc = val_generator_orig.samples\n",
+ "\n",
+ "# Number of batches for multi-GPU exploitation.\n",
+ "# Note: Multi-GPU exploitation for data parallelism splits mini-batches\n",
+ "# into a set of micro-batches to be run in parallel on each GPU, but\n",
+ "# Keras will view the set of micro-batches as a single batch with\n",
+ "# multiple sources of inputs (i.e. Keras will view a set of examples\n",
+ "# being run in parallel as a single example with multiple sources of\n",
+ "# inputs).\n",
+ "train_batches = int(math.ceil(tc/batch_size))\n",
+ "val_batches = int(math.ceil(vc/batch_size))\n",
+ "\n",
+ "# Class counts (just for information)\n",
+ "train_class_counts = np.bincount(train_generator_orig.classes)\n",
+ "val_class_counts = np.bincount(val_generator_orig.classes)\n",
+ "\n",
+ "print(tc, vc)\n",
+ "print(train_batches, val_batches)\n",
+ "print(train_class_counts / np.sum(train_class_counts), val_class_counts / np.sum(val_class_counts))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Generate class weights for training"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "class_counts = np.bincount(train_generator_orig.classes)\n",
+ "class_weights = dict(zip(range(classes), min(class_counts) / class_counts))\n",
+ "print(class_counts)\n",
+ "print(class_weights)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Plot random images (Optional)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "def show_random_image(save_dir):\n",
+ " c = np.random.randint(1, 4)\n",
+ " class_dir = os.path.join(save_dir, str(c))\n",
+ " files = os.listdir(class_dir)\n",
+ " i = np.random.randint(0, len(files))\n",
+ " fname = os.path.join(class_dir, files[i])\n",
+ " print(fname)\n",
+ " img = Image.open(fname)\n",
+ " plt.imshow(img)\n",
+ "\n",
+ "# show_random_image(train_save_dir)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def plot(gen):\n",
+ " r, c = 6, 6\n",
+ " fig, ax = plt.subplots(r, c)\n",
+ " plt.setp(ax, xticks=[], yticks=[])\n",
+ " plt.tight_layout()\n",
+ " x, y, fname = next(gen)\n",
+ " batch_size = x.shape[0]\n",
+ " for i in range(r):\n",
+ " for j in range(c):\n",
+ " if i*c + j < batch_size:\n",
+ " im = x[i*c + j].astype(np.uint8)\n",
+ " if K.image_data_format() == 'channels_first':\n",
+ " im = im.transpose(1,2,0) # (C,H,W) -> (H,W,C)\n",
+ " ax[i][j].imshow(im)\n",
+ " ax[i][j].set_xlabel(y[i*c + j])\n",
+ "\n",
+ "plot(train_generator_orig)\n",
+ "plot(val_generator_orig)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Evaluate previous model checkpoint"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "# NOTE: We could call the `model.evaluate*` methods,\n",
+ "# but that would not allow us to create contingency\n",
+ "# matrices. Instead, we repeatedly loop over batches\n",
+ "# of data, collecting both the true labels and\n",
+ "# predictions. Then, we can compute any metrics\n",
+ "# desired, including 3x3 contingency matrices."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "# def extract_metrics(model, raw_metrics):\n",
+ "# labeled_metrics = list(zip(model.metrics_names, raw_metrics))\n",
+ "# losses = [v for k,v in labeled_metrics if k == \"loss\"]\n",
+ "# accuracies = [v for k,v in labeled_metrics if k.endswith(\"acc\")]\n",
+ "# loss = sum(losses) / num_gpus\n",
+ "# acc = sum(accuracies) / num_gpus\n",
+ "# metrics = {\"loss\": loss, \"acc\": acc}\n",
+ "# return labeled_metrics, metrics\n",
+ "\n",
+ "# raw_metrics = model.evaluate_generator(val_generator, val_samples=32,\n",
+ "# max_q_size=8, nb_worker=1, pickle_safe=False)\n",
+ "\n",
+ "# labeled_metrics, metrics = extract_metrics(model, raw_metrics)\n",
+ "# print(labeled_metrics)\n",
+ "# print(metrics)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Get predictions\n",
+ "for dataset in [(\"train\", p, tc, val_generator)]: #, (\"val\", val_p, vc, val_generator)]:\n",
+ " name, perc, count, gen = dataset\n",
+ "\n",
+ " ys = []\n",
+ " preds = []\n",
+ " fnames = []\n",
+ " batches = math.floor(count / batch_size)\n",
+ " for i in range(batches):\n",
+ " # Get batch.\n",
+ "# x, y = next(gen)\n",
+ " x, y, fname = next(gen)\n",
+ "\n",
+ " # Get predictions\n",
+ " pred = model.predict(x)\n",
+ "\n",
+ " # Store y and predictions\n",
+ " ys.extend(y) # y is always a list of parallel batches, even if only 1 batch\n",
+ " if isinstance(pred, list):\n",
+ " preds.extend(pred)\n",
+ " else:\n",
+ " preds.append(pred)\n",
+ " fnames.extend(fname)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ " # Create DataFrames\n",
+ " y = np.concatenate(ys)\n",
+ " pred = np.concatenate(preds)\n",
+ " fname = np.concatenate(fnames)\n",
+ " y_df = pd.DataFrame(y, columns=[1,2,3])\n",
+ " pred_df = pd.DataFrame(pred, columns=[1,2,3])\n",
+ " fname_df = pd.DataFrame(np.atleast_2d(fname).T, columns=[\"filenames\"])\n",
+ "\n",
+ " # Create class, prediction, slide_num DataFrames\n",
+ " y_class_df = y_df.idxmax(axis=1)\n",
+ " pred_class_df = pred_df.idxmax(axis=1)\n",
+ " y_class_df.name = \"actual\"\n",
+ " pred_class_df.name = \"predicted\"\n",
+ " slide_info_df = fname_df.filenames.str.extract('(?P<class>\\d)\\/\\d+_(?P<slide_num>\\d+)_\\d+.jpeg', expand=True)\n",
+ " slide_info_df[\"class\"] = slide_info_df[\"class\"].astype(int)\n",
+ " slide_info_df[\"slide_num\"] = slide_info_df[\"slide_num\"].astype(int)\n",
+ " df = pd.concat([fname_df, slide_info_df, y_class_df, pred_class_df], axis=1)\n",
+ " \n",
+ " # sanity check\n",
+ " assert np.allclose(df[\"class\"], df.actual)\n",
+ " \n",
+ " # Create Contingency matrix\n",
+ " contingency_mat = pd.crosstab(df.actual, df.predicted)\n",
+ "\n",
+ "# # Save DataFrames\n",
+ "# y_df.to_csv(os.path.join(exp_dir, \"{model_ck}-{perc}%-{data}-y_df.csv\".format(model_ck=model_file[:-5], perc=100*perc, data=name)), header=True)\n",
+ "# pred_df.to_csv(os.path.join(exp_dir, \"{model_ck}-{perc}%-{data}-pred_df.csv\".format(model_ck=model_file[:-5], perc=100*perc, data=name)), header=True)\n",
+ "# df.to_csv(os.path.join(exp_dir, \"{model_ck}-{perc}%-{data}-df.csv\".format(model_ck=model_file[:-5], perc=100*perc, data=name)), header=True)\n",
+ "\n",
+ "# # Save results\n",
+ "# with open(os.path.join(exp_dir, \"{model_ck}-{perc}%-{data}-results.txt\".format(model_ck=model_file[:-5], perc=100*perc, data=name)), 'w') as f:\n",
+ "# print(\"Dataset: {}\".format(name), file=f)\n",
+ "# print(\"Number of samples: {}\".format(len(y_df)), file=f)\n",
+ "# print(contingency_mat, file=f)\n",
+ "# print(\"Accuracy: {}\".format(np.mean(np.equal(y_class, pred_class))), file=f)\n",
+ " print(\"Number of samples: {}\".format(len(y_df)))\n",
+ " print(contingency_mat)\n",
+ " print(\"Accuracy: {}\".format(np.mean(np.equal(y_class_df, pred_class_df))))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "len(y_df), len(pred_df), len(fname_df), len(df)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df2 = df.loc[:, [\"slide_num\", \"actual\", \"predicted\"]]\n",
+ "df2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df3 = df2.groupby(\"slide_num\").mean()\n",
+ "df3[\"predicted_round\"] = df3.predicted.map(round)\n",
+ "df3"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "sum(df3.actual == df3.predicted_round) / len(df3)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "pd.crosstab(df3.actual, df3.predicted_round)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "gb = df2.groupby([\"slide_num\"]) #, \"predicted\"])\n",
+ "gb.describe()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "---"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Read in predictions + true DataFrames and extract metrics"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "# # Read DataFrames\n",
+ "# y_df = pd.read_csv(os.path.join(exp_dir, \"{}-y_df.csv\".format(model_file[:-5])), index_col=0)\n",
+ "# pred_df = pd.read_csv(os.path.join(exp_dir, \"{}-pred_df.csv\".format(model_file[:-5])), index_col=0)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "# # Create Contingency matrix\n",
+ "# y_class = y_df.idxmax(axis=1)\n",
+ "# pred_class = pred_df.idxmax(axis=1)\n",
+ "# y_class.name = \"Actual\"\n",
+ "# pred_class.name = \"Predicted\"\n",
+ "# contingency_mat = pd.crosstab(y_class, pred_class)\n",
+ "\n",
+ "# print(\"Number of samples: {}\".format(len(y_df)))\n",
+ "# print(contingency_mat)\n",
+ "# print(\"Accuracy: {}\".format(np.mean(np.equal(y_class, pred_class))))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true,
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "# # --- Alternate approach with NumPy arrays only\n",
+ "# y_c = np.argmax(y, axis=1) + 1\n",
+ "# pred_c = np.argmax(pred, axis=1) + 1\n",
+ "# y_actu = pd.Series(y_c, name=\"Actual\")\n",
+ "# y_pred = pd.Series(pred_c, name=\"Predicted\")\n",
+ "# contingency_mat = pd.crosstab(y_actu, y_pred)\n",
+ "\n",
+ "# print(\"Number of samples: {}\".format(len(y_c)))\n",
+ "# print(contingency_mat)\n",
+ "# print(\"Accuracy: {}\".format(np.mean(np.equal(y_c, pred_c))))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "collapsed": true
+ },
+ "source": [
+ "---"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Sample images + predictions & write to disk"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "# path_template = os.path.join(\"visualize\", \"{dataset}\", \"Pred_{pred}-Actual_{actual}\")\n",
+ "# for dataset in [\"train\", \"val\"]:\n",
+ "# for i in range(3):\n",
+ "# for j in range(3):\n",
+ "# os.makedirs(path_template.format(dataset=dataset, pred=i+1, actual=j+1), exist_ok=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true,
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "# filename_template = os.path.join(path_template, \"{hash}.jpeg\")\n",
+ "# batches = 8\n",
+ "\n",
+ "# for dataset in [(\"train\", train_generator_orig), (\"val\", val_generator_orig)]:\n",
+ "# name, gen = dataset\n",
+ "# print(name)\n",
+ " \n",
+ "# for i in range(batches):\n",
+ "# # Get batch.\n",
+ "# x_orig, y_orig = next(gen)\n",
+ "# x = preprocess_input(np.copy(x_orig))\n",
+ "# y = y_orig\n",
+ "\n",
+ "# # Get predictions\n",
+ "# raw_preds = model.predict(x)\n",
+ "# raw_metrics = model.evaluate(x, y)\n",
+ "# labeled_metrics, metrics = extract_metrics(model, raw_metrics)\n",
+ "\n",
+ "# # Create contingency matrix\n",
+ "# y = np.argmax(y, axis=1)+1\n",
+ "# preds = np.argmax(raw_preds, axis=1)+1\n",
+ "# y_actu = pd.Series(y, name=\"Actual\")\n",
+ "# y_pred = pd.Series(preds, name=\"Predicted\")\n",
+ "# contingency_mat = pd.crosstab(y_actu, y_pred)\n",
+ "\n",
+ "# # # Output images in directories based on misclassification.\n",
+ "# # def plot(x, y):\n",
+ "# # r, c = 6, 6\n",
+ "# # fig, ax = plt.subplots(r, c)\n",
+ "# # plt.setp(ax, xticks=[], yticks=[])\n",
+ "# # plt.tight_layout()\n",
+ "# # batch_size = x.shape[0]\n",
+ "# # for i in range(r):\n",
+ "# # for j in range(c):\n",
+ "# # if i*c + j < batch_size:\n",
+ "# # ax[i][j].imshow(x[i*c + j].astype(np.uint8))\n",
+ "# # ax[i][j].set_xlabel(\"{preds}-{y}\".format(y=y[i*c + j], preds=preds[i*c + j]))\n",
+ "\n",
+ "# # plot(x_orig, y)\n",
+ "# # plt.show()\n",
+ "\n",
+ "# for n in range(x_orig.shape[0]):\n",
+ "# img = Image.fromarray(x_orig[n].astype(np.uint8), 'RGB')\n",
+ "# filename = filename_template.format(dataset=name, pred=preds[n], actual=y[n], hash=np.random.randint(1e6))\n",
+ "# img.save(filename)\n",
+ "\n",
+ "# print(contingency_mat)\n",
+ "# print(np.mean(y==preds))\n",
+ "# print(labeled_metrics)\n",
+ "# print(metrics)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "---"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Predict"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "x, label, _ = (next(train_generator_orig))\n",
+ "Image.fromarray((x[0]).astype(np.uint8))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "preds = resnet50.predict(preprocess_input(x[0].reshape(1, 224, 224, 3)))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(\"Actual: {}\".format(label[0]))\n",
+ "print(\"Pred: {}\".format(preds[0]))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "---"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Cleanup"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Stop processes cleanly. Otherwise, zombie processes will\n",
+ "# persist and hold onto GPU memory.\n",
+ "try:\n",
+ " pool.terminate()\n",
+ "except:\n",
+ " pass\n",
+ "for p in mp.active_children():\n",
+ " p.terminate()\n",
+ "mp.active_children()"
+ ]
+ }
+ ],
+ "metadata": {
+ "anaconda-cloud": {},
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.6.1"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/532da1bc/projects/breast_cancer/MachineLearning-Keras-ResNet50.ipynb
----------------------------------------------------------------------
diff --git a/projects/breast_cancer/MachineLearning-Keras-ResNet50.ipynb b/projects/breast_cancer/MachineLearning-Keras-ResNet50.ipynb
new file mode 100644
index 0000000..331b666
--- /dev/null
+++ b/projects/breast_cancer/MachineLearning-Keras-ResNet50.ipynb
@@ -0,0 +1,717 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Imports"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%load_ext autoreload\n",
+ "%autoreload 2\n",
+ "%matplotlib inline\n",
+ "\n",
+ "import math\n",
+ "import multiprocessing as mp\n",
+ "import os\n",
+ "\n",
+ "import keras\n",
+ "import keras.backend as K\n",
+ "from keras.applications.resnet50 import ResNet50\n",
+ "from keras.callbacks import ModelCheckpoint, TensorBoard\n",
+ "from keras.initializers import VarianceScaling\n",
+ "from keras.layers import Dense, Dropout, Flatten, GlobalAveragePooling2D, Input, Lambda, merge\n",
+ "from keras.models import Model, load_model\n",
+ "from keras.optimizers import SGD\n",
+ "from keras.preprocessing.image import ImageDataGenerator\n",
+ "from keras.regularizers import l2\n",
+ "from keras.utils import to_categorical\n",
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "from PIL import Image\n",
+ "import tensorflow as tf\n",
+ "\n",
+ "# After move to Keras 2.0 API, need to check if this can still be used.\n",
+ "# from preprocessing.image import ImageDataGenerator # multiprocessing ImageDataGenerator\n",
+ "\n",
+ "plt.rcParams['figure.figsize'] = (10, 10)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Settings"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# os.environ['CUDA_VISIBLE_DEVICES'] = \"\"\n",
+ "size = 224\n",
+ "channels = 3\n",
+ "data_format = 'channels_last' # channels_first is too slow, prob due to unnecessary conversions\n",
+ "classes = 3\n",
+ "p = 0.01\n",
+ "val_p = 0.01\n",
+ "num_gpus = 4\n",
+ "batch_size = 32 * num_gpus # for 2 GPUs, 32/GPU has 1.2x systems speedup over 16/GPU\n",
+ "train_dir = \"train_updated_norm_v3\"\n",
+ "val_dir = \"val_updated_norm_v3\"\n",
+ "new_run = True\n",
+ "experiment_template = \"resnet50-{p}%-{num_gpus}-gpu-{batch_size}-batch-size-{train_dir}-data-{val_p}%-val-sanity\"\n",
+ "experiment = experiment_template.format(p=int(p*100), val_p=int(val_p*100), num_gpus=num_gpus,\n",
+ " batch_size=batch_size, train_dir=train_dir)\n",
+ "print(experiment)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "K.set_image_data_format(data_format)\n",
+ "if data_format == 'channels_first':\n",
+ " input_shape = (channels, size, size)\n",
+ "else:\n",
+ " input_shape = (size, size, channels)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Setup experiment directory"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def get_run_dir(path, new_run):\n",
+ " \"\"\"Create a directory for this training run.\"\"\"\n",
+ " os.makedirs(path, exist_ok=True)\n",
+ " num_experiments = len(os.listdir(path))\n",
+ " if new_run:\n",
+ " run = num_experiments # run 0, 1, 2, ...\n",
+ " else:\n",
+ " run = min(0, num_experiments - 1) # continue training\n",
+ " run_dir = os.path.join(path, str(run))\n",
+ " os.makedirs(run_dir, exist_ok=True)\n",
+ " return run_dir\n",
+ "\n",
+ "def get_experiment_dir(experiment, new_run):\n",
+ " \"\"\"Create an experiment directory for this experiment.\"\"\"\n",
+ " base_dir = os.path.join(\"experiments\", \"keras\", experiment)\n",
+ " exp_dir = get_run_dir(base_dir, new_run)\n",
+ " return exp_dir\n",
+ "\n",
+ "exp_dir = get_experiment_dir(experiment, new_run=new_run)\n",
+ "print(exp_dir)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Create train & val data generators"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "def preprocess_input(x):\n",
+ " \"\"\"\n",
+ " Preprocesses a tensor encoding a batch of images.\n",
+ "\n",
+ " Adapted from keras/applications/imagenet_utils.py\n",
+ "\n",
+ " # Arguments\n",
+ " x: input Numpy tensor, 4D of shape (N, H, W, C).\n",
+ " # Returns\n",
+ " Preprocessed tensor.\n",
+ " \"\"\"\n",
+ " # Zero-center by subtracting mean pixel value per channel\n",
+ " # based on means from a 50%, evenly-distributed sample.\n",
+ " # Means: updated-data norm v3, norm, no-norm original\n",
+ " x[:, :, :, 0] -= 183.36777842 #189.54944625 #194.27633667\n",
+ " x[:, :, :, 1] -= 138.81743141 #152.73427159 #145.3067627\n",
+ " x[:, :, :, 2] -= 166.07406199 #176.89543273 #181.27861023 \n",
+ " x = x[:, :, :, ::-1] # 'RGB'->'BGR' due to pretrained ResNet\n",
+ " return x\n",
+ "\n",
+ "# Multi-GPU exploitation\n",
+ "def split(x, num_splits):\n",
+ " \"\"\"Split batch into K equal-sized batches.\"\"\"\n",
+ " # Split tensors evenly, even if it means throwing away a few examples.\n",
+ " samples = math.floor(len(x) / num_splits)\n",
+ " x_splits = [arr[:samples] for arr in np.array_split(x, num_splits)]\n",
+ " return x_splits\n",
+ "\n",
+ "def gen_preprocessed_batch(batch_generator, num_gpus):\n",
+ " \"\"\"Yield preprocessed batches of x,y data.\"\"\"\n",
+ " for xs, ys in batch_generator:\n",
+ " yield split(preprocess_input(xs), num_gpus), split(ys, num_gpus)\n",
+ "# yield split(xs, num_gpus), split(ys, num_gpus) # for tf aug experiments"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "K.image_data_format()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train_save_dir = \"images/{stage}/{p}\".format(stage=train_dir, p=p)\n",
+ "val_save_dir = \"images/{stage}/{p}\".format(stage=val_dir, p=val_p)\n",
+ "print(train_save_dir, val_save_dir)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Create train & val image generators\n",
+ "#try:\n",
+ "# # For interactive work, kill any existing pool.\n",
+ "# pool.terminate()\n",
+ "#except:\n",
+ "# pass\n",
+ "#pool = mp.Pool(processes=8)\n",
+ "#train_datagen = ImageDataGenerator(pool=pool, horizontal_flip=True, vertical_flip=True,\n",
+ "# rotation_range=180, shear_range=0.1, fill_mode='reflect')\n",
+ "#val_datagen = ImageDataGenerator(pool=pool)\n",
+ "\n",
+ "train_datagen = ImageDataGenerator(horizontal_flip=True, vertical_flip=True) #, samplewise_center=True)\n",
+ " #rotation_range=180, shear_range=0.1, fill_mode='reflect')\n",
+ "val_datagen = ImageDataGenerator()\n",
+ "train_generator_orig = train_datagen.flow_from_directory(train_save_dir, batch_size=batch_size, target_size=(size, size))\n",
+ "val_generator_orig = val_datagen.flow_from_directory(val_save_dir, batch_size=batch_size, target_size=(size, size))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "# Create train & val preprocessed generators\n",
+ "train_generator = gen_preprocessed_batch(train_generator_orig, num_gpus)\n",
+ "val_generator = gen_preprocessed_batch(val_generator_orig, num_gpus)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Get number of batches"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "# Number of examples.\n",
+ "tc = train_generator_orig.samples\n",
+ "vc = val_generator_orig.samples\n",
+ "\n",
+ "# Number of batches for multi-GPU exploitation.\n",
+ "# Note: Multi-GPU exploitation for data parallelism splits mini-batches\n",
+ "# into a set of micro-batches to be run in parallel on each GPU, but\n",
+ "# Keras will view the set of micro-batches as a single batch with\n",
+ "# multiple sources of inputs (i.e. Keras will view a set of examples\n",
+ "# being run in parallel as a single example with multiple sources of\n",
+ "# inputs).\n",
+ "train_batches = int(math.ceil(tc/batch_size))\n",
+ "val_batches = int(math.ceil(vc/batch_size))\n",
+ "\n",
+ "# Class counts (just for information)\n",
+ "train_class_counts = np.bincount(train_generator_orig.classes)\n",
+ "val_class_counts = np.bincount(val_generator_orig.classes)\n",
+ "\n",
+ "print(tc, vc)\n",
+ "print(train_batches, val_batches)\n",
+ "print(train_class_counts / np.sum(train_class_counts), val_class_counts / np.sum(val_class_counts))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Generate class weights for training"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "class_counts = np.bincount(train_generator_orig.classes)\n",
+ "class_weights = dict(zip(range(classes), min(class_counts) / class_counts))\n",
+ "print(class_counts)\n",
+ "print(class_weights)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Plot random images (Optional)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "def show_random_image(save_dir):\n",
+ " c = np.random.randint(1, 4)\n",
+ " class_dir = os.path.join(save_dir, str(c))\n",
+ " files = os.listdir(class_dir)\n",
+ " i = np.random.randint(0, len(files))\n",
+ " fname = os.path.join(class_dir, files[i])\n",
+ " print(fname)\n",
+ " img = Image.open(fname)\n",
+ " plt.imshow(img)\n",
+ "\n",
+ "# show_random_image(train_save_dir)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def plot(gen):\n",
+ " r, c = 6, 6\n",
+ " fig, ax = plt.subplots(r, c)\n",
+ " plt.setp(ax, xticks=[], yticks=[])\n",
+ " plt.tight_layout()\n",
+ " x, y = next(gen)\n",
+ " batch_size = x.shape[0]\n",
+ " for i in range(r):\n",
+ " for j in range(c):\n",
+ " if i*c + j < batch_size:\n",
+ " im = x[i*c + j].astype(np.uint8)\n",
+ " if K.image_data_format() == 'channels_first':\n",
+ " im = im.transpose(1,2,0) # (C,H,W) -> (H,W,C)\n",
+ " ax[i][j].imshow(im)\n",
+ " ax[i][j].set_xlabel(y[i*c + j])\n",
+ "\n",
+ "plot(train_generator_orig)\n",
+ "plot(val_generator_orig)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "collapsed": true
+ },
+ "source": [
+ "# Training\n",
+ "1. Setup ResNet50 pretrained model with new input & output layers.\n",
+ "2. Train new output layers (all others frozen).\n",
+ "3. Fine tune [some subset of the] original layers.\n",
+ "4. Profit."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Setup training metrics & callbacks"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "# Setup training metrics & callbacks\n",
+ "# Careful, TensorBoard callback could OOM with large validation set\n",
+ "# TODO: Add input images to TensorBoard output (maybe as a separate callback)\n",
+ "# TODO: Monitor size of input queues with callbacks\n",
+ "model_filename = os.path.join(exp_dir, \"{val_loss:.2f}-{epoch:02d}.hdf5\")\n",
+ "checkpointer = ModelCheckpoint(model_filename)\n",
+ "tensorboard = TensorBoard(log_dir=exp_dir, write_graph=False)\n",
+ "callbacks = [checkpointer, tensorboard]\n",
+ "metrics = ['accuracy'] #, fmeasure, precision, recall]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Setup ResNet50 model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "## Color augmentation\n",
+ "## TODO: Visualize this in TensorBoard with custom callback every ~100 iterations\n",
+ "#def preprocess(x):\n",
+ "# # import these inside this function so that future model loads\n",
+ "# # will not complain about `tf` not being defined\n",
+ "# import tensorflow as tf\n",
+ "# import keras.backend as K\n",
+ "# \n",
+ "# def augment(img):\n",
+ "# img = tf.image.random_brightness(img, max_delta=64/255)\n",
+ "# img = tf.image.random_saturation(img, lower=0, upper=0.25)\n",
+ "# img = tf.image.random_hue(img, max_delta=0.04)\n",
+ "# img = tf.image.random_contrast(img, lower=0, upper=0.75)\n",
+ "# return img\n",
+ "# \n",
+ "# # Fix dimensions for tf.image ops\n",
+ "# if K.image_data_format() == 'channels_first':\n",
+ "# x = tf.transpose(x, [0,2,3,1]) # (N,C,H,W) -> (N,H,W,C)\n",
+ "# \n",
+ "# # Augment during training.\n",
+ "# x = K.in_train_phase(tf.map_fn(augment, x, swap_memory=True), x)\n",
+ "# \n",
+ "# # Zero-center by subtracting mean pixel value per channel\n",
+ "# # based on means from a 50%, evenly-distributed sample.\n",
+ "# # Means: updated-data norm v3, norm, no-norm original\n",
+ "# x = x - [183.36777842, 138.81743141, 166.07406199]\n",
+ "# x = tf.reverse(x, axis=[-1])\n",
+ "# \n",
+ "# if K.image_data_format() == 'channels_first':\n",
+ "# x = tf.transpose(x, [0,3,1,2]) # (N,H,W,C) -> (N,C,H,W)\n",
+ "# return x"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true,
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "K.clear_session()\n",
+ "\n",
+ "# Create model by replacing classifier of ResNet50 model with new\n",
+ "# classifier specific to the breast cancer problem.\n",
+ "with tf.device(\"/cpu\"):\n",
+ " inputs = Input(shape=input_shape)\n",
+ " x = inputs\n",
+ " #x = Lambda(preprocess)(x)\n",
+ " resnet50_base = ResNet50(include_top=False, input_shape=input_shape, input_tensor=x) #weights=None)\n",
+ " x = Flatten()(resnet50_base.output) # could also use GlobalAveragePooling2D since output is (None, 1, 1, 2048)\n",
+ " x = Dropout(0.5)(x)\n",
+ " # init Dense weights with Gaussian scaled by sqrt(1/fan_in)\n",
+ " preds = Dense(classes, kernel_initializer=VarianceScaling(), activation=\"softmax\")(x)\n",
+ "# resnet50 = Model(input=resnet50_base.input, output=preds, name=\"resnet50\")\n",
+ " resnet50 = Model(inputs=inputs, outputs=preds, name=\"resnet50\")\n",
+ "\n",
+ "# Multi-GPU exploitation via a linear combination of GPU loss functions.\n",
+ "ins = []\n",
+ "outs = []\n",
+ "for i in range(num_gpus):\n",
+ " with tf.device(\"/gpu:{}\".format(i)):\n",
+ " x = Input(shape=input_shape) # split of batch\n",
+ " out = resnet50(x) # run split on shared model\n",
+ " ins.append(x)\n",
+ " outs.append(out)\n",
+ "model = Model(inputs=ins, outputs=outs) # multi-GPU, data-parallel model\n",
+ "\n",
+ "# Freeze all pre-trained ResNet layers.\n",
+ "for layer in resnet50_base.layers:\n",
+ " layer.trainable = False\n",
+ "\n",
+ "# Compile model.\n",
+ "#optim = SGD(lr=0.1, momentum=0.9, decay=0.99, nesterov=True)\n",
+ "#optim = keras.optimizers.RMSprop(lr=0.05)\n",
+ "optim = keras.optimizers.Adam(lr=0.001)\n",
+ "model.compile(optimizer=optim, loss=\"categorical_crossentropy\",\n",
+ " loss_weights=[1/num_gpus]*num_gpus, metrics=metrics)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "# Explore model\n",
+ "# for x in model.inputs + model.outputs + model.metrics_tensors + model.targets:\n",
+ "# print(x.name, x.device) # check that tensor devices exploit multi-GPU\n",
+ "\n",
+ "# for i, layer in enumerate(resnet50.layers):\n",
+ "# print(i, layer.name, layer.input_shape, layer.output_shape)\n",
+ "\n",
+ "# print(model.summary())\n",
+ "print(resnet50.summary())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "# Visualize Model\n",
+ "from IPython.display import SVG\n",
+ "from keras.utils.vis_utils import model_to_dot\n",
+ "SVG(model_to_dot(resnet50).create(prog='dot', format='svg'))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Train new softmax classifier"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "# Dual-GPU speedup: ~1.7-1.8x\n",
+ "# Keras device placement improvements (metrics, losses) (no val or callbacks, full model):\n",
+ "# batch_size=32, 2 gpus, 100 iters, no keras changes: 128s, 108s, 107s\n",
+ "# batch_size=32, 2 gpus, 100 iters, w/ keras changes: 94s, 75s, 75s\n",
+ "# batch_size=32, 1 gpu, 100 iters, w/ keras changes: 148s, 133s, 133s\n",
+ "# batch_size=64, 2 gpus, 50 iters, w/ keras changes: 93s, 74s, 75s\n",
+ "# batch_size=128, 2 gpus, 25 iters, w/ keras changes: 90s, 73s, 74s\n",
+ "epochs = 4\n",
+ "hist1 = model.fit_generator(train_generator, steps_per_epoch=train_batches,\n",
+ " validation_data=val_generator, validation_steps=val_batches,\n",
+ " epochs=epochs, class_weight=class_weights, callbacks=callbacks) #,\n",
+ " #max_q_size=8, nb_worker=1, pickle_safe=False)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Fine-tune model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "# Explore model\n",
+ "# for x in model.inputs + model.outputs + model.metrics_tensors + model.targets:\n",
+ "# print(x.name, x.device) # check that tensor devices exploit multi-GPU\n",
+ "\n",
+ "for i, layer in enumerate(resnet50_base.layers):\n",
+ " print(i, layer.name, layer.input_shape, layer.output_shape)\n",
+ "\n",
+ "# print(model.summary())\n",
+ "# print(model.get_layer(\"resnet50\").summary())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "# Unfreeze some subset of the model and fine-tune by training slowly with low lr.\n",
+ "for layer in resnet50_base.layers[164:]: #[154:]: # unfreeze final 2 residual blocks + exit flow ([154:])\n",
+ " layer.trainable = True\n",
+ "# if hasattr(layer, 'W_regularizer'):\n",
+ "# layer.W_regularizer = l2(1e-4)\n",
+ "\n",
+ "optim = SGD(lr=0.0001, momentum=0.9)\n",
+ "# optim = keras.optimizers.Adam(lr=0.001)\n",
+ "model.compile(optimizer=optim, loss=\"categorical_crossentropy\",\n",
+ " loss_weights=[1/num_gpus]*num_gpus, metrics=metrics)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(model.summary())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "# model.load_weights(os.path.join(\"experiments/keras/resnet50-100%-2-gpu-64-batch-size/0\", \"5.08-08.hdf5\"))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "initial_epoch = epochs\n",
+ "epochs = initial_epoch + 20\n",
+ "hist2 = model.fit_generator(train_generator, steps_per_epoch=train_batches,\n",
+ " validation_data=val_generator, validation_steps=val_batches,\n",
+ " epochs=epochs, initial_epoch=initial_epoch,\n",
+ " class_weight=class_weights, callbacks=callbacks) #,\n",
+ " #max_q_size=8, nb_worker=1, pickle_safe=False)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Evaluate model on validation set"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "raw_metrics = model.evaluate_generator(val_generator, steps=val_batches) #,\n",
+ " #max_q_size=8, nb_worker=1, pickle_safe=False)\n",
+ "labeled_metrics = list(zip(model.metrics_names, raw_metrics))\n",
+ "losses = [v for k,v in labeled_metrics if k == \"loss\"]\n",
+ "accuracies = [v for k,v in labeled_metrics if k.endswith(\"acc\")]\n",
+ "loss = sum(losses) / num_gpus\n",
+ "acc = sum(accuracies) / num_gpus\n",
+ "metrics = {\"loss\": loss, \"acc\": acc}\n",
+ "print(labeled_metrics)\n",
+ "print(metrics)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Save model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "filename = \"{acc:.5}_acc_{loss:.5}_loss_model.hdf5\".format(**metrics)\n",
+ "fullpath = os.path.join(exp_dir, filename)\n",
+ "model.save(fullpath)\n",
+ "print(\"Saved model file to {}\".format(fullpath))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Cleanup"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "# # Stop processes cleanly. Otherwise, zombie processes will\n",
+ "# # persist and hold onto GPU memory.\n",
+ "# try:\n",
+ "# pool.terminate()\n",
+ "# except:\n",
+ "# pass\n",
+ "# for p in mp.active_children():\n",
+ "# p.terminate()\n",
+ "# mp.active_children()"
+ ]
+ }
+ ],
+ "metadata": {
+ "anaconda-cloud": {},
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.6.1"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/532da1bc/projects/breast_cancer/MachineLearning.ipynb
----------------------------------------------------------------------
diff --git a/projects/breast_cancer/MachineLearning.ipynb b/projects/breast_cancer/MachineLearning.ipynb
index 0ac880c..b27116f 100644
--- a/projects/breast_cancer/MachineLearning.ipynb
+++ b/projects/breast_cancer/MachineLearning.ipynb
@@ -2,10 +2,7 @@
"cells": [
{
"cell_type": "markdown",
- "metadata": {
- "deletable": true,
- "editable": true
- },
+ "metadata": {},
"source": [
"# Predicting Breast Cancer Proliferation Scores with Apache Spark and Apache SystemML\n",
"\n",
@@ -15,10 +12,7 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "deletable": true,
- "editable": true
- },
+ "metadata": {},
"source": [
"# Setup"
]
@@ -26,11 +20,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false,
- "deletable": true,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"%load_ext autoreload\n",
@@ -52,9 +42,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
- "deletable": true,
- "editable": true
+ "collapsed": true
},
"outputs": [],
"source": [
@@ -63,10 +51,7 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "deletable": true,
- "editable": true
- },
+ "metadata": {},
"source": [
"# Read in train & val data"
]
@@ -75,9 +60,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": true,
- "deletable": true,
- "editable": true
+ "collapsed": true
},
"outputs": [],
"source": [
@@ -92,11 +75,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false,
- "deletable": true,
- "editable": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"if p < 1:\n",
@@ -114,9 +93,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
- "deletable": true,
- "editable": true
+ "collapsed": true
},
"outputs": [],
"source": [
@@ -129,9 +106,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
- "deletable": true,
- "editable": true
+ "collapsed": true
},
"outputs": [],
"source": [
@@ -142,10 +117,7 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "deletable": true,
- "editable": true
- },
+ "metadata": {},
"source": [
"# Extract X and Y matrices"
]
@@ -154,9 +126,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
- "deletable": true,
- "editable": true
+ "collapsed": true
},
"outputs": [],
"source": [
@@ -171,10 +141,7 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "deletable": true,
- "editable": true
- },
+ "metadata": {},
"source": [
"# Convert to SystemML Matrices\n",
"Note: This allows for reuse of the matrices on multiple\n",
@@ -188,9 +155,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
- "deletable": true,
- "editable": true
+ "collapsed": true
},
"outputs": [],
"source": [
@@ -216,10 +181,7 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "deletable": true,
- "editable": true
- },
+ "metadata": {},
"source": [
"# Trigger Caching (Optional)\n",
"Note: This will take a while and is not necessary, but doing it\n",
@@ -232,9 +194,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
- "deletable": true,
- "editable": true
+ "collapsed": true
},
"outputs": [],
"source": [
@@ -252,10 +212,7 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "deletable": true,
- "editable": true
- },
+ "metadata": {},
"source": [
"# Save Matrices (Optional)"
]
@@ -264,9 +221,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": true,
- "deletable": true,
- "editable": true
+ "collapsed": true
},
"outputs": [],
"source": [
@@ -282,30 +237,21 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "deletable": true,
- "editable": true
- },
+ "metadata": {},
"source": [
"---"
]
},
{
"cell_type": "markdown",
- "metadata": {
- "deletable": true,
- "editable": true
- },
+ "metadata": {},
"source": [
"# Softmax Classifier"
]
},
{
"cell_type": "markdown",
- "metadata": {
- "deletable": true,
- "editable": true
- },
+ "metadata": {},
"source": [
"## Sanity Check: Overfit Small Portion"
]
@@ -314,14 +260,12 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
- "deletable": true,
- "editable": true
+ "collapsed": true
},
"outputs": [],
"source": [
"script = \"\"\"\n",
- "source(\"softmax_clf.dml\") as clf\n",
+ "source(\"breastcancer/softmax_clf.dml\") as clf\n",
"\n",
"# Hyperparameters & Settings\n",
"lr = 1e-2 # learning rate\n",
@@ -343,10 +287,7 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "deletable": true,
- "editable": true
- },
+ "metadata": {},
"source": [
"## Train"
]
@@ -355,14 +296,12 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
- "deletable": true,
- "editable": true
+ "collapsed": true
},
"outputs": [],
"source": [
"script = \"\"\"\n",
- "source(\"softmax_clf.dml\") as clf\n",
+ "source(\"breastcancer/softmax_clf.dml\") as clf\n",
"\n",
"# Hyperparameters & Settings\n",
"lr = 5e-7 # learning rate\n",
@@ -383,10 +322,7 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "deletable": true,
- "editable": true
- },
+ "metadata": {},
"source": [
"## Eval"
]
@@ -395,14 +331,12 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
- "deletable": true,
- "editable": true
+ "collapsed": true
},
"outputs": [],
"source": [
"script = \"\"\"\n",
- "source(\"softmax_clf.dml\") as clf\n",
+ "source(\"breastcancer/softmax_clf.dml\") as clf\n",
"\n",
"# Eval\n",
"probs = clf::predict(X, W, b)\n",
@@ -418,10 +352,7 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "deletable": true,
- "editable": true
- },
+ "metadata": {},
"source": [
"---"
]
@@ -429,9 +360,7 @@
{
"cell_type": "markdown",
"metadata": {
- "collapsed": true,
- "deletable": true,
- "editable": true
+ "collapsed": true
},
"source": [
"# LeNet-like ConvNet"
@@ -439,10 +368,7 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "deletable": true,
- "editable": true
- },
+ "metadata": {},
"source": [
"## Sanity Check: Overfit Small Portion"
]
@@ -451,14 +377,12 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
- "deletable": true,
- "editable": true
+ "collapsed": true
},
"outputs": [],
"source": [
"script = \"\"\"\n",
- "source(\"convnet.dml\") as clf\n",
+ "source(\"breastcancer/convnet.dml\") as clf\n",
"\n",
"# Hyperparameters & Settings\n",
"lr = 1e-2 # learning rate\n",
@@ -484,10 +408,7 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "deletable": true,
- "editable": true
- },
+ "metadata": {},
"source": [
"## Hyperparameter Search"
]
@@ -496,14 +417,12 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
- "deletable": true,
- "editable": true
+ "collapsed": true
},
"outputs": [],
"source": [
"script = \"\"\"\n",
- "source(\"convnet.dml\") as clf\n",
+ "source(\"breastcancer/convnet.dml\") as clf\n",
"\n",
"dir = \"models/lenet-cnn/hyperparam-search/\"\n",
"\n",
@@ -543,41 +462,67 @@
},
{
"cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Train"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
"metadata": {
- "deletable": true,
- "editable": true
+ "collapsed": true
},
+ "outputs": [],
"source": [
- "## Train"
+ "ml.setStatistics(True)\n",
+ "ml.setExplain(True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "# sc.setLogLevel(\"OFF\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
- "deletable": true,
- "editable": true
+ "collapsed": true,
+ "scrolled": false
},
"outputs": [],
"source": [
"script = \"\"\"\n",
- "source(\"convnet.dml\") as clf\n",
+ "source(\"breastcancer/convnet_distrib_sgd.dml\") as clf\n",
"\n",
"# Hyperparameters & Settings\n",
"lr = 0.00205 # learning rate\n",
"mu = 0.632 # momentum\n",
"decay = 0.99 # learning rate decay constant\n",
"lambda = 0.00385\n",
- "batch_size = 32\n",
+ "batch_size = 1\n",
+ "parallel_batches = 19\n",
"epochs = 1\n",
- "log_interval = 10\n",
+ "log_interval = 1\n",
"dir = \"models/lenet-cnn/train/\"\n",
+ "n = 50 #1216 # limit on number of samples (for debugging)\n",
+ "X = X[1:n,]\n",
+ "Y = Y[1:n,]\n",
+ "X_val = X_val[1:n,]\n",
+ "Y_val = Y_val[1:n,]\n",
"\n",
"# Train\n",
"[Wc1, bc1, Wc2, bc2, Wc3, bc3, Wa1, ba1, Wa2, ba2] =\n",
" clf::train(X, Y, X_val, Y_val, C, Hin, Win, lr, mu, decay,\n",
- " lambda, batch_size, epochs, log_interval, dir)\n",
+ " lambda, batch_size, parallel_batches, epochs,\n",
+ " log_interval, dir)\n",
"\"\"\"\n",
"outputs = (\"Wc1\", \"bc1\", \"Wc2\", \"bc2\", \"Wc3\", \"bc3\",\n",
" \"Wa1\", \"ba1\", \"Wa2\", \"ba2\")\n",
@@ -585,15 +530,53 @@
" C=c, Hin=size, Win=size)\n",
" .output(*outputs))\n",
"outs = ml.execute(script).get(*outputs)\n",
- "Wc1, bc1, Wc2, bc2, Wc3, bc3, Wa1, ba1, Wa2, ba2 = outs"
+ "Wc1, bc1, Wc2, bc2, Wc3, bc3, Wa1, ba1, Wa2, ba2 = outs\n",
+ "Wc1, bc1, Wc2, bc2, Wc3, bc3, Wa1, ba1, Wa2, ba2"
]
},
{
- "cell_type": "markdown",
+ "cell_type": "code",
+ "execution_count": null,
"metadata": {
- "deletable": true,
- "editable": true
+ "collapsed": true
},
+ "outputs": [],
+ "source": [
+ "script = \"\"\"\n",
+ "source(\"breastcancer/convnet_distrib_sgd.dml\") as clf\n",
+ "\n",
+ "# Hyperparameters & Settings\n",
+ "lr = 0.00205 # learning rate\n",
+ "mu = 0.632 # momentum\n",
+ "decay = 0.99 # learning rate decay constant\n",
+ "lambda = 0.00385\n",
+ "batch_size = 1\n",
+ "parallel_batches = 19\n",
+ "epochs = 1\n",
+ "log_interval = 1\n",
+ "dir = \"models/lenet-cnn/train/\"\n",
+ "\n",
+ "# Dummy data\n",
+ "[X, Y, C, Hin, Win] = clf::generate_dummy_data(50) #1216)\n",
+ "[X_val, Y_val, C, Hin, Win] = clf::generate_dummy_data(100)\n",
+ "\n",
+ "# Train\n",
+ "[Wc1, bc1, Wc2, bc2, Wc3, bc3, Wa1, ba1, Wa2, ba2] =\n",
+ " clf::train(X, Y, X_val, Y_val, C, Hin, Win, lr, mu, decay,\n",
+ " lambda, batch_size, parallel_batches, epochs,\n",
+ " log_interval, dir)\n",
+ "\"\"\"\n",
+ "outputs = (\"Wc1\", \"bc1\", \"Wc2\", \"bc2\", \"Wc3\", \"bc3\",\n",
+ " \"Wa1\", \"ba1\", \"Wa2\", \"ba2\")\n",
+ "script = dml(script).output(*outputs)\n",
+ "outs = ml.execute(script).get(*outputs)\n",
+ "Wc1, bc1, Wc2, bc2, Wc3, bc3, Wa1, ba1, Wa2, ba2 = outs\n",
+ "Wc1, bc1, Wc2, bc2, Wc3, bc3, Wa1, ba1, Wa2, ba2"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
"source": [
"## Eval"
]
@@ -602,14 +585,12 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
- "deletable": true,
- "editable": true
+ "collapsed": true
},
"outputs": [],
"source": [
"script = \"\"\"\n",
- "source(\"convnet.dml\") as clf\n",
+ "source(\"breastcancer/convnet_distrib_sgd.dml\") as clf\n",
"\n",
"# Eval\n",
"probs = clf::predict(X, C, Hin, Win, Wc1, bc1, Wc2, bc2, Wc3, bc3, Wa1, ba1, Wa2, ba2)\n",
@@ -629,9 +610,94 @@
"loss, acc, loss_val, acc_val = ml.execute(script).get(*outputs)\n",
"loss, acc, loss_val, acc_val"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "script = \"\"\"\n",
+ "source(\"breastcancer/convnet_distrib_sgd.dml\") as clf\n",
+ "\n",
+ "# Dummy data\n",
+ "[X, Y, C, Hin, Win] = clf::generate_dummy_data(1216)\n",
+ "[X_val, Y_val, C, Hin, Win] = clf::generate_dummy_data(100)\n",
+ "\n",
+ "# Eval\n",
+ "probs = clf::predict(X, C, Hin, Win, Wc1, bc1, Wc2, bc2, Wc3, bc3, Wa1, ba1, Wa2, ba2)\n",
+ "[loss, accuracy] = clf::eval(probs, Y)\n",
+ "probs_val = clf::predict(X_val, C, Hin, Win, Wc1, bc1, Wc2, bc2, Wc3, bc3, Wa1, ba1, Wa2, ba2)\n",
+ "[loss_val, accuracy_val] = clf::eval(probs_val, Y_val)\n",
+ "\"\"\"\n",
+ "outputs = (\"loss\", \"accuracy\", \"loss_val\", \"accuracy_val\")\n",
+ "script = (dml(script).input(Wc1=Wc1, bc1=bc1,\n",
+ " Wc2=Wc2, bc2=bc2,\n",
+ " Wc3=Wc3, bc3=bc3,\n",
+ " Wa1=Wa1, ba1=ba1,\n",
+ " Wa2=Wa2, ba2=ba2)\n",
+ " .output(*outputs))\n",
+ "loss, acc, loss_val, acc_val = ml.execute(script).get(*outputs)\n",
+ "loss, acc, loss_val, acc_val"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "collapsed": true
+ },
+ "source": [
+ "---"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "# script = \"\"\"\n",
+ "# N = 102400 # num examples\n",
+ "# C = 3 # num input channels\n",
+ "# Hin = 256 # input height\n",
+ "# Win = 256 # input width\n",
+ "# X = rand(rows=N, cols=C*Hin*Win, pdf=\"normal\")\n",
+ "# \"\"\"\n",
+ "# outputs = \"X\"\n",
+ "# script = dml(script).output(*outputs)\n",
+ "# thisX = ml.execute(script).get(*outputs)\n",
+ "# thisX"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "# script = \"\"\"\n",
+ "# f = function(matrix[double] X) return(matrix[double] Y) {\n",
+ "# if (1==1) {}\n",
+ "# a = as.scalar(rand(rows=1, cols=1))\n",
+ "# Y = X * a\n",
+ "# }\n",
+ "# Y = f(X)\n",
+ "# \"\"\"\n",
+ "# outputs = \"Y\"\n",
+ "# script = dml(script).input(X=thisX).output(*outputs)\n",
+ "# thisY = ml.execute(script).get(*outputs)\n",
+ "# thisY"
+ ]
}
],
"metadata": {
+ "anaconda-cloud": {},
"kernelspec": {
"display_name": "Python 3 + Spark 2.x + SystemML",
"language": "python",
@@ -647,7 +713,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.6.0"
+ "version": "3.6.1"
}
},
"nbformat": 4,