You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by du...@apache.org on 2017/09/07 23:02:38 UTC
systemml git commit: [MINOR] Fixes for the breast cancer project.
Repository: systemml
Updated Branches:
refs/heads/master c00029a7b -> 4d376637a
[MINOR] Fixes for the breast cancer project.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/4d376637
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/4d376637
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/4d376637
Branch: refs/heads/master
Commit: 4d376637a99891e53ea93f650fb9341fc19b99f9
Parents: c00029a
Author: Mike Dusenberry <mw...@us.ibm.com>
Authored: Thu Sep 7 16:01:45 2017 -0700
Committer: Mike Dusenberry <mw...@us.ibm.com>
Committed: Thu Sep 7 16:01:45 2017 -0700
----------------------------------------------------------------------
projects/breast_cancer/MachineLearning-Keras-ResNet50.ipynb | 3 +--
projects/breast_cancer/breastcancer/preprocessing.py | 4 +++-
projects/breast_cancer/preprocess.py | 2 +-
3 files changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/4d376637/projects/breast_cancer/MachineLearning-Keras-ResNet50.ipynb
----------------------------------------------------------------------
diff --git a/projects/breast_cancer/MachineLearning-Keras-ResNet50.ipynb b/projects/breast_cancer/MachineLearning-Keras-ResNet50.ipynb
index bafa74a..b99a51d 100644
--- a/projects/breast_cancer/MachineLearning-Keras-ResNet50.ipynb
+++ b/projects/breast_cancer/MachineLearning-Keras-ResNet50.ipynb
@@ -28,12 +28,11 @@
"from keras.applications.resnet50 import ResNet50\n",
"from keras.callbacks import ModelCheckpoint, TensorBoard\n",
"from keras.initializers import VarianceScaling\n",
- "from keras.layers import Dense, Dropout, Flatten, GlobalAveragePooling2D, Input, Lambda, merge\n",
+ "from keras.layers import Dense, Dropout, Flatten, GlobalAveragePooling2D, Input, Lambda\n",
"from keras.models import Model, load_model\n",
"from keras.optimizers import SGD\n",
"from keras.preprocessing.image import ImageDataGenerator\n",
"from keras.regularizers import l2\n",
- "from keras.utils import to_categorical\n",
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"import pandas as pd\n",
http://git-wip-us.apache.org/repos/asf/systemml/blob/4d376637/projects/breast_cancer/breastcancer/preprocessing.py
----------------------------------------------------------------------
diff --git a/projects/breast_cancer/breastcancer/preprocessing.py b/projects/breast_cancer/breastcancer/preprocessing.py
index 763bde5..0e4e91f 100644
--- a/projects/breast_cancer/breastcancer/preprocessing.py
+++ b/projects/breast_cancer/breastcancer/preprocessing.py
@@ -71,6 +71,8 @@ def open_slide(slide_num, folder, training):
slide = openslide.open_slide(filename)
except OpenSlideError:
slide = None
+ except FileNotFoundError:
+ slide = None
return slide
@@ -586,7 +588,7 @@ def preprocess(spark, slide_nums, folder="data", training=True, tile_size=1024,
# Append labels
labels_df = get_labels_df(folder)
samples_with_labels = (samples.map(
- lambda tup: (tup[0], int(labels_df.at[tup[0],"tumor_score"]),
+ lambda tup: (int(tup[0]), int(labels_df.at[tup[0],"tumor_score"]),
float(labels_df.at[tup[0],"molecular_score"]), Vectors.dense(tup[1]))))
df = samples_with_labels.toDF(["slide_num", "tumor_score", "molecular_score", "sample"])
df = df.select(df.slide_num.astype("int"), df.tumor_score.astype("int"),
http://git-wip-us.apache.org/repos/asf/systemml/blob/4d376637/projects/breast_cancer/preprocess.py
----------------------------------------------------------------------
diff --git a/projects/breast_cancer/preprocess.py b/projects/breast_cancer/preprocess.py
index e90fe8c..71055ad 100644
--- a/projects/breast_cancer/preprocess.py
+++ b/projects/breast_cancer/preprocess.py
@@ -33,7 +33,7 @@ import pandas as pd
from sklearn.model_selection import train_test_split
from pyspark.sql import SparkSession
-from breastcancer.preprocessing import add_row_indices, get_labels_df, preprocess, save
+from breastcancer.preprocessing import add_row_indices, get_labels_df, preprocess, save, sample
# Create new SparkSession