You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@submarine.apache.org by li...@apache.org on 2020/10/09 13:00:49 UTC
[submarine] branch master updated: SUBMARINE-655. Add a deepfm
example in Jupyter notebook
This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 3881dd1 SUBMARINE-655. Add a deepfm example in Jupyter notebook
3881dd1 is described below
commit 3881dd1b5775ab9418796c2bada914d30d030c38
Author: Kevin Su <pi...@apache.org>
AuthorDate: Tue Oct 6 09:37:32 2020 +0800
SUBMARINE-655. Add a deepfm example in Jupyter notebook
### What is this PR for?
To enhance the user experience of a notebook, we could add a deepfm example under the working directory (default is "/home/jovyan/") in the jupyter notebook.
[deepfm_example.ipynb](https://github.com/pingsutw/hadoop-submarine/blob/8632f4ecdb3c38609f559e5e3a3eafb46cda9bd5/submarine-sdk/pysubmarine/example/deepfm_example.ipynb)
### What type of PR is it?
[Improvement]
### Todos
* [ ] - Task
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-655
### How should this be tested?
https://travis-ci.org/github/pingsutw/hadoop-submarine/builds/733169607
### Screenshots (if appropriate)

### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Kevin Su <pi...@apache.org>
Closes #430 from pingsutw/SUBMARINE-655 and squashes the following commits:
fac5d81 [Kevin Su] SUBMARINE-655. Add a deepfm example in Jupyter notebook
---
dev-support/docker-images/jupyter/Dockerfile | 1 +
submarine-sdk/pysubmarine/example/deepfm.json | 35 ++++
.../pysubmarine/example/deepfm_example.ipynb | 215 +++++++++++++++++++++
submarine-sdk/pysubmarine/setup.py | 2 +-
4 files changed, 252 insertions(+), 1 deletion(-)
diff --git a/dev-support/docker-images/jupyter/Dockerfile b/dev-support/docker-images/jupyter/Dockerfile
index 7e46168..dbcfb60 100644
--- a/dev-support/docker-images/jupyter/Dockerfile
+++ b/dev-support/docker-images/jupyter/Dockerfile
@@ -96,6 +96,7 @@ RUN pip install notebook==6.1.3 && \
git clone https://github.com/apache/submarine && \
pip install submarine/submarine-sdk/pysubmarine && \
cp submarine/submarine-sdk/pysubmarine/example/submarine_experiment_sdk.ipynb $HOME && \
+ cp -r submarine/submarine-sdk/pysubmarine/example/{data,deepfm_example.ipynb,deepfm.json} $HOME && \
rm submarine -rf
diff --git a/submarine-sdk/pysubmarine/example/deepfm.json b/submarine-sdk/pysubmarine/example/deepfm.json
new file mode 100644
index 0000000..ece078e
--- /dev/null
+++ b/submarine-sdk/pysubmarine/example/deepfm.json
@@ -0,0 +1,35 @@
+{
+ "input": {
+ "train_data": ["./data/tr.libsvm"],
+ "valid_data": ["./data/va.libsvm"],
+ "test_data": ["./data/te.libsvm"],
+ "type": "libsvm"
+ },
+ "output": {
+ "save_model_dir": "./experiment",
+ "metric": "auc"
+ },
+ "training": {
+ "batch_size" : 512,
+ "field_size": 39,
+ "num_epochs": 3,
+ "feature_size": 117581,
+ "embedding_size": 256,
+ "learning_rate": 0.0005,
+ "batch_norm_decay": 0.9,
+ "l2_reg": 0.0001,
+ "deep_layers": [400, 400, 400],
+ "dropout": [0.3, 0.3, 0.3],
+ "batch_norm": false,
+ "optimizer": "adam",
+ "log_steps": 10,
+ "seed": 77,
+ "mode": "local"
+ },
+ "resource": {
+ "num_cpu": 4,
+ "num_gpu": 0,
+ "num_thread": 0
+ }
+}
+
diff --git a/submarine-sdk/pysubmarine/example/deepfm_example.ipynb b/submarine-sdk/pysubmarine/example/deepfm_example.ipynb
new file mode 100644
index 0000000..232baf0
--- /dev/null
+++ b/submarine-sdk/pysubmarine/example/deepfm_example.ipynb
@@ -0,0 +1,215 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Sample for running TensorFlow DeepFM example\n",
+ "\n",
+ "The notebook shows how to use Submarine ctr library to train and evaluate a model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "{\r\n",
+ " \"input\": {\r\n",
+ " \"train_data\": [\"./data/tr.libsvm\"],\r\n",
+ " \"valid_data\": [\"./data/va.libsvm\"],\r\n",
+ " \"test_data\": [\"./data/te.libsvm\"],\r\n",
+ " \"type\": \"libsvm\"\r\n",
+ " },\r\n",
+ " \"output\": {\r\n",
+ " \"save_model_dir\": \"./experiment\",\r\n",
+ " \"metric\": \"auc\"\r\n",
+ " },\r\n",
+ " \"training\": {\r\n",
+ " \"batch_size\" : 512,\r\n",
+ " \"field_size\": 39,\r\n",
+ " \"num_epochs\": 3,\r\n",
+ " \"feature_size\": 117581,\r\n",
+ " \"embedding_size\": 256,\r\n",
+ " \"learning_rate\": 0.0005,\r\n",
+ " \"batch_norm_decay\": 0.9,\r\n",
+ " \"l2_reg\": 0.0001,\r\n",
+ " \"deep_layers\": [400, 400, 400],\r\n",
+ " \"dropout\": [0.3, 0.3, 0.3],\r\n",
+ " \"batch_norm\": false,\r\n",
+ " \"optimizer\": \"adam\",\r\n",
+ " \"log_steps\": 10,\r\n",
+ " \"seed\": 77,\r\n",
+ " \"mode\": \"local\"\r\n",
+ " },\r\n",
+ " \"resource\": {\r\n",
+ " \"num_cpu\": 4,\r\n",
+ " \"num_gpu\": 0,\r\n",
+ " \"num_thread\": 0\r\n",
+ " }\r\n",
+ "}\r\n",
+ "\r\n"
+ ]
+ }
+ ],
+ "source": [
+ "!cat deepfm.json"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from submarine.ml.tensorflow.model import DeepFM"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Model parameters : {'output': {'save_model_dir': './experiment', 'metric': 'auc'}, 'training': {'batch_size': 512, 'field_size': 39, 'num_epochs': 3, 'feature_size': 117581, 'embedding_size': 256, 'learning_rate': 0.0005, 'batch_norm_decay': 0.9, 'l2_reg': 0.0001, 'deep_layers': [400, 400, 400], 'dropout': [0.3, 0.3, 0.3], 'batch_norm': False, 'optimizer': 'adam', 'log_steps': 10, 'num_threads': 4, 'num_gpu': 0, 'seed': 77, 'mode': 'local'}, 'resource': {'num_cpu': 4, 'num_gpu': 0 [...]
+ "From /home/kevin/git/submarine/submarine-sdk/pysubmarine/submarine/utils/tf_utils.py:60: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
+ "\n",
+ "Using config: {'_model_dir': './experiment', '_tf_random_seed': None, '_save_summary_steps': 10, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': device_count {\n",
+ " key: \"CPU\"\n",
+ " value: 4\n",
+ "}\n",
+ "device_count {\n",
+ " key: \"GPU\"\n",
+ " value: 0\n",
+ "}\n",
+ ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 10, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f7fef82e6a0>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster' [...]
+ "Not using Distribute Coordinator.\n",
+ "Running training and evaluation locally (non-distributed).\n",
+ "Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps None or save_checkpoints_secs 600.\n",
+ "From /home/kevin/anaconda3/envs/tf/lib/python3.6/site-packages/tensorflow_core/python/training/training_util.py:236: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
+ "Instructions for updating:\n",
+ "Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts.\n",
+ "From /home/kevin/anaconda3/envs/tf/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.string_to_number is deprecated. Please use tf.strings.to_number instead.\n",
+ "\n",
+ "Calling model_fn.\n",
+ "From /home/kevin/git/submarine/submarine-sdk/pysubmarine/submarine/ml/tensorflow/model/base_tf_model.py:114: The name tf.set_random_seed is deprecated. Please use tf.compat.v1.set_random_seed instead.\n",
+ "\n",
+ "\n",
+ "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
+ "For more information, please see:\n",
+ " * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
+ " * https://github.com/tensorflow/addons\n",
+ " * https://github.com/tensorflow/io (for I/O related ops)\n",
+ "If you depend on functionality not listed there, please file an issue.\n",
+ "\n",
+ "From /home/kevin/git/submarine/submarine-sdk/pysubmarine/submarine/ml/tensorflow/layers/core.py:109: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
+ "\n",
+ "From /home/kevin/git/submarine/submarine-sdk/pysubmarine/submarine/ml/tensorflow/layers/core.py:110: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n",
+ "\n",
+ "From /home/kevin/anaconda3/envs/tf/lib/python3.6/site-packages/tensorflow_core/contrib/layers/python/layers/layers.py:1866: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
+ "Instructions for updating:\n",
+ "Please use `layer.__call__` method instead.\n",
+ "From /home/kevin/git/submarine/submarine-sdk/pysubmarine/submarine/ml/tensorflow/layers/core.py:82: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
+ "Instructions for updating:\n",
+ "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
+ "Large dropout rate: 0.7 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n",
+ "Large dropout rate: 0.7 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n",
+ "Large dropout rate: 0.7 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n",
+ "From /home/kevin/git/submarine/submarine-sdk/pysubmarine/submarine/utils/tf_utils.py:102: The name tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY is deprecated. Please use tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY instead.\n",
+ "\n",
+ "From /home/kevin/anaconda3/envs/tf/lib/python3.6/site-packages/tensorflow_core/python/ops/nn_impl.py:183: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
+ "Instructions for updating:\n",
+ "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
+ "From /home/kevin/git/submarine/submarine-sdk/pysubmarine/submarine/utils/tf_utils.py:119: The name tf.metrics.auc is deprecated. Please use tf.compat.v1.metrics.auc instead.\n",
+ "\n",
+ "From /home/kevin/anaconda3/envs/tf/lib/python3.6/site-packages/tensorflow_core/python/ops/metrics_impl.py:808: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
+ "Instructions for updating:\n",
+ "Deprecated in favor of operator or tf.math.divide.\n",
+ "From /home/kevin/git/submarine/submarine-sdk/pysubmarine/submarine/ml/tensorflow/optimizer.py:35: The name tf.train.AdamOptimizer is deprecated. Please use tf.compat.v1.train.AdamOptimizer instead.\n",
+ "\n",
+ "From /home/kevin/git/submarine/submarine-sdk/pysubmarine/submarine/utils/tf_utils.py:131: The name tf.train.get_global_step is deprecated. Please use tf.compat.v1.train.get_global_step instead.\n",
+ "\n",
+ "Done calling model_fn.\n",
+ "Create CheckpointSaverHook.\n",
+ "Graph was finalized.\n",
+ "Running local_init_op.\n",
+ "Done running local_init_op.\n",
+ "Saving checkpoints for 0 into ./experiment/model.ckpt.\n",
+ "loss = 2.311297, step = 1\n",
+ "Saving checkpoints for 6 into ./experiment/model.ckpt.\n",
+ "Calling model_fn.\n",
+ "Done calling model_fn.\n",
+ "Starting evaluation at 2020-10-06T09:35:12Z\n",
+ "Graph was finalized.\n",
+ "Restoring parameters from ./experiment/model.ckpt-6\n",
+ "Running local_init_op.\n",
+ "Done running local_init_op.\n",
+ "Finished evaluation at 2020-10-06-09:35:27\n",
+ "Saving dict for global step 6: auc = 0.5862471, global_step = 6, loss = 1.8989911\n",
+ "Saving 'checkpoint_path' summary for global step 6: ./experiment/model.ckpt-6\n",
+ "Loss for final step: 4.241601.\n",
+ "Calling model_fn.\n",
+ "Done calling model_fn.\n",
+ "Starting evaluation at 2020-10-06T09:35:27Z\n",
+ "Graph was finalized.\n",
+ "Restoring parameters from ./experiment/model.ckpt-6\n",
+ "Running local_init_op.\n",
+ "Done running local_init_op.\n",
+ "Finished evaluation at 2020-10-06-09:35:29\n",
+ "Saving dict for global step 6: auc = 0.5862471, global_step = 6, loss = 1.8989911\n",
+ "Saving 'checkpoint_path' summary for global step 6: ./experiment/model.ckpt-6\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Model metrics : {'auc': 0.5862471, 'loss': 1.8989911, 'global_step': 6}\n"
+ ]
+ }
+ ],
+ "source": [
+ "model = DeepFM(json_path='deepfm.json')\n",
+ "model.train()\n",
+ "result = model.evaluate()\n",
+ "print(\"Model metrics : \", result)"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.6.12"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
diff --git a/submarine-sdk/pysubmarine/setup.py b/submarine-sdk/pysubmarine/setup.py
index b339533..efd2d69 100644
--- a/submarine-sdk/pysubmarine/setup.py
+++ b/submarine-sdk/pysubmarine/setup.py
@@ -28,7 +28,7 @@ setup(
packages=find_packages(exclude=['tests', 'tests.*']),
install_requires=[
'six>=1.10.0',
- 'numpy',
+ 'numpy==1.18.5',
'pandas',
'sqlalchemy',
'sqlparse',
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@submarine.apache.org
For additional commands, e-mail: dev-help@submarine.apache.org