You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@submarine.apache.org by by...@apache.org on 2021/06/29 09:16:08 UTC
[submarine] branch master updated: SUBMARINE-859. Mlflow sqlite
database should be replaced with mysql
This is an automated email from the ASF dual-hosted git repository.
byronhsu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new e8e2711 SUBMARINE-859. Mlflow sqlite database should be replaced with mysql
e8e2711 is described below
commit e8e2711164b7b3b36594e7f5fc7662d9edd74dde
Author: jeff-901 <b0...@ntu.edu.com>
AuthorDate: Mon Jun 28 20:40:25 2021 +0800
SUBMARINE-859. Mlflow sqlite database should be replaced with mysql
### What is this PR for?
Mlflow backend sqlite database is replaced by mysql database to overcome the deadlock issue.
### What type of PR is it?
Improvement
### Todos
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-859
### How should this be tested?
### Screenshots (if appropriate)
### Questions:
* Do the license files need updating? No
* Are there breaking changes for older versions? No
* Does this need new documentation? No
Author: jeff-901 <b0...@ntu.edu.com>
Signed-off-by: byronhsu <by...@apache.org>
Closes #614 from jeff-901/SUBMARINE-859 and squashes the following commits:
c3700103 [jeff-901] merge
a366ec83 [jeff-901] sql format
bfa0e871 [jeff-901] edit sql format
bc68101f [jeff-901] add mysql for mlflow backend
4f15d133 [jeff-901] for travis
00220038 [jeff-901] for travis
6f47fde5 [jeff-901] edit sql format
0b400633 [jeff-901] add mysql for mlflow backend
35f1e945 [jeff-901] for travis
e0293dea [jeff-901] for travis
eb0adac5 [jeff-901] edit remain url
85650f0c [jeff-901] edit mlflow dockerfile s3 url
ed1cfe28 [jeff-901] delete fix ip
---
dev-support/database/init-database.sh | 5 +
dev-support/database/mlflow.sql | 152 +++++++++++++++++++++++++
dev-support/docker-images/database/startup.sh | 4 +
dev-support/docker-images/mlflow/Dockerfile | 2 +-
dev-support/mini-submarine/conf/setup-mysql.sh | 5 +
5 files changed, 167 insertions(+), 1 deletion(-)
diff --git a/dev-support/database/init-database.sh b/dev-support/database/init-database.sh
index f9bf047..968b1b3 100755
--- a/dev-support/database/init-database.sh
+++ b/dev-support/database/init-database.sh
@@ -38,3 +38,8 @@ mysql -e "CREATE DATABASE IF NOT EXISTS metastore;"
mysql -e "CREATE USER IF NOT EXISTS 'metastore'@'%' IDENTIFIED BY 'password';"
mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'metastore'@'%';"
mysql -e "use metastore; source ./metastore.sql; show tables;"
+
+mysql -e "CREATE DATABASE IF NOT EXISTS mlflow;"
+mysql -e "CREATE USER IF NOT EXISTS 'mlflow'@'%' IDENTIFIED BY 'password';"
+mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'mlflow'@'%';"
+mysql -e "use mlflow; source ./mlflow.sql; show tables;"
\ No newline at end of file
diff --git a/dev-support/database/mlflow.sql b/dev-support/database/mlflow.sql
new file mode 100644
index 0000000..9cfa441
--- /dev/null
+++ b/dev-support/database/mlflow.sql
@@ -0,0 +1,152 @@
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+-- http://www.apache.org/licenses/LICENSE-2.0
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+
+DROP TABLE IF EXISTS `alembic_version`;
+CREATE TABLE `alembic_version` (
+ `version_num` VARCHAR(32) NOT NULL,
+ CONSTRAINT `alembic_version_pkc` PRIMARY KEY (`version_num`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+DROP TABLE IF EXISTS `experiments`;
+CREATE TABLE `experiments` (
+ `experiment_id` INTEGER NOT NULL,
+ `name` VARCHAR(256) NOT NULL,
+ `artifact_location` VARCHAR(256),
+ `lifecycle_stage` VARCHAR(32),
+ CONSTRAINT `experiment_pk` PRIMARY KEY (`experiment_id`),
+ UNIQUE (`name`),
+ CONSTRAINT `experiments_lifecycle_stage` CHECK (`lifecycle_stage` IN ('active', 'deleted'))
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+DROP TABLE IF EXISTS `registered_models`;
+CREATE TABLE `registered_models` (
+ `name` VARCHAR(256) NOT NULL,
+ `creation_time` BIGINT,
+ `last_updated_time` BIGINT,
+ `description` VARCHAR(5000),
+ CONSTRAINT `registered_model_pk` PRIMARY KEY (`name`),
+ UNIQUE (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+DROP TABLE IF EXISTS `experiment_tags`;
+CREATE TABLE `experiment_tags` (
+ `key` VARCHAR(250) NOT NULL,
+ `value` VARCHAR(5000),
+ `experiment_id` INTEGER NOT NULL,
+ CONSTRAINT `experiment_tag_pk` PRIMARY KEY (`key`, `experiment_id`),
+ FOREIGN KEY(`experiment_id`) REFERENCES `experiments` (`experiment_id`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+DROP TABLE IF EXISTS `model_versions`;
+CREATE TABLE `model_versions` (
+ `name` VARCHAR(256) NOT NULL,
+ `version` INTEGER NOT NULL,
+ `creation_time` BIGINT,
+ `last_updated_time` BIGINT,
+ `description` VARCHAR(5000),
+ `user_id` VARCHAR(256),
+ `current_stage` VARCHAR(20),
+ `source` VARCHAR(500),
+ `run_id` VARCHAR(32),
+ `status` VARCHAR(20),
+ `status_message` VARCHAR(500),
+ `run_link` VARCHAR(500),
+ CONSTRAINT `model_version_pk` PRIMARY KEY (`name`, `version`),
+ FOREIGN KEY(`name`) REFERENCES `registered_models` (`name`) ON UPDATE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+DROP TABLE IF EXISTS `registered_model_tags`;
+CREATE TABLE `registered_model_tags` (
+ `key` VARCHAR(250) NOT NULL,
+ `value` VARCHAR(5000),
+ `name` VARCHAR(256) NOT NULL,
+ CONSTRAINT `registered_model_tag_pk` PRIMARY KEY (`key`, `name`),
+ FOREIGN KEY(`name`) REFERENCES `registered_models` (`name`) ON UPDATE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+DROP TABLE IF EXISTS `runs`;
+CREATE TABLE runs (
+ `run_uuid` VARCHAR(32) NOT NULL,
+ `name` VARCHAR(250),
+ `source_type` VARCHAR(20),
+ `source_name` VARCHAR(500),
+ `entry_point_name` VARCHAR(50),
+ `user_id` VARCHAR(256),
+ `status` VARCHAR(9),
+ `start_time` BIGINT,
+ `end_time` BIGINT,
+ `source_version` VARCHAR(50),
+ `lifecycle_stage` VARCHAR(20),
+ `artifact_uri` VARCHAR(200),
+ `experiment_id` INTEGER,
+ CONSTRAINT `run_pk` PRIMARY KEY (`run_uuid`),
+ FOREIGN KEY(`experiment_id`) REFERENCES `experiments` (`experiment_id`),
+ CONSTRAINT `source_type` CHECK (`source_type` IN ('NOTEBOOK', 'JOB', 'LOCAL', 'UNKNOWN', 'PROJECT')),
+ CONSTRAINT `runs_lifecycle_stage` CHECK (`lifecycle_stage` IN ('active', 'deleted')),
+ CHECK (status IN ('SCHEDULED', 'FAILED', 'FINISHED', 'RUNNING', 'KILLED'))
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+DROP TABLE IF EXISTS `latest_metrics`;
+CREATE TABLE `latest_metrics` (
+ `key` VARCHAR(250) NOT NULL,
+ `value` FLOAT NOT NULL,
+ `timestamp` BIGINT,
+ `step` BIGINT NOT NULL,
+ `is_nan` BOOLEAN NOT NULL,
+ `run_uuid` VARCHAR(32) NOT NULL,
+ CONSTRAINT `latest_metric_pk` PRIMARY KEY (`key`, `run_uuid`),
+ FOREIGN KEY(`run_uuid`) REFERENCES `runs` (`run_uuid`),
+ CHECK (`is_nan` IN (0, 1))
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+DROP TABLE IF EXISTS `metrics`;
+CREATE TABLE `metrics` (
+ `key` VARCHAR(250) NOT NULL,
+ `value` FLOAT NOT NULL,
+ `timestamp` BIGINT NOT NULL,
+ `run_uuid` VARCHAR(32) NOT NULL,
+ `step` BIGINT DEFAULT '0' NOT NULL,
+ `is_nan` BOOLEAN DEFAULT '0' NOT NULL,
+ CONSTRAINT `metric_pk` PRIMARY KEY (`key`, `timestamp`, `step`, `run_uuid`, `value`, `is_nan`),
+ FOREIGN KEY(`run_uuid`) REFERENCES `runs` (`run_uuid`),
+ CHECK (`is_nan` IN (0, 1))
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+DROP TABLE IF EXISTS `model_version_tags`;
+CREATE TABLE `model_version_tags` (
+ `key` VARCHAR(250) NOT NULL,
+ `value` VARCHAR(5000),
+ `name` VARCHAR(256) NOT NULL,
+ `version` INTEGER NOT NULL,
+ CONSTRAINT `model_version_tag_pk` PRIMARY KEY (`key`, `name`, `version`),
+ FOREIGN KEY(`name`, version) REFERENCES `model_versions` (`name`, `version`) ON UPDATE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+DROP TABLE IF EXISTS `params`;
+CREATE TABLE `params` (
+ `key` VARCHAR(250) NOT NULL,
+ `value` VARCHAR(250) NOT NULL,
+ `run_uuid` VARCHAR(32) NOT NULL,
+ CONSTRAINT `param_pk` PRIMARY KEY (`key`, `run_uuid`),
+ FOREIGN KEY(`run_uuid`) REFERENCES runs (`run_uuid`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+DROP TABLE IF EXISTS `tags`;
+CREATE TABLE `tags` (
+ `key` VARCHAR(250) NOT NULL,
+ `value` VARCHAR(5000),
+ `run_uuid` VARCHAR(32) NOT NULL,
+ CONSTRAINT `tag_pk` PRIMARY KEY (`key`, `run_uuid`),
+ FOREIGN KEY(`run_uuid`) REFERENCES `runs` (`run_uuid`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
diff --git a/dev-support/docker-images/database/startup.sh b/dev-support/docker-images/database/startup.sh
index 884fab6..ef5a591 100755
--- a/dev-support/docker-images/database/startup.sh
+++ b/dev-support/docker-images/database/startup.sh
@@ -31,4 +31,8 @@ CREATE DATABASE metastore;
CREATE USER 'metastore'@'%' IDENTIFIED BY 'password';
GRANT ALL PRIVILEGES ON * . * TO 'metastore'@'%';
use metastore; source /tmp/database/metastore.sql;
+CREATE DATABASE mlflow;
+CREATE USER 'mlflow'@'%' IDENTIFIED BY 'password';
+GRANT ALL PRIVILEGES ON * . * TO 'mlflow'@'%';
+use mlflow; source /tmp/database/mlflow.sql;
EOF
diff --git a/dev-support/docker-images/mlflow/Dockerfile b/dev-support/docker-images/mlflow/Dockerfile
index ea000f2..258e594 100644
--- a/dev-support/docker-images/mlflow/Dockerfile
+++ b/dev-support/docker-images/mlflow/Dockerfile
@@ -30,7 +30,7 @@ RUN wget https://dl.min.io/client/mc/release/linux-amd64/mc && chmod +x mc
ENV MLFLOW_S3_ENDPOINT_URL http://submarine-minio-service:9000
ENV AWS_ACCESS_KEY_ID submarine_minio
ENV AWS_SECRET_ACCESS_KEY submarine_minio
-ENV BACKEND_URI sqlite:///store.db
+ENV BACKEND_URI mysql+pymysql://mlflow:password@localhost:3306/mlflow
EXPOSE 5000
diff --git a/dev-support/mini-submarine/conf/setup-mysql.sh b/dev-support/mini-submarine/conf/setup-mysql.sh
index d59632b..57d68ad 100755
--- a/dev-support/mini-submarine/conf/setup-mysql.sh
+++ b/dev-support/mini-submarine/conf/setup-mysql.sh
@@ -36,3 +36,8 @@ mysql -e "CREATE DATABASE metastore;"
mysql -e "CREATE USER 'metastore'@'%' IDENTIFIED BY 'password';"
mysql -e "GRANT ALL PRIVILEGES ON * . * TO 'metastore'@'%';"
mysql -e "use metastore; source /home/yarn/database/metastore.sql;"
+
+mysql -e "CREATE DATABASE mlflow;"
+mysql -e "CREATE USER 'mlflow'@'%' IDENTIFIED BY 'password';"
+mysql -e "GRANT ALL PRIVILEGES ON * . * TO 'mlflow'@'%';"
+mysql -e "use mlflow; source /home/yarn/database/mlflow.sql;"
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@submarine.apache.org
For additional commands, e-mail: dev-help@submarine.apache.org