You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficcontrol.apache.org by mi...@apache.org on 2016/11/01 19:16:19 UTC
[1/2] incubator-trafficcontrol git commit: add docker setup to run
pgloader
Repository: incubator-trafficcontrol
Updated Branches:
refs/heads/psql 0e1ebd7c5 -> 91021764b
add docker setup to run pgloader
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/commit/d8913e9e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/tree/d8913e9e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/diff/d8913e9e
Branch: refs/heads/psql
Commit: d8913e9e64f70ecee5ad58d780145579fbf99bca
Parents: 0e1ebd7
Author: Dan Kirkwood <da...@gmail.com>
Authored: Mon Oct 17 15:43:46 2016 -0600
Committer: Jeremy Mitchell <mi...@gmail.com>
Committed: Tue Nov 1 13:15:28 2016 -0600
----------------------------------------------------------------------
.../app/db/pg-migration/Dockerfile-pgloader | 8 ++++
.../app/db/pg-migration/Dockerfile-postgres | 9 +++++
traffic_ops/app/db/pg-migration/README.md | 19 ++++++++++
.../app/db/pg-migration/docker-compose.yml | 40 ++++++++++++++++++++
.../app/db/pg-migration/mysql/conf.d/mysql.cnf | 8 ++++
traffic_ops/app/db/pg-migration/mysql/run.sh | 8 ++++
traffic_ops/app/db/pg-migration/runpgloader.sh | 13 +++++++
7 files changed, 105 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/d8913e9e/traffic_ops/app/db/pg-migration/Dockerfile-pgloader
----------------------------------------------------------------------
diff --git a/traffic_ops/app/db/pg-migration/Dockerfile-pgloader b/traffic_ops/app/db/pg-migration/Dockerfile-pgloader
new file mode 100644
index 0000000..5bd4aab
--- /dev/null
+++ b/traffic_ops/app/db/pg-migration/Dockerfile-pgloader
@@ -0,0 +1,8 @@
+FROM dimitri/pgloader
+MAINTAINER Dan Kirkwood <da...@apache.org>
+
+# install nc to check that postgres and mysql are ready..
+RUN apt-get -y install netcat
+
+ADD runpgloader.sh .
+CMD ./runpgloader.sh
http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/d8913e9e/traffic_ops/app/db/pg-migration/Dockerfile-postgres
----------------------------------------------------------------------
diff --git a/traffic_ops/app/db/pg-migration/Dockerfile-postgres b/traffic_ops/app/db/pg-migration/Dockerfile-postgres
new file mode 100644
index 0000000..e5dcb30
--- /dev/null
+++ b/traffic_ops/app/db/pg-migration/Dockerfile-postgres
@@ -0,0 +1,9 @@
+FROM postgres:9.6
+
+MAINTAINER Dan Kirkwood
+
+# Adjust PostgreSQL configuration so that remote connections to the
+# database are possible.
+RUN echo "host all all 0.0.0.0/0 md5" >> $PGDATA/pg_hba.conf
+RUN echo "listen_addresses='*'" >> $PGDATA/postgresql.conf
+EXPOSE "5432:5432"
http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/d8913e9e/traffic_ops/app/db/pg-migration/README.md
----------------------------------------------------------------------
diff --git a/traffic_ops/app/db/pg-migration/README.md b/traffic_ops/app/db/pg-migration/README.md
new file mode 100644
index 0000000..ed31e54
--- /dev/null
+++ b/traffic_ops/app/db/pg-migration/README.md
@@ -0,0 +1,19 @@
+
+# Converting existing mysql `traffic_ops` database to postgres
+
+* Requires `docker-engine` and `docker-compose`.
+
+* Ensure database is up-to-date with latest `traffic_ops` migrations for `mysql` (last 1.x release of `traffic_ops`)
+ * `cd /opt/traffic_ops/app; ./db/admin.pl --env=production upgrade`
+
+* Get a database dump from `traffic_ops`
+ * `Tools->DB Dump`
+
+* In development environment, `cd traffic_ops/app/db/pg-migration`.
+
+* Move the `mysql` database dump file into `./mysql/initdb.d` directory. The file must have a `.sql` suffix.
+
+* `docker-compose down -v && docker-compose build && docker-compose up`
+
+* Postgres is still running in a docker container -- dump the database to a file:
+ `docker exec -it pgmigration_postgres_host_1 pg_dump -Utraffic_ops >pg.sql`
http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/d8913e9e/traffic_ops/app/db/pg-migration/docker-compose.yml
----------------------------------------------------------------------
diff --git a/traffic_ops/app/db/pg-migration/docker-compose.yml b/traffic_ops/app/db/pg-migration/docker-compose.yml
new file mode 100644
index 0000000..2d0a0b8
--- /dev/null
+++ b/traffic_ops/app/db/pg-migration/docker-compose.yml
@@ -0,0 +1,40 @@
+version: '2'
+services:
+ mysql_host:
+ image: mysql:5.6
+ restart: "no"
+ environment:
+ - MYSQL_DATABASE=traffic_ops_db
+ - MYSQL_RANDOM_ROOT_PASSWORD=yes
+ - MYSQL_PASSWORD=twelve
+ - MYSQL_USER=to_user
+ volumes:
+ - ./mysql/conf.d:/etc/mysql/conf.d
+ - ./mysql/initdb.d:/docker-entrypoint-initdb.d
+
+ postgres_host:
+ build:
+ dockerfile: pg-migration/Dockerfile-postgres
+ context: ..
+ restart: "no"
+ environment:
+ - POSTGRES_DB=traffic_ops
+ - POSTGRES_PASSWORD=twelve
+ - POSTGRES_USER=traffic_ops
+ ports:
+ - "5432:5432"
+
+ pgloader:
+ build:
+ dockerfile: Dockerfile-pgloader
+ context: .
+ restart: "no"
+ environment:
+ - MYSQL_HOST=mysql_host
+ - MYSQL_DATABASE=traffic_ops_db
+ - MYSQL_PASSWORD=twelve
+ - MYSQL_USER=to_user
+ - POSTGRES_HOST=postgres_host
+ - POSTGRES_DB=traffic_ops
+ - POSTGRES_PASSWORD=twelve
+ - POSTGRES_USER=traffic_ops
http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/d8913e9e/traffic_ops/app/db/pg-migration/mysql/conf.d/mysql.cnf
----------------------------------------------------------------------
diff --git a/traffic_ops/app/db/pg-migration/mysql/conf.d/mysql.cnf b/traffic_ops/app/db/pg-migration/mysql/conf.d/mysql.cnf
new file mode 100644
index 0000000..18dc029
--- /dev/null
+++ b/traffic_ops/app/db/pg-migration/mysql/conf.d/mysql.cnf
@@ -0,0 +1,8 @@
+# This is custom config file attached from docker host
+
+[mysql]
+default_character_set = utf8
+
+[mysqld]
+character_set_server = utf8 # If you prefer utf8
+collation_server = utf8_general_ci
http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/d8913e9e/traffic_ops/app/db/pg-migration/mysql/run.sh
----------------------------------------------------------------------
diff --git a/traffic_ops/app/db/pg-migration/mysql/run.sh b/traffic_ops/app/db/pg-migration/mysql/run.sh
new file mode 100644
index 0000000..50c149d
--- /dev/null
+++ b/traffic_ops/app/db/pg-migration/mysql/run.sh
@@ -0,0 +1,8 @@
+docker run \
+ --name mysql \
+ -p 3306:3306 \
+ -v $(pwd)/mysql/conf.d:/etc/mysql/conf.d \
+ -v $(pwd)/mysql/initdb.d:/docker-entrypoint-initdb.d \
+ -d mysql
+
+
http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/d8913e9e/traffic_ops/app/db/pg-migration/runpgloader.sh
----------------------------------------------------------------------
diff --git a/traffic_ops/app/db/pg-migration/runpgloader.sh b/traffic_ops/app/db/pg-migration/runpgloader.sh
new file mode 100755
index 0000000..3ae3055
--- /dev/null
+++ b/traffic_ops/app/db/pg-migration/runpgloader.sh
@@ -0,0 +1,13 @@
+#!/bin/bash -x
+
+# make sure postgres and mysql ports are both active
+echo "POSTGRES_HOST=$POSTGRES_HOST MYSQL_HOST=$MYSQL_HOST"
+for c in "$POSTGRES_HOST 5432" "$MYSQL_HOST 3306"; do
+ while true; do
+ echo Waiting for $c
+ sleep 3
+ nc -z $c && break
+ done
+done
+
+pgloader -v --cast 'type tinyint to smallint drop typemod' mysql://$MYSQL_USER:$MYSQL_PASSWORD@$MYSQL_HOST/traffic_ops_db postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB
[2/2] incubator-trafficcontrol git commit: This closes #14
Posted by mi...@apache.org.
This closes #14
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/commit/91021764
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/tree/91021764
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/diff/91021764
Branch: refs/heads/psql
Commit: 91021764b983b6ab0e0cb2366e45b58669b35938
Parents: d8913e9
Author: Jeremy Mitchell <mi...@gmail.com>
Authored: Tue Nov 1 13:16:08 2016 -0600
Committer: Jeremy Mitchell <mi...@gmail.com>
Committed: Tue Nov 1 13:16:08 2016 -0600
----------------------------------------------------------------------
----------------------------------------------------------------------