You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by yw...@apache.org on 2019/10/29 05:52:19 UTC
[bigtop] branch cnb updated: BIGTOP-3250: Upgrade Presto docker
image
This is an automated email from the ASF dual-hosted git repository.
ywkim pushed a commit to branch cnb
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/cnb by this push:
new 9a40032 BIGTOP-3250: Upgrade Presto docker image
9a40032 is described below
commit 9a40032bb65443db6e0e696f67802386d75b8c1c
Author: Youngwoo Kim <yw...@apache.org>
AuthorDate: Wed Oct 16 17:04:29 2019 +0900
BIGTOP-3250: Upgrade Presto docker image
---
presto3-minio/coordinator/Dockerfile | 6 ------
presto3-minio/coordinator/README.md | 8 --------
presto3-minio/coordinator/config.properties | 8 --------
presto3-minio/coordinator/json | 2 --
presto3-minio/test.sh | 19 -------------------
presto3-minio/worker/Dockerfile | 5 -----
presto3-minio/worker/README.md | 6 ------
presto3-minio/worker/json | 2 --
{presto3-minio => prestosql}/README.md | 0
prestosql/docker/Makefile | 8 ++++++++
prestosql/docker/README.md | 6 ++++++
prestosql/docker/coordinator/Dockerfile | 14 ++++++++++++++
prestosql/docker/coordinator/README.md | 7 +++++++
prestosql/docker/coordinator/config.properties | 5 +++++
prestosql/docker/coordinator/jvm.config | 12 ++++++++++++
.../docker/coordinator}/minio.properties | 0
prestosql/docker/coordinator/node.properties | 3 +++
prestosql/docker/worker/Dockerfile | 12 ++++++++++++
prestosql/docker/worker/README.md | 6 ++++++
.../docker}/worker/config.properties | 3 ---
prestosql/docker/worker/jvm.config | 12 ++++++++++++
.../docker/worker}/minio.properties | 0
prestosql/docker/worker/node.properties | 3 +++
.../presto-deployment-master.yaml | 2 +-
.../presto-deployment-worker.yaml | 4 ++--
{presto3-minio => prestosql}/presto-deployment.yaml | 2 +-
{presto3-minio => prestosql}/service.yaml | 0
27 files changed, 92 insertions(+), 63 deletions(-)
diff --git a/presto3-minio/coordinator/Dockerfile b/presto3-minio/coordinator/Dockerfile
deleted file mode 100755
index ac9c71b..0000000
--- a/presto3-minio/coordinator/Dockerfile
+++ /dev/null
@@ -1,6 +0,0 @@
-FROM wlan0/presto-benchmark-environment:coordinator
-COPY ./json /resources/nyc_taxi_data_3M.json
-COPY ./minio.properties /resources/presto-server-302-e.11/etc/catalog/minio.properties
-
-COPY config.properties /resources/presto-server-302-e.11/etc/config.properties
-CMD cat /resources/presto-server-302-e.11/etc/config.properties ; /resources/presto-server-302-e.11/bin/launcher start && cat /resources/presto-server-302-e.11/etc/config.properties && echo "Hardcoded to s3://minio:9000 , using 'minio' and 'minio123' as the access credentials..." && sleep 60 && cat /resources/presto-server-302-e.11/etc/config.properties && tail -f ./resources/presto-server-302-e.11/var/log/server.log
diff --git a/presto3-minio/coordinator/README.md b/presto3-minio/coordinator/README.md
deleted file mode 100755
index 3423f65..0000000
--- a/presto3-minio/coordinator/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-# Use squash !
-
-Building this image, make sure you use squash so you can gut the 12 GB file :)
-
-
-docker build --squash -t jayunit100/wlan0-presto:latest ./
-
-
diff --git a/presto3-minio/coordinator/config.properties b/presto3-minio/coordinator/config.properties
deleted file mode 100755
index 6a214ae..0000000
--- a/presto3-minio/coordinator/config.properties
+++ /dev/null
@@ -1,8 +0,0 @@
-coordinator=true
-node-scheduler.include-coordinator=true
-http-server.http.port=8080
-query.max-memory=1GB
-query.max-memory-per-node=1GB
-query.max-total-memory-per-node=1GB
-discovery-server.enabled=true
-discovery.uri=http://presto:8080
diff --git a/presto3-minio/coordinator/json b/presto3-minio/coordinator/json
deleted file mode 100755
index 6906dcd..0000000
--- a/presto3-minio/coordinator/json
+++ /dev/null
@@ -1,2 +0,0 @@
-{"id":"3418178","vendor_id":"1","pickup_datetime":"2014-03-13 20:17:16","dropoff_datetime":"2014-03-13 20:22:29","store_and_fwd_flag":"N","rate_code_id":"1","pickup_longitude":"-73.9873046875","pickup_latitude":"40.691890716552734","dropoff_longitude":"-73.975677490234375","dropoff_latitude":"40.696006774902344","passenger_count":"1","trip_distance":"0.90","fare_amount":"5.5","extra":"0.5","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","ehail_fee":"","improvement_surcharge":"","tota [...]
-
diff --git a/presto3-minio/test.sh b/presto3-minio/test.sh
deleted file mode 100755
index be3ffff..0000000
--- a/presto3-minio/test.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-echo "ok to cleanup?"
-read x
-
-kubectl delete ns minio
-helm delete minio --purge
-
-
-
-kubectl create ns minio ;
-# Sets up minio passwords...
-kubectl -n minio create secret generic my-minio-secret --from-literal=accesskey=minio --from-literal=secretkey=minio123
-
-# Install minio
-helm install --set existingSecret=my-minio-secret stable/minio --namespace=minio --name=minio
-
-echo "installing via helm: minio"
-
-# Install presto w/ minio configured...
-kubectl create -f ./ -n minio
diff --git a/presto3-minio/worker/Dockerfile b/presto3-minio/worker/Dockerfile
deleted file mode 100755
index 6dcf74e..0000000
--- a/presto3-minio/worker/Dockerfile
+++ /dev/null
@@ -1,5 +0,0 @@
-FROM wlan0/presto-benchmark-environment:coordinator
-COPY ./json /resources/nyc_taxi_data_3M.json
-COPY minio.properties /resources/presto-server-302-e.11/etc/catalog/minio.properties
-COPY config.properties /resources/presto-server-302-e.11/etc/config.properties
-CMD cat /resources/presto-server-302-e.11/etc/config.properties ; /resources/presto-server-302-e.11/bin/launcher start && cat /resources/presto-server-302-e.11/etc/config.properties && echo "Hardcoded to s3://minio:9000 , using 'minio' and 'minio123' as the access credentials..." && sleep 60 && cat /resources/presto-server-302-e.11/etc/config.properties && tail -f ./resources/presto-server-302-e.11/var/log/server.log
diff --git a/presto3-minio/worker/README.md b/presto3-minio/worker/README.md
deleted file mode 100755
index d5e6e16..0000000
--- a/presto3-minio/worker/README.md
+++ /dev/null
@@ -1,6 +0,0 @@
-# Use squash !
-
-Building this image, make sure you use squash so you can gut the 12 GB file :)
-
-
-docker build --squash -t jayunit100/wlan0-presto-worker:latest ./
diff --git a/presto3-minio/worker/json b/presto3-minio/worker/json
deleted file mode 100755
index 6906dcd..0000000
--- a/presto3-minio/worker/json
+++ /dev/null
@@ -1,2 +0,0 @@
-{"id":"3418178","vendor_id":"1","pickup_datetime":"2014-03-13 20:17:16","dropoff_datetime":"2014-03-13 20:22:29","store_and_fwd_flag":"N","rate_code_id":"1","pickup_longitude":"-73.9873046875","pickup_latitude":"40.691890716552734","dropoff_longitude":"-73.975677490234375","dropoff_latitude":"40.696006774902344","passenger_count":"1","trip_distance":"0.90","fare_amount":"5.5","extra":"0.5","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","ehail_fee":"","improvement_surcharge":"","tota [...]
-
diff --git a/presto3-minio/README.md b/prestosql/README.md
similarity index 100%
rename from presto3-minio/README.md
rename to prestosql/README.md
diff --git a/prestosql/docker/Makefile b/prestosql/docker/Makefile
new file mode 100644
index 0000000..aa5c1ac
--- /dev/null
+++ b/prestosql/docker/Makefile
@@ -0,0 +1,8 @@
+PRESTO_VERSION := latest
+
+.PHONY: build
+
+build:
+ docker build --squash --build-arg PRESTO_IMAGE=prestosql/presto:${PRESTO_VERSION} -t bigtop/prestosql-coordinator:${PRESTO_VERSION} coordinator
+ docker build --squash --build-arg PRESTO_IMAGE=prestosql/presto:${PRESTO_VERSION} -t bigtop/prestosql-worker:${PRESTO_VERSION} worker
+
diff --git a/prestosql/docker/README.md b/prestosql/docker/README.md
new file mode 100644
index 0000000..71bd24c
--- /dev/null
+++ b/prestosql/docker/README.md
@@ -0,0 +1,6 @@
+Buil Docker images for PrestoSQL
+
+```
+$ make build
+
+```
diff --git a/prestosql/docker/coordinator/Dockerfile b/prestosql/docker/coordinator/Dockerfile
new file mode 100755
index 0000000..b294453
--- /dev/null
+++ b/prestosql/docker/coordinator/Dockerfile
@@ -0,0 +1,14 @@
+ARG PRESTO_IMAGE=prestosql/presto:latest
+FROM $PRESTO_IMAGE
+
+LABEL maintainer="Apache Bogtop community <de...@bigtop.apache.org>"
+LABEL url="https://bigtop.apache.org"
+
+ENV PRESTO_HOME=/usr/lib/presto
+ENV PRESTO_DATA=/data/presto
+
+COPY minio.properties $PRESTO_HOME/etc/catalog/minio.properties
+COPY config.properties $PRESTO_HOME/etc/config.properties
+COPY jvm.config $PRESTO_HOME/etc/jvm.config
+COPY node.properties $PRESTO_HOME/etc/node.properties
+
diff --git a/prestosql/docker/coordinator/README.md b/prestosql/docker/coordinator/README.md
new file mode 100755
index 0000000..6a6686f
--- /dev/null
+++ b/prestosql/docker/coordinator/README.md
@@ -0,0 +1,7 @@
+# Use squash !
+
+Building this image, make sure you use squash so you can gut the 12 GB file :)
+
+docker build --squash -t bigtop/prestosql-coordinator:320 --build-arg PRESTO_IMAGE=prestosql/presto:320 .
+
+
diff --git a/prestosql/docker/coordinator/config.properties b/prestosql/docker/coordinator/config.properties
new file mode 100755
index 0000000..826221f
--- /dev/null
+++ b/prestosql/docker/coordinator/config.properties
@@ -0,0 +1,5 @@
+coordinator=true
+node-scheduler.include-coordinator=false
+http-server.http.port=8080
+discovery-server.enabled=true
+discovery.uri=http://presto:8080
diff --git a/prestosql/docker/coordinator/jvm.config b/prestosql/docker/coordinator/jvm.config
new file mode 100644
index 0000000..b01949e
--- /dev/null
+++ b/prestosql/docker/coordinator/jvm.config
@@ -0,0 +1,12 @@
+-server
+-Xmx1G
+-XX:-UseBiasedLocking
+-XX:+UseG1GC
+-XX:G1HeapRegionSize=32M
+-XX:+ExplicitGCInvokesConcurrent
+-XX:+HeapDumpOnOutOfMemoryError
+-XX:+UseGCOverheadLimit
+-XX:+ExitOnOutOfMemoryError
+-XX:ReservedCodeCacheSize=256M
+-Djdk.attach.allowAttachSelf=true
+-Djdk.nio.maxCachedBufferSize=2000000
diff --git a/presto3-minio/worker/minio.properties b/prestosql/docker/coordinator/minio.properties
similarity index 100%
rename from presto3-minio/worker/minio.properties
rename to prestosql/docker/coordinator/minio.properties
diff --git a/prestosql/docker/coordinator/node.properties b/prestosql/docker/coordinator/node.properties
new file mode 100644
index 0000000..efcdedc
--- /dev/null
+++ b/prestosql/docker/coordinator/node.properties
@@ -0,0 +1,3 @@
+node.environment=docker
+node.data-dir=/data/presto
+plugin.dir=/usr/lib/presto/plugin
diff --git a/prestosql/docker/worker/Dockerfile b/prestosql/docker/worker/Dockerfile
new file mode 100755
index 0000000..46f42ed
--- /dev/null
+++ b/prestosql/docker/worker/Dockerfile
@@ -0,0 +1,12 @@
+ARG PRESTO_IMAGE=prestosql/presto:latest
+FROM $PRESTO_IMAGE
+
+LABEL maintainer="Apache Bogtop community <de...@bigtop.apache.org>"
+LABEL url="https://bigtop.apache.org"
+
+ENV PRESTO_HOME=/usr/lib/presto
+
+COPY minio.properties $PRESTO_HOME/etc/catalog/minio.properties
+COPY config.properties $PRESTO_HOME/etc/config.properties
+COPY jvm.config $PRESTO_HOME/etc/jvm.config
+COPY node.properties $PRESTO_HOME/etc/node.properties
diff --git a/prestosql/docker/worker/README.md b/prestosql/docker/worker/README.md
new file mode 100755
index 0000000..7037ce1
--- /dev/null
+++ b/prestosql/docker/worker/README.md
@@ -0,0 +1,6 @@
+# Use squash !
+
+Building this image, make sure you use squash so you can gut the 12 GB file :)
+
+
+docker build --squash -t bigtop/prestosql-worker:320 --build-arg PRESTO_IMAGE=prestosql/presto:320 .
diff --git a/presto3-minio/worker/config.properties b/prestosql/docker/worker/config.properties
similarity index 59%
rename from presto3-minio/worker/config.properties
rename to prestosql/docker/worker/config.properties
index 0a089be..22636e4 100755
--- a/presto3-minio/worker/config.properties
+++ b/prestosql/docker/worker/config.properties
@@ -1,7 +1,4 @@
coordinator=false
node-scheduler.include-coordinator=false
http-server.http.port=8080
-query.max-memory=1GB
-query.max-memory-per-node=1GB
-query.max-total-memory-per-node=1GB
discovery.uri=http://presto-master:8080
diff --git a/prestosql/docker/worker/jvm.config b/prestosql/docker/worker/jvm.config
new file mode 100644
index 0000000..b01949e
--- /dev/null
+++ b/prestosql/docker/worker/jvm.config
@@ -0,0 +1,12 @@
+-server
+-Xmx1G
+-XX:-UseBiasedLocking
+-XX:+UseG1GC
+-XX:G1HeapRegionSize=32M
+-XX:+ExplicitGCInvokesConcurrent
+-XX:+HeapDumpOnOutOfMemoryError
+-XX:+UseGCOverheadLimit
+-XX:+ExitOnOutOfMemoryError
+-XX:ReservedCodeCacheSize=256M
+-Djdk.attach.allowAttachSelf=true
+-Djdk.nio.maxCachedBufferSize=2000000
diff --git a/presto3-minio/coordinator/minio.properties b/prestosql/docker/worker/minio.properties
similarity index 100%
rename from presto3-minio/coordinator/minio.properties
rename to prestosql/docker/worker/minio.properties
diff --git a/prestosql/docker/worker/node.properties b/prestosql/docker/worker/node.properties
new file mode 100644
index 0000000..efcdedc
--- /dev/null
+++ b/prestosql/docker/worker/node.properties
@@ -0,0 +1,3 @@
+node.environment=docker
+node.data-dir=/data/presto
+plugin.dir=/usr/lib/presto/plugin
diff --git a/presto3-minio/presto-deployment-master.yaml b/prestosql/presto-deployment-master.yaml
similarity index 88%
rename from presto3-minio/presto-deployment-master.yaml
rename to prestosql/presto-deployment-master.yaml
index 332785f..05d1f56 100755
--- a/presto3-minio/presto-deployment-master.yaml
+++ b/prestosql/presto-deployment-master.yaml
@@ -14,7 +14,7 @@ spec:
app: presto
spec:
containers:
- - image: jayunit100/wlan0-presto-coordinator:latest
+ - image: bigtop/prestosql-coordinator:latest
name: presto-w
ports:
- containerPort: 8080
diff --git a/presto3-minio/presto-deployment-worker.yaml b/prestosql/presto-deployment-worker.yaml
similarity index 84%
rename from presto3-minio/presto-deployment-worker.yaml
rename to prestosql/presto-deployment-worker.yaml
index a0d2cdd..d6abb38 100755
--- a/presto3-minio/presto-deployment-worker.yaml
+++ b/prestosql/presto-deployment-worker.yaml
@@ -14,11 +14,11 @@ spec:
app: presto-worker
spec:
containers:
- - image: jayunit100/wlan0-presto-worker:latest
+ - image: bigtop/prestosql-worker:latest
name: presto-w
resources:
requests:
- memory: "6G"
+ memory: "2G"
ports:
- containerPort: 8080
hostname: presto
diff --git a/presto3-minio/presto-deployment.yaml b/prestosql/presto-deployment.yaml
similarity index 88%
rename from presto3-minio/presto-deployment.yaml
rename to prestosql/presto-deployment.yaml
index 3e3c7da..905f683 100644
--- a/presto3-minio/presto-deployment.yaml
+++ b/prestosql/presto-deployment.yaml
@@ -14,7 +14,7 @@ spec:
io.kompose.service: presto
spec:
containers:
- - image: jayunit100/wlan0-presto-worker:latest
+ - image: bigtop/bigtop-prestosql-worker:latest
name: presto-w
ports:
- containerPort: 8080
diff --git a/presto3-minio/service.yaml b/prestosql/service.yaml
similarity index 100%
rename from presto3-minio/service.yaml
rename to prestosql/service.yaml