You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2019/11/12 05:00:37 UTC
[incubator-pinot] 01/01: Adding pinot presto docker image
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch pinot-presto-docker-image
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit b55aa4929761d4e70825b3b38c2dc0cc080cf6f7
Author: Xiang Fu <fx...@gmail.com>
AuthorDate: Mon Nov 11 20:59:31 2019 -0800
Adding pinot presto docker image
---
docker/images/pinot-presto/Dockerfile | 59 +++++++++
docker/images/pinot-presto/README.md | 61 +++++++++
docker/images/pinot-presto/docker-build.sh | 47 +++++++
docker/images/pinot-presto/etc/config.properties | 27 ++++
docker/images/pinot-presto/etc/jvm.config | 8 ++
docker/images/pinot-presto/etc/log.properties | 20 +++
docker/images/pinot-presto/etc/node.properties | 22 ++++
kubernetes/examples/helm/pinot-presto-cli.sh | 31 +++++
kubernetes/examples/helm/presto-coordinator.yaml | 158 +++++++++++++++++++++++
kubernetes/examples/helm/superset.yaml | 2 +-
10 files changed, 434 insertions(+), 1 deletion(-)
diff --git a/docker/images/pinot-presto/Dockerfile b/docker/images/pinot-presto/Dockerfile
new file mode 100644
index 0000000..23965f9
--- /dev/null
+++ b/docker/images/pinot-presto/Dockerfile
@@ -0,0 +1,59 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+FROM openjdk:8
+
+LABEL MAINTAINER=dev@pinot.apache.org
+
+ARG PRESTO_BRANCH=master
+ARG PRESTO_GIT_URL="https://github.com/prestodb/presto.git"
+RUN echo "Trying to build Presto image from [ ${PRESTO_GIT_URL} ] on branch [ ${PRESTO_BRANCH} ]"
+ENV PRESTO_HOME=/home/presto
+ENV PRESTO_BUILD_DIR=/home/presto/build
+
+# extra dependency for running launcher
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends \
+ vim wget curl git maven && \
+ rm -rf /var/lib/apt/lists/*
+
+RUN groupadd -g 999 presto && \
+ useradd -r -u 999 -g presto --create-home --shell /bin/bash presto
+USER presto
+
+RUN git clone ${PRESTO_GIT_URL} ${PRESTO_BUILD_DIR} && \
+ cd ${PRESTO_BUILD_DIR} && \
+ git checkout ${PRESTO_BRANCH} && \
+ ./mvnw clean install -DskipTests && \
+ mkdir -p ${PRESTO_HOME}/data && \
+ cp -r presto-server/target/presto-server-*/presto-server-*/* ${PRESTO_HOME} && \
+ mkdir -p ${PRESTO_HOME}/plugin/pinot && cp -r presto-pinot/target/presto-pinot-*/* ${PRESTO_HOME}/plugin/pinot/ && \
+ mvn dependency:purge-local-repository -DactTransitively=false -DreResolve=false --fail-at-end && \
+ rm -rf ${PRESTO_BUILD_DIR}
+
+COPY etc ${PRESTO_HOME}/etc
+EXPOSE 8080
+
+VOLUME ["${PRESTO_HOME}/etc", "${PRESTO_HOME}/data"]
+
+WORKDIR ${PRESTO_HOME}
+
+ENTRYPOINT ["./bin/launcher"]
+
+CMD ["run"]
\ No newline at end of file
diff --git a/docker/images/pinot-presto/README.md b/docker/images/pinot-presto/README.md
new file mode 100644
index 0000000..8a39b75
--- /dev/null
+++ b/docker/images/pinot-presto/README.md
@@ -0,0 +1,61 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+# Presto
+
+Docker image for [Presto](https://github.com/prestodb/presto) with Pinot integration.
+
+This docker build project is specialized for Pinot.
+
+## How to build
+
+```bash
+./docker-build.sh
+```
+
+You can also build directly with `docker build` command by setting arguments:
+```bash
+docker build \
+ --build-arg PRESTO_BRANCH=master \
+ --tag fx19880617/pinot-presto:0.1 \
+ --target build .
+```
+## How to push
+
+```bash
+docker push fx19880617/pinot-presto:0.1
+```
+
+## Configuration
+
+Follow the [instructions](https://prestodb.io/docs/current/installation/deployment.html) provided by Presto for writing your own configuration files under `etc` directory.
+
+## Volumes
+
+The image defines two data volumes: one for mounting configuration into the container, and one for data.
+
+The configuration volume is located alternatively at `/home/presto/etc`, which contains all the configuration and plugins.
+
+The data volume is located at `/home/presto/data`.
+
+## Kubernetes Examples
+
+Please refer to [`presto-coordinator.yaml`](../../../kubernetes/examples/helm/prest-coordinator.yaml) as k8s deployment example.
diff --git a/docker/images/pinot-presto/docker-build.sh b/docker/images/pinot-presto/docker-build.sh
new file mode 100755
index 0000000..c84f22e
--- /dev/null
+++ b/docker/images/pinot-presto/docker-build.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+if [[ "$#" -gt 0 ]]
+then
+ DOCKER_TAG=$1
+else
+ DOCKER_TAG="pinot-presto:latest"
+ echo "Not specified a Docker Tag, using default tag: ${DOCKER_TAG}."
+fi
+
+if [[ "$#" -gt 1 ]]
+then
+ PRESTO_BRANCH=$2
+else
+ PRESTO_BRANCH=master
+ echo "Not specified a Presto branch to build, using default branch: ${PRESTO_BRANCH}."
+fi
+
+if [[ "$#" -gt 2 ]]
+then
+ PRESTO_GIT_URL=$3
+else
+ PRESTO_GIT_URL="https://github.com/prestodb/presto.git"
+fi
+
+echo "Trying to build Pinot Presto docker image from Git URL: [ ${PRESTO_GIT_URL} ] on branch: [ ${PRESTO_BRANCH} ] and tag it as: [ ${DOCKER_TAG} ]."
+
+docker build --no-cache -t ${DOCKER_TAG} --build-arg PRESTO_BRANCH=${PRESTO_BRANCH} --build-arg PRESTO_GIT_URL=${PRESTO_GIT_URL} -f Dockerfile .
diff --git a/docker/images/pinot-presto/etc/config.properties b/docker/images/pinot-presto/etc/config.properties
new file mode 100644
index 0000000..5a553e3
--- /dev/null
+++ b/docker/images/pinot-presto/etc/config.properties
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+coordinator=true
+node-scheduler.include-coordinator=true
+http-server.http.port=8080
+query.max-memory=5GB
+query.max-memory-per-node=1GB
+query.max-total-memory-per-node=2GB
+discovery-server.enabled=true
+discovery.uri=http://presto-coordinator:8080
\ No newline at end of file
diff --git a/docker/images/pinot-presto/etc/jvm.config b/docker/images/pinot-presto/etc/jvm.config
new file mode 100644
index 0000000..5985b68
--- /dev/null
+++ b/docker/images/pinot-presto/etc/jvm.config
@@ -0,0 +1,8 @@
+-server
+-Xmx16G
+-XX:+UseG1GC
+-XX:G1HeapRegionSize=32M
+-XX:+UseGCOverheadLimit
+-XX:+ExplicitGCInvokesConcurrent
+-XX:+HeapDumpOnOutOfMemoryError
+-XX:+ExitOnOutOfMemoryError
\ No newline at end of file
diff --git a/docker/images/pinot-presto/etc/log.properties b/docker/images/pinot-presto/etc/log.properties
new file mode 100644
index 0000000..7ae1016
--- /dev/null
+++ b/docker/images/pinot-presto/etc/log.properties
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+com.facebook.presto=INFO
\ No newline at end of file
diff --git a/docker/images/pinot-presto/etc/node.properties b/docker/images/pinot-presto/etc/node.properties
new file mode 100644
index 0000000..7da7c40
--- /dev/null
+++ b/docker/images/pinot-presto/etc/node.properties
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+node.environment=production
+node.id=presto-coordinator
+node.data-dir=/home/presto/data
\ No newline at end of file
diff --git a/kubernetes/examples/helm/pinot-presto-cli.sh b/kubernetes/examples/helm/pinot-presto-cli.sh
new file mode 100755
index 0000000..9195a77
--- /dev/null
+++ b/kubernetes/examples/helm/pinot-presto-cli.sh
@@ -0,0 +1,31 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+if [[ ! -f "/tmp/presto-cli" ]]; then
+ curl -L https://repo1.maven.org/maven2/com/facebook/presto/presto-cli/0.228/presto-cli-0.228-executable.jar -o /tmp/presto-cli
+ chmod +x /tmp/presto-cli
+fi
+
+if [[ $(nc -z localhost 18080) != 0 ]]; then
+ kubectl port-forward service/presto-coordinator 18080:8080 -n pinot-quickstart > /dev/null &
+fi
+
+/tmp/presto-cli --server localhost:18080 --catalog pinot --schema default
+pkill -f "kubectl port-forward service/presto-coordinator 18080:8080 -n pinot-quickstart"
+
diff --git a/kubernetes/examples/helm/presto-coordinator.yaml b/kubernetes/examples/helm/presto-coordinator.yaml
new file mode 100644
index 0000000..7edd084
--- /dev/null
+++ b/kubernetes/examples/helm/presto-coordinator.yaml
@@ -0,0 +1,158 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: presto-catalog
+ namespace: pinot-quickstart
+data:
+ pinot.properties: |-
+ connector.name=pinot
+ pinot.controller-urls=pinot-controller:9000
+ pinot.controller-rest-service=pinot-controller:9000
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: presto-etc
+ namespace: pinot-quickstart
+data:
+ config.properties: |-
+ coordinator=true
+ node-scheduler.include-coordinator=true
+ http-server.http.port=8080
+ query.max-memory=5GB
+ query.max-memory-per-node=1GB
+ query.max-total-memory-per-node=2GB
+ discovery-server.enabled=true
+ discovery.uri=http://presto-coordinator:8080
+
+ jvm.config: |-
+ -server
+ -Xmx16G
+ -XX:+UseG1GC
+ -XX:G1HeapRegionSize=32M
+ -XX:+UseGCOverheadLimit
+ -XX:+ExplicitGCInvokesConcurrent
+ -XX:+HeapDumpOnOutOfMemoryError
+ -XX:+ExitOnOutOfMemoryError
+
+ log.properties: |-
+ com.facebook.presto=INFO
+
+ node.properties: |-
+ node.environment=production
+ node.id=presto-coordinator
+ node.data-dir=/home/presto/data
+
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: presto-coordinator
+ namespace: pinot-quickstart
+spec:
+ selector:
+ matchLabels:
+ app: presto-coordinator
+ serviceName: presto-coordinator
+ replicas: 1
+ updateStrategy:
+ type: RollingUpdate
+ podManagementPolicy: Parallel
+ template:
+ metadata:
+ labels:
+ app: presto-coordinator
+ spec:
+ terminationGracePeriodSeconds: 30
+ securityContext:
+ runAsGroup: 1000
+ fsGroup: 1000
+ containers:
+ - image: fx19880617/pinot-presto:latest
+ imagePullPolicy: Always
+ name: presto-coordinator
+ args: [ "run" ]
+ ports:
+ - containerPort: 8080
+ protocol: TCP
+ resources:
+ requests:
+ memory: 4Gi
+ volumeMounts:
+ - name: presto-data
+ mountPath: "/home/presto/data"
+ - name: presto-catalog
+ mountPath: "/home/presto/etc/catalog"
+ readOnly: true
+ - name: presto-etc
+ mountPath: "/home/presto/etc/config.properties"
+ subPath: config.properties
+ readOnly: true
+ - name: presto-etc
+ mountPath: "/home/presto/etc/log.properties"
+ subPath: log.properties
+ readOnly: true
+ - name: presto-etc
+ mountPath: "/home/presto/etc/node.properties"
+ subPath: node.properties
+ readOnly: true
+ - name: presto-etc
+ mountPath: "/home/presto/etc/jvm.config"
+ subPath: jvm.config
+ readOnly: true
+ nodeSelector: {}
+ restartPolicy: Always
+ volumes:
+ - name: presto-catalog
+ configMap:
+ name: presto-catalog
+ - name: presto-etc
+ configMap:
+ name: presto-etc
+ volumeClaimTemplates:
+ - metadata:
+ name: presto-data
+ annotations:
+ pv.beta.kubernetes.io/gid: "1000"
+ pv.beta.kubernetes.io/groups: "1000"
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ storageClassName: "default"
+ resources:
+ requests:
+ storage: 5Gi
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: presto-coordinator
+ namespace: pinot-quickstart
+spec:
+ ports:
+ # [podname].presto-coordinator.pinot-quickstart.svc.cluster.local
+ - port: 8080
+ clusterIP: None
+ selector:
+ app: presto-coordinator
diff --git a/kubernetes/examples/helm/superset.yaml b/kubernetes/examples/helm/superset.yaml
index 91e4630..fa8137a 100644
--- a/kubernetes/examples/helm/superset.yaml
+++ b/kubernetes/examples/helm/superset.yaml
@@ -2302,7 +2302,7 @@ spec:
spec:
accessModes:
- ReadWriteOnce
- storageClassName: "standard"
+ storageClassName: "default"
resources:
requests:
storage: 1Gi
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org