You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by ni...@apache.org on 2022/09/09 18:01:55 UTC
[pulsar] branch master updated: [build][sql] Make pulsar-trino-disto self-contained (#17062)
This is an automated email from the ASF dual-hosted git repository.
nicoloboschi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 743b0380785 [build][sql] Make pulsar-trino-disto self-contained (#17062)
743b0380785 is described below
commit 743b03807852251cdb4f26073656ee6ae087826e
Author: tison <wa...@gmail.com>
AuthorDate: Sat Sep 10 02:01:46 2022 +0800
[build][sql] Make pulsar-trino-disto self-contained (#17062)
---
.github/workflows/ci-owasp-dep-check.yaml | 2 +-
bin/pulsar | 45 ++++++++++++++--------
distribution/server/src/assemble/bin.xml | 4 +-
docker-compose/kitchen-sink/docker-compose.yml | 2 +-
docker/pulsar/Dockerfile | 6 +--
.../presto-distribution/src/assembly/assembly.xml | 4 ++
.../main/resources/conf}/catalog/pulsar.properties | 0
.../src/main/resources/conf}/config.properties | 0
.../src/main/resources/conf}/jvm.config | 0
.../src/main/resources/conf}/log.properties | 0
src/check-binary-license.sh | 8 ++--
.../docker-images/latest-version-image/Dockerfile | 2 +-
.../scripts/run-presto-worker.sh | 2 +-
.../containers/PrestoWorkerContainer.java | 6 +--
.../integration/topologies/PulsarCluster.java | 2 +-
15 files changed, 49 insertions(+), 34 deletions(-)
diff --git a/.github/workflows/ci-owasp-dep-check.yaml b/.github/workflows/ci-owasp-dep-check.yaml
index 8cee7349b84..3ae868507f5 100644
--- a/.github/workflows/ci-owasp-dep-check.yaml
+++ b/.github/workflows/ci-owasp-dep-check.yaml
@@ -87,7 +87,7 @@ jobs:
docker rmi $(docker images -q) -f
df -h
- # Projects dependent on flume, hdfs, hbase, and presto currently excluded from the scan.
+ # Projects dependent on flume, hdfs, hbase, and trino currently excluded from the scan.
- name: run "clean verify" to trigger dependency check
if: ${{ steps.changes.outputs.poms == 'true' }}
run: mvn -q -B -ntp clean verify -PskipDocker,owasp-dependency-check -DskipTests -pl '!pulsar-sql,!distribution/io,!distribution/offloaders,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb'
diff --git a/bin/pulsar b/bin/pulsar
index f8bc941f9e3..52676818fce 100755
--- a/bin/pulsar
+++ b/bin/pulsar
@@ -33,7 +33,6 @@ DEFAULT_PROXY_CONF=$PULSAR_HOME/conf/proxy.conf
DEFAULT_STANDALONE_CONF=$PULSAR_HOME/conf/standalone.conf
DEFAULT_WEBSOCKET_CONF=$PULSAR_HOME/conf/websocket.conf
DEFAULT_LOG_CONF=$PULSAR_HOME/conf/log4j2.yaml
-DEFAULT_PULSAR_PRESTO_CONF=${PULSAR_HOME}/conf/presto
DEFAULT_FUNCTIONS_LOG_CONF=$PULSAR_HOME/conf/functions_log4j2.xml
# functions related variables
@@ -46,7 +45,8 @@ PY_INSTANCE_FILE=${PULSAR_PY_INSTANCE_FILE:-"${DEFAULT_PY_INSTANCE_FILE}"}
DEFAULT_FUNCTIONS_EXTRA_DEPS_DIR=$PULSAR_HOME/instances/deps
FUNCTIONS_EXTRA_DEPS_DIR=${PULSAR_FUNCTIONS_EXTRA_DEPS_DIR:-"${DEFAULT_FUNCTIONS_EXTRA_DEPS_DIR}"}
SQL_HOME=$PULSAR_HOME/pulsar-sql
-PRESTO_HOME=${PULSAR_HOME}/lib/presto
+TRINO_HOME=${PULSAR_HOME}/trino
+DEFAULT_PULSAR_TRINO_CONF=${TRINO_HOME}/conf
pulsar_help() {
cat <<EOF
@@ -90,7 +90,7 @@ Environment variables:
PULSAR_PROXY_CONF Configuration file for Pulsar proxy (default: $DEFAULT_PROXY_CONF)
PULSAR_WORKER_CONF Configuration file for functions worker (default: $DEFAULT_WORKER_CONF)
PULSAR_STANDALONE_CONF Configuration file for standalone (default: $DEFAULT_STANDALONE_CONF)
- PULSAR_PRESTO_CONF Configuration directory for Pulsar Presto (default: $DEFAULT_PULSAR_PRESTO_CONF)
+ PULSAR_TRINO_CONF Configuration directory for Pulsar SQL (default: $DEFAULT_PULSAR_TRINO_CONF)
PULSAR_EXTRA_OPTS Extra options to be passed to the jvm
PULSAR_EXTRA_CLASSPATH Add extra paths to the pulsar classpath
PULSAR_PID_DIR Folder where the pulsar server PID file should be stored
@@ -177,17 +177,17 @@ if [ ! -f "${PY_INSTANCE_FILE}" ]; then
PY_INSTANCE_FILE=${BUILT_PY_INSTANCE_FILE}
fi
-# find pulsar sql presto distribution location
-check_presto_libraries() {
- if [ ! -d "${PRESTO_HOME}" ]; then
-
- BUILT_PRESTO_HOME="${SQL_HOME}/presto-distribution/target/pulsar-presto-distribution"
- if [ ! -d "${BUILT_PRESTO_HOME}" ]; then
- echo "\nCouldn't find presto distribution.";
+# find pulsar sql trino distribution location
+check_trino_libraries() {
+ if [ ! -d "${TRINO_HOME}" ]; then
+ BUILT_TRINO_HOME="${SQL_HOME}/presto-distribution/target/pulsar-presto-distribution"
+ if [ ! -d "${BUILT_TRINO_HOME}" ]; then
+ echo "\nCouldn't find trino distribution.";
echo "Make sure you've run 'mvn package'\n";
exit 1;
fi
- PRESTO_HOME=${BUILT_PRESTO_HOME}
+ TRINO_HOME=${BUILT_TRINO_HOME}
+ PULSAR_TRINO_CONF=${BUILT_TRINO_HOME}/conf
fi
}
@@ -257,8 +257,19 @@ if [ -z "$PULSAR_LOG_CONF" ]; then
PULSAR_LOG_CONF=$DEFAULT_LOG_CONF
fi
-if [ -z "$PULSAR_PRESTO_CONF" ]; then
- PULSAR_PRESTO_CONF=$DEFAULT_PULSAR_PRESTO_CONF
+if [ -z "$PULSAR_TRINO_CONF" ]; then
+ # TODO: As PIP-200 accepted, this compatibility is not promised. Refactor when we drop this b/w compatibility.
+ if [ -z "$PULSAR_PRESTO_CONF" ]; then
+ PULSAR_TRINO_CONF=$DEFAULT_PULSAR_TRINO_CONF
+ else
+ PULSAR_TRINO_CONF=$PULSAR_PRESTO_CONF
+ fi
+ if [ ! -d "${PULSAR_TRINO_CONF}" ]; then
+ FALLBACK_PULSAR_PRESTO_CONF=${PULSAR_HOME}/conf/presto
+ if [ -d "${FALLBACK_PULSAR_PRESTO_CONF}" ]; then
+ PULSAR_TRINO_CONF=$FALLBACK_PULSAR_PRESTO_CONF
+ fi
+ fi
fi
if [ -z "$FUNCTIONS_LOG_CONF"]; then
@@ -387,11 +398,11 @@ elif [ $COMMAND == "broker-tool" ]; then
elif [ $COMMAND == "compact-topic" ]; then
exec $JAVA $OPTS org.apache.pulsar.compaction.CompactorTool --broker-conf $PULSAR_BROKER_CONF $@
elif [ $COMMAND == "sql" ]; then
- check_presto_libraries
- exec $JAVA -cp "${PRESTO_HOME}/lib/*" io.trino.cli.Trino --server localhost:8081 "${@}"
+ check_trino_libraries
+ exec $JAVA -cp "${TRINO_HOME}/lib/*" io.trino.cli.Trino --server localhost:8081 "${@}"
elif [ $COMMAND == "sql-worker" ]; then
- check_presto_libraries
- exec python3 ${PRESTO_HOME}/bin/launcher.py --etc-dir ${PULSAR_PRESTO_CONF} "${@}"
+ check_trino_libraries
+ exec python3 ${TRINO_HOME}/bin/launcher.py --etc-dir ${PULSAR_TRINO_CONF} "${@}"
elif [ $COMMAND == "tokens" ]; then
exec $JAVA $OPTS org.apache.pulsar.utils.auth.tokens.TokensCliUtils $@
elif [ $COMMAND == "version" ]; then
diff --git a/distribution/server/src/assemble/bin.xml b/distribution/server/src/assemble/bin.xml
index 12c443dc6c5..895ae2dfe17 100644
--- a/distribution/server/src/assemble/bin.xml
+++ b/distribution/server/src/assemble/bin.xml
@@ -62,7 +62,7 @@
</fileSet>
<fileSet>
<directory>${basedir}/../../pulsar-sql/presto-distribution/target/pulsar-presto-distribution</directory>
- <outputDirectory>lib/presto</outputDirectory>
+ <outputDirectory>trino</outputDirectory>
<excludes>
<exclude>bin</exclude>
<exclude>bin/**</exclude>
@@ -70,7 +70,7 @@
</fileSet>
<fileSet>
<directory>${basedir}/../../pulsar-sql/presto-distribution/target/pulsar-presto-distribution</directory>
- <outputDirectory>lib/presto</outputDirectory>
+ <outputDirectory>trino</outputDirectory>
<includes>
<include>bin/**</include>
</includes>
diff --git a/docker-compose/kitchen-sink/docker-compose.yml b/docker-compose/kitchen-sink/docker-compose.yml
index 779e8d82806..fab7c5b9f1c 100644
--- a/docker-compose/kitchen-sink/docker-compose.yml
+++ b/docker-compose/kitchen-sink/docker-compose.yml
@@ -368,7 +368,7 @@ services:
image: apachepulsar/pulsar-all:latest
restart: on-failure
command: >
- bash -c "bin/apply-config-from-env-with-prefix.py SQL_PREFIX_ conf/presto/catalog/pulsar.properties && \
+ bash -c "bin/apply-config-from-env-with-prefix.py SQL_PREFIX_ trino/conf/catalog/pulsar.properties && \
bin/apply-config-from-env.py conf/pulsar_env.sh && \
bin/watch-znode.py -z $$zookeeperServers -p /initialized-$$clusterName -w && \
exec bin/pulsar sql-worker run"
diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile
index 7ed9b454bf5..0089fe56acc 100644
--- a/docker/pulsar/Dockerfile
+++ b/docker/pulsar/Dockerfile
@@ -42,9 +42,9 @@ RUN for SUBDIRECTORY in conf data download logs; do \
chmod -R g+w /pulsar/$SUBDIRECTORY; \
done
-# Presto writes logs to this directory (at least during tests), so we need to give the process permission
-# to create those log directories. This should be removed when presto is removed.
-RUN chmod g+w /pulsar/lib/presto
+# Trino writes logs to this directory (at least during tests), so we need to give the process permission
+# to create those log directories. This should be removed when Trino is removed.
+RUN chmod g+w /pulsar/trino
### Create 2nd stage from Ubuntu image
### and add OpenJDK and Python dependencies (for Pulsar functions)
diff --git a/pulsar-sql/presto-distribution/src/assembly/assembly.xml b/pulsar-sql/presto-distribution/src/assembly/assembly.xml
index 4f1bac20dce..96c0421c715 100644
--- a/pulsar-sql/presto-distribution/src/assembly/assembly.xml
+++ b/pulsar-sql/presto-distribution/src/assembly/assembly.xml
@@ -46,6 +46,10 @@
<directory>${basedir}/../presto-pulsar-plugin/target/pulsar-presto-connector/</directory>
<outputDirectory>plugin/</outputDirectory>
</fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/conf/</directory>
+ <outputDirectory>conf/</outputDirectory>
+ </fileSet>
</fileSets>
<dependencySets>
<dependencySet>
diff --git a/conf/presto/catalog/pulsar.properties b/pulsar-sql/presto-distribution/src/main/resources/conf/catalog/pulsar.properties
similarity index 100%
rename from conf/presto/catalog/pulsar.properties
rename to pulsar-sql/presto-distribution/src/main/resources/conf/catalog/pulsar.properties
diff --git a/conf/presto/config.properties b/pulsar-sql/presto-distribution/src/main/resources/conf/config.properties
similarity index 100%
rename from conf/presto/config.properties
rename to pulsar-sql/presto-distribution/src/main/resources/conf/config.properties
diff --git a/conf/presto/jvm.config b/pulsar-sql/presto-distribution/src/main/resources/conf/jvm.config
similarity index 100%
rename from conf/presto/jvm.config
rename to pulsar-sql/presto-distribution/src/main/resources/conf/jvm.config
diff --git a/conf/presto/log.properties b/pulsar-sql/presto-distribution/src/main/resources/conf/log.properties
similarity index 100%
rename from conf/presto/log.properties
rename to pulsar-sql/presto-distribution/src/main/resources/conf/log.properties
diff --git a/src/check-binary-license.sh b/src/check-binary-license.sh
index bdf55b3a0b1..64e5bd4320a 100755
--- a/src/check-binary-license.sh
+++ b/src/check-binary-license.sh
@@ -41,7 +41,7 @@ if [ -z $TARBALL ]; then
exit 1
fi
-JARS=$(tar -tf $TARBALL | grep '\.jar' | grep -v 'lib/presto/' | grep -v '/examples/' | grep -v '/instances/' | sed 's!.*/!!' | sort)
+JARS=$(tar -tf $TARBALL | grep '\.jar' | grep -v 'trino/' | grep -v '/examples/' | grep -v '/instances/' | sed 's!.*/!!' | sort)
LICENSEPATH=$(tar -tf $TARBALL | awk '/^[^\/]*\/LICENSE/')
LICENSE=$(tar -O -xf $TARBALL "$LICENSEPATH")
@@ -96,7 +96,7 @@ done
if [ "$NO_PRESTO" -ne 1 ]; then
# check pulsar sql jars
- JARS=$(tar -tf $TARBALL | grep '\.jar' | grep 'lib/presto/' | grep -v pulsar-client | grep -v bouncy-castle-bc | grep -v pulsar-metadata | grep -v 'managed-ledger' | grep -v 'pulsar-client-admin' | grep -v 'pulsar-client-api' | grep -v 'pulsar-functions-api' | grep -v 'pulsar-presto-connector-original' | grep -v 'pulsar-presto-distribution' | grep -v 'pulsar-common' | grep -v 'pulsar-functions-proto' | grep -v 'pulsar-functions-utils' | grep -v 'pulsar-io-core' | grep -v 'pulsar-tran [...]
+ JARS=$(tar -tf $TARBALL | grep '\.jar' | grep 'trino/' | grep -v pulsar-client | grep -v bouncy-castle-bc | grep -v pulsar-metadata | grep -v 'managed-ledger' | grep -v 'pulsar-client-admin' | grep -v 'pulsar-client-api' | grep -v 'pulsar-functions-api' | grep -v 'pulsar-presto-connector-original' | grep -v 'pulsar-presto-distribution' | grep -v 'pulsar-common' | grep -v 'pulsar-functions-proto' | grep -v 'pulsar-functions-utils' | grep -v 'pulsar-io-core' | grep -v 'pulsar-transacti [...]
if [ -n "$JARS" ]; then
LICENSEPATH=$(tar -tf $TARBALL | awk '/^[^\/]*\/lib\/presto\/LICENSE/')
LICENSE=$(tar -O -xf $TARBALL "$LICENSEPATH")
@@ -111,7 +111,7 @@ if [ "$NO_PRESTO" -ne 1 ]; then
echo "$LICENSE" | grep -q $J
if [ $? != 0 ]; then
- echo $J unaccounted for in lib/presto/LICENSE
+ echo $J unaccounted for in trino/LICENSE
EXIT=1
fi
done
@@ -120,7 +120,7 @@ if [ "$NO_PRESTO" -ne 1 ]; then
for J in $LICENSEJARS; do
echo "$JARS" | grep -q $J
if [ $? != 0 ]; then
- echo $J mentioned in lib/presto/LICENSE, but not bundled
+ echo $J mentioned in trino/LICENSE, but not bundled
EXIT=2
fi
done
diff --git a/tests/docker-images/latest-version-image/Dockerfile b/tests/docker-images/latest-version-image/Dockerfile
index 8793c4b4b5f..f153a2723ad 100644
--- a/tests/docker-images/latest-version-image/Dockerfile
+++ b/tests/docker-images/latest-version-image/Dockerfile
@@ -89,7 +89,7 @@ COPY scripts/init-cluster.sh scripts/run-global-zk.sh scripts/run-local-zk.sh \
scripts/run-standalone.sh \
/pulsar/bin/
-COPY conf/presto/jvm.config /pulsar/conf/presto/
+COPY conf/presto/jvm.config /pulsar/trino/conf
# copy python test examples
RUN mkdir -p /pulsar/instances/deps
diff --git a/tests/docker-images/latest-version-image/scripts/run-presto-worker.sh b/tests/docker-images/latest-version-image/scripts/run-presto-worker.sh
index 87b393a37dc..8c934cbf173 100755
--- a/tests/docker-images/latest-version-image/scripts/run-presto-worker.sh
+++ b/tests/docker-images/latest-version-image/scripts/run-presto-worker.sh
@@ -18,7 +18,7 @@
# under the License.
#
-bin/apply-config-from-env-with-prefix.py SQL_PREFIX_ conf/presto/catalog/pulsar.properties && \
+bin/apply-config-from-env-with-prefix.py SQL_PREFIX_ trino/conf/catalog/pulsar.properties && \
bin/apply-config-from-env.py conf/pulsar_env.sh
if [ -z "$NO_AUTOSTART" ]; then
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PrestoWorkerContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PrestoWorkerContainer.java
index 2bb696580f0..2119ba54d55 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PrestoWorkerContainer.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PrestoWorkerContainer.java
@@ -43,11 +43,11 @@ public class PrestoWorkerContainer extends PulsarContainer<PrestoWorkerContainer
@Override
protected void afterStart() {
DockerUtils.runCommandAsyncWithLogging(this.dockerClient, this.getContainerId(),
- "tail", "-f", "/pulsar/lib/presto/var/log/launcher.log");
+ "tail", "-f", "/pulsar/trino/var/log/launcher.log");
DockerUtils.runCommandAsyncWithLogging(this.dockerClient, this.getContainerId(),
"tail", "-f", "/var/log/pulsar/presto_worker.log");
DockerUtils.runCommandAsyncWithLogging(this.dockerClient, this.getContainerId(),
- "tail", "-f", "/pulsar/lib/presto/var/log/server.log");
+ "tail", "-f", "/pulsar/trino/var/log/server.log");
}
@Override
@@ -57,7 +57,7 @@ public class PrestoWorkerContainer extends PulsarContainer<PrestoWorkerContainer
DockerUtils.dumpContainerDirToTargetCompressed(
getDockerClient(),
getContainerId(),
- "/pulsar/lib/presto/var/log"
+ "/pulsar/trino/var/log"
);
}
}
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java
index d204735ea0c..3b94808360b 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java
@@ -438,7 +438,7 @@ public class PulsarCluster {
.withEnv("pulsar.web-service-url", "http://pulsar-broker-0:8080")
.withEnv("SQL_PREFIX_pulsar.max-message-size", "" + spec.maxMessageSize)
.withClasspathResourceMapping(
- resourcePath, "/pulsar/conf/presto/config.properties", BindMode.READ_WRITE);
+ resourcePath, "/pulsar/trino/conf/config.properties", BindMode.READ_WRITE);
if (spec.queryLastMessage) {
container.withEnv("pulsar.bookkeeper-use-v2-protocol", "false")
.withEnv("pulsar.bookkeeper-explicit-interval", "10");