You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ko...@apache.org on 2023/11/09 05:33:14 UTC
(arrow-flight-sql-postgresql) branch main updated: Provide Docker images (#159)
This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-flight-sql-postgresql.git
The following commit(s) were added to refs/heads/main by this push:
new f04789b Provide Docker images (#159)
f04789b is described below
commit f04789b130411582b02a7b5ea9944e705b3f1e29
Author: Sutou Kouhei <ko...@clear-code.com>
AuthorDate: Thu Nov 9 14:33:10 2023 +0900
Provide Docker images (#159)
Closes #158
---
.gitignore => .dockerignore | 17 +------
.github/workflows/package.yaml | 39 +++++++++++++++-
.gitignore | 1 +
package/docker/apache-arrow-flight-sql-enable.sh | 52 ++++++++++++++++++++++
.../postgresql-15-pgdg/Dockerfile | 34 +++++++-------
.../apt/ubuntu-jammy-amd64/Dockerfile | 1 +
.../postgresql-16-pgdg/Dockerfile | 34 +++++++-------
.../apt/ubuntu-jammy-amd64/Dockerfile | 1 +
src/afs.cc | 1 +
test/helper/sandbox.rb | 6 ++-
10 files changed, 139 insertions(+), 47 deletions(-)
diff --git a/.gitignore b/.dockerignore
similarity index 72%
copy from .gitignore
copy to .dockerignore
index 9eff26e..ce0600c 100644
--- a/.gitignore
+++ b/.dockerignore
@@ -15,18 +15,5 @@
# specific language governing permissions and limitations
# under the License.
-/*.tar.gz
-/.cache/
-/Gemfile.lock
-/compile_commands.json
-/dev/release/.env
-/dev/release/apache-rat-0.13.jar
-/dev/release/dist
-/dev/release/filtered_rat.txt
-/dev/release/rat.xml
-/doc/build/
-/package/*/*.tar.gz
-/package/*/apt/build.sh
-/package/*/apt/env.sh
-/package/*/apt/repositories/
-/package/*/apt/tmp/
+*
+!package/docker/
diff --git a/.github/workflows/package.yaml b/.github/workflows/package.yaml
index b723efc..d96442d 100644
--- a/.github/workflows/package.yaml
+++ b/.github/workflows/package.yaml
@@ -124,6 +124,26 @@ jobs:
ARCHITECTURE=${PACKAGE_TARGET##*-}
echo "ARCHITECTURE=${ARCHITECTURE}" >> $GITHUB_ENV
+ case ${PACKAGE_TARGET} in
+ debian-bookworm-*)
+ DOCKER_IMAGE_BUILD=yes
+ DOCKER_TAG_PREFIX="ghcr.io/${GITHUB_REPOSITORY}:"
+ DOCKER_TAG_SUFFIX="-bookworm-${POSTGRESQL_VERSION}"
+ DOCKER_TAGS="${DOCKER_VERSION}"
+ if [ "${GITHUB_REF_TYPE}" = "tag" ]; then
+ DOCKER_TAGS="${DOCKER_TAG_PREFIX}${GITHUB_REF_NAME}${DOCKER_TAG_SUFFIX}"
+ DOCKER_TAGS+=",${DOCKER_TAG_PREFIX}latest${DOCKER_TAG_SUFFIX}"
+ else
+ DOCKER_TAGS="${DOCKER_TAG_PREFIX}main${DOCKER_TAG_SUFFIX}"
+ fi
+ echo "DOCKER_TAGS=${DOCKER_TAGS}" >> $GITHUB_ENV
+ ;;
+ *)
+ DOCKER_IMAGE_BUILD=no
+ ;;
+ esac
+ echo "DOCKER_IMAGE_BUILD=${DOCKER_IMAGE_BUILD}" >> $GITHUB_ENV
+
source_archive=$(echo ${BASE_NAME}-*.tar.gz)
VERSION=${source_archive#${BASE_NAME}-}
VERSION=${VERSION%.tar.gz}
@@ -172,10 +192,13 @@ jobs:
${{ matrix.target }}.tar.gz
env:
GH_TOKEN: ${{ github.token }}
- - name: Push Docker image
+ - name: Push Docker image for building packages
run: |
pushd ${BASE_NAME}/package/${PACKAGE}
rake docker:push || :
+ cp \
+ ${TASK_NAMESPACE}/repositories/*/pool/*/*/*/*/*.deb \
+ ../docker/
popd
- name: Test
run: |
@@ -190,3 +213,17 @@ jobs:
local \
package/${PACKAGE}/${TASK_NAMESPACE}/repositories
popd
+ - uses: docker/setup-qemu-action@v3
+ if: env.DOCKER_IMAGE_BUILD == 'yes'
+ - uses: docker/setup-buildx-action@v3
+ if: env.DOCKER_IMAGE_BUILD == 'yes'
+ id: buildx
+ - uses: docker/build-push-action@v5
+ if: env.DOCKER_IMAGE_BUILD == 'yes'
+ id: build-push
+ with:
+ context: ${{ env.BASE_NAME }}
+ file: ${{ env.BASE_NAME }}/package/${{ env.PACKAGE }}/Dockerfile
+ platforms: linux/${{ env.ARCHITECTURE }}
+ push: ${{ github.event_name == 'push' }}
+ tags: ${{ env.DOCKER_TAGS }}
diff --git a/.gitignore b/.gitignore
index 9eff26e..b055998 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,3 +30,4 @@
/package/*/apt/env.sh
/package/*/apt/repositories/
/package/*/apt/tmp/
+/package/docker/*.deb
diff --git a/package/docker/apache-arrow-flight-sql-enable.sh b/package/docker/apache-arrow-flight-sql-enable.sh
new file mode 100755
index 0000000..b0013a9
--- /dev/null
+++ b/package/docker/apache-arrow-flight-sql-enable.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -eux
+
+psql=(psql \
+ --no-align \
+ --no-psqlrc \
+ --tuples-only)
+select_shared_preload_libraries=<<SQL
+SELECT setting
+ FROM pg_file_settings
+ WHERE
+ name ='shared_preload_libraries' AND
+ applied
+SQL
+shared_preload_libraries=$("${psql[@]}" --command "${select_shared_preload_libraries}")
+if [ -n "${shared_preload_libraries}" ]; then
+ shared_preload_libraries+=","
+fi
+shared_preload_libraries+="arrow_flight_sql"
+echo "shared_preload_libraries = '${shared_preload_libraries}'" | \
+ tee -a "${PGDATA}/postgresql.conf"
+
+ssl=$(psql \
+ --no-psqlrc \
+ --tuples-only \
+ --no-align \
+ --command "SELECT setting FROM pg_settings WHERE name ='ssl'")
+if [ "${ssl}" = "on" ]; then
+ uri="grpc+tls://0.0.0.0:15432"
+else
+ uri="grpc://0.0.0.0:15432"
+fi
+echo "arrow_flight_sql.uri = '${url}'" | \
+ tee -a "${PGDATA}/postgresql.conf"
diff --git a/.gitignore b/package/postgresql-15-pgdg/Dockerfile
similarity index 50%
copy from .gitignore
copy to package/postgresql-15-pgdg/Dockerfile
index 9eff26e..29c31be 100644
--- a/.gitignore
+++ b/package/postgresql-15-pgdg/Dockerfile
@@ -15,18 +15,22 @@
# specific language governing permissions and limitations
# under the License.
-/*.tar.gz
-/.cache/
-/Gemfile.lock
-/compile_commands.json
-/dev/release/.env
-/dev/release/apache-rat-0.13.jar
-/dev/release/dist
-/dev/release/filtered_rat.txt
-/dev/release/rat.xml
-/doc/build/
-/package/*/*.tar.gz
-/package/*/apt/build.sh
-/package/*/apt/env.sh
-/package/*/apt/repositories/
-/package/*/apt/tmp/
+FROM postgres:15-bookworm
+
+COPY package/docker/postgresql-15-pgdg-apache-arrow-flight-sql_*_amd64.deb ./
+RUN \
+ apt update && \
+ apt install -y -V lsb-release wget && \
+ wget https://apache.jfrog.io/artifactory/arrow/debian/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && \
+ apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && \
+ rm apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && \
+ apt update && \
+ apt install -y -V \
+ ./postgresql-15-pgdg-apache-arrow-flight-sql_*.deb && \
+ rm -f postgresql-15-pgdg-apache-arrow-flight-sql_*.deb && \
+ apt clean && \
+ rm -rf /var/lib/apt/lists/*
+
+COPY package/docker/apache-arrow-flight-sql-enable.sh /docker-entrypoint-initdb.d/
+
+EXPOSE 15432
diff --git a/package/postgresql-15-pgdg/apt/ubuntu-jammy-amd64/Dockerfile b/package/postgresql-15-pgdg/apt/ubuntu-jammy-amd64/Dockerfile
index 3254391..174e15e 100644
--- a/package/postgresql-15-pgdg/apt/ubuntu-jammy-amd64/Dockerfile
+++ b/package/postgresql-15-pgdg/apt/ubuntu-jammy-amd64/Dockerfile
@@ -60,5 +60,6 @@ RUN \
libkrb5-dev \
meson \
ninja-build \
+ pkg-config \
postgresql-server-dev-15 && \
apt clean
diff --git a/.gitignore b/package/postgresql-16-pgdg/Dockerfile
similarity index 50%
copy from .gitignore
copy to package/postgresql-16-pgdg/Dockerfile
index 9eff26e..bc621ee 100644
--- a/.gitignore
+++ b/package/postgresql-16-pgdg/Dockerfile
@@ -15,18 +15,22 @@
# specific language governing permissions and limitations
# under the License.
-/*.tar.gz
-/.cache/
-/Gemfile.lock
-/compile_commands.json
-/dev/release/.env
-/dev/release/apache-rat-0.13.jar
-/dev/release/dist
-/dev/release/filtered_rat.txt
-/dev/release/rat.xml
-/doc/build/
-/package/*/*.tar.gz
-/package/*/apt/build.sh
-/package/*/apt/env.sh
-/package/*/apt/repositories/
-/package/*/apt/tmp/
+FROM postgres:16-bookworm
+
+COPY package/docker/postgresql-16-pgdg-apache-arrow-flight-sql_*_amd64.deb ./
+RUN \
+ apt update && \
+ apt install -y -V lsb-release wget && \
+ wget https://apache.jfrog.io/artifactory/arrow/debian/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && \
+ apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && \
+ rm apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && \
+ apt update && \
+ apt install -y -V \
+ ./postgresql-16-pgdg-apache-arrow-flight-sql_*.deb && \
+ rm -f postgresql-16-pgdg-apache-arrow-flight-sql_*.deb && \
+ apt clean && \
+ rm -rf /var/lib/apt/lists/*
+
+COPY package/docker/apache-arrow-flight-sql-enable.sh /docker-entrypoint-initdb.d/
+
+EXPOSE 15432
diff --git a/package/postgresql-16-pgdg/apt/ubuntu-jammy-amd64/Dockerfile b/package/postgresql-16-pgdg/apt/ubuntu-jammy-amd64/Dockerfile
index 4e8a6bb..18105ff 100644
--- a/package/postgresql-16-pgdg/apt/ubuntu-jammy-amd64/Dockerfile
+++ b/package/postgresql-16-pgdg/apt/ubuntu-jammy-amd64/Dockerfile
@@ -60,5 +60,6 @@ RUN \
libkrb5-dev \
meson \
ninja-build \
+ pkg-config \
postgresql-server-dev-16 && \
apt clean
diff --git a/src/afs.cc b/src/afs.cc
index cd76824..9bf1ad5 100644
--- a/src/afs.cc
+++ b/src/afs.cc
@@ -77,6 +77,7 @@ extern "C"
# define AFS_FUNC __func__
#endif
+// #define AFS_DEBUG
#ifdef AFS_DEBUG
# define P(...) ereport(DEBUG5, errmsg_internal(__VA_ARGS__))
#else
diff --git a/test/helper/sandbox.rb b/test/helper/sandbox.rb
index 53969e5..6bd0623 100644
--- a/test/helper/sandbox.rb
+++ b/test/helper/sandbox.rb
@@ -161,6 +161,7 @@ module Helper
end
conf.puts("logging_collector = on")
conf.puts("log_filename = '#{@log_base_name}'")
+ conf.puts("log_min_messages = debug5") if ENV["AFS_DEBUG"] == "yes"
conf.puts("shared_preload_libraries = " +
"'#{shared_preload_libraries.join(",")}'")
conf.puts("arrow_flight_sql.uri = '#{@flight_sql_uri}'")
@@ -375,7 +376,10 @@ module Helper
begin
yield
ensure
- stop_postgres if @postgresql
+ if @postgresql
+ stop_postgres
+ puts(@postgresql.read_log) unless passed?
+ end
end
end