You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by al...@apache.org on 2021/05/16 11:36:48 UTC
[arrow-rs] branch master updated: Remove old release scripts (#293)
This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new 1fc48da Remove old release scripts (#293)
1fc48da is described below
commit 1fc48daab0d915fcd26c919d2fc47a7b057bd13e
Author: Andrew Lamb <an...@nerdnetworks.org>
AuthorDate: Sun May 16 07:36:39 2021 -0400
Remove old release scripts (#293)
* Remove old release scripts
* Add rat files back in
---
dev/release/.env.example | 48 -
dev/release/.gitignore | 21 -
dev/release/01-prepare-test.rb | 665 -------
dev/release/01-prepare.sh | 291 ---
dev/release/02-source-test.rb | 146 --
dev/release/02-source.sh | 162 --
dev/release/03-binary-submit.sh | 46 -
dev/release/04-binary-download.sh | 38 -
dev/release/05-binary-upload.sh | 137 --
dev/release/README.md | 24 -
dev/release/Rakefile | 37 -
dev/release/VERIFY.md | 76 -
dev/release/binary-common.sh | 86 -
dev/release/binary-task.rb | 1909 --------------------
dev/release/binary/.dockerignore | 18 -
dev/release/binary/Dockerfile | 68 -
dev/release/binary/runner.sh | 36 -
dev/release/check-rat-report.py | 59 -
dev/release/download_rc_binaries.py | 173 --
dev/release/post-01-upload.sh | 71 -
dev/release/post-02-binary.sh | 95 -
dev/release/post-03-website.sh | 266 ---
dev/release/post-04-ruby.sh | 55 -
dev/release/post-05-js.sh | 48 -
dev/release/post-06-csharp.sh | 59 -
dev/release/post-07-rust.sh | 74 -
dev/release/post-08-remove-rc.sh | 50 -
dev/release/post-09-docs.sh | 68 -
dev/release/post-10-python.sh | 44 -
dev/release/post-11-java.sh | 69 -
dev/release/run-test.rb | 31 -
dev/release/setup-gpg-agent.sh | 24 -
dev/release/test-helper.rb | 96 -
dev/release/verify-apt.sh | 150 --
dev/release/verify-release-candidate-wheels.bat | 107 --
dev/release/verify-release-candidate.bat | 130 --
dev/release/verify-release-candidate.sh | 808 ---------
dev/release/verify-yum.sh | 154 --
dev/tasks/README.md | 19 -
..._version10.2numpy1.17python3.6.____cpython.yaml | 70 -
..._version10.2numpy1.17python3.7.____cpython.yaml | 70 -
..._version10.2numpy1.17python3.8.____cpython.yaml | 70 -
..._version10.2numpy1.19python3.9.____cpython.yaml | 70 -
..._versionNonenumpy1.17python3.6.____cpython.yaml | 70 -
..._versionNonenumpy1.17python3.7.____cpython.yaml | 70 -
..._versionNonenumpy1.17python3.8.____cpython.yaml | 70 -
..._versionNonenumpy1.19python3.9.____cpython.yaml | 70 -
.../linux_aarch64_python3.6.____cpython.yaml | 71 -
.../linux_aarch64_python3.7.____cpython.yaml | 71 -
.../linux_aarch64_python3.8.____cpython.yaml | 71 -
.../linux_aarch64_python3.9.____cpython.yaml | 71 -
.../osx_64_numpy1.17python3.6.____cpython.yaml | 65 -
.../osx_64_numpy1.17python3.7.____cpython.yaml | 65 -
.../osx_64_numpy1.17python3.8.____cpython.yaml | 65 -
.../osx_64_numpy1.19python3.9.____cpython.yaml | 65 -
.../osx_arm64_python3.8.____cpython.yaml | 65 -
.../osx_arm64_python3.9.____cpython.yaml | 65 -
.../.ci_support/r/linux_64_r_base3.6.yaml | 22 -
.../.ci_support/r/linux_64_r_base4.0.yaml | 22 -
.../.ci_support/r/osx_64_r_base3.6.yaml | 26 -
.../.ci_support/r/osx_64_r_base4.0.yaml | 26 -
.../.ci_support/r/win_64_r_base3.6.yaml | 12 -
.../.ci_support/r/win_64_r_base4.0.yaml | 12 -
..._versionNonenumpy1.17python3.6.____cpython.yaml | 55 -
..._versionNonenumpy1.17python3.7.____cpython.yaml | 55 -
..._versionNonenumpy1.17python3.8.____cpython.yaml | 55 -
..._versionNonenumpy1.19python3.9.____cpython.yaml | 55 -
dev/tasks/conda-recipes/.scripts/logging_utils.sh | 30 -
dev/tasks/conda-recipes/README.md | 67 -
dev/tasks/conda-recipes/arrow-cpp/LLVM_LICENSE.txt | 68 -
dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat | 54 -
dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat | 44 -
dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh | 95 -
dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh | 49 -
dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 302 ----
dev/tasks/conda-recipes/azure.clean.yml | 28 -
dev/tasks/conda-recipes/azure.linux.yml | 38 -
dev/tasks/conda-recipes/azure.osx.yml | 80 -
dev/tasks/conda-recipes/azure.win.yml | 77 -
dev/tasks/conda-recipes/build_steps.sh | 55 -
dev/tasks/conda-recipes/clean.py | 80 -
dev/tasks/conda-recipes/conda-forge.yml | 1 -
dev/tasks/conda-recipes/drone-steps.sh | 29 -
dev/tasks/conda-recipes/drone.yml | 43 -
dev/tasks/conda-recipes/parquet-cpp/meta.yaml | 51 -
dev/tasks/conda-recipes/r-arrow/bld.bat | 9 -
dev/tasks/conda-recipes/r-arrow/build.sh | 3 -
dev/tasks/conda-recipes/r-arrow/build_win.sh | 8 -
dev/tasks/conda-recipes/r-arrow/configure.win | 8 -
dev/tasks/conda-recipes/r-arrow/install.libs.R | 5 -
dev/tasks/conda-recipes/r-arrow/meta.yaml | 66 -
dev/tasks/conda-recipes/run_docker_build.sh | 77 -
dev/tasks/cpp-examples/github.linux.yml | 46 -
dev/tasks/docker-tests/azure.linux.yml | 52 -
dev/tasks/docker-tests/circle.linux.yml | 51 -
dev/tasks/docker-tests/github.linux.yml | 42 -
dev/tasks/gandiva-jars/README.md | 29 -
dev/tasks/gandiva-jars/build-cpp-linux.sh | 73 -
dev/tasks/gandiva-jars/build-cpp-osx.sh | 49 -
dev/tasks/gandiva-jars/build-java.sh | 64 -
dev/tasks/gandiva-jars/github.linux.yml | 47 -
dev/tasks/gandiva-jars/github.osx.yml | 46 -
dev/tasks/homebrew-formulae/apache-arrow.rb | 69 -
.../homebrew-formulae/autobrew/apache-arrow.rb | 88 -
dev/tasks/homebrew-formulae/github.macos.yml | 56 -
dev/tasks/linux-packages/.gitignore | 28 -
dev/tasks/linux-packages/README.md | 40 -
dev/tasks/linux-packages/Rakefile | 234 ---
.../apache-arrow-apt-source/Rakefile | 64 -
.../apt/debian-bullseye/Dockerfile | 40 -
.../apt/debian-buster/Dockerfile | 41 -
.../apt/ubuntu-bionic/Dockerfile | 41 -
.../apt/ubuntu-focal/Dockerfile | 41 -
.../apt/ubuntu-groovy/Dockerfile | 41 -
.../apt/ubuntu-xenial/Dockerfile | 41 -
.../debian/apache-arrow-apt-source.install | 2 -
.../apache-arrow-apt-source/debian/changelog | 0
.../apache-arrow-apt-source/debian/compat | 1 -
.../apache-arrow-apt-source/debian/control | 23 -
.../apache-arrow-apt-source/debian/copyright | 26 -
.../apache-arrow-apt-source/debian/rules | 37 -
.../apache-arrow-apt-source/debian/source/format | 1 -
.../linux-packages/apache-arrow-release/Rakefile | 66 -
.../apache-arrow-release/yum/Apache-Arrow.repo | 37 -
.../yum/apache-arrow-release.spec.in | 110 --
.../apache-arrow-release/yum/centos-7/Dockerfile | 28 -
.../apache-arrow-release/yum/centos-8/Dockerfile | 28 -
dev/tasks/linux-packages/apache-arrow/Rakefile | 120 --
.../apache-arrow/apt/debian-bullseye-arm64/from | 18 -
.../apache-arrow/apt/debian-bullseye/Dockerfile | 81 -
.../apt/debian-bullseye/qemu-dummy-static | 33 -
.../apache-arrow/apt/debian-buster-arm64/from | 18 -
.../apache-arrow/apt/debian-buster/Dockerfile | 85 -
.../apt/debian-buster/qemu-dummy-static | 33 -
.../apache-arrow/apt/ubuntu-bionic-arm64/from | 18 -
.../apache-arrow/apt/ubuntu-bionic/Dockerfile | 88 -
.../apt/ubuntu-bionic/qemu-dummy-static | 33 -
.../apache-arrow/apt/ubuntu-focal-arm64/from | 18 -
.../apache-arrow/apt/ubuntu-focal/Dockerfile | 78 -
.../apt/ubuntu-focal/qemu-dummy-static | 33 -
.../apache-arrow/apt/ubuntu-groovy-arm64/from | 18 -
.../apache-arrow/apt/ubuntu-groovy/Dockerfile | 79 -
.../apt/ubuntu-groovy/qemu-dummy-static | 33 -
.../linux-packages/apache-arrow/debian/changelog | 111 --
.../linux-packages/apache-arrow/debian/compat | 1 -
.../linux-packages/apache-arrow/debian/control.in | 583 ------
.../linux-packages/apache-arrow/debian/copyright | 193 --
.../apache-arrow/debian/gir1.2-arrow-1.0.install | 1 -
.../debian/gir1.2-arrow-cuda-1.0.install | 1 -
.../debian/gir1.2-arrow-dataset-1.0.install | 1 -
.../apache-arrow/debian/gir1.2-gandiva-1.0.install | 1 -
.../apache-arrow/debian/gir1.2-parquet-1.0.install | 1 -
.../apache-arrow/debian/gir1.2-plasma-1.0.install | 1 -
.../apache-arrow/debian/libarrow-cuda-dev.install | 6 -
.../debian/libarrow-cuda-glib-dev.install | 4 -
.../debian/libarrow-cuda-glib400.install | 1 -
.../apache-arrow/debian/libarrow-cuda400.install | 1 -
.../debian/libarrow-dataset-dev.install | 6 -
.../debian/libarrow-dataset-glib-dev.install | 4 -
.../debian/libarrow-dataset-glib-doc.doc-base | 9 -
.../debian/libarrow-dataset-glib-doc.install | 1 -
.../debian/libarrow-dataset-glib-doc.links | 3 -
.../debian/libarrow-dataset-glib400.install | 1 -
.../debian/libarrow-dataset400.install | 1 -
.../apache-arrow/debian/libarrow-dev.install | 21 -
.../debian/libarrow-flight-dev.install | 6 -
.../apache-arrow/debian/libarrow-flight400.install | 1 -
.../apache-arrow/debian/libarrow-glib-dev.install | 6 -
.../apache-arrow/debian/libarrow-glib-doc.doc-base | 9 -
.../apache-arrow/debian/libarrow-glib-doc.install | 2 -
.../apache-arrow/debian/libarrow-glib-doc.links | 3 -
.../apache-arrow/debian/libarrow-glib400.install | 1 -
.../debian/libarrow-python-dev.install | 6 -
.../debian/libarrow-python-flight-dev.install | 6 -
.../debian/libarrow-python-flight400.install | 1 -
.../apache-arrow/debian/libarrow-python400.install | 1 -
.../apache-arrow/debian/libarrow400.install | 1 -
.../apache-arrow/debian/libgandiva-dev.install | 7 -
.../debian/libgandiva-glib-dev.install | 4 -
.../debian/libgandiva-glib-doc.doc-base | 9 -
.../debian/libgandiva-glib-doc.install | 1 -
.../apache-arrow/debian/libgandiva-glib-doc.links | 3 -
.../apache-arrow/debian/libgandiva-glib400.install | 1 -
.../apache-arrow/debian/libgandiva400.install | 1 -
.../apache-arrow/debian/libparquet-dev.install | 7 -
.../debian/libparquet-glib-dev.install | 4 -
.../debian/libparquet-glib-doc.doc-base | 9 -
.../debian/libparquet-glib-doc.install | 1 -
.../apache-arrow/debian/libparquet-glib-doc.links | 3 -
.../apache-arrow/debian/libparquet-glib400.install | 1 -
.../apache-arrow/debian/libparquet400.install | 1 -
.../apache-arrow/debian/libplasma-dev.install | 7 -
.../apache-arrow/debian/libplasma-glib-dev.install | 4 -
.../debian/libplasma-glib-doc.doc-base | 9 -
.../apache-arrow/debian/libplasma-glib-doc.install | 1 -
.../apache-arrow/debian/libplasma-glib-doc.links | 3 -
.../apache-arrow/debian/libplasma-glib400.install | 1 -
.../apache-arrow/debian/libplasma400.install | 1 -
.../apache-arrow/debian/patches/series | 0
.../debian/plasma-store-server.install | 1 -
dev/tasks/linux-packages/apache-arrow/debian/rules | 103 --
.../apache-arrow/debian/source/format | 1 -
dev/tasks/linux-packages/apache-arrow/debian/watch | 2 -
.../linux-packages/apache-arrow/yum/arrow.spec.in | 802 --------
.../apache-arrow/yum/centos-7/Dockerfile | 62 -
.../apache-arrow/yum/centos-7/qemu-dummy-static | 33 -
.../apache-arrow/yum/centos-8-aarch64/from | 18 -
.../apache-arrow/yum/centos-8/Dockerfile | 65 -
.../apache-arrow/yum/centos-8/qemu-dummy-static | 33 -
dev/tasks/linux-packages/apt/build.sh | 115 --
dev/tasks/linux-packages/github.linux.amd64.yml | 100 -
dev/tasks/linux-packages/helper.rb | 70 -
dev/tasks/linux-packages/package-task.rb | 622 -------
dev/tasks/linux-packages/travis.linux.arm64.yml | 149 --
dev/tasks/linux-packages/yum/build.sh | 157 --
dev/tasks/macros.jinja | 198 --
dev/tasks/nightlies.sample.yml | 68 -
dev/tasks/nuget-packages/github.linux.yml | 43 -
dev/tasks/python-sdist/github.yml | 45 -
dev/tasks/python-wheels/github.linux.amd64.yml | 48 -
dev/tasks/python-wheels/github.osx.yml | 133 --
dev/tasks/python-wheels/github.windows.yml | 53 -
dev/tasks/python-wheels/travis.linux.arm64.yml | 73 -
dev/tasks/r/azure.linux.yml | 74 -
dev/tasks/r/github.devdocs.yml | 92 -
dev/tasks/r/github.linux.cran.yml | 79 -
dev/tasks/r/github.linux.version.compatibility.yml | 109 --
dev/tasks/r/github.linux.versions.yml | 80 -
dev/tasks/r/github.macos-linux.local.yml | 88 -
dev/tasks/r/github.macos.autobrew.yml | 78 -
dev/tasks/tasks.yml | 1703 -----------------
dev/tasks/vcpkg-tests/cpp-build-vcpkg.bat | 90 -
dev/tasks/vcpkg-tests/github.windows.yml | 63 -
dev/tasks/verify-rc/github.linux.yml | 75 -
dev/tasks/verify-rc/github.macos.yml | 50 -
dev/tasks/verify-rc/github.win.yml | 45 -
236 files changed, 18560 deletions(-)
diff --git a/dev/release/.env.example b/dev/release/.env.example
deleted file mode 100644
index 0126cdd..0000000
--- a/dev/release/.env.example
+++ /dev/null
@@ -1,48 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# The GPG key ID to sign artifacts. The GPG key ID must be registered
-# to both of the followings:
-#
-# * https://dist.apache.org/repos/dist/dev/arrow/KEYS
-# * https://dist.apache.org/repos/dist/release/arrow/KEYS
-#
-# See these files how to import your GPG key ID to these files.
-#
-# You must set this.
-#GPG_KEY_ID=08D3564B7C6A9CAFBFF6A66791D18FCF079F8007
-
-# The Bintray repository where artifacts are uploaded.
-# You can use your Bintray repository such as kou/arrow for test.
-BINTRAY_REPOSITORY=apache/arrow
-
-# The Bintray repository where released artifacts exist.
-# The released artifacts are used to build APT/Yum repository.
-# The Bintray repository isn't changed. (Download only. No upload.)
-#
-# Normally, you don't need to change this.
-SOURCE_BINTRAY_REPOSITORY=apache/arrow
-
-# The Bintray user name to upload artifacts to Bintray.
-#
-# You must set this.
-#BINTRAY_USER=kou
-
-# The Bintray API key to upload artifacts to Bintray.
-#
-# You must set this.
-#BINTRAY_API_KEY=secret
diff --git a/dev/release/.gitignore b/dev/release/.gitignore
deleted file mode 100644
index f3d708a..0000000
--- a/dev/release/.gitignore
+++ /dev/null
@@ -1,21 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-/.env
-/binary/id_rsa
-/binary/id_rsa.pub
-/binary/tmp/
diff --git a/dev/release/01-prepare-test.rb b/dev/release/01-prepare-test.rb
deleted file mode 100644
index b316ad2..0000000
--- a/dev/release/01-prepare-test.rb
+++ /dev/null
@@ -1,665 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-class PrepareTest < Test::Unit::TestCase
- include GitRunnable
- include VersionDetectable
-
- def setup
- @current_commit = git_current_commit
- detect_versions
-
- top_dir = Pathname(__dir__).parent.parent
- @original_git_repository = top_dir + ".git"
- Dir.mktmpdir do |dir|
- @test_git_repository = Pathname(dir) + "arrow"
- git("clone", @original_git_repository.to_s, @test_git_repository.to_s)
- Dir.chdir(@test_git_repository) do
- @tag_name = "apache-arrow-#{@release_version}"
- @release_branch = "release-#{@release_version}-rc0"
- @script = "dev/release/01-prepare.sh"
- git("checkout", "-b", @release_branch, @current_commit)
- yield
- end
- FileUtils.rm_rf(@test_git_repository)
- end
- end
-
- def omit_on_release_branch
- omit("Not for release branch") if on_release_branch?
- end
-
- def prepare(*targets)
- if targets.last.is_a?(Hash)
- additional_env = targets.pop
- else
- additional_env = {}
- end
- env = {"PREPARE_DEFAULT" => "0"}
- targets.each do |target|
- env["PREPARE_#{target}"] = "1"
- end
- env = env.merge(additional_env)
- sh(env, @script, @release_version, @next_version, "0")
- end
-
- def parse_patch(patch)
- diffs = []
- in_hunk = false
- patch.each_line do |line|
- case line
- when /\A--- a\//
- path = $POSTMATCH.chomp
- diffs << {path: path, hunks: []}
- in_hunk = false
- when /\A@@/
- in_hunk = true
- diffs.last[:hunks] << []
- when /\A[-+]/
- next unless in_hunk
- diffs.last[:hunks].last << line.chomp
- end
- end
- diffs.sort_by do |diff|
- diff[:path]
- end
- end
-
- def test_linux_packages
- user = "Arrow Developers"
- email = "dev@arrow.apache.org"
- prepare("LINUX_PACKAGES",
- "DEBFULLNAME" => user,
- "DEBEMAIL" => email)
- changes = parse_patch(git("log", "-n", "1", "-p"))
- sampled_changes = changes.collect do |change|
- {
- path: change[:path],
- sampled_hunks: change[:hunks].collect(&:first),
- # sampled_hunks: change[:hunks],
- }
- end
- base_dir = "dev/tasks/linux-packages"
- today = Time.now.utc.strftime("%a %b %d %Y")
- expected_changes = [
- {
- path: "#{base_dir}/apache-arrow-apt-source/debian/changelog",
- sampled_hunks: [
- "+apache-arrow-apt-source (#{@release_version}-1) " +
- "unstable; urgency=low",
- ],
- },
- {
- path:
- "#{base_dir}/apache-arrow-release/yum/apache-arrow-release.spec.in",
- sampled_hunks: [
- "+* #{today} #{user} <#{email}> - #{@release_version}-1",
- ],
- },
- {
- path: "#{base_dir}/apache-arrow/debian/changelog",
- sampled_hunks: [
- "+apache-arrow (#{@release_version}-1) unstable; urgency=low",
- ],
- },
- {
- path: "#{base_dir}/apache-arrow/yum/arrow.spec.in",
- sampled_hunks: [
- "+* #{today} #{user} <#{email}> - #{@release_version}-1",
- ],
- },
- ]
- assert_equal(expected_changes, sampled_changes)
- end
-
- def test_version_pre_tag
- omit_on_release_branch
- prepare("VERSION_PRE_TAG")
- assert_equal([
- {
- path: "c_glib/meson.build",
- hunks: [
- ["-version = '#{@snapshot_version}'",
- "+version = '#{@release_version}'"],
- ],
- },
- {
- path: "ci/scripts/PKGBUILD",
- hunks: [
- ["-pkgver=#{@previous_version}.9000",
- "+pkgver=#{@release_version}"],
- ],
- },
- {
- path: "cpp/CMakeLists.txt",
- hunks: [
- ["-set(ARROW_VERSION \"#{@snapshot_version}\")",
- "+set(ARROW_VERSION \"#{@release_version}\")"],
- ],
- },
- {
- path: "cpp/vcpkg.json",
- hunks: [
- ["- \"version-string\": \"#{@snapshot_version}\",",
- "+ \"version-string\": \"#{@release_version}\","],
- ],
- },
- {
- path: "csharp/Directory.Build.props",
- hunks: [
- ["- <Version>#{@snapshot_version}</Version>",
- "+ <Version>#{@release_version}</Version>"],
- ],
- },
- {
- path: "dev/tasks/homebrew-formulae/apache-arrow.rb",
- hunks: [
- ["- url \"https://www.apache.org/dyn/closer.lua?path=arrow/arrow-#{@snapshot_version}/apache-arrow-#{@snapshot_version}.tar.gz\"",
- "+ url \"https://www.apache.org/dyn/closer.lua?path=arrow/arrow-#{@release_version}/apache-arrow-#{@release_version}.tar.gz\""],
- ],
- },
- {
- path: "dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb",
- hunks: [
- ["- url \"https://www.apache.org/dyn/closer.lua?path=arrow/arrow-#{@previous_version}.9000/apache-arrow-#{@previous_version}.9000.tar.gz\"",
- "+ url \"https://www.apache.org/dyn/closer.lua?path=arrow/arrow-#{@release_version}/apache-arrow-#{@release_version}.tar.gz\""],
- ],
- },
- {
- path: "js/package.json",
- hunks: [
- ["- \"version\": \"#{@snapshot_version}\"",
- "+ \"version\": \"#{@release_version}\""]
- ],
- },
- {
- path: "matlab/CMakeLists.txt",
- hunks: [
- ["-set(MLARROW_VERSION \"#{@snapshot_version}\")",
- "+set(MLARROW_VERSION \"#{@release_version}\")"],
- ],
- },
- {
- path: "python/setup.py",
- hunks: [
- ["-default_version = '#{@snapshot_version}'",
- "+default_version = '#{@release_version}'"],
- ],
- },
- {
- path: "r/DESCRIPTION",
- hunks: [
- ["-Version: #{@previous_version}.9000",
- "+Version: #{@release_version}"],
- ],
- },
- {
- path: "r/NEWS.md",
- hunks: [
- ["-\# arrow #{@previous_version}.9000",
- "+\# arrow #{@release_version}"],
- ],
- },
- {
- path: "ruby/red-arrow-cuda/lib/arrow-cuda/version.rb",
- hunks: [
- ["- VERSION = \"#{@snapshot_version}\"",
- "+ VERSION = \"#{@release_version}\""],
- ],
- },
- {
- path: "ruby/red-arrow-dataset/lib/arrow-dataset/version.rb",
- hunks: [
- ["- VERSION = \"#{@snapshot_version}\"",
- "+ VERSION = \"#{@release_version}\""],
- ],
- },
- {
- path: "ruby/red-arrow/lib/arrow/version.rb",
- hunks: [
- ["- VERSION = \"#{@snapshot_version}\"",
- "+ VERSION = \"#{@release_version}\""],
- ],
- },
- {
- path: "ruby/red-gandiva/lib/gandiva/version.rb",
- hunks: [
- ["- VERSION = \"#{@snapshot_version}\"",
- "+ VERSION = \"#{@release_version}\""],
- ],
- },
- {
- path: "ruby/red-parquet/lib/parquet/version.rb",
- hunks: [
- ["- VERSION = \"#{@snapshot_version}\"",
- "+ VERSION = \"#{@release_version}\""],
- ],
- },
- {
- path: "ruby/red-plasma/lib/plasma/version.rb",
- hunks: [
- ["- VERSION = \"#{@snapshot_version}\"",
- "+ VERSION = \"#{@release_version}\""],
- ],
- },
- {
- path: "rust/arrow-flight/Cargo.toml",
- hunks: [
- ["-version = \"#{@snapshot_version}\"",
- "+version = \"#{@release_version}\""],
- ["-arrow = { path = \"../arrow\", version = \"#{@snapshot_version}\" }",
- "+arrow = { path = \"../arrow\", version = \"#{@release_version}\" }"],
- ],
- },
- {
- path: "rust/arrow-pyarrow-integration-testing/Cargo.toml",
- hunks: [
- ["-version = \"#{@snapshot_version}\"",
- "+version = \"#{@release_version}\""],
- ["-arrow = { path = \"../arrow\", version = \"#{@snapshot_version}\" }",
- "+arrow = { path = \"../arrow\", version = \"#{@release_version}\" }"],
- ],
- },
- {
- path: "rust/arrow/Cargo.toml",
- hunks: [
- ["-version = \"#{@snapshot_version}\"",
- "+version = \"#{@release_version}\""],
- ],
- },
- {
- path: "rust/benchmarks/Cargo.toml",
- hunks: [
- ["-version = \"#{@snapshot_version}\"",
- "+version = \"#{@release_version}\""],
- ],
- },
- {
- path: "rust/datafusion-examples/Cargo.toml",
- hunks: [
- ["-version = \"#{@snapshot_version}\"",
- "+version = \"#{@release_version}\""],
- ],
- },
- {
- path: "rust/datafusion/Cargo.toml",
- hunks: [
- ["-version = \"#{@snapshot_version}\"",
- "+version = \"#{@release_version}\""],
- ["-arrow = { path = \"../arrow\", version = \"#{@snapshot_version}\", features = [\"prettyprint\"] }",
- "-parquet = { path = \"../parquet\", version = \"#{@snapshot_version}\", features = [\"arrow\"] }",
- "+arrow = { path = \"../arrow\", version = \"#{@release_version}\", features = [\"prettyprint\"] }",
- "+parquet = { path = \"../parquet\", version = \"#{@release_version}\", features = [\"arrow\"] }"],
- ],
- },
- {
- path: "rust/datafusion/README.md",
- hunks: [
- ["-datafusion = \"#{@snapshot_version}\"",
- "+datafusion = \"#{@release_version}\""],
- ],
- },
- {
- path: "rust/integration-testing/Cargo.toml",
- hunks: [
- ["-version = \"#{@snapshot_version}\"",
- "+version = \"#{@release_version}\""],
- ],
- },
- {
- path: "rust/parquet/Cargo.toml",
- hunks: [
- ["-version = \"#{@snapshot_version}\"",
- "+version = \"#{@release_version}\""],
- ["-arrow = { path = \"../arrow\", version = \"#{@snapshot_version}\", optional = true }",
- "+arrow = { path = \"../arrow\", version = \"#{@release_version}\", optional = true }"],
- ["-arrow = { path = \"../arrow\", version = \"#{@snapshot_version}\" }",
- "+arrow = { path = \"../arrow\", version = \"#{@release_version}\" }"]
- ],
- },
- {
- path: "rust/parquet/README.md",
- hunks: [
- ["-parquet = \"#{@snapshot_version}\"",
- "+parquet = \"#{@release_version}\""],
- ["-See [crate documentation](https://docs.rs/crate/parquet/#{@snapshot_version}) on available API.",
- "+See [crate documentation](https://docs.rs/crate/parquet/#{@release_version}) on available API."],
- ],
- },
- {
- path: "rust/parquet_derive/Cargo.toml",
- hunks: [
- ["-version = \"#{@snapshot_version}\"",
- "+version = \"#{@release_version}\""],
- ["-parquet = { path = \"../parquet\", version = \"#{@snapshot_version}\" }",
- "+parquet = { path = \"../parquet\", version = \"#{@release_version}\" }"],
- ],
- },
- {
- path: "rust/parquet_derive/README.md",
- hunks: [
- ["-parquet = \"#{@snapshot_version}\"",
- "-parquet_derive = \"#{@snapshot_version}\"",
- "+parquet = \"#{@release_version}\"",
- "+parquet_derive = \"#{@release_version}\""],
- ],
- },
- {
- path: "rust/parquet_derive_test/Cargo.toml",
- hunks: [
- ["-version = \"#{@snapshot_version}\"",
- "+version = \"#{@release_version}\""],
- ["-parquet = { path = \"../parquet\", version = \"#{@snapshot_version}\" }",
- "-parquet_derive = { path = \"../parquet_derive\", version = \"#{@snapshot_version}\" }",
- "+parquet = { path = \"../parquet\", version = \"#{@release_version}\" }",
- "+parquet_derive = { path = \"../parquet_derive\", version = \"#{@release_version}\" }"],
- ],
- },
- ],
- parse_patch(git("log", "-n", "1", "-p")))
- end
-
- def test_version_post_tag
- if on_release_branch?
- prepare("VERSION_POST_TAG")
- else
- prepare("VERSION_PRE_TAG",
- "VERSION_POST_TAG")
- end
- assert_equal([
- {
- path: "c_glib/meson.build",
- hunks: [
- ["-version = '#{@release_version}'",
- "+version = '#{@next_snapshot_version}'"],
- ],
- },
- {
- path: "ci/scripts/PKGBUILD",
- hunks: [
- ["-pkgver=#{@release_version}",
- "+pkgver=#{@release_version}.9000"],
- ],
- },
- {
- path: "cpp/CMakeLists.txt",
- hunks: [
- ["-set(ARROW_VERSION \"#{@release_version}\")",
- "+set(ARROW_VERSION \"#{@next_snapshot_version}\")"],
- ],
- },
- {
- path: "cpp/vcpkg.json",
- hunks: [
- ["- \"version-string\": \"#{@release_version}\",",
- "+ \"version-string\": \"#{@next_snapshot_version}\","],
- ],
- },
- {
- path: "csharp/Directory.Build.props",
- hunks: [
- ["- <Version>#{@release_version}</Version>",
- "+ <Version>#{@next_snapshot_version}</Version>"],
- ],
- },
- {
- path: "dev/tasks/homebrew-formulae/apache-arrow.rb",
- hunks: [
- ["- url \"https://www.apache.org/dyn/closer.lua?path=arrow/arrow-#{@release_version}/apache-arrow-#{@release_version}.tar.gz\"",
- "+ url \"https://www.apache.org/dyn/closer.lua?path=arrow/arrow-#{@next_snapshot_version}/apache-arrow-#{@next_snapshot_version}.tar.gz\""],
- ],
- },
- {
- path: "dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb",
- hunks: [
- ["- url \"https://www.apache.org/dyn/closer.lua?path=arrow/arrow-#{@release_version}/apache-arrow-#{@release_version}.tar.gz\"",
- "+ url \"https://www.apache.org/dyn/closer.lua?path=arrow/arrow-#{@release_version}.9000/apache-arrow-#{@release_version}.9000.tar.gz\""],
- ],
- },
- {
- path: "js/package.json",
- hunks: [
- ["- \"version\": \"#{@release_version}\"",
- "+ \"version\": \"#{@next_snapshot_version}\""],
- ],
- },
- {
- path: "matlab/CMakeLists.txt",
- hunks: [
- ["-set(MLARROW_VERSION \"#{@release_version}\")",
- "+set(MLARROW_VERSION \"#{@next_snapshot_version}\")"],
- ],
- },
- {
- path: "python/setup.py",
- hunks: [
- ["-default_version = '#{@release_version}'",
- "+default_version = '#{@next_snapshot_version}'"],
- ],
- },
- {
- path: "r/DESCRIPTION",
- hunks: [
- ["-Version: #{@release_version}",
- "+Version: #{@release_version}.9000"],
- ],
- },
- {
- path: "r/NEWS.md",
- # Note that these are additions only, no replacement
- hunks: [
- ["+# arrow #{@release_version}.9000",
- "+"],
- ],
- },
- {
- path: "ruby/red-arrow-cuda/lib/arrow-cuda/version.rb",
- hunks: [
- ["- VERSION = \"#{@release_version}\"",
- "+ VERSION = \"#{@next_snapshot_version}\""],
- ],
- },
- {
- path: "ruby/red-arrow-dataset/lib/arrow-dataset/version.rb",
- hunks: [
- ["- VERSION = \"#{@release_version}\"",
- "+ VERSION = \"#{@next_snapshot_version}\""],
- ],
- },
- {
- path: "ruby/red-arrow/lib/arrow/version.rb",
- hunks: [
- ["- VERSION = \"#{@release_version}\"",
- "+ VERSION = \"#{@next_snapshot_version}\""],
- ],
- },
- {
- path: "ruby/red-gandiva/lib/gandiva/version.rb",
- hunks: [
- ["- VERSION = \"#{@release_version}\"",
- "+ VERSION = \"#{@next_snapshot_version}\""],
- ],
- },
- {
- path: "ruby/red-parquet/lib/parquet/version.rb",
- hunks: [
- ["- VERSION = \"#{@release_version}\"",
- "+ VERSION = \"#{@next_snapshot_version}\""],
- ],
- },
- {
- path: "ruby/red-plasma/lib/plasma/version.rb",
- hunks: [
- ["- VERSION = \"#{@release_version}\"",
- "+ VERSION = \"#{@next_snapshot_version}\""],
- ],
- },
- {
- path: "rust/arrow-flight/Cargo.toml",
- hunks: [
- ["-version = \"#{@release_version}\"",
- "+version = \"#{@next_snapshot_version}\""],
- ["-arrow = { path = \"../arrow\", version = \"#{@release_version}\" }",
- "+arrow = { path = \"../arrow\", version = \"#{@next_snapshot_version}\" }"],
- ],
- },
- {
- path: "rust/arrow-pyarrow-integration-testing/Cargo.toml",
- hunks: [
- ["-version = \"#{@release_version}\"",
- "+version = \"#{@next_snapshot_version}\""],
- ["-arrow = { path = \"../arrow\", version = \"#{@release_version}\" }",
- "+arrow = { path = \"../arrow\", version = \"#{@next_snapshot_version}\" }"],
- ],
- },
- {
- path: "rust/arrow/Cargo.toml",
- hunks: [
- ["-version = \"#{@release_version}\"",
- "+version = \"#{@next_snapshot_version}\""],
- ],
- },
- {
- path: "rust/benchmarks/Cargo.toml",
- hunks: [
- ["-version = \"#{@release_version}\"",
- "+version = \"#{@next_snapshot_version}\""],
- ],
- },
- {
- path: "rust/datafusion-examples/Cargo.toml",
- hunks: [
- ["-version = \"#{@release_version}\"",
- "+version = \"#{@next_snapshot_version}\""],
- ],
- },
- {
- path: "rust/datafusion/Cargo.toml",
- hunks: [
- ["-version = \"#{@release_version}\"",
- "+version = \"#{@next_snapshot_version}\""],
- ["-arrow = { path = \"../arrow\", version = \"#{@release_version}\", features = [\"prettyprint\"] }",
- "-parquet = { path = \"../parquet\", version = \"#{@release_version}\", features = [\"arrow\"] }",
- "+arrow = { path = \"../arrow\", version = \"#{@next_snapshot_version}\", features = [\"prettyprint\"] }",
- "+parquet = { path = \"../parquet\", version = \"#{@next_snapshot_version}\", features = [\"arrow\"] }"],
- ],
- },
- {
- path: "rust/datafusion/README.md",
- hunks: [
- ["-datafusion = \"#{@release_version}\"",
- "+datafusion = \"#{@next_snapshot_version}\""],
- ],
- },
- {
- path: "rust/integration-testing/Cargo.toml",
- hunks: [
- ["-version = \"#{@release_version}\"",
- "+version = \"#{@next_snapshot_version}\""],
- ],
- },
- {
- path: "rust/parquet/Cargo.toml",
- hunks: [
- ["-version = \"#{@release_version}\"",
- "+version = \"#{@next_snapshot_version}\""],
- ["-arrow = { path = \"../arrow\", version = \"#{@release_version}\", optional = true }",
- "+arrow = { path = \"../arrow\", version = \"#{@next_snapshot_version}\", optional = true }"],
- ["-arrow = { path = \"../arrow\", version = \"#{@release_version}\" }",
- "+arrow = { path = \"../arrow\", version = \"#{@next_snapshot_version}\" }"]
- ],
- },
- {
- path: "rust/parquet/README.md",
- hunks: [
- ["-parquet = \"#{@release_version}\"",
- "+parquet = \"#{@next_snapshot_version}\""],
- ["-See [crate documentation](https://docs.rs/crate/parquet/#{@release_version}) on available API.",
- "+See [crate documentation](https://docs.rs/crate/parquet/#{@next_snapshot_version}) on available API."],
- ],
- },
- {
- path: "rust/parquet_derive/Cargo.toml",
- hunks: [
- ["-version = \"#{@release_version}\"",
- "+version = \"#{@next_snapshot_version}\""],
- ["-parquet = { path = \"../parquet\", version = \"#{@release_version}\" }",
- "+parquet = { path = \"../parquet\", version = \"#{@next_snapshot_version}\" }"],
- ],
- },
- {
- path: "rust/parquet_derive/README.md",
- hunks: [
- ["-parquet = \"#{@release_version}\"",
- "-parquet_derive = \"#{@release_version}\"",
- "+parquet = \"#{@next_snapshot_version}\"",
- "+parquet_derive = \"#{@next_snapshot_version}\""],
- ],
- },
- {
- path: "rust/parquet_derive_test/Cargo.toml",
- hunks: [
- ["-version = \"#{@release_version}\"",
- "+version = \"#{@next_snapshot_version}\""],
- ["-parquet = { path = \"../parquet\", version = \"#{@release_version}\" }",
- "-parquet_derive = { path = \"../parquet_derive\", version = \"#{@release_version}\" }",
- "+parquet = { path = \"../parquet\", version = \"#{@next_snapshot_version}\" }",
- "+parquet_derive = { path = \"../parquet_derive\", version = \"#{@next_snapshot_version}\" }"],
- ],
- },
- ],
- parse_patch(git("log", "-n", "1", "-p")))
- end
-
- def test_deb_package_names
- prepare("DEB_PACKAGE_NAMES")
- changes = parse_patch(git("log", "-n", "1", "-p"))
- sampled_changes = changes.collect do |change|
- first_hunk = change[:hunks][0]
- first_removed_line = first_hunk.find {|line| line.start_with?("-")}
- first_added_line = first_hunk.find {|line| line.start_with?("+")}
- {
- sampled_diff: [first_removed_line, first_added_line],
- path: change[:path],
- }
- end
- expected_changes = [
- {
- sampled_diff: [
- "-dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib#{@so_version}.install",
- "+dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib#{@next_so_version}.install",
- ],
- path: "dev/release/rat_exclude_files.txt"
- },
- {
- sampled_diff: [
- "-Package: libarrow#{@so_version}",
- "+Package: libarrow#{@next_so_version}",
- ],
- path: "dev/tasks/linux-packages/apache-arrow/debian/control.in"
- },
- {
- sampled_diff: [
- "- - libarrow-glib#{@so_version}-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb",
- "+ - libarrow-glib#{@next_so_version}-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb",
- ],
- path: "dev/tasks/tasks.yml",
- },
- ]
- assert_equal(expected_changes, sampled_changes)
- end
-end
diff --git a/dev/release/01-prepare.sh b/dev/release/01-prepare.sh
deleted file mode 100755
index 80703c2..0000000
--- a/dev/release/01-prepare.sh
+++ /dev/null
@@ -1,291 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-set -ue
-
-SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-if [ "$#" -ne 3 ]; then
- echo "Usage: $0 <version> <next_version> <rc-num>"
- exit 1
-fi
-
-update_versions() {
- local base_version=$1
- local next_version=$2
- local type=$3
-
- case ${type} in
- release)
- local version=${base_version}
- local r_version=${base_version}
- ;;
- snapshot)
- local version=${next_version}-SNAPSHOT
- local r_version=${base_version}.9000
- ;;
- esac
-
- cd "${SOURCE_DIR}/../../c_glib"
- sed -i.bak -E -e \
- "s/^version = '.+'/version = '${version}'/" \
- meson.build
- rm -f meson.build.bak
- git add meson.build
- cd -
-
- cd "${SOURCE_DIR}/../../ci/scripts"
- sed -i.bak -E -e \
- "s/^pkgver=.+/pkgver=${r_version}/" \
- PKGBUILD
- rm -f PKGBUILD.bak
- git add PKGBUILD
- cd -
-
- cd "${SOURCE_DIR}/../../cpp"
- sed -i.bak -E -e \
- "s/^set\(ARROW_VERSION \".+\"\)/set(ARROW_VERSION \"${version}\")/" \
- CMakeLists.txt
- rm -f CMakeLists.txt.bak
- git add CMakeLists.txt
-
- sed -i.bak -E -e \
- "s/\"version-string\": \".+\"/\"version-string\": \"${version}\"/" \
- vcpkg.json
- rm -f vcpkg.json.bak
- git add vcpkg.json
- cd -
-
- cd "${SOURCE_DIR}/../../csharp"
- sed -i.bak -E -e \
- "s/^ <Version>.+<\/Version>/ <Version>${version}<\/Version>/" \
- Directory.Build.props
- rm -f Directory.Build.props.bak
- git add Directory.Build.props
- cd -
-
- cd "${SOURCE_DIR}/../../dev/tasks/homebrew-formulae"
- sed -i.bak -E -e \
- "s/arrow-[0-9.]+[0-9]+/arrow-${r_version}/g" \
- autobrew/apache-arrow.rb
- rm -f autobrew/apache-arrow.rb.bak
- git add autobrew/apache-arrow.rb
- sed -i.bak -E -e \
- "s/arrow-[0-9.\-]+[0-9SNAPHOT]+/arrow-${version}/g" \
- apache-arrow.rb
- rm -f apache-arrow.rb.bak
- git add apache-arrow.rb
- cd -
-
- cd "${SOURCE_DIR}/../../js"
- sed -i.bak -E -e \
- "s/^ \"version\": \".+\"/ \"version\": \"${version}\"/" \
- package.json
- rm -f package.json.bak
- git add package.json
- cd -
-
- cd "${SOURCE_DIR}/../../matlab"
- sed -i.bak -E -e \
- "s/^set\(MLARROW_VERSION \".+\"\)/set(MLARROW_VERSION \"${version}\")/" \
- CMakeLists.txt
- rm -f CMakeLists.txt.bak
- git add CMakeLists.txt
- cd -
-
- cd "${SOURCE_DIR}/../../python"
- sed -i.bak -E -e \
- "s/^default_version = '.+'/default_version = '${version}'/" \
- setup.py
- rm -f setup.py.bak
- git add setup.py
- cd -
-
- cd "${SOURCE_DIR}/../../r"
- sed -i.bak -E -e \
- "s/^Version: .+/Version: ${r_version}/" \
- DESCRIPTION
- rm -f DESCRIPTION.bak
- git add DESCRIPTION
- if [ ${type} = "snapshot" ]; then
- # Add a news entry for the new dev version
- echo "dev"
- sed -i.bak -E -e \
- "0,/^# arrow /s/^(# arrow .+)/# arrow ${r_version}\n\n\1/" \
- NEWS.md
- else
- # Replace dev version with release version
- echo "release"
- sed -i.bak -E -e \
- "0,/^# arrow /s/^# arrow .+/# arrow ${r_version}/" \
- NEWS.md
- fi
- rm -f NEWS.md.bak
- git add NEWS.md
- cd -
-
- cd "${SOURCE_DIR}/../../ruby"
- sed -i.bak -E -e \
- "s/^ VERSION = \".+\"/ VERSION = \"${version}\"/g" \
- */*/*/version.rb
- rm -f */*/*/version.rb.bak
- git add */*/*/version.rb
- cd -
-
- cd "${SOURCE_DIR}/../../rust"
- sed -i.bak -E \
- -e "s/^version = \".+\"/version = \"${version}\"/g" \
- -e "s/^(arrow = .* version = )\".*\"(( .*)|(, features = .*)|(, optional = .*))$/\\1\"${version}\"\\2/g" \
- -e "s/^(arrow-flight = .* version = )\".+\"( .*)/\\1\"${version}\"\\2/g" \
- -e "s/^(parquet = .* version = )\".*\"(( .*)|(, features = .*))$/\\1\"${version}\"\\2/g" \
- -e "s/^(parquet_derive = .* version = )\".*\"(( .*)|(, features = .*))$/\\1\"${version}\"\\2/g" \
- */Cargo.toml
- rm -f */Cargo.toml.bak
- git add */Cargo.toml
-
- sed -i.bak -E \
- -e "s/^([^ ]+) = \".+\"/\\1 = \"${version}\"/g" \
- -e "s,docs\.rs/crate/([^/]+)/[^)]+,docs.rs/crate/\\1/${version},g" \
- */README.md
- rm -f */README.md.bak
- git add */README.md
- cd -
-}
-
-############################## Pre-Tag Commits ##############################
-
-version=$1
-next_version=$2
-next_version_snapshot="${next_version}-SNAPSHOT"
-rc_number=$3
-
-release_tag="apache-arrow-${version}"
-release_branch="release-${version}"
-release_candidate_branch="release-${version}-rc${rc_number}"
-
-: ${PREPARE_DEFAULT:=1}
-: ${PREPARE_CHANGELOG:=${PREPARE_DEFAULT}}
-: ${PREPARE_LINUX_PACKAGES:=${PREPARE_DEFAULT}}
-: ${PREPARE_VERSION_PRE_TAG:=${PREPARE_DEFAULT}}
-: ${PREPARE_BRANCH:=${PREPARE_DEFAULT}}
-: ${PREPARE_TAG:=${PREPARE_DEFAULT}}
-: ${PREPARE_VERSION_POST_TAG:=${PREPARE_DEFAULT}}
-: ${PREPARE_DEB_PACKAGE_NAMES:=${PREPARE_DEFAULT}}
-
-if [ ${PREPARE_TAG} -gt 0 ]; then
- if [ $(git tag -l "${release_tag}") ]; then
- echo "Delete existing git tag $release_tag"
- git tag -d "${release_tag}"
- fi
-fi
-
-if [ ${PREPARE_BRANCH} -gt 0 ]; then
- if [[ $(git branch -l "${release_candidate_branch}") ]]; then
- next_rc_number=$(($rc_number+1))
- echo "Branch ${release_candidate_branch} already exists, so create a new release candidate:"
- echo "1. Checkout the master branch for major releases and maint-<version> for patch releases."
- echo "2. Execute the script again with bumped RC number."
- echo "Commands:"
- echo " git checkout master"
- echo " dev/release/01-prepare.sh ${version} ${next_version} ${next_rc_number}"
- exit 1
- fi
-
- echo "Create local branch ${release_candidate_branch} for release candidate ${rc_number}"
- git checkout -b ${release_candidate_branch}
-fi
-
-############################## Pre-Tag Commits ##############################
-
-if [ ${PREPARE_CHANGELOG} -gt 0 ]; then
- echo "Updating changelog for $version"
- # Update changelog
- archery release changelog add $version
- git add ${SOURCE_DIR}/../../CHANGELOG.md
- git commit -m "[Release] Update CHANGELOG.md for $version"
-fi
-
-if [ ${PREPARE_LINUX_PACKAGES} -gt 0 ]; then
- echo "Updating .deb/.rpm changelogs for $version"
- cd $SOURCE_DIR/../tasks/linux-packages
- rake \
- version:update \
- ARROW_RELEASE_TIME="$(date +%Y-%m-%dT%H:%M:%S%z)" \
- ARROW_VERSION=${version}
- git add */debian*/changelog */yum/*.spec.in
- git commit -m "[Release] Update .deb/.rpm changelogs for $version"
- cd -
-fi
-
-if [ ${PREPARE_VERSION_PRE_TAG} -gt 0 ]; then
- echo "Prepare release ${version} on tag ${release_tag} then reset to version ${next_version_snapshot}"
-
- update_versions "${version}" "${next_version}" "release"
- git commit -m "[Release] Update versions for ${version}"
-fi
-
-############################## Tag the Release ##############################
-
-if [ ${PREPARE_TAG} -gt 0 ]; then
- git tag -a "${release_tag}" -m "[Release] Apache Arrow Release ${version}"
-fi
-
-############################## Post-Tag Commits #############################
-
-if [ ${PREPARE_VERSION_POST_TAG} -gt 0 ]; then
- echo "Updating versions for ${next_version_snapshot}"
- update_versions "${version}" "${next_version}" "snapshot"
- git commit -m "[Release] Update versions for ${next_version_snapshot}"
-fi
-
-if [ ${PREPARE_DEB_PACKAGE_NAMES} -gt 0 ]; then
- echo "Updating .deb package names for ${next_version}"
- so_version() {
- local version=$1
- local major_version=$(echo $version | sed -E -e 's/^([0-9]+)\.[0-9]+\.[0-9]+$/\1/')
- local minor_version=$(echo $version | sed -E -e 's/^[0-9]+\.([0-9]+)\.[0-9]+$/\1/')
- expr ${major_version} \* 100 + ${minor_version}
- }
- deb_lib_suffix=$(so_version $version)
- next_deb_lib_suffix=$(so_version $next_version)
- if [ "${deb_lib_suffix}" != "${next_deb_lib_suffix}" ]; then
- cd $SOURCE_DIR/../tasks/linux-packages/apache-arrow
- for target in debian*/lib*${deb_lib_suffix}.install; do
- git mv \
- ${target} \
- $(echo $target | sed -e "s/${deb_lib_suffix}/${next_deb_lib_suffix}/")
- done
- deb_lib_suffix_substitute_pattern="s/(lib(arrow|gandiva|parquet|plasma)[-a-z]*)${deb_lib_suffix}/\\1${next_deb_lib_suffix}/g"
- sed -i.bak -E -e "${deb_lib_suffix_substitute_pattern}" debian*/control*
- rm -f debian*/control*.bak
- git add debian*/control*
- cd -
- cd $SOURCE_DIR/../tasks/
- sed -i.bak -E -e "${deb_lib_suffix_substitute_pattern}" tasks.yml
- rm -f tasks.yml.bak
- git add tasks.yml
- cd -
- cd $SOURCE_DIR
- sed -i.bak -E -e "${deb_lib_suffix_substitute_pattern}" rat_exclude_files.txt
- rm -f rat_exclude_files.txt.bak
- git add rat_exclude_files.txt
- git commit -m "[Release] Update .deb package names for $next_version"
- cd -
- fi
-fi
diff --git a/dev/release/02-source-test.rb b/dev/release/02-source-test.rb
deleted file mode 100644
index 7d92881..0000000
--- a/dev/release/02-source-test.rb
+++ /dev/null
@@ -1,146 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-class SourceTest < Test::Unit::TestCase
- include GitRunnable
- include VersionDetectable
-
- def setup
- @current_commit = git_current_commit
- detect_versions
- @tag_name = "apache-arrow-#{@release_version}"
- @script = File.expand_path("dev/release/02-source.sh")
-
- Dir.mktmpdir do |dir|
- Dir.chdir(dir) do
- yield
- end
- end
- end
-
- def source(*targets)
- env = {
- "SOURCE_DEFAULT" => "0",
- "release_hash" => @current_commit,
- }
- targets.each do |target|
- env["SOURCE_#{target}"] = "1"
- end
- output = sh(env, @script, @release_version, "0")
- sh("tar", "xf", "#{@tag_name}.tar.gz")
- output
- end
-
- def test_symbolic_links
- source
- Dir.chdir(@tag_name) do
- assert_equal([],
- Find.find(".").find_all {|path| File.symlink?(path)})
- end
- end
-
- def test_csharp_git_commit_information
- source
- Dir.chdir("#{@tag_name}/csharp") do
- FileUtils.mv("dummy.git", "../.git")
- sh("dotnet", "pack", "-c", "Release")
- FileUtils.mv("../.git", "dummy.git")
- Dir.chdir("artifacts/Apache.Arrow/Release") do
- sh("unzip", "Apache.Arrow.#{@snapshot_version}.nupkg")
- FileUtils.chmod(0400, "Apache.Arrow.nuspec")
- nuspec = REXML::Document.new(File.read("Apache.Arrow.nuspec"))
- nuspec_repository = nuspec.elements["package/metadata/repository"]
- attributes = {}
- nuspec_repository.attributes.each do |key, value|
- attributes[key] = value
- end
- assert_equal({
- "type" => "git",
- "url" => "https://github.com/apache/arrow",
- "commit" => @current_commit,
- },
- attributes)
- end
- end
- end
-
- def test_python_version
- source
- Dir.chdir("#{@tag_name}/python") do
- sh("python3", "setup.py", "sdist")
- if on_release_branch?
- pyarrow_source_archive = "dist/pyarrow-#{@release_version}.tar.gz"
- else
- pyarrow_source_archive = "dist/pyarrow-#{@release_version}a0.tar.gz"
- end
- assert_equal([pyarrow_source_archive],
- Dir.glob("dist/pyarrow-*.tar.gz"))
- end
- end
-
- def test_vote
- jira_url = "https://issues.apache.org/jira"
- jql_conditions = [
- "project = ARROW",
- "status in (Resolved, Closed)",
- "fixVersion = #{@release_version}",
- ]
- jql = jql_conditions.join(" AND ")
- n_resolved_issues = nil
- search_url = URI("#{jira_url}/rest/api/2/search?jql=#{CGI.escape(jql)}")
- search_url.open do |response|
- n_resolved_issues = JSON.parse(response.read)["total"]
- end
- output = source("VOTE")
- assert_equal(<<-VOTE.strip, output[/^-+$(.+?)^-+$/m, 1].strip)
-To: dev@arrow.apache.org
-Subject: [VOTE] Release Apache Arrow #{@release_version} - RC0
-
-Hi,
-
-I would like to propose the following release candidate (RC0) of Apache
-Arrow version #{@release_version}. This is a release consisting of #{n_resolved_issues}
-resolved JIRA issues[1].
-
-This release candidate is based on commit:
-#{@current_commit} [2]
-
-The source release rc0 is hosted at [3].
-The binary artifacts are hosted at [4][5][6][7].
-The changelog is located at [8].
-
-Please download, verify checksums and signatures, run the unit tests,
-and vote on the release. See [9] for how to validate a release candidate.
-
-The vote will be open for at least 72 hours.
-
-[ ] +1 Release this as Apache Arrow #{@release_version}
-[ ] +0
-[ ] -1 Do not release this as Apache Arrow #{@release_version} because...
-
-[1]: https://issues.apache.org/jira/issues/?jql=project%20%3D%20ARROW%20AND%20status%20in%20%28Resolved%2C%20Closed%29%20AND%20fixVersion%20%3D%20#{@release_version}
-[2]: https://github.com/apache/arrow/tree/#{@current_commit}
-[3]: https://dist.apache.org/repos/dist/dev/arrow/apache-arrow-#{@release_version}-rc0
-[4]: https://bintray.com/apache/arrow/centos-rc/#{@release_version}-rc0
-[5]: https://bintray.com/apache/arrow/debian-rc/#{@release_version}-rc0
-[6]: https://bintray.com/apache/arrow/python-rc/#{@release_version}-rc0
-[7]: https://bintray.com/apache/arrow/ubuntu-rc/#{@release_version}-rc0
-[8]: https://github.com/apache/arrow/blob/#{@current_commit}/CHANGELOG.md
-[9]: https://cwiki.apache.org/confluence/display/ARROW/How+to+Verify+Release+Candidates
- VOTE
- end
-end
diff --git a/dev/release/02-source.sh b/dev/release/02-source.sh
deleted file mode 100755
index 89850e7..0000000
--- a/dev/release/02-source.sh
+++ /dev/null
@@ -1,162 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-set -e
-
-: ${SOURCE_DEFAULT:=1}
-: ${SOURCE_RAT:=${SOURCE_DEFAULT}}
-: ${SOURCE_UPLOAD:=${SOURCE_DEFAULT}}
-: ${SOURCE_VOTE:=${SOURCE_DEFAULT}}
-
-SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)"
-
-if [ "$#" -ne 2 ]; then
- echo "Usage: $0 <version> <rc-num>"
- exit
-fi
-
-version=$1
-rc=$2
-
-tag=apache-arrow-${version}
-tagrc=${tag}-rc${rc}
-rc_url="https://dist.apache.org/repos/dist/dev/arrow/${tagrc}"
-
-echo "Preparing source for tag ${tag}"
-
-: ${release_hash:=$(cd "${SOURCE_TOP_DIR}" && git rev-list --max-count=1 ${tag})}
-
-if [ ${SOURCE_UPLOAD} -gt 0 ]; then
- if [ -z "$release_hash" ]; then
- echo "Cannot continue: unknown git tag: $tag"
- exit
- fi
-fi
-
-echo "Using commit $release_hash"
-
-tarball=${tag}.tar.gz
-
-rm -rf ${tag}
-# be conservative and use the release hash, even though git produces the same
-# archive (identical hashes) using the scm tag
-(cd "${SOURCE_TOP_DIR}" && \
- git archive ${release_hash} --prefix ${tag}/) | \
- tar xf -
-
-# Resolve all hard and symbolic links
-rm -rf ${tag}.tmp
-mv ${tag} ${tag}.tmp
-cp -R -L ${tag}.tmp ${tag}
-rm -rf ${tag}.tmp
-
-# Create a dummy .git/ directory to download the source files from GitHub with Source Link in C#.
-dummy_git=${tag}/csharp/dummy.git
-mkdir ${dummy_git}
-pushd ${dummy_git}
-echo ${release_hash} > HEAD
-echo '[remote "origin"] url = https://github.com/apache/arrow.git' >> config
-mkdir objects refs
-popd
-
-# Create new tarball from modified source directory
-tar czf ${tarball} ${tag}
-rm -rf ${tag}
-
-if [ ${SOURCE_RAT} -gt 0 ]; then
- "${SOURCE_DIR}/run-rat.sh" ${tarball}
-fi
-
-if [ ${SOURCE_UPLOAD} -gt 0 ]; then
- # sign the archive
- gpg --armor --output ${tarball}.asc --detach-sig ${tarball}
- shasum -a 256 $tarball > ${tarball}.sha256
- shasum -a 512 $tarball > ${tarball}.sha512
-
- # check out the arrow RC folder
- svn co --depth=empty https://dist.apache.org/repos/dist/dev/arrow tmp
-
- # add the release candidate for the tag
- mkdir -p tmp/${tagrc}
-
- # copy the rc tarball into the tmp dir
- cp ${tarball}* tmp/${tagrc}
-
- # commit to svn
- svn add tmp/${tagrc}
- svn ci -m "Apache Arrow ${version} RC${rc}" tmp/${tagrc}
-
- # clean up
- rm -rf tmp
-
- echo "Success! The release candidate is available here:"
- echo " ${rc_url}"
- echo ""
- echo "Commit SHA1: ${release_hash}"
- echo ""
-fi
-
-if [ ${SOURCE_VOTE} -gt 0 ]; then
- echo "The following draft email has been created to send to the"
- echo "dev@arrow.apache.org mailing list"
- echo ""
- echo "---------------------------------------------------------"
- jira_url="https://issues.apache.org/jira"
- jql="project%20%3D%20ARROW%20AND%20status%20in%20%28Resolved%2C%20Closed%29%20AND%20fixVersion%20%3D%20${version}"
- n_resolved_issues=$(curl "${jira_url}/rest/api/2/search/?jql=${jql}" | jq ".total")
- cat <<MAIL
-To: dev@arrow.apache.org
-Subject: [VOTE] Release Apache Arrow ${version} - RC${rc}
-
-Hi,
-
-I would like to propose the following release candidate (RC${rc}) of Apache
-Arrow version ${version}. This is a release consisting of ${n_resolved_issues}
-resolved JIRA issues[1].
-
-This release candidate is based on commit:
-${release_hash} [2]
-
-The source release rc${rc} is hosted at [3].
-The binary artifacts are hosted at [4][5][6][7].
-The changelog is located at [8].
-
-Please download, verify checksums and signatures, run the unit tests,
-and vote on the release. See [9] for how to validate a release candidate.
-
-The vote will be open for at least 72 hours.
-
-[ ] +1 Release this as Apache Arrow ${version}
-[ ] +0
-[ ] -1 Do not release this as Apache Arrow ${version} because...
-
-[1]: ${jira_url}/issues/?jql=${jql}
-[2]: https://github.com/apache/arrow/tree/${release_hash}
-[3]: ${rc_url}
-[4]: https://bintray.com/apache/arrow/centos-rc/${version}-rc${rc}
-[5]: https://bintray.com/apache/arrow/debian-rc/${version}-rc${rc}
-[6]: https://bintray.com/apache/arrow/python-rc/${version}-rc${rc}
-[7]: https://bintray.com/apache/arrow/ubuntu-rc/${version}-rc${rc}
-[8]: https://github.com/apache/arrow/blob/${release_hash}/CHANGELOG.md
-[9]: https://cwiki.apache.org/confluence/display/ARROW/How+to+Verify+Release+Candidates
-MAIL
- echo "---------------------------------------------------------"
-fi
diff --git a/dev/release/03-binary-submit.sh b/dev/release/03-binary-submit.sh
deleted file mode 100755
index 1bdbc20..0000000
--- a/dev/release/03-binary-submit.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -e
-
-if [ "$#" -ne 2 ]; then
- echo "Usage: $0 <version> <rc-num>"
- exit
-fi
-
-version=$1
-rc_number=$2
-version_with_rc="${version}-rc${rc_number}"
-crossbow_job_prefix="release-${version_with_rc}"
-
-release_tag="apache-arrow-${version}"
-release_candidate_branch="release-${version}-rc${rc_number}"
-
-: ${GIT_REMOTE:="origin"}
-
-git checkout ${release_candidate_branch}
-git push -u ${GIT_REMOTE} ${release_candidate_branch}
-
-# archery will submit a job with id: "${crossbow_job_prefix}-0" unless there
-# are jobs submitted with the same prefix (the integer at the end is auto
-# incremented)
-archery crossbow submit \
- --job-prefix ${crossbow_job_prefix} \
- --arrow-version ${version_with_rc} \
- --group packaging
diff --git a/dev/release/04-binary-download.sh b/dev/release/04-binary-download.sh
deleted file mode 100755
index d0b61b0..0000000
--- a/dev/release/04-binary-download.sh
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-set -e
-
-if [ "$#" -ne 2 ]; then
- echo "Usage: $0 <version> <rc-num>"
- exit
-fi
-
-version=$1
-rc_number=$2
-version_with_rc="${version}-rc${rc_number}"
-crossbow_job_prefix="release-${version_with_rc}"
-
-# archery will submit a job with id: "${crossbow_job_prefix}-0" unless there
-# are jobs submitted with the same prefix (the integer at the end is auto
-# incremented)
-: ${CROSSBOW_JOB_ID:="${crossbow_job_prefix}-0"}
-
-archery crossbow download-artifacts ${CROSSBOW_JOB_ID}
diff --git a/dev/release/05-binary-upload.sh b/dev/release/05-binary-upload.sh
deleted file mode 100755
index 4a360c2..0000000
--- a/dev/release/05-binary-upload.sh
+++ /dev/null
@@ -1,137 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -e
-set -u
-set -o pipefail
-
-SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-if [ "$#" -ne 2 ]; then
- echo "Usage: $0 <version> <rc-num>"
- exit
-fi
-
-version=$1
-rc=$2
-
-version_with_rc="${version}-rc${rc}"
-crossbow_job_prefix="release-${version_with_rc}"
-crossbow_package_dir="${SOURCE_DIR}/../../packages"
-
-: ${CROSSBOW_JOB_ID:="${crossbow_job_prefix}-0"}
-artifact_dir="${crossbow_package_dir}/${CROSSBOW_JOB_ID}"
-
-if [ ! -e "$artifact_dir" ]; then
- echo "$artifact_dir does not exist"
- exit 1
-fi
-
-if [ ! -d "$artifact_dir" ]; then
- echo "$artifact_dir is not a directory"
- exit 1
-fi
-
-cd "${SOURCE_DIR}"
-
-: ${BINTRAY_REPOSITORY_CUSTOM:=${BINTRAY_REPOSITORY:-}}
-: ${SOURCE_BINTRAY_REPOSITORY_CUSTOM:=${SOURCE_BINTRAY_REPOSITORY:-}}}
-
-if [ ! -f .env ]; then
- echo "You must create $(pwd)/.env"
- echo "You can use $(pwd)/.env.example as template"
- exit 1
-fi
-. .env
-
-if [ -n "${BINTRAY_REPOSITORY_CUSTOM}" ]; then
- BINTRAY_REPOSITORY=${BINTRAY_REPOSITORY_CUSTOM}
-fi
-
-if [ -n "${SOURCE_BINTRAY_REPOSITORY_CUSTOM}" ]; then
- SOURCE_BINTRAY_REPOSITORY=${SOURCE_BINTRAY_REPOSITORY_CUSTOM}
-fi
-
-. binary-common.sh
-
-# By default upload all artifacts.
-# To deactivate one category, deactivate the category and all of its dependents.
-# To explicitly select one category, set UPLOAD_DEFAULT=0 UPLOAD_X=1.
-: ${UPLOAD_DEFAULT:=1}
-: ${UPLOAD_CENTOS_RPM:=${UPLOAD_DEFAULT}}
-: ${UPLOAD_CENTOS_YUM:=${UPLOAD_DEFAULT}}
-: ${UPLOAD_DEBIAN_APT:=${UPLOAD_DEFAULT}}
-: ${UPLOAD_DEBIAN_DEB:=${UPLOAD_DEFAULT}}
-: ${UPLOAD_NUGET:=${UPLOAD_DEFAULT}}
-: ${UPLOAD_PYTHON:=${UPLOAD_DEFAULT}}
-: ${UPLOAD_UBUNTU_APT:=${UPLOAD_DEFAULT}}
-: ${UPLOAD_UBUNTU_DEB:=${UPLOAD_DEFAULT}}
-
-rake_tasks=()
-apt_targets=()
-yum_targets=()
-if [ ${UPLOAD_DEBIAN_DEB} -gt 0 ]; then
- rake_tasks+=(deb)
- apt_targets+=(debian)
-fi
-if [ ${UPLOAD_DEBIAN_APT} -gt 0 ]; then
- rake_tasks+=(apt:rc)
- apt_targets+=(debian)
-fi
-if [ ${UPLOAD_UBUNTU_DEB} -gt 0 ]; then
- rake_tasks+=(deb)
- apt_targets+=(ubuntu)
-fi
-if [ ${UPLOAD_UBUNTU_APT} -gt 0 ]; then
- rake_tasks+=(apt:rc)
- apt_targets+=(ubuntu)
-fi
-if [ ${UPLOAD_CENTOS_RPM} -gt 0 ]; then
- rake_tasks+=(rpm)
- yum_targets+=(centos)
-fi
-if [ ${UPLOAD_CENTOS_YUM} -gt 0 ]; then
- rake_tasks+=(yum:rc)
- yum_targets+=(centos)
-fi
-if [ ${UPLOAD_NUGET} -gt 0 ]; then
- rake_tasks+=(nuget:rc)
-fi
-if [ ${UPLOAD_PYTHON} -gt 0 ]; then
- rake_tasks+=(python:rc)
-fi
-rake_tasks+=(summary:rc)
-
-tmp_dir=binary/tmp
-mkdir -p "${tmp_dir}"
-source_artifacts_dir="${tmp_dir}/artifacts"
-rm -rf "${source_artifacts_dir}"
-cp -a "${artifact_dir}" "${source_artifacts_dir}"
-
-docker_run \
- ./runner.sh \
- rake \
- "${rake_tasks[@]}" \
- APT_TARGETS=$(IFS=,; echo "${apt_targets[*]}") \
- ARTIFACTS_DIR="${tmp_dir}/artifacts" \
- BINTRAY_REPOSITORY=${BINTRAY_REPOSITORY} \
- RC=${rc} \
- SOURCE_BINTRAY_REPOSITORY=${SOURCE_BINTRAY_REPOSITORY} \
- VERSION=${version} \
- YUM_TARGETS=$(IFS=,; echo "${yum_targets[*]}")
diff --git a/dev/release/README.md b/dev/release/README.md
deleted file mode 100644
index 0a9cc3e..0000000
--- a/dev/release/README.md
+++ /dev/null
@@ -1,24 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-## Release management scripts
-
-To learn more, see the project wiki:
-
-https://cwiki.apache.org/confluence/display/ARROW/Release+Management+Guide
diff --git a/dev/release/Rakefile b/dev/release/Rakefile
deleted file mode 100644
index ff57bad..0000000
--- a/dev/release/Rakefile
+++ /dev/null
@@ -1,37 +0,0 @@
-# -*- ruby -*-
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-require_relative "binary-task"
-
-if File.exist?(".env")
- File.open(".env") do |env|
- env.each_line do |line|
- case line.strip
- when /\A#/
- when /\A([^=]+)=(.*)\z/
- key = $1
- value = $2
- ENV[key] ||= value
- end
- end
- end
-end
-
-binary_task = BinaryTask.new
-binary_task.define
diff --git a/dev/release/VERIFY.md b/dev/release/VERIFY.md
deleted file mode 100644
index 5b441ac..0000000
--- a/dev/release/VERIFY.md
+++ /dev/null
@@ -1,76 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-# Verifying Arrow releases
-
-## Windows
-
-We've provided a convenience script for verifying the C++ and Python builds on
-Windows. Read the comments in `verify-release-candidate.bat` for instructions.
-
-## Linux and macOS
-
-We've provided a convenience script for verifying the C++, Python, C
-GLib, Java and JavaScript builds on Linux and macOS. Read the comments in
-`verify-release-candidate.sh` for instructions.
-
-### C GLib
-
-You need the followings to verify C GLib build:
-
- * GLib
- * GObject Introspection
- * Ruby (not EOL-ed version is required)
- * gobject-introspection gem
- * test-unit gem
-
-You can install them by the followings on Debian GNU/Linux and Ubuntu:
-
-```console
-% sudo apt install -y -V libgirepository1.0-dev ruby-dev
-% sudo gem install gobject-introspection test-unit
-```
-
-You can install them by the followings on CentOS:
-
-```console
-% sudo yum install -y gobject-introspection-devel
-% git clone https://github.com/sstephenson/rbenv.git ~/.rbenv
-% git clone https://github.com/sstephenson/ruby-build.git ~/.rbenv/plugins/ruby-build
-% echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bash_profile
-% echo 'eval "$(rbenv init -)"' >> ~/.bash_profile
-% exec ${SHELL} --login
-% sudo yum install -y gcc make patch openssl-devel readline-devel zlib-devel
-% rbenv install 2.4.2
-% rbenv global 2.4.2
-% gem install gobject-introspection test-unit
-```
-
-You can install them by the followings on macOS:
-
-```console
-% brew install -y gobject-introspection
-% gem install gobject-introspection test-unit
-```
-
-You need to set `PKG_CONFIG_PATH` to find libffi on macOS:
-
-```console
-% export PKG_CONFIG_PATH=$(brew --prefix libffi)/lib/pkgconfig:$PKG_CONFIG_PATH
-```
diff --git a/dev/release/binary-common.sh b/dev/release/binary-common.sh
deleted file mode 100644
index 7c66e37..0000000
--- a/dev/release/binary-common.sh
+++ /dev/null
@@ -1,86 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-docker_image_name=apache-arrow/release-binary
-gpg_agent_extra_socket="$(gpgconf --list-dirs agent-extra-socket)"
-if [ $(uname) = "Darwin" ]; then
- docker_uid=10000
- docker_gid=10000
-else
- docker_uid=$(id -u)
- docker_gid=$(id -g)
-fi
-docker_ssh_key="${SOURCE_DIR}/binary/id_rsa"
-
-if [ ! -f "${docker_ssh_key}" ]; then
- ssh-keygen -N "" -f "${docker_ssh_key}"
-fi
-
-docker_gpg_ssh() {
- local ssh_port=$1
- shift
- local known_hosts_file=$(mktemp -t "arrow-binary-gpg-ssh-known-hosts.XXXXX")
- local exit_code=
- if ssh \
- -o StrictHostKeyChecking=no \
- -o UserKnownHostsFile=${known_hosts_file} \
- -i "${docker_ssh_key}" \
- -p ${ssh_port} \
- -R "/home/arrow/.gnupg/S.gpg-agent:${gpg_agent_extra_socket}" \
- arrow@127.0.0.1 \
- "$@"; then
- exit_code=$?;
- else
- exit_code=$?;
- fi
- rm -f ${known_hosts_file}
- return ${exit_code}
-}
-
-docker_run() {
- local container_id_dir=$(mktemp -d -t "arrow-binary-gpg-container.XXXXX")
- local container_id_file=${container_id_dir}/id
- docker \
- run \
- --cidfile ${container_id_file} \
- --detach \
- --publish-all \
- --rm \
- --volume "$PWD":/host \
- ${docker_image_name} \
- bash -c "
-if [ \$(id -u) -ne ${docker_uid} ]; then
- usermod --uid ${docker_uid} arrow
- chown -R arrow: ~arrow
-fi
-/usr/sbin/sshd -D
-"
- local container_id=$(cat ${container_id_file})
- local ssh_port=$(docker port ${container_id} | grep -E -o '[0-9]+$')
- # Wait for sshd available
- while ! docker_gpg_ssh ${ssh_port} : > /dev/null 2>&1; do
- sleep 0.1
- done
- gpg --export ${GPG_KEY_ID} | docker_gpg_ssh ${ssh_port} gpg --import
- docker_gpg_ssh ${ssh_port} "$@"
- docker kill ${container_id}
- rm -rf ${container_id_dir}
-}
-
-docker build -t ${docker_image_name} "${SOURCE_DIR}/binary"
-
-chmod go-rwx "${docker_ssh_key}"
diff --git a/dev/release/binary-task.rb b/dev/release/binary-task.rb
deleted file mode 100644
index 42bc1fe..0000000
--- a/dev/release/binary-task.rb
+++ /dev/null
@@ -1,1909 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-require "digest/sha2"
-require "io/console"
-require "json"
-require "net/http"
-require "pathname"
-require "tempfile"
-require "thread"
-require "time"
-
-class BinaryTask
- include Rake::DSL
-
- class ThreadPool
- def initialize(use_case, &worker)
- @n_workers = choose_n_workers(use_case)
- @worker = worker
- @jobs = Thread::Queue.new
- @workers = @n_workers.times.collect do
- Thread.new do
- loop do
- job = @jobs.pop
- break if job.nil?
- @worker.call(job)
- end
- end
- end
- end
-
- def <<(job)
- @jobs << job
- end
-
- def join
- @n_workers.times do
- @jobs << nil
- end
- @workers.each(&:join)
- end
-
- private
- def choose_n_workers(use_case)
- case use_case
- when :bintray
- # Too many workers cause Bintray error.
- 6
- when :gpg
- # Too many workers cause gpg-agent error.
- 2
- else
- raise "Unknown use case: #{use_case}"
- end
- end
- end
-
- class ProgressReporter
- def initialize(label, count_max=0)
- @label = label
- @count_max = count_max
-
- @mutex = Thread::Mutex.new
-
- @time_start = Time.now
- @time_previous = Time.now
- @count_current = 0
- @count_previous = 0
- end
-
- def advance
- @mutex.synchronize do
- @count_current += 1
-
- return if @count_max.zero?
-
- time_current = Time.now
- if time_current - @time_previous <= 1
- return
- end
-
- show_progress(time_current)
- end
- end
-
- def increment_max
- @mutex.synchronize do
- @count_max += 1
- end
- end
-
- def finish
- @mutex.synchronize do
- return if @count_max.zero?
- show_progress(Time.now)
- $stderr.puts
- end
- end
-
- private
- def show_progress(time_current)
- n_finishes = @count_current - @count_previous
- throughput = n_finishes.to_f / (time_current - @time_previous)
- @time_previous = time_current
- @count_previous = @count_current
-
- message = build_message(time_current, throughput)
- $stderr.print("\r#{message}") if message
- end
-
- def build_message(time_current, throughput)
- percent = (@count_current / @count_max.to_f) * 100
- formatted_count = "[%s/%s]" % [
- format_count(@count_current),
- format_count(@count_max),
- ]
- elapsed_second = time_current - @time_start
- if throughput.zero?
- rest_second = 0
- else
- rest_second = (@count_max - @count_current) / throughput
- end
- separator = " - "
- progress = "%5.1f%% %s %s %s %s" % [
- percent,
- formatted_count,
- format_time_interval(elapsed_second),
- format_time_interval(rest_second),
- format_throughput(throughput),
- ]
- label = @label
-
- width = guess_terminal_width
- return "#{label}#{separator}#{progress}" if width.nil?
-
- return nil if progress.size > width
-
- label_width = width - progress.size - separator.size
- if label.size > label_width
- ellipsis = "..."
- shorten_label_width = label_width - ellipsis.size
- if shorten_label_width < 1
- return progress
- else
- label = label[0, shorten_label_width] + ellipsis
- end
- end
- "#{label}#{separator}#{progress}"
- end
-
- def format_count(count)
- "%d" % count
- end
-
- def format_time_interval(interval)
- if interval < 60
- "00:00:%02d" % interval
- elsif interval < (60 * 60)
- minute, second = interval.divmod(60)
- "00:%02d:%02d" % [minute, second]
- elsif interval < (60 * 60 * 24)
- minute, second = interval.divmod(60)
- hour, minute = minute.divmod(60)
- "%02d:%02d:%02d" % [hour, minute, second]
- else
- minute, second = interval.divmod(60)
- hour, minute = minute.divmod(60)
- day, hour = hour.divmod(24)
- "%dd %02d:%02d:%02d" % [day, hour, minute, second]
- end
- end
-
- def format_throughput(throughput)
- "%2d/s" % throughput
- end
-
- def guess_terminal_width
- guess_terminal_width_from_io ||
- guess_terminal_width_from_command ||
- guess_terminal_width_from_env ||
- 80
- end
-
- def guess_terminal_width_from_io
- if IO.respond_to?(:console) and IO.console
- IO.console.winsize[1]
- elsif $stderr.respond_to?(:winsize)
- begin
- $stderr.winsize[1]
- rescue SystemCallError
- nil
- end
- else
- nil
- end
- end
-
- def guess_terminal_width_from_command
- IO.pipe do |input, output|
- begin
- pid = spawn("tput", "cols", {:out => output, :err => output})
- rescue SystemCallError
- return nil
- end
-
- output.close
- _, status = Process.waitpid2(pid)
- return nil unless status.success?
-
- result = input.read.chomp
- begin
- Integer(result, 10)
- rescue ArgumentError
- nil
- end
- end
- end
-
- def guess_terminal_width_from_env
- env = ENV["COLUMNS"] || ENV["TERM_WIDTH"]
- return nil if env.nil?
-
- begin
- Integer(env, 10)
- rescue ArgumentError
- nil
- end
- end
- end
-
- class BintrayClient
- class Error < StandardError
- attr_reader :request
- attr_reader :response
- def initialize(request, response, message)
- @request = request
- @response = response
- super(message)
- end
- end
-
- def initialize(options={})
- @options = options
- repository = @options[:repository]
- @subject, @repository = repository.split("/", 2) if repository
- @package = @options[:package]
- @version = @options[:version]
- @user = @options[:user]
- @api_key = @options[:api_key]
- end
-
- def request(method, headers, *components, &block)
- url = build_request_url(*components)
- http = Net::HTTP.new(url.host, url.port)
- http.set_debug_output($stderr) if ENV["DEBUG"]
- http.use_ssl = true
- http.start do |http|
- request = build_request(method, url, headers, &block)
- http.request(request) do |response|
- case response
- when Net::HTTPSuccess
- return JSON.parse(response.body)
- else
- message = "failed to request: "
- message << "#{url}: #{request.method}: "
- message << "#{response.message} #{response.code}:\n"
- message << response.body
- raise Error.new(request, response, message)
- end
- end
- end
- end
-
- def repository
- request(:get,
- {},
- "repos",
- @subject,
- @repository)
- end
-
- def create_repository
- request(:post,
- {},
- "repos",
- @subject,
- @repository) do
- request = {
- "name" => @repository,
- "desc" => "Apache Arrow",
- }
- JSON.generate(request)
- end
- end
-
- def ensure_repository
- begin
- repository
- rescue Error => error
- case error.response
- when Net::HTTPNotFound
- create_repository
- else
- raise
- end
- end
- end
-
- def package
- request(:get,
- {},
- "packages",
- @subject,
- @repository,
- @package)
- end
-
- def package_versions
- begin
- package["versions"]
- rescue Error => error
- case error.response
- when Net::HTTPNotFound
- []
- else
- raise
- end
- end
- end
-
- def create_package(description)
- request(:post,
- {},
- "packages",
- @subject,
- @repository) do
- request = {
- "name" => @package,
- "desc" => description,
- "licenses" => ["Apache-2.0"],
- "vcs_url" => "https://github.com/apache/arrow.git",
- "website_url" => "https://arrow.apache.org/",
- "issue_tracker_url" => "https://issues.apache.org/jira/browse/ARROW",
- "github_repo" => "apache/arrow",
- "public_download_numbers" => true,
- "public_stats" => true,
- }
- JSON.generate(request)
- end
- end
-
- def ensure_package(description)
- begin
- package
- rescue Error => error
- case error.response
- when Net::HTTPNotFound
- create_package(description)
- else
- raise
- end
- end
- end
-
- def create_version(description)
- request(:post,
- {},
- "packages",
- @subject,
- @repository,
- @package,
- "versions") do
- request = {
- "name" => @version,
- "desc" => description,
- }
- JSON.generate(request)
- end
- end
-
- def ensure_version(version, description)
- return if package["versions"].include?(version)
- create_version(description)
- end
-
- def files
- request(:get,
- {},
- "packages",
- @subject,
- @repository,
- @package,
- "versions",
- @version,
- "files")
- end
-
- def upload(path, destination_path)
- sha256 = Digest::SHA256.file(path).hexdigest
- headers = {
- "X-Bintray-Override" => "1",
- "X-Bintray-Package" => @package,
- "X-Bintray-Publish" => "1",
- "X-Bintray-Version" => @version,
- "X-Checksum-Sha2" => sha256,
- "Content-Length" => File.size(path).to_s,
- }
- File.open(path, "rb") do |input|
- request(:put,
- headers,
- "content",
- @subject,
- @repository,
- destination_path) do
- input
- end
- end
- end
-
- def delete(path)
- request(:delete,
- {},
- "content",
- @subject,
- @repository,
- path)
- end
-
- private
- def build_request_url(*components)
- if components.last.is_a?(Hash)
- parameters = components.pop
- else
- parameters = nil
- end
- path = components.join("/")
- url = "https://bintray.com/api/v1/#{path}"
- if parameters
- separator = "?"
- parameters.each do |key, value|
- url << "#{separator}#{CGI.escape(key)}=#{CGI.escape(value)}"
- separator = "&"
- end
- end
- URI(url)
- end
-
- def build_request(method, url, headers, &block)
- case method
- when :get
- request = Net::HTTP::Get.new(url, headers)
- when :post
- request = Net::HTTP::Post.new(url, headers)
- when :put
- request = Net::HTTP::Put.new(url, headers)
- when :delete
- request = Net::HTTP::Delete.new(url, headers)
- else
- raise "unsupported HTTP method: #{method.inspect}"
- end
- request.basic_auth(@user, @api_key) if @user and @api_key
- if block_given?
- request["Content-Type"] = "application/json"
- body = yield
- if body.is_a?(String)
- request.body = body
- else
- request.body_stream = body
- end
- end
- request
- end
- end
-
- module HashChekable
- def same_hash?(path, sha256)
- return false unless File.exist?(path)
- Digest::SHA256.file(path).hexdigest == sha256
- end
- end
-
- class BintrayDownloader
- include HashChekable
-
- def initialize(repository:,
- distribution:,
- version:,
- rc: nil,
- destination:,
- user:,
- api_key:)
- @repository = repository
- @distribution = distribution
- @version = version
- @rc = rc
- @destination = destination
- @user = user
- @api_key = api_key
- end
-
- def download
- client.ensure_repository
-
- progress_label = "Downloading: #{package} #{full_version}"
- progress_reporter = ProgressReporter.new(progress_label)
- pool = ThreadPool.new(:bintray) do |path, output_path|
- download_file(path, output_path)
- progress_reporter.advance
- end
- target_files.each do |file|
- path = file["path"]
- path_without_package = path.split("/", 2)[1..-1].join("/")
- output_path = "#{@destination}/#{path_without_package}"
- yield(output_path)
- sha256 = file["sha256"]
- next if same_hash?(output_path, sha256)
- output_dir = File.dirname(output_path)
- FileUtils.mkdir_p(output_dir)
- progress_reporter.increment_max
- pool << [path, output_path]
- end
- pool.join
- progress_reporter.finish
- end
-
- private
- def package
- if @rc
- "#{@distribution}-rc"
- else
- @distribution
- end
- end
-
- def full_version
- if @rc
- "#{@version}-rc#{@rc}"
- else
- @version
- end
- end
-
- def client(options={})
- default_options = {
- repository: @repository,
- package: package,
- version: full_version,
- user: @user,
- api_key: @api_key,
- }
- BintrayClient.new(default_options.merge(options))
- end
-
- def target_files
- begin
- client.files
- rescue BintrayClient::Error
- []
- end
- end
-
- def download_file(path, output_path)
- max_n_retries = 5
- n_retries = 0
- url = URI("https://dl.bintray.com/#{@repository}/#{path}")
- begin
- download_url(url, output_path)
- rescue OpenSSL::OpenSSLError,
- SocketError,
- SystemCallError,
- Timeout::Error => error
- n_retries += 1
- if n_retries <= max_n_retries
- $stderr.puts
- $stderr.puts("Retry #{n_retries}: #{url}: " +
- "#{error.class}: #{error.message}")
- retry
- else
- raise
- end
- end
- end
-
- def download_url(url, output_path)
- loop do
- http = Net::HTTP.new(url.host, url.port)
- http.set_debug_output($stderr) if ENV["DEBUG"]
- http.use_ssl = true
- http.start do |http|
- request = Net::HTTP::Get.new(url)
- http.request(request) do |response|
- case response
- when Net::HTTPSuccess
- save_response(response, output_path)
- return
- when Net::HTTPRedirection
- url = URI(response["Location"])
- when Net::HTTPNotFound
- $stderr.puts(build_download_error_message(url, response))
- return
- else
- raise build_download_error_message(url, response)
- end
- end
- end
- end
- end
-
- def save_response(response, output_path)
- File.open(output_path, "wb") do |output|
- response.read_body do |chunk|
- output.print(chunk)
- end
- end
- last_modified = response["Last-Modified"]
- if last_modified
- FileUtils.touch(output_path, mtime: Time.rfc2822(last_modified))
- end
- end
-
- def build_download_error_message(url, response)
- message = "failed to download: "
- message << "#{url}: #{response.message} #{response.code}:\n"
- message << response.body
- message
- end
- end
-
- class BintrayUploader
- include HashChekable
-
- def initialize(repository:,
- distribution:,
- distribution_label:,
- version:,
- rc: nil,
- source:,
- destination_prefix: "",
- user:,
- api_key:)
- @repository = repository
- @distribution = distribution
- @distribution_label = distribution_label
- @version = version
- @rc = rc
- @source = source
- @destination_prefix = destination_prefix
- @user = user
- @api_key = api_key
- end
-
- def upload
- client.ensure_repository
- client.ensure_package(package_description)
- client.ensure_version(full_version, version_description)
-
- progress_label = "Uploading: #{package} #{full_version}"
- progress_reporter = ProgressReporter.new(progress_label)
- pool = ThreadPool.new(:bintray) do |path, relative_path|
- upload_file(path, relative_path)
- progress_reporter.advance
- end
-
- files = existing_files
- source = Pathname(@source)
- source.glob("**/*") do |path|
- next if path.directory?
- destination_path =
- "#{package}/#{@destination_prefix}#{path.relative_path_from(source)}"
- file = files[destination_path]
- next if file and same_hash?(path.to_s, file["sha256"])
- progress_reporter.increment_max
- pool << [path, destination_path]
- end
- pool.join
- progress_reporter.finish
- end
-
- private
- def package
- if @rc
- "#{@distribution}-rc"
- else
- @distribution
- end
- end
-
- def full_version
- if @rc
- "#{@version}-rc#{@rc}"
- else
- @version
- end
- end
-
- def package_description
- if @rc
- release_type = "RC"
- else
- release_type = "Release"
- end
- case @distribution
- when "debian", "ubuntu"
- "#{release_type} deb packages for #{@distribution_label}"
- when "centos"
- "#{release_type} RPM packages for #{@distribution_label}"
- else
- "#{release_type} binaries for #{@distribution_label}"
- end
- end
-
- def version_description
- if @rc
- "Apache Arrow #{@version} RC#{@rc} for #{@distribution_label}"
- else
- "Apache Arrow #{@version} for #{@distribution_label}"
- end
- end
-
- def client
- BintrayClient.new(repository: @repository,
- package: package,
- version: full_version,
- user: @user,
- api_key: @api_key)
- end
-
- def existing_files
- files = {}
- client.files.each do |file|
- files[file["path"]] = file
- end
- files
- end
-
- def upload_file(path, destination_path)
- max_n_retries = 3
- n_retries = 0
- begin
- begin
- client.upload(path, destination_path)
- rescue BintrayClient::Error => error
- case error.response
- when Net::HTTPConflict
- n_retries += 1
- if n_retries <= max_n_retries
- client.delete(destination_path)
- retry
- else
- $stderr.puts(error)
- end
- else
- $stderr.puts(error)
- end
- end
- rescue OpenSSL::OpenSSLError,
- SocketError,
- SystemCallError,
- Timeout::Error => error
- n_retries += 1
- if n_retries <= max_n_retries
- $stderr.puts
- $stderr.puts("Retry #{n_retries}: #{path}: " +
- "#{error.class}: #{error.message}")
- retry
- else
- raise
- end
- end
- end
- end
-
- def define
- define_apt_tasks
- define_yum_tasks
- define_python_tasks
- define_nuget_tasks
- define_summary_tasks
- end
-
- private
- def env_value(name)
- value = ENV[name]
- value = yield(name) if value.nil? and block_given?
- raise "Specify #{name} environment variable" if value.nil?
- value
- end
-
- def verbose?
- ENV["VERBOSE"] == "yes"
- end
-
- def default_output
- if verbose?
- nil
- else
- IO::NULL
- end
- end
-
- def gpg_key_id
- env_value("GPG_KEY_ID")
- end
-
- def shorten_gpg_key_id(id)
- id[-8..-1]
- end
-
- def rpm_gpg_key_package_name(id)
- "gpg-pubkey-#{shorten_gpg_key_id(id).downcase}"
- end
-
- def bintray_user
- env_value("BINTRAY_USER")
- end
-
- def bintray_api_key
- env_value("BINTRAY_API_KEY")
- end
-
- def bintray_repository
- env_value("BINTRAY_REPOSITORY")
- end
-
- def source_bintray_repository
- env_value("SOURCE_BINTRAY_REPOSITORY") do
- bintray_repository
- end
- end
-
- def artifacts_dir
- env_value("ARTIFACTS_DIR")
- end
-
- def version
- env_value("VERSION")
- end
-
- def rc
- env_value("RC")
- end
-
- def full_version
- "#{version}-rc#{rc}"
- end
-
- def valid_sign?(path, sign_path)
- IO.pipe do |input, output|
- begin
- sh({"LANG" => "C"},
- "gpg",
- "--verify",
- sign_path,
- path,
- out: default_output,
- err: output,
- verbose: false)
- rescue
- return false
- end
- output.close
- /Good signature/ === input.read
- end
- end
-
- def sign(source_path, destination_path)
- if File.exist?(destination_path)
- return if valid_sign?(source_path, destination_path)
- rm(destination_path, verbose: false)
- end
- sh("gpg",
- "--detach-sig",
- "--local-user", gpg_key_id,
- "--output", destination_path,
- source_path,
- out: default_output,
- verbose: verbose?)
- end
-
- def sha512(source_path, destination_path)
- if File.exist?(destination_path)
- sha512 = File.read(destination_path).split[0]
- return if Digest::SHA512.file(source_path).hexdigest == sha512
- end
- absolute_destination_path = File.expand_path(destination_path)
- Dir.chdir(File.dirname(source_path)) do
- sh("shasum",
- "--algorithm", "512",
- File.basename(source_path),
- out: absolute_destination_path,
- verbose: verbose?)
- end
- end
-
- def sign_dir(label, dir)
- progress_label = "Signing: #{label}"
- progress_reporter = ProgressReporter.new(progress_label)
-
- target_paths = []
- Pathname(dir).glob("**/*") do |path|
- next if path.directory?
- case path.extname
- when ".asc", ".sha512"
- next
- end
- progress_reporter.increment_max
- target_paths << path.to_s
- end
- target_paths.each do |path|
- sign(path, "#{path}.asc")
- sha512(path, "#{path}.sha512")
- progress_reporter.advance
- end
- progress_reporter.finish
- end
-
- def download_distribution(distribution,
- destination,
- with_source_repository: false)
- existing_paths = {}
- Pathname(destination).glob("**/*") do |path|
- next if path.directory?
- existing_paths[path.to_s] = true
- end
- if with_source_repository
- source_client = BintrayClient.new(repository: source_bintray_repository,
- package: distribution,
- user: bintray_user,
- api_key: bintray_api_key)
- source_client.package_versions[0, 10].each do |source_version|
- downloader = BintrayDownloader.new(repository: source_bintray_repository,
- distribution: distribution,
- version: source_version,
- destination: destination,
- user: bintray_user,
- api_key: bintray_api_key)
- downloader.download do |output_path|
- existing_paths.delete(output_path)
- end
- end
- end
- downloader = BintrayDownloader.new(repository: bintray_repository,
- distribution: distribution,
- version: version,
- rc: rc,
- destination: destination,
- user: bintray_user,
- api_key: bintray_api_key)
- downloader.download do |output_path|
- existing_paths.delete(output_path)
- end
- existing_paths.each_key do |path|
- rm_f(path, verbose: verbose?)
- end
- end
-
- def same_content?(path1, path2)
- File.exist?(path1) and
- File.exist?(path2) and
- Digest::SHA256.file(path1) == Digest::SHA256.file(path2)
- end
-
- def copy_artifact(source_path,
- destination_path,
- progress_reporter)
- return if same_content?(source_path, destination_path)
- progress_reporter.increment_max
- destination_dir = File.dirname(destination_path)
- unless File.exist?(destination_dir)
- mkdir_p(destination_dir, verbose: verbose?)
- end
- cp(source_path, destination_path, verbose: verbose?)
- progress_reporter.advance
- end
-
- def tmp_dir
- "binary/tmp"
- end
-
- def rc_dir
- "#{tmp_dir}/rc"
- end
-
- def release_dir
- "#{tmp_dir}/release"
- end
-
- def deb_dir
- "#{rc_dir}/deb/#{full_version}"
- end
-
- def apt_repository_label
- "Apache Arrow"
- end
-
- def apt_repository_description
- "Apache Arrow packages"
- end
-
- def apt_rc_repositories_dir
- "#{rc_dir}/apt/repositories"
- end
-
- def apt_release_repositories_dir
- "#{release_dir}/apt/repositories"
- end
-
- def available_apt_targets
- [
- ["debian", "buster", "main"],
- ["debian", "bullseye", "main"],
- ["ubuntu", "bionic", "main"],
- ["ubuntu", "focal", "main"],
- ["ubuntu", "groovy", "main"],
- ]
- end
-
- def apt_distribution_label(distribution)
- case distribution
- when "debian"
- "Debian"
- when "ubuntu"
- "Ubuntu"
- else
- distribution
- end
- end
-
- def apt_targets
- env_apt_targets = (ENV["APT_TARGETS"] || "").split(",")
- if env_apt_targets.empty?
- available_apt_targets
- else
- available_apt_targets.select do |distribution, code_name, component|
- env_apt_targets.any? do |env_apt_target|
- env_apt_target.start_with?("#{distribution}-#{code_name}")
- end
- end
- end
- end
-
- def apt_distributions
- apt_targets.collect(&:first).uniq
- end
-
- def apt_architectures
- [
- "amd64",
- "arm64",
- ]
- end
-
- def define_deb_tasks
- directory deb_dir
-
- namespace :deb do
- desc "Copy deb packages"
- task :copy => deb_dir do
- apt_targets.each do |distribution, code_name, component|
- progress_label = "Copying: #{distribution} #{code_name}"
- progress_reporter = ProgressReporter.new(progress_label)
-
- source_dir_prefix = "#{artifacts_dir}/#{distribution}-#{code_name}"
- Dir.glob("#{source_dir_prefix}*/**/*") do |path|
- next if File.directory?(path)
- base_name = File.basename(path)
- if base_name.start_with?("apache-arrow-archive-keyring")
- package_name = "apache-arrow-archive-keyring"
- else
- package_name = "apache-arrow"
- end
- distribution_dir = [
- deb_dir,
- distribution,
- ].join("/")
- destination_path = [
- distribution_dir,
- "pool",
- code_name,
- component,
- package_name[0],
- package_name,
- base_name,
- ].join("/")
- copy_artifact(path,
- destination_path,
- progress_reporter)
- case base_name
- when /\A[^_]+-archive-keyring_.*\.deb\z/
- latest_archive_keyring_package_path = [
- distribution_dir,
- "#{package_name}-latest-#{code_name}.deb"
- ].join("/")
- copy_artifact(path,
- latest_archive_keyring_package_path,
- progress_reporter)
- end
- end
- progress_reporter.finish
- end
- end
-
- desc "Sign deb packages"
- task :sign => deb_dir do
- apt_distributions.each do |distribution|
- distribution_dir = "#{deb_dir}/#{distribution}"
- Dir.glob("#{distribution_dir}/**/*.dsc") do |path|
- begin
- sh({"LANG" => "C"},
- "gpg",
- "--verify",
- path,
- out: IO::NULL,
- err: IO::NULL,
- verbose: false)
- rescue
- sh("debsign",
- "--no-re-sign",
- "-k#{gpg_key_id}",
- path,
- out: default_output,
- verbose: verbose?)
- end
- end
- sign_dir(distribution, distribution_dir)
- end
- end
-
- desc "Upload deb packages"
- task :upload do
- apt_distributions.each do |distribution|
- distribution_dir = "#{deb_dir}/#{distribution}"
- distribution_label = apt_distribution_label(distribution)
- uploader = BintrayUploader.new(repository: bintray_repository,
- distribution: distribution,
- distribution_label: distribution_label,
- version: version,
- rc: rc,
- source: distribution_dir,
- user: bintray_user,
- api_key: bintray_api_key)
- uploader.upload
- end
- end
- end
-
- desc "Release deb packages"
- deb_tasks = [
- "deb:copy",
- "deb:sign",
- "deb:upload",
- ]
- task :deb => deb_tasks
- end
-
- def generate_apt_release(dists_dir, code_name, component, architecture)
- dir = "#{dists_dir}/#{component}/"
- if architecture == "source"
- dir << architecture
- else
- dir << "binary-#{architecture}"
- end
-
- mkdir_p(dir, verbose: verbose?)
- File.open("#{dir}/Release", "w") do |release|
- release.puts(<<-RELEASE)
-Archive: #{code_name}
-Component: #{component}
-Origin: #{apt_repository_label}
-Label: #{apt_repository_label}
-Architecture: #{architecture}
- RELEASE
- end
- end
-
- def generate_apt_ftp_archive_generate_conf(code_name, component)
- conf = <<-CONF
-Dir::ArchiveDir ".";
-Dir::CacheDir ".";
-TreeDefault::Directory "pool/#{code_name}/#{component}";
-TreeDefault::SrcDirectory "pool/#{code_name}/#{component}";
-Default::Packages::Extensions ".deb";
-Default::Packages::Compress ". gzip xz";
-Default::Sources::Compress ". gzip xz";
-Default::Contents::Compress "gzip";
- CONF
-
- apt_architectures.each do |architecture|
- conf << <<-CONF
-
-BinDirectory "dists/#{code_name}/#{component}/binary-#{architecture}" {
- Packages "dists/#{code_name}/#{component}/binary-#{architecture}/Packages";
- Contents "dists/#{code_name}/#{component}/Contents-#{architecture}";
- SrcPackages "dists/#{code_name}/#{component}/source/Sources";
-};
- CONF
- end
-
- conf << <<-CONF
-
-Tree "dists/#{code_name}" {
- Sections "#{component}";
- Architectures "#{apt_architectures.join(" ")} source";
-};
- CONF
-
- conf
- end
-
- def generate_apt_ftp_archive_release_conf(code_name, component)
- <<-CONF
-APT::FTPArchive::Release::Origin "#{apt_repository_label}";
-APT::FTPArchive::Release::Label "#{apt_repository_label}";
-APT::FTPArchive::Release::Architectures "#{apt_architectures.join(" ")}";
-APT::FTPArchive::Release::Codename "#{code_name}";
-APT::FTPArchive::Release::Suite "#{code_name}";
-APT::FTPArchive::Release::Components "#{component}";
-APT::FTPArchive::Release::Description "#{apt_repository_description}";
- CONF
- end
-
- def apt_update(repositories_dir)
- apt_targets.each do |distribution, code_name, component|
- base_dir = "#{repositories_dir}/#{distribution}"
- pool_dir = "#{base_dir}/pool/#{code_name}"
- next unless File.exist?(pool_dir)
- dists_dir = "#{base_dir}/dists/#{code_name}"
- rm_rf(dists_dir, verbose: verbose?)
- generate_apt_release(dists_dir, code_name, component, "source")
- apt_architectures.each do |architecture|
- generate_apt_release(dists_dir, code_name, component, architecture)
- end
-
- generate_conf_file = Tempfile.new("apt-ftparchive-generate.conf")
- File.open(generate_conf_file.path, "w") do |conf|
- conf.puts(generate_apt_ftp_archive_generate_conf(code_name,
- component))
- end
- cd(base_dir, verbose: verbose?) do
- sh("apt-ftparchive",
- "generate",
- generate_conf_file.path,
- out: default_output,
- verbose: verbose?)
- end
-
- Dir.glob("#{dists_dir}/Release*") do |release|
- rm_f(release, verbose: verbose?)
- end
- Dir.glob("#{base_dir}/*.db") do |db|
- rm_f(db, verbose: verbose?)
- end
- release_conf_file = Tempfile.new("apt-ftparchive-release.conf")
- File.open(release_conf_file.path, "w") do |conf|
- conf.puts(generate_apt_ftp_archive_release_conf(code_name,
- component))
- end
- release_file = Tempfile.new("apt-ftparchive-release")
- sh("apt-ftparchive",
- "-c", release_conf_file.path,
- "release",
- dists_dir,
- out: release_file.path,
- verbose: verbose?)
- release_path = "#{dists_dir}/Release"
- signed_release_path = "#{release_path}.gpg"
- in_release_path = "#{dists_dir}/InRelease"
- mv(release_file.path, release_path, verbose: verbose?)
- chmod(0644, release_path, verbose: verbose?)
- sh("gpg",
- "--sign",
- "--detach-sign",
- "--armor",
- "--local-user", gpg_key_id,
- "--output", signed_release_path,
- release_path,
- out: default_output,
- verbose: verbose?)
- sh("gpg",
- "--clear-sign",
- "--local-user", gpg_key_id,
- "--output", in_release_path,
- release_path,
- out: default_output,
- verbose: verbose?)
- end
- end
-
- def define_apt_rc_tasks
- directory apt_rc_repositories_dir
-
- namespace :apt do
- namespace :rc do
- desc "Download deb files for RC APT repositories"
- task :download => apt_rc_repositories_dir do
- apt_distributions.each do |distribution|
- download_distribution(distribution,
- "#{apt_rc_repositories_dir}/#{distribution}",
- with_source_repository: true)
- end
- end
-
- desc "Update RC APT repositories"
- task :update do
- apt_update(apt_rc_repositories_dir)
- apt_targets.each do |distribution, code_name, component|
- base_dir = "#{apt_rc_repositories_dir}/#{distribution}"
- dists_dir = "#{base_dir}/dists/#{code_name}"
- next unless File.exist?(dists_dir)
- sign_dir("#{distribution} #{code_name}",
- dists_dir)
- end
- end
-
- desc "Upload RC APT repositories"
- task :upload => apt_rc_repositories_dir do
- apt_distributions.each do |distribution|
- dists_dir = "#{apt_rc_repositories_dir}/#{distribution}/dists"
- distribution_label = apt_distribution_label(distribution)
- uploader = BintrayUploader.new(repository: bintray_repository,
- distribution: distribution,
- distribution_label: distribution_label,
- version: version,
- rc: rc,
- source: dists_dir,
- destination_prefix: "dists/",
- user: bintray_user,
- api_key: bintray_api_key)
- uploader.upload
- end
- end
- end
-
- desc "Release RC APT repositories"
- apt_rc_tasks = [
- "apt:rc:download",
- "apt:rc:update",
- "apt:rc:upload",
- ]
- task :rc => apt_rc_tasks
- end
- end
-
- def define_apt_release_tasks
- directory apt_release_repositories_dir
-
- namespace :apt do
- namespace :release do
- desc "Download RC APT repositories"
- task :download => apt_release_repositories_dir do
- apt_distributions.each do |distribution|
- distribution_dir = "#{apt_release_repositories_dir}/#{distribution}"
- download_distribution(distribution, distribution_dir)
- end
- end
-
- desc "Upload release APT repositories"
- task :upload => apt_release_repositories_dir do
- apt_distributions.each do |distribution|
- distribution_dir = "#{apt_release_repositories_dir}/#{distribution}"
- distribution_label = apt_distribution_label(distribution)
- uploader = BintrayUploader.new(repository: bintray_repository,
- distribution: distribution,
- distribution_label: distribution_label,
- version: version,
- source: distribution_dir,
- user: bintray_user,
- api_key: bintray_api_key)
- uploader.upload
- end
- end
- end
-
- desc "Release APT repositories"
- apt_release_tasks = [
- "apt:release:download",
- "apt:release:upload",
- ]
- task :release => apt_release_tasks
- end
- end
-
- def define_apt_tasks
- define_deb_tasks
- define_apt_rc_tasks
- define_apt_release_tasks
- end
-
- def rpm_dir
- "#{rc_dir}/rpm/#{full_version}"
- end
-
- def yum_rc_repositories_dir
- "#{rc_dir}/yum/repositories"
- end
-
- def yum_release_repositories_dir
- "#{release_dir}/yum/repositories"
- end
-
- def available_yum_targets
- [
- ["centos", "7"],
- ["centos", "8"],
- ]
- end
-
- def yum_distribution_label(distribution)
- case distribution
- when "centos"
- "CentOS"
- else
- distribution
- end
- end
-
- def yum_targets
- env_yum_targets = (ENV["YUM_TARGETS"] || "").split(",")
- if env_yum_targets.empty?
- available_yum_targets
- else
- available_yum_targets.select do |distribution, distribution_version|
- env_yum_targets.any? do |env_yum_target|
- env_yum_target.start_with?("#{distribution}-#{distribution_version}")
- end
- end
- end
- end
-
- def yum_distributions
- yum_targets.collect(&:first).uniq
- end
-
- def yum_architectures
- [
- "aarch64",
- "x86_64",
- ]
- end
-
- def signed_rpm?(rpm)
- IO.pipe do |input, output|
- system("rpm", "--checksig", rpm, out: output)
- output.close
- signature = input.gets.sub(/\A#{Regexp.escape(rpm)}: /, "")
- signature.split.include?("signatures")
- end
- end
-
- def sign_rpms(directory)
- thread_pool = ThreadPool.new(:gpg) do |rpm|
- unless signed_rpm?(rpm)
- sh("rpm",
- "-D", "_gpg_name #{gpg_key_id}",
- "-D", "__gpg_check_password_cmd /bin/true true",
- "--resign",
- rpm,
- out: default_output,
- verbose: verbose?)
- end
- end
- Dir.glob("#{directory}/**/*.rpm") do |rpm|
- thread_pool << rpm
- end
- thread_pool.join
- end
-
- def rpm_sign(directory)
- unless system("rpm", "-q",
- rpm_gpg_key_package_name(gpg_key_id),
- out: IO::NULL)
- gpg_key = Tempfile.new(["apache-arrow-binary", ".asc"])
- sh("gpg",
- "--armor",
- "--export", gpg_key_id,
- out: gpg_key.path,
- verbose: verbose?)
- sh("rpm",
- "--import", gpg_key.path,
- out: default_output,
- verbose: verbose?)
- gpg_key.close!
- end
-
- yum_targets.each do |distribution, distribution_version|
- source_dir = [
- directory,
- distribution,
- distribution_version,
- ].join("/")
- sign_rpms(source_dir)
- end
- end
-
- def define_rpm_tasks
- directory rpm_dir
-
- namespace :rpm do
- desc "Copy RPM packages"
- task :copy => rpm_dir do
- yum_targets.each do |distribution, distribution_version|
- progress_label = "Copying: #{distribution} #{distribution_version}"
- progress_reporter = ProgressReporter.new(progress_label)
-
- destination_prefix = [
- rpm_dir,
- distribution,
- distribution_version,
- ].join("/")
- source_dir_prefix =
- "#{artifacts_dir}/#{distribution}-#{distribution_version}"
- Dir.glob("#{source_dir_prefix}*/**/*") do |path|
- next if File.directory?(path)
- base_name = File.basename(path)
- type = base_name.split(".")[-2]
- destination_paths = []
- case type
- when "src"
- destination_paths << [
- destination_prefix,
- "Source",
- "SPackages",
- base_name,
- ].join("/")
- when "noarch"
- yum_architectures.each do |architecture|
- destination_paths << [
- destination_prefix,
- architecture,
- "Packages",
- base_name,
- ].join("/")
- end
- else
- destination_paths << [
- destination_prefix,
- type,
- "Packages",
- base_name,
- ].join("/")
- end
- destination_paths.each do |destination_path|
- copy_artifact(path,
- destination_path,
- progress_reporter)
- end
- case base_name
- when /\A(apache-arrow-release)-.*\.noarch\.rpm\z/
- package_name = $1
- latest_release_package_path = [
- destination_prefix,
- "#{package_name}-latest.rpm"
- ].join("/")
- copy_artifact(path,
- latest_release_package_path,
- progress_reporter)
- end
- end
-
- progress_reporter.finish
- end
- end
-
- desc "Sign RPM packages"
- task :sign do
- rpm_sign(rpm_dir)
- yum_targets.each do |distribution, distribution_version|
- source_dir = [
- rpm_dir,
- distribution,
- distribution_version,
- ].join("/")
- sign_dir("#{distribution}-#{distribution_version}",
- source_dir)
- end
- end
-
- desc "Upload RPM packages"
- task :upload do
- yum_distributions.each do |distribution|
- distribution_dir = "#{rpm_dir}/#{distribution}"
- distribution_label = yum_distribution_label(distribution)
- uploader = BintrayUploader.new(repository: bintray_repository,
- distribution: distribution,
- distribution_label: distribution_label,
- version: version,
- rc: rc,
- source: distribution_dir,
- user: bintray_user,
- api_key: bintray_api_key)
- uploader.upload
- end
- end
- end
-
- desc "Release RPM packages"
- rpm_tasks = [
- "rpm:copy",
- "rpm:sign",
- "rpm:upload",
- ]
- task :rpm => rpm_tasks
- end
-
- def yum_update(repositories_dir)
- yum_distributions.each do |distribution|
- distribution_dir = "#{repositories_dir}/#{distribution}"
- Dir.glob("#{distribution_dir}/**/repodata") do |repodata|
- rm_rf(repodata, verbose: verbose?)
- end
- end
-
- yum_targets.each do |distribution, distribution_version|
- base_dir = [
- repositories_dir,
- distribution,
- distribution_version,
- ].join("/")
- base_dir = Pathname(base_dir)
- next unless base_dir.directory?
- base_dir.glob("*") do |arch_dir|
- next unless arch_dir.directory?
- sh(ENV["CREATEREPO"] || "createrepo",
- "--update",
- arch_dir.to_s,
- out: default_output,
- verbose: verbose?)
- end
- end
- end
-
- def define_yum_rc_tasks
- directory yum_rc_repositories_dir
-
- namespace :yum do
- namespace :rc do
- desc "Download RPM files for RC Yum repositories"
- task :download => yum_rc_repositories_dir do
- yum_distributions.each do |distribution|
- distribution_dir = "#{yum_rc_repositories_dir}/#{distribution}"
- download_distribution(distribution,
- distribution_dir,
- with_source_repository: true)
- end
- end
-
- desc "Update RC Yum repositories"
- task :update => yum_rc_repositories_dir do
- yum_update(yum_rc_repositories_dir)
- yum_targets.each do |distribution, distribution_version|
- base_dir = [
- yum_rc_repositories_dir,
- distribution,
- distribution_version,
- ].join("/")
- base_dir = Pathname(base_dir)
- next unless base_dir.directory?
- base_dir.glob("*") do |arch_dir|
- next unless arch_dir.directory?
- sign_label =
- "#{distribution}-#{distribution_version} #{arch_dir.basename}"
- sign_dir(sign_label,
- arch_dir.to_s)
- end
- end
- end
-
- desc "Upload RC Yum repositories"
- task :upload => yum_rc_repositories_dir do
- yum_targets.each do |distribution, distribution_version|
- distribution_label = yum_distribution_label(distribution)
- base_dir = [
- yum_rc_repositories_dir,
- distribution,
- distribution_version,
- ].join("/")
- base_dir = Pathname(base_dir)
- base_dir.glob("**/repodata") do |repodata_dir|
- relative_dir = [
- distribution_version,
- repodata_dir.relative_path_from(base_dir).to_s
- ].join("/")
- uploader =
- BintrayUploader.new(repository: bintray_repository,
- distribution: distribution,
- distribution_label: distribution_label,
- version: version,
- rc: rc,
- source: repodata_dir.to_s,
- destination_prefix: "#{relative_dir}/",
- user: bintray_user,
- api_key: bintray_api_key)
- uploader.upload
- end
- end
- end
- end
-
- desc "Release RC Yum packages"
- yum_rc_tasks = [
- "yum:rc:download",
- "yum:rc:update",
- "yum:rc:upload",
- ]
- task :rc => yum_rc_tasks
- end
- end
-
- def define_yum_release_tasks
- directory yum_release_repositories_dir
-
- namespace :yum do
- namespace :release do
- desc "Download RC Yum repositories"
- task :download => yum_release_repositories_dir do
- yum_distributions.each do |distribution|
- distribution_dir = "#{yum_release_repositories_dir}/#{distribution}"
- download_distribution(distribution, distribution_dir)
- end
- end
-
- desc "Upload release Yum repositories"
- task :upload => yum_release_repositories_dir do
- yum_distributions.each do |distribution|
- distribution_dir = "#{yum_release_repositories_dir}/#{distribution}"
- distribution_label = yum_distribution_label(distribution)
- uploader = BintrayUploader.new(repository: bintray_repository,
- distribution: distribution,
- distribution_label: distribution_label,
- version: version,
- source: distribution_dir,
- user: bintray_user,
- api_key: bintray_api_key)
- uploader.upload
- end
- end
- end
-
- desc "Release Yum packages"
- yum_release_tasks = [
- "yum:release:download",
- "yum:release:upload",
- ]
- task :release => yum_release_tasks
- end
- end
-
- def define_yum_tasks
- define_rpm_tasks
- define_yum_rc_tasks
- define_yum_release_tasks
- end
-
- def define_generic_data_rc_tasks(label,
- id,
- rc_dir,
- target_files_glob)
- directory rc_dir
-
- namespace id do
- namespace :rc do
- desc "Copy #{label} packages"
- task :copy => rc_dir do
- progress_label = "Copying: #{label}"
- progress_reporter = ProgressReporter.new(progress_label)
-
- Pathname(artifacts_dir).glob(target_files_glob) do |path|
- next if path.directory?
- destination_path = [
- rc_dir,
- path.basename.to_s,
- ].join("/")
- copy_artifact(path, destination_path, progress_reporter)
- end
-
- progress_reporter.finish
- end
-
- desc "Sign #{label} packages"
- task :sign => rc_dir do
- sign_dir(label, rc_dir)
- end
-
- desc "Upload #{label} packages"
- task :upload do
- uploader = BintrayUploader.new(repository: bintray_repository,
- distribution: id.to_s,
- distribution_label: label,
- version: version,
- rc: rc,
- source: rc_dir,
- destination_prefix: "#{full_version}/",
- user: bintray_user,
- api_key: bintray_api_key)
- uploader.upload
- end
- end
-
- desc "Release RC #{label} packages"
- rc_tasks = [
- "#{id}:rc:copy",
- "#{id}:rc:sign",
- "#{id}:rc:upload",
- ]
- task :rc => rc_tasks
- end
- end
-
- def define_generic_data_release_tasks(label, id, release_dir)
- directory release_dir
-
- namespace id do
- namespace :release do
- desc "Download RC #{label} packages"
- task :download => release_dir do
- download_distribution(id.to_s, release_dir)
- end
-
- desc "Upload release #{label} packages"
- task :upload => release_dir do
- packages_dir = "#{release_dir}/#{full_version}"
- uploader = BintrayUploader.new(repository: bintray_repository,
- distribution: id.to_s,
- distribution_label: label,
- version: version,
- source: packages_dir,
- destination_prefix: "#{version}/",
- user: bintray_user,
- api_key: bintray_api_key)
- uploader.upload
- end
- end
-
- desc "Release #{label} packages"
- release_tasks = [
- "#{id}:release:download",
- "#{id}:release:upload",
- ]
- task :release => release_tasks
- end
- end
-
- def define_generic_data_tasks(label,
- id,
- rc_dir,
- release_dir,
- target_files_glob)
- define_generic_data_rc_tasks(label, id, rc_dir, target_files_glob)
- define_generic_data_release_tasks(label, id, release_dir)
- end
-
- def define_python_tasks
- define_generic_data_tasks("Python",
- :python,
- "#{rc_dir}/python/#{full_version}",
- "#{release_dir}/python/#{full_version}",
- "{conda-*,wheel-*,python-sdist}/**/*")
- end
-
- def define_nuget_tasks
- define_generic_data_tasks("NuGet",
- :nuget,
- "#{rc_dir}/nuget/#{full_version}",
- "#{release_dir}/nuget/#{full_version}",
- "nuget/**/*")
- end
-
- def define_summary_tasks
- namespace :summary do
- desc "Show RC summary"
- task :rc do
- puts(<<-SUMMARY)
-Success! The release candidate binaries are available here:
- https://bintray.com/#{bintray_repository}/debian-rc/#{full_version}
- https://bintray.com/#{bintray_repository}/ubuntu-rc/#{full_version}
- https://bintray.com/#{bintray_repository}/centos-rc/#{full_version}
- https://bintray.com/#{bintray_repository}/python-rc/#{full_version}
- https://bintray.com/#{bintray_repository}/nuget-rc/#{full_version}
- SUMMARY
- end
-
- desc "Show release summary"
- task :release do
- puts(<<-SUMMARY)
-Success! The release binaries are available here:
- https://bintray.com/#{bintray_repository}/debian/#{version}
- https://bintray.com/#{bintray_repository}/ubuntu/#{version}
- https://bintray.com/#{bintray_repository}/centos/#{version}
- https://bintray.com/#{bintray_repository}/python/#{version}
- https://bintray.com/#{bintray_repository}/nuget/#{version}
- SUMMARY
- end
- end
- end
-end
diff --git a/dev/release/binary/.dockerignore b/dev/release/binary/.dockerignore
deleted file mode 100644
index f2c46d8..0000000
--- a/dev/release/binary/.dockerignore
+++ /dev/null
@@ -1,18 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-/tmp/
diff --git a/dev/release/binary/Dockerfile b/dev/release/binary/Dockerfile
deleted file mode 100644
index 26ef3f9..0000000
--- a/dev/release/binary/Dockerfile
+++ /dev/null
@@ -1,68 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-FROM ubuntu:18.04
-
-ENV DEBIAN_FRONTEND noninteractive
-
-ARG DEBUG
-
-RUN \
- quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \
- apt update ${quiet} && \
- apt install -y -V ${quiet} \
- apt-utils \
- createrepo \
- devscripts \
- gpg \
- locales \
- openssh-server \
- rake \
- rpm \
- ruby \
- sudo && \
- apt clean && \
- rm -rf /var/lib/apt/lists/*
-
-RUN locale-gen en_US.UTF-8
-
-RUN mkdir -p /run/sshd
-RUN echo "StreamLocalBindUnlink yes" >> /etc/ssh/sshd_config
-
-ENV ARROW_USER arrow
-ENV ARROW_UID 10000
-
-RUN \
- groupadd --gid ${ARROW_UID} ${ARROW_USER} && \
- useradd --uid ${ARROW_UID} --gid ${ARROW_UID} --create-home ${ARROW_USER} && \
- mkdir -p /home/arrow/.gnupg /home/arrow/.ssh && \
- chown -R arrow: /home/arrow/.gnupg /home/arrow/.ssh && \
- chmod -R og-rwx /home/arrow/.gnupg /home/arrow/.ssh && \
- echo "${ARROW_USER} ALL=(ALL:ALL) NOPASSWD:ALL" | \
- EDITOR=tee visudo -f /etc/sudoers.d/arrow
-
-COPY id_rsa.pub /home/arrow/.ssh/authorized_keys
-RUN \
- chown -R arrow: /home/arrow/.ssh && \
- chmod -R og-rwx /home/arrow/.ssh
-
-COPY runner.sh /home/arrow/runner.sh
-RUN \
- chown -R arrow: /home/arrow/runner.sh && \
- chmod +x /home/arrow/runner.sh
-
-EXPOSE 22
diff --git a/dev/release/binary/runner.sh b/dev/release/binary/runner.sh
deleted file mode 100755
index 5cf5033..0000000
--- a/dev/release/binary/runner.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -u
-
-export LANG=C
-
-target_dir=/host/binary/tmp
-original_owner=$(stat --format=%u ${target_dir})
-original_group=$(stat --format=%g ${target_dir})
-
-sudo -H chown -R ${USER}: ${target_dir}
-restore_owner() {
- sudo -H chown -R ${original_owner}:${original_group} ${target_dir}
-}
-trap restore_owner EXIT
-
-cd /host
-
-"$@"
diff --git a/dev/release/check-rat-report.py b/dev/release/check-rat-report.py
deleted file mode 100644
index e30d72b..0000000
--- a/dev/release/check-rat-report.py
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/usr/bin/python
-##############################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-##############################################################################
-import fnmatch
-import re
-import sys
-import xml.etree.ElementTree as ET
-
-if len(sys.argv) != 3:
- sys.stderr.write("Usage: %s exclude_globs.lst rat_report.xml\n" %
- sys.argv[0])
- sys.exit(1)
-
-exclude_globs_filename = sys.argv[1]
-xml_filename = sys.argv[2]
-
-globs = [line.strip() for line in open(exclude_globs_filename, "r")]
-
-tree = ET.parse(xml_filename)
-root = tree.getroot()
-resources = root.findall('resource')
-
-all_ok = True
-for r in resources:
- approvals = r.findall('license-approval')
- if not approvals or approvals[0].attrib['name'] == 'true':
- continue
- clean_name = re.sub('^[^/]+/', '', r.attrib['name'])
- excluded = False
- for g in globs:
- if fnmatch.fnmatch(clean_name, g):
- excluded = True
- break
- if not excluded:
- sys.stdout.write("NOT APPROVED: %s (%s): %s\n" % (
- clean_name, r.attrib['name'], approvals[0].attrib['name']))
- all_ok = False
-
-if not all_ok:
- sys.exit(1)
-
-print('OK')
-sys.exit(0)
diff --git a/dev/release/download_rc_binaries.py b/dev/release/download_rc_binaries.py
deleted file mode 100755
index 5ed8ece..0000000
--- a/dev/release/download_rc_binaries.py
+++ /dev/null
@@ -1,173 +0,0 @@
-#!/usr/bin/env python
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import re
-
-import argparse
-import concurrent.futures as cf
-import functools
-import hashlib
-import json
-import os
-import subprocess
-import urllib.request
-
-
-BINTRAY_API_ROOT = "https://bintray.com/api/v1"
-BINTRAY_DL_ROOT = "https://dl.bintray.com"
-BINTRAY_REPO = os.getenv('BINTRAY_REPOSITORY', 'apache/arrow')
-DEFAULT_PARALLEL_DOWNLOADS = 8
-
-
-class Bintray:
-
- def __init__(self, repo=BINTRAY_REPO):
- self.repo = repo
-
- def get_file_list(self, package, version):
- url = os.path.join(BINTRAY_API_ROOT, 'packages', self.repo, package,
- 'versions', version, 'files')
- request = urllib.request.urlopen(url).read()
- return json.loads(request)
-
- def download_files(self, files, dest=None, num_parallel=None,
- re_match=None):
- """
- Download files from Bintray in parallel. If file already exists, will
- overwrite if the checksum does not match what Bintray says it should be
-
- Parameters
- ----------
- files : List[Dict]
- File listing from Bintray
- dest : str, default None
- Defaults to current working directory
- num_parallel : int, default 8
- Number of files to download in parallel. If set to None, uses
- default
- """
- if dest is None:
- dest = os.getcwd()
- if num_parallel is None:
- num_parallel = DEFAULT_PARALLEL_DOWNLOADS
-
- if re_match is not None:
- regex = re.compile(re_match)
- files = [x for x in files if regex.match(x['path'])]
-
- if num_parallel == 1:
- for path in files:
- self._download_file(dest, path)
- else:
- parallel_map_terminate_early(
- functools.partial(self._download_file, dest),
- files,
- num_parallel
- )
-
- def _download_file(self, dest, info):
- relpath = info['path']
-
- base, filename = os.path.split(relpath)
-
- dest_dir = os.path.join(dest, base)
- os.makedirs(dest_dir, exist_ok=True)
-
- dest_path = os.path.join(dest_dir, filename)
-
- if os.path.exists(dest_path):
- with open(dest_path, 'rb') as f:
- sha256sum = hashlib.sha256(f.read()).hexdigest()
- if sha256sum == info['sha256']:
- print('Local file {} sha256 matches, skipping'
- .format(dest_path))
- return
- else:
- print('Local file sha256 does not match, overwriting')
-
- print("Downloading {} to {}".format(relpath, dest_path))
-
- bintray_abspath = os.path.join(BINTRAY_DL_ROOT, self.repo, relpath)
-
- cmd = [
- 'curl', '--fail', '--location', '--retry', '5',
- '--output', dest_path, bintray_abspath
- ]
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- stdout, stderr = proc.communicate()
- if proc.returncode != 0:
- raise Exception("Downloading {} failed\nstdout: {}\nstderr: {}"
- .format(relpath, stdout, stderr))
-
-
-def parallel_map_terminate_early(f, iterable, num_parallel):
- tasks = []
- with cf.ProcessPoolExecutor(num_parallel) as pool:
- for v in iterable:
- tasks.append(pool.submit(functools.partial(f, v)))
-
- for task in cf.as_completed(tasks):
- if task.exception() is not None:
- e = task.exception()
- for task in tasks:
- task.cancel()
- raise e
-
-
-ARROW_PACKAGE_TYPES = ['centos', 'debian', 'nuget', 'python', 'ubuntu']
-
-
-def download_rc_binaries(version, rc_number, re_match=None, dest=None,
- num_parallel=None, target_package_type=None):
- bintray = Bintray()
-
- version_string = '{}-rc{}'.format(version, rc_number)
- if target_package_type:
- package_types = [target_package_type]
- else:
- package_types = ARROW_PACKAGE_TYPES
- for package_type in package_types:
- files = bintray.get_file_list('{}-rc'.format(package_type),
- version_string)
- bintray.download_files(files, re_match=re_match, dest=dest,
- num_parallel=num_parallel)
-
-
-if __name__ == '__main__':
- parser = argparse.ArgumentParser(
- description='Download release candidate binaries'
- )
- parser.add_argument('version', type=str, help='The version number')
- parser.add_argument('rc_number', type=int,
- help='The release candidate number, e.g. 0, 1, etc')
- parser.add_argument('-e', '--regexp', type=str, default=None,
- help=('Regular expression to match on file names '
- 'to only download certain files'))
- parser.add_argument('--dest', type=str, default=os.getcwd(),
- help='The output folder for the downloaded files')
- parser.add_argument('--num_parallel', type=int, default=8,
- help='The number of concurrent downloads to do')
- parser.add_argument('--package_type', type=str, default=None,
- help='The package type to be downloaded')
- args = parser.parse_args()
-
- download_rc_binaries(args.version, args.rc_number, dest=args.dest,
- re_match=args.regexp, num_parallel=args.num_parallel,
- target_package_type=args.package_type)
diff --git a/dev/release/post-01-upload.sh b/dev/release/post-01-upload.sh
deleted file mode 100755
index 4f8053d..0000000
--- a/dev/release/post-01-upload.sh
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-set -e
-set -u
-
-if [ "$#" -ne 2 ]; then
- echo "Usage: $0 <version> <rc-num>"
- exit
-fi
-
-version=$1
-rc=$2
-
-tmp_dir=tmp-apache-arrow-dist
-
-echo "Recreate temporary directory: ${tmp_dir}"
-rm -rf ${tmp_dir}
-mkdir -p ${tmp_dir}
-
-echo "Clone dev dist repository"
-svn \
- co \
- https://dist.apache.org/repos/dist/dev/arrow/apache-arrow-${version}-rc${rc} \
- ${tmp_dir}/dev
-
-echo "Clone release dist repository"
-svn co https://dist.apache.org/repos/dist/release/arrow ${tmp_dir}/release
-
-echo "Copy ${version}-rc${rc} to release working copy"
-release_version=arrow-${version}
-mkdir -p ${tmp_dir}/release/${release_version}
-cp -r ${tmp_dir}/dev/* ${tmp_dir}/release/${release_version}/
-svn add ${tmp_dir}/release/${release_version}
-
-echo "Keep only the three most recent versions"
-old_releases=$(
- svn ls ${tmp_dir}/release/ | \
- grep '^arrow-' | \
- sort --version-sort --reverse | \
- tail -n +4
-)
-for old_release_version in $old_releases; do
- echo "Remove old release ${old_release_version}"
- svn delete ${tmp_dir}/release/${old_release_version}
-done
-
-echo "Commit release"
-svn ci -m "Apache Arrow ${version}" ${tmp_dir}/release
-
-echo "Clean up"
-rm -rf ${tmp_dir}
-
-echo "Success! The release is available here:"
-echo " https://dist.apache.org/repos/dist/release/arrow/${release_version}"
diff --git a/dev/release/post-02-binary.sh b/dev/release/post-02-binary.sh
deleted file mode 100755
index 9f531af..0000000
--- a/dev/release/post-02-binary.sh
+++ /dev/null
@@ -1,95 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -e
-set -o pipefail
-
-SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-if [ "$#" -ne 2 ]; then
- echo "Usage: $0 <version> <rc-num>"
- exit
-fi
-
-version=$1
-rc=$2
-
-cd "${SOURCE_DIR}"
-
-: ${BINTRAY_REPOSITORY_CUSTOM:=${BINTRAY_REPOSITORY:-}}
-
-if [ ! -f .env ]; then
- echo "You must create $(pwd)/.env"
- echo "You can use $(pwd)/.env.example as template"
- exit 1
-fi
-. .env
-
-if [ -n "${BINTRAY_REPOSITORY_CUSTOM}" ]; then
- BINTRAY_REPOSITORY=${BINTRAY_REPOSITORY_CUSTOM}
-fi
-
-. binary-common.sh
-
-# By default deploy all artifacts.
-# To deactivate one category, deactivate the category and all of its dependents.
-# To explicitly select one category, set DEPLOY_DEFAULT=0 DEPLOY_X=1.
-: ${DEPLOY_DEFAULT:=1}
-: ${DEPLOY_CENTOS:=${DEPLOY_DEFAULT}}
-: ${DEPLOY_DEBIAN:=${DEPLOY_DEFAULT}}
-: ${DEPLOY_NUGET:=${DEPLOY_DEFAULT}}
-: ${DEPLOY_PYTHON:=${DEPLOY_DEFAULT}}
-: ${DEPLOY_UBUNTU:=${DEPLOY_DEFAULT}}
-
-rake_tasks=()
-apt_targets=()
-yum_targets=()
-if [ ${DEPLOY_DEBIAN} -gt 0 ]; then
- rake_tasks+=(apt:release)
- apt_targets+=(debian)
-fi
-if [ ${DEPLOY_UBUNTU} -gt 0 ]; then
- rake_tasks+=(apt:release)
- apt_targets+=(ubuntu)
-fi
-if [ ${DEPLOY_CENTOS} -gt 0 ]; then
- rake_tasks+=(yum:release)
- yum_targets+=(centos)
-fi
-if [ ${DEPLOY_NUGET} -gt 0 ]; then
- rake_tasks+=(nuget:release)
-fi
-if [ ${DEPLOY_PYTHON} -gt 0 ]; then
- rake_tasks+=(python:release)
-fi
-rake_tasks+=(summary:release)
-
-tmp_dir=binary/tmp
-mkdir -p "${tmp_dir}"
-
-docker_run \
- ./runner.sh \
- rake \
- "${rake_tasks[@]}" \
- APT_TARGETS=$(IFS=,; echo "${apt_targets[*]}") \
- ARTIFACTS_DIR="${tmp_dir}/artifacts" \
- BINTRAY_REPOSITORY=${BINTRAY_REPOSITORY} \
- RC=${rc} \
- VERSION=${version} \
- YUM_TARGETS=$(IFS=,; echo "${yum_targets[*]}")
diff --git a/dev/release/post-03-website.sh b/dev/release/post-03-website.sh
deleted file mode 100755
index b427142..0000000
--- a/dev/release/post-03-website.sh
+++ /dev/null
@@ -1,266 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -e
-set -u
-
-SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-ARROW_DIR="${SOURCE_DIR}/../.."
-ARROW_SITE_DIR="${ARROW_DIR}/../arrow-site"
-
-if [ "$#" -ne 2 ]; then
- echo "Usage: $0 <previous-version> <version>"
- exit 1
-fi
-
-previous_version=$1
-version=$2
-
-branch_name=release-note-${version}
-release_dir="${ARROW_SITE_DIR}/_release"
-announce_file="${release_dir}/${version}.md"
-versions_yml="${ARROW_SITE_DIR}/_data/versions.yml"
-
-pushd "${ARROW_SITE_DIR}"
-git checkout master
-git checkout -b ${branch_name}
-popd
-
-pushd "${ARROW_DIR}"
-
-release_date=$(LANG=C date "+%-d %B %Y")
-previous_tag_date=$(git log -n 1 --pretty=%aI apache-arrow-${previous_version})
-rough_previous_release_date=$(date --date "${previous_tag_date}" +%s)
-rough_release_date=$(date +%s)
-rough_n_development_months=$((
- (${rough_release_date} - ${rough_previous_release_date}) / (60 * 60 * 24 * 30)
-))
-
-git_tag=apache-arrow-${version}
-git_range=apache-arrow-${previous_version}..${git_tag}
-
-committers_command_line="git shortlog -csn ${git_range}"
-contributors_command_line="git shortlog -sn ${git_range}"
-
-committers=$(${committers_command_line})
-contributors=$(${contributors_command_line})
-
-n_commits=$(git log --pretty=oneline ${git_range} | wc -l)
-n_contributors=$(${contributors_command_line} | wc -l)
-
-git_tag_hash=$(git log -n 1 --pretty=%H ${git_tag})
-
-popd
-
-pushd "${ARROW_SITE_DIR}"
-
-# Add announce for the current version
-cat <<ANNOUNCE > "${announce_file}"
----
-layout: default
-title: Apache Arrow ${version} Release
-permalink: /release/${version}.html
----
-<!--
-{% comment %}
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to you under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-{% endcomment %}
--->
-
-# Apache Arrow ${version} (${release_date})
-
-This is a major release covering more than ${rough_n_development_months} months of development.
-
-## Download
-
-* [**Source Artifacts**][1]
-* **Binary Artifacts**
- * [For CentOS][2]
- * [For Debian][3]
- * [For Python][4]
- * [For Ubuntu][5]
-* [Git tag][6]
-
-## Contributors
-
-This release includes ${n_commits} commits from ${n_contributors} distinct contributors.
-
-\`\`\`console
-$ ${contributors_command_line}
-ANNOUNCE
-
-echo "${contributors}" >> "${announce_file}"
-
-cat <<ANNOUNCE >> "${announce_file}"
-\`\`\`
-
-## Patch Committers
-
-The following Apache committers merged contributed patches to the repository.
-
-\`\`\`console
-$ ${committers_command_line}
-ANNOUNCE
-
-echo "${committers}" >> "${announce_file}"
-
-cat <<ANNOUNCE >> "${announce_file}"
-\`\`\`
-
-## Changelog
-
-ANNOUNCE
-
-archery release changelog generate ${version} | \
- sed -e 's/^#/##/g' >> "${announce_file}"
-
-cat <<ANNOUNCE >> "${announce_file}"
-[1]: https://www.apache.org/dyn/closer.lua/arrow/arrow-${version}/
-[2]: https://bintray.com/apache/arrow/centos/${version}/
-[3]: https://bintray.com/apache/arrow/debian/${version}/
-[4]: https://bintray.com/apache/arrow/python/${version}/
-[5]: https://bintray.com/apache/arrow/ubuntu/${version}/
-[6]: https://github.com/apache/arrow/releases/tag/apache-arrow-${version}
-ANNOUNCE
-git add "${announce_file}"
-
-
-# Update index
-pushd "${release_dir}"
-
-index_file=index.md
-rm -f ${index_file}
-announce_files="$(ls | sort --version-sort --reverse)"
-cat <<INDEX > ${index_file}
----
-layout: default
-title: Releases
-permalink: /release/index.html
----
-<!--
-{% comment %}
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to you under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-{% endcomment %}
--->
-
-# Apache Arrow Releases
-
-Navigate to the release page for downloads and the changelog.
-
-INDEX
-
-i=0
-for md_file in ${announce_files}; do
- i=$((i + 1))
- title=$(grep '^# Apache Arrow' ${md_file} | sed -e 's/^# Apache Arrow //')
- echo "* [${title}][${i}]" >> ${index_file}
-done
-echo >> ${index_file}
-
-i=0
-for md_file in ${announce_files}; do
- i=$((i + 1))
- html_file=$(echo ${md_file} | sed -e 's/md$/html/')
- echo "[${i}]: {{ site.baseurl }}/release/${html_file}" >> ${index_file}
-done
-
-git add ${index_file}
-
-popd
-
-
-# Update versions.yml
-pinned_version=$(echo ${version} | sed -e 's/\.[^.]*$/.*/')
-
-apache_download_url=https://downloads.apache.org
-
-cat <<YAML > "${versions_yml}"
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to you under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Database of the current version
-#
-current:
- number: '${version}'
- pinned_number: '${pinned_version}'
- date: '${release_date}'
- git-tag: '${git_tag_hash}'
- github-tag-link: 'https://github.com/apache/arrow/releases/tag/${git_tag}'
- release-notes: 'https://arrow.apache.org/release/${version}.html'
- mirrors: 'https://www.apache.org/dyn/closer.lua/arrow/arrow-${version}/'
- tarball-name: 'apache-arrow-${version}.tar.gz'
- tarball-url: 'https://www.apache.org/dyn/closer.lua?action=download&filename=arrow/arrow-${version}/apache-arrow-${version}.tar.gz'
- java-artifacts: 'http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%22${version}%22'
- asc: '${apache_download_url}/arrow/arrow-${version}/apache-arrow-${version}.tar.gz.asc'
- sha256: '${apache_download_url}/arrow/arrow-${version}/apache-arrow-${version}.tar.gz.sha256'
- sha512: '${apache_download_url}/arrow/arrow-${version}/apache-arrow-${version}.tar.gz.sha512'
-YAML
-git add "${versions_yml}"
-
-git commit -m "[Website] Add release note for ${version}"
-git push -u origin ${branch_name}
-
-github_url=$(git remote get-url origin | \
- sed \
- -e 's,^git@github.com:,https://github.com/,' \
- -e 's,\.git$,,')
-
-echo "Success!"
-echo "Create a pull request:"
-echo " ${github_url}/pull/new/${branch_name}"
-
-popd
diff --git a/dev/release/post-04-ruby.sh b/dev/release/post-04-ruby.sh
deleted file mode 100755
index 7bc42aa..0000000
--- a/dev/release/post-04-ruby.sh
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/bin/bash
-# -*- indent-tabs-mode: nil; sh-indentation: 2; sh-basic-offset: 2 -*-
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-set -e
-set -o pipefail
-
-SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-if [ "$#" -ne 1 ]; then
- echo "Usage: $0 <version>"
- exit
-fi
-
-version=$1
-archive_name=apache-arrow-${version}
-tar_gz=${archive_name}.tar.gz
-
-rm -f ${tar_gz}
-curl \
- --remote-name \
- --fail \
- https://downloads.apache.org/arrow/arrow-${version}/${tar_gz}
-rm -rf ${archive_name}
-tar xf ${tar_gz}
-modules=()
-for module in ${archive_name}/ruby/red-*; do
- pushd ${module}
- rake release
- modules+=($(basename ${module}))
- popd
-done
-rm -rf ${archive_name}
-rm -f ${tar_gz}
-
-echo "Success! The released RubyGems are available here:"
-for module in ${modules[@]}; do
- echo " https://rubygems.org/gems/${module}/versions/${version}"
-done
diff --git a/dev/release/post-05-js.sh b/dev/release/post-05-js.sh
deleted file mode 100755
index 3df07e4..0000000
--- a/dev/release/post-05-js.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/bin/bash
-# -*- indent-tabs-mode: nil; sh-indentation: 2; sh-basic-offset: 2 -*-
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-set -e
-
-SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-if [ "$#" -ne 1 ]; then
- echo "Usage: $0 <version>"
- exit
-fi
-
-version=$1
-archive_name=apache-arrow-${version}
-tar_gz=${archive_name}.tar.gz
-
-rm -f ${tar_gz}
-curl \
- --remote-name \
- --fail \
- https://downloads.apache.org/arrow/arrow-${version}/${tar_gz}
-rm -rf ${archive_name}
-tar xf ${tar_gz}
-pushd ${archive_name}/js
-./npm-release.sh
-popd
-rm -rf ${archive_name}
-rm -f ${tar_gz}
-
-echo "Success! The released npm packages are available here:"
-echo " https://www.npmjs.com/package/apache-arrow/v/${version}"
diff --git a/dev/release/post-06-csharp.sh b/dev/release/post-06-csharp.sh
deleted file mode 100755
index e957202..0000000
--- a/dev/release/post-06-csharp.sh
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/bin/bash
-# -*- indent-tabs-mode: nil; sh-indentation: 2; sh-basic-offset: 2 -*-
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-set -eux
-
-SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-if [ "$#" -ne 1 ]; then
- echo "Usage: $0 <version>"
- exit
-fi
-
-version=$1
-
-if [ -z "${NUGET_API_KEY}" ]; then
- echo "NUGET_API_KEY is empty"
- exit 1
-fi
-
-base_names=()
-base_names+=(Apache.Arrow.${version})
-base_names+=(Apache.Arrow.Flight.${version})
-base_names+=(Apache.Arrow.Flight.AspNetCore.${version})
-for base_name in ${base_names[@]}; do
- for extension in nupkg snupkg; do
- path=${base_name}.${extension}
- rm -f ${path}
- curl \
- --fail \
- --location \
- --remote-name \
- https://apache.bintray.com/arrow/nuget/${version}/${path}
- done
- dotnet nuget push \
- ${base_name}.nupkg \
- -k ${NUGET_API_KEY} \
- -s https://api.nuget.org/v3/index.json
-done
-
-echo "Success! The released NuGet package is available here:"
-echo " https://www.nuget.org/packages/Apache.Arrow/${version}"
diff --git a/dev/release/post-07-rust.sh b/dev/release/post-07-rust.sh
deleted file mode 100755
index 3c94607..0000000
--- a/dev/release/post-07-rust.sh
+++ /dev/null
@@ -1,74 +0,0 @@
-#!/bin/bash
-# -*- indent-tabs-mode: nil; sh-indentation: 2; sh-basic-offset: 2 -*-
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-set -e
-set -o pipefail
-
-SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-if [ "$#" -ne 1 ]; then
- echo "Usage: $0 <version>"
- exit
-fi
-
-version=$1
-
-: ${INSTALL_RUST:=no}
-
-if [ "${INSTALL_RUST}" == "yes" ]; then
- export RUSTUP_HOME="$(pwd)/release-rustup"
- export CARGO_HOME="${RUSTUP_HOME}"
- rm -rf "${RUSTUP_HOME}"
- curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path
- export PATH="${RUSTUP_HOME}/bin:$PATH"
- source "${RUSTUP_HOME}/env"
- rustup default stable
- cargo login
-fi
-
-archive_name=apache-arrow-${version}
-tar_gz=${archive_name}.tar.gz
-rm -f ${tar_gz}
-curl \
- --remote-name \
- --fail \
- https://downloads.apache.org/arrow/arrow-${version}/${tar_gz}
-rm -rf ${archive_name}
-tar xf ${tar_gz}
-modules=()
-for cargo_toml in ${archive_name}/rust/*/Cargo.toml; do
- module_dir=$(dirname ${cargo_toml})
- pushd ${module_dir}
- cargo publish --allow-dirty
- modules+=($(basename ${module_dir}))
- popd
-done
-popd
-rm -rf ${archive_name}
-rm -f ${tar_gz}
-
-if [ "${INSTALL_RUST}" == "yes" ]; then
- rm -rf "${RUSTUP_HOME}"
-fi
-
-echo "Success! The released packages are available here:"
-for module in ${modules[@]}; do
- echo " https://crates.io/crates/${module}/${version}"
-done
diff --git a/dev/release/post-08-remove-rc.sh b/dev/release/post-08-remove-rc.sh
deleted file mode 100755
index a028613..0000000
--- a/dev/release/post-08-remove-rc.sh
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-set -e
-set -u
-set -o pipefail
-
-SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-if [ "$#" -ne 1 ]; then
- echo "Usage: $0 <version>"
- exit
-fi
-
-version=$1
-
-base_url=https://dist.apache.org/repos/dist/dev/arrow
-pattern="^apache-arrow-${version}-rc"
-paths=$()
-if svn ls ${base_url} | grep "${pattern}" > /dev/null 2>&1; then
- rc_paths=$(svn ls ${base_url} | grep "${pattern}")
- rc_urls=()
- for rc_path in ${rc_paths}; do
- rc_urls+=(${base_url}/${rc_path})
- done
- svn rm --message "Remove RC for ${version}" ${rc_urls[@]}
- echo "Removed RC artifacts:"
- for rc_url in ${rc_urls[@]}; do
- echo " ${rc_url}"
- done
-else
- echo "No RC artifacts at ${base_url}"
-fi
diff --git a/dev/release/post-09-docs.sh b/dev/release/post-09-docs.sh
deleted file mode 100755
index c9f75b4..0000000
--- a/dev/release/post-09-docs.sh
+++ /dev/null
@@ -1,68 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -e
-set -u
-
-SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-ARROW_DIR="${SOURCE_DIR}/../.."
-ARROW_SITE_DIR="${ARROW_DIR}/../arrow-site"
-
-if [ "$#" -ne 1 ]; then
- echo "Usage: $0 <version>"
- exit 1
-fi
-
-version=$1
-release_tag="apache-arrow-${version}"
-branch_name=release-docs-${version}
-
-pushd "${ARROW_SITE_DIR}"
-git checkout asf-site
-git checkout -b ${branch_name}
-rm -rf docs/*
-git checkout docs/c_glib/index.html
-popd
-
-pushd "${ARROW_DIR}"
-git checkout "${release_tag}"
-
-archery docker run \
- -v "${ARROW_SITE_DIR}/docs:/build/docs" \
- -e ARROW_DOCS_VERSION="${version}" \
- -e UBUNTU=20.10 \
- ubuntu-docs
-
-: ${PUSH:=1}
-
-if [ ${PUSH} -gt 0 ]; then
- pushd "${ARROW_SITE_DIR}"
- git add docs
- git commit -m "[Website] Update documentations for ${version}"
- git push -u origin ${branch_name}
- github_url=$(git remote get-url origin | \
- sed \
- -e 's,^git@github.com:,https://github.com/,' \
- -e 's,\.git$,,')
- popd
-
- echo "Success!"
- echo "Create a pull request:"
- echo " ${github_url}/pull/new/${branch_name}"
-fi
diff --git a/dev/release/post-10-python.sh b/dev/release/post-10-python.sh
deleted file mode 100755
index 0f7a480..0000000
--- a/dev/release/post-10-python.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -e
-set -o pipefail
-
-SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-if [ "$#" -ne 2 ]; then
- echo "Usage: $0 <version> <rc-num>"
- exit
-fi
-
-version=$1
-rc=$2
-
-tmp=$(mktemp -d -t "arrow-post-python.XXXXX")
-${PYTHON:-python} \
- "${SOURCE_DIR}/download_rc_binaries.py" \
- ${version} \
- ${rc} \
- --dest="${tmp}" \
- --package_type=python
-twine upload ${tmp}/python-rc/${version}-rc${rc}/*.{whl,tar.gz}
-rm -rf "${tmp}"
-
-echo "Success! The released PyPI packages are available here:"
-echo " https://pypi.org/project/pyarrow/${version}"
diff --git a/dev/release/post-11-java.sh b/dev/release/post-11-java.sh
deleted file mode 100755
index d9dc32a..0000000
--- a/dev/release/post-11-java.sh
+++ /dev/null
@@ -1,69 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -e
-set -o pipefail
-
-SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-if [ "$#" -ne 1 ]; then
- echo "Usage: $0 <version>"
- exit
-fi
-
-version=$1
-archive_name=apache-arrow-${version}
-tar_gz=${archive_name}.tar.gz
-
-rm -f ${tar_gz}
-curl \
- --remote-name \
- --fail \
- https://downloads.apache.org/arrow/arrow-${version}/${tar_gz}
-rm -rf ${archive_name}
-tar xf ${tar_gz}
-
-# build the jni bindings similarly like the 01-perform.sh does
-mkdir -p ${archive_name}/cpp/java-build
-pushd ${archive_name}/cpp/java-build
-cmake \
- -DARROW_GANDIVA=ON \
- -DARROW_GANDIVA_JAVA=ON \
- -DARROW_JNI=ON \
- -DARROW_ORC=ON \
- -DCMAKE_BUILD_TYPE=release \
- -G Ninja \
- ..
-ninja
-popd
-
-# go in the java subfolder
-pushd ${archive_name}/java
-# stage the artifacts using both the apache-release and arrow-jni profiles
-mvn -Papache-release,arrow-jni -Darrow.cpp.build.dir=$(realpath ../cpp/java-build) deploy
-popd
-
-echo "Success! The maven artifacts have been stated. Proceed with the following steps:"
-echo "1. Login to the apache repository: https://repository.apache.org/#stagingRepositories"
-echo "2. Select the arrow staging repository you just just created: orgapachearrow-100x"
-echo "3. Click the \"close\" button"
-echo "4. Once validation has passed, click the \"release\" button"
-echo ""
-echo "Note, that you must set up Maven to be able to publish to Apache's repositories."
-echo "Read more at https://www.apache.org/dev/publishing-maven-artifacts.html."
diff --git a/dev/release/run-test.rb b/dev/release/run-test.rb
deleted file mode 100755
index 90df39b..0000000
--- a/dev/release/run-test.rb
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/usr/bin/env ruby
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-$VERBOSE = true
-
-require "pathname"
-
-test_dir = Pathname.new(__dir__)
-
-require "test-unit"
-require_relative "test-helper"
-
-ENV["TEST_UNIT_MAX_DIFF_TARGET_STRING_SIZE"] = "10000"
-
-exit(Test::Unit::AutoRunner.run(true, test_dir.to_s))
diff --git a/dev/release/setup-gpg-agent.sh b/dev/release/setup-gpg-agent.sh
deleted file mode 100644
index 3e31d0e..0000000
--- a/dev/release/setup-gpg-agent.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-# source me
-eval $(gpg-agent --daemon --allow-preset-passphrase)
-gpg --use-agent -s LICENSE.txt
-rm -rf LICENSE.txt.gpg
diff --git a/dev/release/test-helper.rb b/dev/release/test-helper.rb
deleted file mode 100644
index 8a272dd..0000000
--- a/dev/release/test-helper.rb
+++ /dev/null
@@ -1,96 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-require "English"
-require "cgi/util"
-require "fileutils"
-require "find"
-require "json"
-require "open-uri"
-require "rexml/document"
-require "tempfile"
-require "tmpdir"
-
-module CommandRunnable
- class Error < StandardError
- end
-
- def sh(*command_line, check_result: true)
- if command_line[0].is_a?(Hash)
- env = command_line.shift
- else
- env = {}
- end
- stdout = Tempfile.new("command-stdout.log")
- stderr = Tempfile.new("command-stderr.log")
- success = system(env, *command_line, out: stdout.path, err: stderr.path)
- if check_result
- unless success
- message = "Failed to run: #{command_line.join(" ")}\n"
- message << "stdout:\n #{stdout.read}\n"
- message << "stderr:\n #{stderr.read}"
- raise Error, message
- end
- end
- stdout.read
- end
-end
-
-module GitRunnable
- include CommandRunnable
-
- def git(*args)
- if args[0].is_a?(Hash)
- env = args.shift
- else
- env = {}
- end
- sh(env, "git", *args)
- end
-
- def git_current_commit
- git("rev-parse", "HEAD").chomp
- end
-
- def git_tags
- git("tags").lines(chomp: true)
- end
-end
-
-module VersionDetectable
- def detect_versions
- top_dir = Pathname(__dir__).parent.parent
- cpp_cmake_lists = top_dir + "cpp" + "CMakeLists.txt"
- @snapshot_version = cpp_cmake_lists.read[/ARROW_VERSION "(.+?)"/, 1]
- @release_version = @snapshot_version.gsub(/-SNAPSHOT\z/, "")
- @so_version = compute_so_version(@release_version)
- @next_version = @release_version.gsub(/\A\d+/) {|major| major.succ}
- @next_snapshot_version = "#{@next_version}-SNAPSHOT"
- @next_so_version = compute_so_version(@next_version)
- r_description = top_dir + "r" + "DESCRIPTION"
- @previous_version = r_description.read[/^Version: (.+?)\.9000$/, 1]
- end
-
- def compute_so_version(version)
- major, minor, _patch = version.split(".")
- Integer(major, 10) * 100 + Integer(minor, 10)
- end
-
- def on_release_branch?
- @snapshot_version == @release_version
- end
-end
diff --git a/dev/release/verify-apt.sh b/dev/release/verify-apt.sh
deleted file mode 100755
index e7b87a3..0000000
--- a/dev/release/verify-apt.sh
+++ /dev/null
@@ -1,150 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -exu
-
-if [ $# -lt 2 ]; then
- echo "Usage: $0 VERSION rc"
- echo " $0 VERSION release"
- echo " $0 VERSION local"
- echo " e.g.: $0 0.13.0 rc # Verify 0.13.0 RC"
- echo " e.g.: $0 0.13.0 release # Verify 0.13.0"
- echo " e.g.: $0 0.13.0-dev20210203 local # Verify 0.13.0-dev20210203 on local"
- exit 1
-fi
-
-VERSION="$1"
-TYPE="$2"
-
-local_prefix="/arrow/dev/tasks/linux-packages"
-
-export DEBIAN_FRONTEND=noninteractive
-
-apt update
-apt install -y -V \
- curl \
- lsb-release
-
-code_name="$(lsb_release --codename --short)"
-distribution="$(lsb_release --id --short | tr 'A-Z' 'a-z')"
-artifactory_base_url="https://apache.jfrog.io/artifactory/arrow/${distribution}"
-if [ "${TYPE}" = "rc" ]; then
- artifactory_base_url+="-rc"
-fi
-
-have_flight=yes
-have_plasma=yes
-workaround_missing_packages=()
-case "${distribution}-${code_name}" in
- debian-*)
- sed \
- -i"" \
- -e "s/ main$/ main contrib non-free/g" \
- /etc/apt/sources.list
- ;;
-esac
-if [ "$(arch)" = "aarch64" ]; then
- have_plasma=no
-fi
-
-if [ "${TYPE}" = "local" ]; then
- case "${VERSION}" in
- *-dev*)
- package_version="$(echo "${VERSION}" | sed -e 's/-dev\(.*\)$/~dev\1/g')"
- ;;
- *-rc*)
- package_version="$(echo "${VERSION}" | sed -e 's/-rc.*$//g')"
- ;;
- *)
- package_version="${VERSION}"
- ;;
- esac
- package_version+="-1"
- apt_source_path="${local_prefix}/apt/repositories"
- apt_source_path+="/${distribution}/pool/${code_name}/main"
- apt_source_path+="/a/apache-arrow-apt-source"
- apt_source_path+="/apache-arrow-apt-source_${package_version}_all.deb"
- apt install -y -V "${apt_source_path}"
-else
- package_version="${VERSION}-1"
- apt_source_base_name="apache-arrow-apt-source-latest-${code_name}.deb"
- curl \
- --output "${apt_source_base_name}" \
- "${artifactory_base_url}/${apt_source_base_name}"
- apt install -y -V "./${apt_source_base_name}"
-fi
-
-if [ "${TYPE}" = "local" ]; then
- sed \
- -i"" \
- -e "s,^URIs: .*$,URIs: file://${local_prefix}/apt/repositories/${distribution},g" \
- /etc/apt/sources.list.d/apache-arrow.sources
- keys="${local_prefix}/KEYS"
- if [ -f "${keys}" ]; then
- gpg \
- --no-default-keyring \
- --keyring /usr/share/keyrings/apache-arrow-apt-source.gpg \
- --import "${keys}"
- fi
-else
- if [ "${TYPE}" = "rc" ]; then
- sed \
- -i"" \
- -e "s,^URIs: \\(.*\\)/,URIs: \\1-rc/,g" \
- /etc/apt/sources.list.d/apache-arrow.sources
- fi
-fi
-
-apt update
-
-apt install -y -V libarrow-glib-dev=${package_version}
-required_packages=()
-required_packages+=(cmake)
-required_packages+=(g++)
-required_packages+=(git)
-required_packages+=(${workaround_missing_packages[@]})
-apt install -y -V ${required_packages[@]}
-mkdir -p build
-cp -a /arrow/cpp/examples/minimal_build build
-pushd build/minimal_build
-cmake .
-make -j$(nproc)
-./arrow_example
-popd
-
-apt install -y -V libarrow-glib-dev=${package_version}
-apt install -y -V libarrow-glib-doc=${package_version}
-
-if [ "${have_flight}" = "yes" ]; then
- apt install -y -V libarrow-flight-dev=${package_version}
-fi
-
-apt install -y -V libarrow-python-dev=${package_version}
-
-if [ "${have_plasma}" = "yes" ]; then
- apt install -y -V libplasma-glib-dev=${package_version}
- apt install -y -V libplasma-glib-doc=${package_version}
- apt install -y -V plasma-store-server=${package_version}
-fi
-
-apt install -y -V libgandiva-glib-dev=${package_version}
-apt install -y -V libgandiva-glib-doc=${package_version}
-
-apt install -y -V libparquet-glib-dev=${package_version}
-apt install -y -V libparquet-glib-doc=${package_version}
diff --git a/dev/release/verify-release-candidate-wheels.bat b/dev/release/verify-release-candidate-wheels.bat
deleted file mode 100644
index 2b57113..0000000
--- a/dev/release/verify-release-candidate-wheels.bat
+++ /dev/null
@@ -1,107 +0,0 @@
-@rem Licensed to the Apache Software Foundation (ASF) under one
-@rem or more contributor license agreements. See the NOTICE file
-@rem distributed with this work for additional information
-@rem regarding copyright ownership. The ASF licenses this file
-@rem to you under the Apache License, Version 2.0 (the
-@rem "License"); you may not use this file except in compliance
-@rem with the License. You may obtain a copy of the License at
-@rem
-@rem http://www.apache.org/licenses/LICENSE-2.0
-@rem
-@rem Unless required by applicable law or agreed to in writing,
-@rem software distributed under the License is distributed on an
-@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-@rem KIND, either express or implied. See the License for the
-@rem specific language governing permissions and limitations
-@rem under the License.
-
-@rem This script downloads and installs all Windows wheels for a release
-@rem candidate into temporary conda environments and makes sure that imports
-@rem work
-
-@rem To run the script:
-@rem verify-release-candidate-wheels.bat VERSION RC_NUM
-
-@echo on
-
-set _CURRENT_DIR=%CD%
-set _VERIFICATION_DIR=C:\tmp\arrow-verify-release-wheels
-
-if not exist "C:\tmp\" mkdir C:\tmp
-if exist %_VERIFICATION_DIR% rd %_VERIFICATION_DIR% /s /q
-if not exist %_VERIFICATION_DIR% mkdir %_VERIFICATION_DIR%
-
-cd %_VERIFICATION_DIR%
-
-@rem clone Arrow repository to obtain test requirements
-set GIT_ENV_PATH=%_VERIFICATION_DIR%\_git
-call conda create -p %GIT_ENV_PATH% ^
- --no-shortcuts -f -q -y git ^
- || EXIT /B 1
-call activate %GIT_ENV_PATH%
-
-git clone https://github.com/apache/arrow.git || EXIT /B 1
-pushd arrow
-git submodule update --init
-popd
-
-call deactivate
-
-set ARROW_TEST_DATA=%cd%\arrow\testing\data
-
-CALL :verify_wheel 3.6 %1 %2 m
-if errorlevel 1 GOTO error
-
-CALL :verify_wheel 3.7 %1 %2 m
-if errorlevel 1 GOTO error
-
-CALL :verify_wheel 3.8 %1 %2
-if errorlevel 1 GOTO error
-
-:done
-cd %_CURRENT_DIR%
-
-EXIT /B %ERRORLEVEL%
-
-:error
-call deactivate
-cd %_CURRENT_DIR%
-
-EXIT /B 1
-
-@rem a batch function to verify a single wheel
-:verify_wheel
-
-set PY_VERSION=%1
-set ARROW_VERSION=%2
-set RC_NUMBER=%3
-set ABI_TAG=%4
-set PY_VERSION_NO_PERIOD=%PY_VERSION:.=%
-
-set CONDA_ENV_PATH=%_VERIFICATION_DIR%\_verify-wheel-%PY_VERSION%
-call conda create -p %CONDA_ENV_PATH% ^
- --no-shortcuts -f -q -y python=%PY_VERSION% ^
- || EXIT /B 1
-call activate %CONDA_ENV_PATH%
-
-set WHEEL_FILENAME=pyarrow-%ARROW_VERSION%-cp%PY_VERSION_NO_PERIOD%-cp%PY_VERSION_NO_PERIOD%%ABI_TAG%-win_amd64.whl
-
-@rem Requires GNU Wget for Windows
-wget --no-check-certificate -O %WHEEL_FILENAME% https://bintray.com/apache/arrow/download_file?file_path=python-rc%%2F%ARROW_VERSION%-rc%RC_NUMBER%%%2F%WHEEL_FILENAME% || EXIT /B 1
-
-pip install %WHEEL_FILENAME% || EXIT /B 1
-
-pip install -r arrow/python/requirements-test.txt || EXIT /B 1
-
-py.test %CONDA_ENV_PATH%\Lib\site-packages\pyarrow --pdb -v || EXIT /B 1
-
-python -c "import pyarrow" || EXIT /B 1
-python -c "import pyarrow.parquet" || EXIT /B 1
-python -c "import pyarrow.flight" || EXIT /B 1
-python -c "import pyarrow.dataset" || EXIT /B 1
-
-:done
-
-call deactivate
-
-EXIT /B 0
diff --git a/dev/release/verify-release-candidate.bat b/dev/release/verify-release-candidate.bat
deleted file mode 100644
index bef78fc..0000000
--- a/dev/release/verify-release-candidate.bat
+++ /dev/null
@@ -1,130 +0,0 @@
-@rem Licensed to the Apache Software Foundation (ASF) under one
-@rem or more contributor license agreements. See the NOTICE file
-@rem distributed with this work for additional information
-@rem regarding copyright ownership. The ASF licenses this file
-@rem to you under the Apache License, Version 2.0 (the
-@rem "License"); you may not use this file except in compliance
-@rem with the License. You may obtain a copy of the License at
-@rem
-@rem http://www.apache.org/licenses/LICENSE-2.0
-@rem
-@rem Unless required by applicable law or agreed to in writing,
-@rem software distributed under the License is distributed on an
-@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-@rem KIND, either express or implied. See the License for the
-@rem specific language governing permissions and limitations
-@rem under the License.
-
-@rem To run the script:
-@rem verify-release-candidate.bat VERSION RC_NUM
-
-@echo on
-
-if not exist "C:\tmp\" mkdir C:\tmp
-if exist "C:\tmp\arrow-verify-release" rd C:\tmp\arrow-verify-release /s /q
-if not exist "C:\tmp\arrow-verify-release" mkdir C:\tmp\arrow-verify-release
-
-set _VERIFICATION_DIR=C:\tmp\arrow-verify-release
-set _VERIFICATION_DIR_UNIX=C:/tmp/arrow-verify-release
-set _VERIFICATION_CONDA_ENV=%_VERIFICATION_DIR%\conda-env
-set _DIST_URL=https://dist.apache.org/repos/dist/dev/arrow
-set _TARBALL=apache-arrow-%1.tar.gz
-set ARROW_SOURCE=%_VERIFICATION_DIR%\apache-arrow-%1
-set INSTALL_DIR=%_VERIFICATION_DIR%\install
-
-@rem Requires GNU Wget for Windows
-wget --no-check-certificate -O %_TARBALL% %_DIST_URL%/apache-arrow-%1-rc%2/%_TARBALL% || exit /B 1
-
-tar xf %_TARBALL% -C %_VERIFICATION_DIR_UNIX%
-
-set PYTHON=3.6
-
-@rem Using call with conda.bat seems necessary to avoid terminating the batch
-@rem script execution
-call conda create --no-shortcuts -c conda-forge -f -q -y -p %_VERIFICATION_CONDA_ENV% ^
- --file=ci\conda_env_cpp.yml ^
- --file=ci\conda_env_python.yml ^
- git ^
- python=%PYTHON% ^
- || exit /B 1
-
-call activate %_VERIFICATION_CONDA_ENV% || exit /B 1
-
-set GENERATOR=Visual Studio 15 2017 Win64
-set CONFIGURATION=release
-
-pushd %ARROW_SOURCE%
-
-set ARROW_HOME=%INSTALL_DIR%
-set PARQUET_HOME=%INSTALL_DIR%
-set PATH=%INSTALL_DIR%\bin;%PATH%
-
-@rem Build and test Arrow C++ libraries
-mkdir %ARROW_SOURCE%\cpp\build
-pushd %ARROW_SOURCE%\cpp\build
-
-@rem This is the path for Visual Studio Community 2017
-call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64
-
-@rem NOTE(wesm): not using Ninja for now to be able to more easily control the
-@rem generator used
-
-cmake -G "%GENERATOR%" ^
- -DARROW_BOOST_USE_SHARED=ON ^
- -DARROW_BUILD_STATIC=OFF ^
- -DARROW_BUILD_TESTS=ON ^
- -DARROW_CXXFLAGS="/MP" ^
- -DARROW_DATASET=ON ^
- -DARROW_FLIGHT=ON ^
- -DARROW_MIMALLOC=ON ^
- -DARROW_PARQUET=ON ^
- -DARROW_PYTHON=ON ^
- -DARROW_WITH_BROTLI=ON ^
- -DARROW_WITH_BZ2=ON ^
- -DARROW_WITH_LZ4=ON ^
- -DARROW_WITH_SNAPPY=ON ^
- -DARROW_WITH_ZLIB=ON ^
- -DARROW_WITH_ZSTD=ON ^
- -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
- -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^
- -DCMAKE_UNITY_BUILD=ON ^
- -DGTest_SOURCE=BUNDLED ^
- .. || exit /B
-
-cmake --build . --target INSTALL --config Release || exit /B 1
-
-@rem NOTE(wesm): Building googletest is flaky for me with ninja. Building it
-@rem first fixes the problem
-
-@rem ninja googletest_ep || exit /B 1
-@rem ninja install || exit /B 1
-
-@rem Get testing datasets for Parquet unit tests
-git clone https://github.com/apache/parquet-testing.git %_VERIFICATION_DIR%\parquet-testing
-set PARQUET_TEST_DATA=%_VERIFICATION_DIR%\parquet-testing\data
-
-git clone https://github.com/apache/arrow-testing.git %_VERIFICATION_DIR%\arrow-testing
-set ARROW_TEST_DATA=%_VERIFICATION_DIR%\arrow-testing\data
-
-@rem Needed so python-test.exe works
-set PYTHONPATH_ORIGINAL=%PYTHONPATH%
-set PYTHONPATH=%CONDA_PREFIX%\Lib;%CONDA_PREFIX%\Lib\site-packages;%CONDA_PREFIX%\DLLs;%CONDA_PREFIX%;%PYTHONPATH%
-ctest -VV || exit /B 1
-set PYTHONPATH=%PYTHONPATH_ORIGINAL%
-popd
-
-@rem Build and import pyarrow
-pushd %ARROW_SOURCE%\python
-
-pip install -r requirements-test.txt || exit /B 1
-
-set PYARROW_CMAKE_GENERATOR=%GENERATOR%
-set PYARROW_WITH_FLIGHT=1
-set PYARROW_WITH_PARQUET=1
-set PYARROW_WITH_DATASET=1
-python setup.py build_ext --inplace --bundle-arrow-cpp bdist_wheel || exit /B 1
-pytest pyarrow -v -s --enable-parquet || exit /B 1
-
-popd
-
-call deactivate
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
deleted file mode 100755
index 3fc926f..0000000
--- a/dev/release/verify-release-candidate.sh
+++ /dev/null
@@ -1,808 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-# Requirements
-# - Ruby >= 2.3
-# - Maven >= 3.3.9
-# - JDK >=7
-# - gcc >= 4.8
-# - Node.js >= 11.12 (best way is to use nvm)
-# - Go >= 1.11
-#
-# If using a non-system Boost, set BOOST_ROOT and add Boost libraries to
-# LD_LIBRARY_PATH.
-#
-# To reuse build artifacts between runs set ARROW_TMPDIR environment variable to
-# a directory where the temporary files should be placed to, note that this
-# directory is not cleaned up automatically.
-
-case $# in
- 3) ARTIFACT="$1"
- VERSION="$2"
- RC_NUMBER="$3"
- case $ARTIFACT in
- source|binaries|wheels) ;;
- *) echo "Invalid argument: '${ARTIFACT}', valid options are \
-'source', 'binaries', or 'wheels'"
- exit 1
- ;;
- esac
- ;;
- *) echo "Usage: $0 source|binaries X.Y.Z RC_NUMBER"
- exit 1
- ;;
-esac
-
-set -e
-set -x
-set -o pipefail
-
-SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
-ARROW_DIR="$(dirname $(dirname ${SOURCE_DIR}))"
-
-detect_cuda() {
- if ! (which nvcc && which nvidia-smi) > /dev/null; then
- return 1
- fi
-
- local n_gpus=$(nvidia-smi --list-gpus | wc -l)
- return $((${n_gpus} < 1))
-}
-
-# Build options for the C++ library
-
-if [ -z "${ARROW_CUDA:-}" ] && detect_cuda; then
- ARROW_CUDA=ON
-fi
-: ${ARROW_CUDA:=OFF}
-: ${ARROW_FLIGHT:=ON}
-: ${ARROW_GANDIVA:=ON}
-
-ARROW_DIST_URL='https://dist.apache.org/repos/dist/dev/arrow'
-
-download_dist_file() {
- curl \
- --silent \
- --show-error \
- --fail \
- --location \
- --remote-name $ARROW_DIST_URL/$1
-}
-
-download_rc_file() {
- download_dist_file apache-arrow-${VERSION}-rc${RC_NUMBER}/$1
-}
-
-import_gpg_keys() {
- download_dist_file KEYS
- gpg --import KEYS
-}
-
-fetch_archive() {
- local dist_name=$1
- download_rc_file ${dist_name}.tar.gz
- download_rc_file ${dist_name}.tar.gz.asc
- download_rc_file ${dist_name}.tar.gz.sha256
- download_rc_file ${dist_name}.tar.gz.sha512
- gpg --verify ${dist_name}.tar.gz.asc ${dist_name}.tar.gz
- shasum -a 256 -c ${dist_name}.tar.gz.sha256
- shasum -a 512 -c ${dist_name}.tar.gz.sha512
-}
-
-verify_dir_artifact_signatures() {
- # verify the signature and the checksums of each artifact
- find $1 -name '*.asc' | while read sigfile; do
- artifact=${sigfile/.asc/}
- gpg --verify $sigfile $artifact || exit 1
-
- # go into the directory because the checksum files contain only the
- # basename of the artifact
- pushd $(dirname $artifact)
- base_artifact=$(basename $artifact)
- if [ -f $base_artifact.sha256 ]; then
- shasum -a 256 -c $base_artifact.sha256 || exit 1
- fi
- shasum -a 512 -c $base_artifact.sha512 || exit 1
- popd
- done
-}
-
-test_binary() {
- local download_dir=binaries
- mkdir -p ${download_dir}
-
- python $SOURCE_DIR/download_rc_binaries.py $VERSION $RC_NUMBER \
- --dest=${download_dir}
-
- verify_dir_artifact_signatures ${download_dir}
-}
-
-test_apt() {
- for target in "debian:buster" \
- "arm64v8/debian:buster" \
- "ubuntu:bionic" \
- "arm64v8/ubuntu:bionic" \
- "ubuntu:focal" \
- "arm64v8/ubuntu:focal" \
- "ubuntu:groovy" \
- "arm64v8/ubuntu:groovy"; do \
- case "${target}" in
- arm64v8/*)
- if [ "$(arch)" = "aarch64" -o -e /usr/bin/qemu-aarch64-static ]; then
- : # OK
- else
- continue
- fi
- ;;
- esac
- if ! docker run --rm -v "${SOURCE_DIR}"/../..:/arrow:delegated \
- "${target}" \
- /arrow/dev/release/verify-apt.sh \
- "${VERSION}" \
- "rc" \
- "${BINTRAY_REPOSITORY}"; then
- echo "Failed to verify the APT repository for ${target}"
- exit 1
- fi
- done
-}
-
-test_yum() {
- for target in "centos:7" \
- "centos:8" \
- "arm64v8/centos:8"; do
- case "${target}" in
- arm64v8/*)
- if [ "$(arch)" = "aarch64" -o -e /usr/bin/qemu-aarch64-static ]; then
- : # OK
- else
- continue
- fi
- ;;
- esac
- if ! docker run --rm -v "${SOURCE_DIR}"/../..:/arrow:delegated \
- "${target}" \
- /arrow/dev/release/verify-yum.sh \
- "${VERSION}" \
- "rc" \
- "${BINTRAY_REPOSITORY}"; then
- echo "Failed to verify the Yum repository for ${target}"
- exit 1
- fi
- done
-}
-
-
-setup_tempdir() {
- cleanup() {
- if [ "${TEST_SUCCESS}" = "yes" ]; then
- rm -fr "${ARROW_TMPDIR}"
- else
- echo "Failed to verify release candidate. See ${ARROW_TMPDIR} for details."
- fi
- }
-
- if [ -z "${ARROW_TMPDIR}" ]; then
- # clean up automatically if ARROW_TMPDIR is not defined
- ARROW_TMPDIR=$(mktemp -d -t "$1.XXXXX")
- trap cleanup EXIT
- else
- # don't clean up automatically
- mkdir -p "${ARROW_TMPDIR}"
- fi
-}
-
-setup_miniconda() {
- # Setup short-lived miniconda for Python and integration tests
- if [ "$(uname)" == "Darwin" ]; then
- if [ "$(uname -m)" == "arm64" ]; then
- MINICONDA_URL=https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-arm64.sh
- else
- MINICONDA_URL=https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
- fi
- else
- MINICONDA_URL=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
- fi
-
- MINICONDA=$PWD/test-miniconda
-
- if [ ! -d "${MINICONDA}" ]; then
- # Setup miniconda only if the directory doesn't exist yet
- wget -O miniconda.sh $MINICONDA_URL
- bash miniconda.sh -b -p $MINICONDA
- rm -f miniconda.sh
- fi
- echo "Installed miniconda at ${MINICONDA}"
-
- . $MINICONDA/etc/profile.d/conda.sh
-
- conda create -n arrow-test -y -q -c conda-forge \
- python=3.8 \
- nomkl \
- numpy \
- pandas \
- cython
- conda activate arrow-test
- echo "Using conda environment ${CONDA_PREFIX}"
-}
-
-# Build and test Java (Requires newer Maven -- I used 3.3.9)
-
-test_package_java() {
- pushd java
-
- mvn test
- mvn package
-
- popd
-}
-
-# Build and test C++
-
-test_and_install_cpp() {
- mkdir -p cpp/build
- pushd cpp/build
-
- ARROW_CMAKE_OPTIONS="
-${ARROW_CMAKE_OPTIONS:-}
--DCMAKE_INSTALL_PREFIX=$ARROW_HOME
--DCMAKE_INSTALL_LIBDIR=lib
--DARROW_FLIGHT=${ARROW_FLIGHT}
--DARROW_PLASMA=ON
--DARROW_ORC=ON
--DARROW_PYTHON=ON
--DARROW_GANDIVA=${ARROW_GANDIVA}
--DARROW_PARQUET=ON
--DARROW_DATASET=ON
--DPARQUET_REQUIRE_ENCRYPTION=ON
--DARROW_VERBOSE_THIRDPARTY_BUILD=ON
--DARROW_WITH_BZ2=ON
--DARROW_WITH_ZLIB=ON
--DARROW_WITH_ZSTD=ON
--DARROW_WITH_LZ4=ON
--DARROW_WITH_SNAPPY=ON
--DARROW_WITH_BROTLI=ON
--DARROW_BOOST_USE_SHARED=ON
--DCMAKE_BUILD_TYPE=release
--DARROW_BUILD_TESTS=ON
--DARROW_BUILD_INTEGRATION=ON
--DARROW_CUDA=${ARROW_CUDA}
--DARROW_DEPENDENCY_SOURCE=AUTO
-"
- cmake $ARROW_CMAKE_OPTIONS ..
-
- make -j$NPROC install
-
- # TODO: ARROW-5036: plasma-serialization_tests broken
- # TODO: ARROW-5054: libgtest.so link failure in flight-server-test
- LD_LIBRARY_PATH=$PWD/release:$LD_LIBRARY_PATH ctest \
- --exclude-regex "plasma-serialization_tests" \
- -j$NPROC \
- --output-on-failure \
- -L unittest
- popd
-}
-
-test_csharp() {
- pushd csharp
-
- local csharp_bin=${PWD}/bin
- mkdir -p ${csharp_bin}
-
- if which dotnet > /dev/null 2>&1; then
- if ! which sourcelink > /dev/null 2>&1; then
- local dotnet_tools_dir=$HOME/.dotnet/tools
- if [ -d "${dotnet_tools_dir}" ]; then
- PATH="${dotnet_tools_dir}:$PATH"
- fi
- fi
- else
- local dotnet_version=3.1.405
- local dotnet_platform=
- case "$(uname)" in
- Linux)
- dotnet_platform=linux
- ;;
- Darwin)
- dotnet_platform=macos
- ;;
- esac
- local dotnet_download_thank_you_url=https://dotnet.microsoft.com/download/thank-you/dotnet-sdk-${dotnet_version}-${dotnet_platform}-x64-binaries
- local dotnet_download_url=$( \
- curl --location ${dotnet_download_thank_you_url} | \
- grep 'window\.open' | \
- grep -E -o '[^"]+' | \
- sed -n 2p)
- curl ${dotnet_download_url} | \
- tar xzf - -C ${csharp_bin}
- PATH=${csharp_bin}:${PATH}
- fi
-
- dotnet test
- mv dummy.git ../.git
- dotnet pack -c Release
- mv ../.git dummy.git
-
- if ! which sourcelink > /dev/null 2>&1; then
- dotnet tool install --tool-path ${csharp_bin} sourcelink
- PATH=${csharp_bin}:${PATH}
- if ! sourcelink --help > /dev/null 2>&1; then
- export DOTNET_ROOT=${csharp_bin}
- fi
- fi
-
- sourcelink test artifacts/Apache.Arrow/Release/netstandard1.3/Apache.Arrow.pdb
- sourcelink test artifacts/Apache.Arrow/Release/netcoreapp2.1/Apache.Arrow.pdb
-
- popd
-}
-
-# Build and test Python
-
-test_python() {
- pushd python
-
- pip install -r requirements-build.txt -r requirements-test.txt
-
- export PYARROW_WITH_DATASET=1
- export PYARROW_WITH_PARQUET=1
- export PYARROW_WITH_PLASMA=1
- if [ "${ARROW_CUDA}" = "ON" ]; then
- export PYARROW_WITH_CUDA=1
- fi
- if [ "${ARROW_FLIGHT}" = "ON" ]; then
- export PYARROW_WITH_FLIGHT=1
- fi
- if [ "${ARROW_GANDIVA}" = "ON" ]; then
- export PYARROW_WITH_GANDIVA=1
- fi
-
- python setup.py build_ext --inplace
- pytest pyarrow -v --pdb
-
- popd
-}
-
-test_glib() {
- pushd c_glib
-
- pip install meson
-
- meson build --prefix=$ARROW_HOME --libdir=lib
- ninja -C build
- ninja -C build install
-
- export GI_TYPELIB_PATH=$ARROW_HOME/lib/girepository-1.0:$GI_TYPELIB_PATH
-
- if ! bundle --version; then
- gem install --no-document bundler
- fi
-
- bundle install --path vendor/bundle
- bundle exec ruby test/run-test.rb
-
- popd
-}
-
-test_js() {
- pushd js
-
- if [ "${INSTALL_NODE}" -gt 0 ]; then
- export NVM_DIR="`pwd`/.nvm"
- mkdir -p $NVM_DIR
- curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | \
- PROFILE=/dev/null bash
- [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
-
- nvm install --lts
- npm install -g yarn
- fi
-
- yarn --frozen-lockfile
- yarn run-s clean:all lint build
- yarn test
- popd
-}
-
-test_ruby() {
- pushd ruby
-
- local modules="red-arrow red-plasma red-gandiva red-parquet"
- if [ "${ARROW_CUDA}" = "ON" ]; then
- modules="${modules} red-arrow-cuda"
- fi
-
- for module in ${modules}; do
- pushd ${module}
- bundle install --path vendor/bundle
- bundle exec ruby test/run-test.rb
- popd
- done
-
- popd
-}
-
-test_go() {
- local VERSION=1.14.1
- local ARCH=amd64
-
- if [ "$(uname)" == "Darwin" ]; then
- local OS=darwin
- else
- local OS=linux
- fi
-
- local GO_ARCHIVE=go$VERSION.$OS-$ARCH.tar.gz
- wget https://dl.google.com/go/$GO_ARCHIVE
-
- mkdir -p local-go
- tar -xzf $GO_ARCHIVE -C local-go
- rm -f $GO_ARCHIVE
-
- export GOROOT=`pwd`/local-go/go
- export GOPATH=`pwd`/local-go/gopath
- export PATH=$GOROOT/bin:$GOPATH/bin:$PATH
-
- pushd go/arrow
-
- go get -v ./...
- go test ./...
- go clean -modcache
-
- popd
-}
-
-test_rust() {
- # install rust toolchain in a similar fashion like test-miniconda
- export RUSTUP_HOME=$PWD/test-rustup
- export CARGO_HOME=$PWD/test-rustup
-
- curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path
-
- export PATH=$RUSTUP_HOME/bin:$PATH
- source $RUSTUP_HOME/env
-
- # build and test rust
- pushd rust
-
- # raises on any formatting errors
- rustup component add rustfmt --toolchain stable
- cargo +stable fmt --all -- --check
-
- # we are targeting Rust nightly for releases
- rustup default nightly
-
- # use local modules because we don't publish modules to crates.io yet
- sed \
- -i.bak \
- -E \
- -e 's/^arrow = "([^"]*)"/arrow = { version = "\1", path = "..\/arrow" }/g' \
- -e 's/^parquet = "([^"]*)"/parquet = { version = "\1", path = "..\/parquet" }/g' \
- */Cargo.toml
-
- # raises on any warnings
- RUSTFLAGS="-D warnings" cargo build
- cargo test
-
- popd
-}
-
-# Run integration tests
-test_integration() {
- JAVA_DIR=$PWD/java
- CPP_BUILD_DIR=$PWD/cpp/build
-
- export ARROW_JAVA_INTEGRATION_JAR=$JAVA_DIR/tools/target/arrow-tools-$VERSION-jar-with-dependencies.jar
- export ARROW_CPP_EXE_PATH=$CPP_BUILD_DIR/release
-
- pip install -e dev/archery
-
- INTEGRATION_TEST_ARGS=""
-
- if [ "${ARROW_FLIGHT}" = "ON" ]; then
- INTEGRATION_TEST_ARGS="${INTEGRATION_TEST_ARGS} --run-flight"
- fi
-
- # Flight integration test executable have runtime dependency on
- # release/libgtest.so
- LD_LIBRARY_PATH=$ARROW_CPP_EXE_PATH:$LD_LIBRARY_PATH \
- archery integration \
- --with-cpp=${TEST_INTEGRATION_CPP} \
- --with-java=${TEST_INTEGRATION_JAVA} \
- --with-js=${TEST_INTEGRATION_JS} \
- --with-go=${TEST_INTEGRATION_GO} \
- $INTEGRATION_TEST_ARGS
-}
-
-clone_testing_repositories() {
- # Clone testing repositories if not cloned already
- if [ ! -d "arrow-testing" ]; then
- git clone https://github.com/apache/arrow-testing.git
- fi
- if [ ! -d "parquet-testing" ]; then
- git clone https://github.com/apache/parquet-testing.git
- fi
- export ARROW_TEST_DATA=$PWD/arrow-testing/data
- export PARQUET_TEST_DATA=$PWD/parquet-testing/data
-}
-
-test_source_distribution() {
- export ARROW_HOME=$ARROW_TMPDIR/install
- export PARQUET_HOME=$ARROW_TMPDIR/install
- export LD_LIBRARY_PATH=$ARROW_HOME/lib:${LD_LIBRARY_PATH:-}
- export PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig:${PKG_CONFIG_PATH:-}
-
- if [ "$(uname)" == "Darwin" ]; then
- NPROC=$(sysctl -n hw.ncpu)
- else
- NPROC=$(nproc)
- fi
-
- clone_testing_repositories
-
- if [ ${TEST_JAVA} -gt 0 ]; then
- test_package_java
- fi
- if [ ${TEST_CPP} -gt 0 ]; then
- test_and_install_cpp
- fi
- if [ ${TEST_CSHARP} -gt 0 ]; then
- test_csharp
- fi
- if [ ${TEST_PYTHON} -gt 0 ]; then
- test_python
- fi
- if [ ${TEST_GLIB} -gt 0 ]; then
- test_glib
- fi
- if [ ${TEST_RUBY} -gt 0 ]; then
- test_ruby
- fi
- if [ ${TEST_JS} -gt 0 ]; then
- test_js
- fi
- if [ ${TEST_GO} -gt 0 ]; then
- test_go
- fi
- if [ ${TEST_RUST} -gt 0 ]; then
- test_rust
- fi
- if [ ${TEST_INTEGRATION} -gt 0 ]; then
- test_integration
- fi
-}
-
-test_binary_distribution() {
- : ${BINTRAY_REPOSITORY:=apache/arrow}
-
- if [ ${TEST_BINARY} -gt 0 ]; then
- test_binary
- fi
- if [ ${TEST_APT} -gt 0 ]; then
- test_apt
- fi
- if [ ${TEST_YUM} -gt 0 ]; then
- test_yum
- fi
-}
-
-check_python_imports() {
- python << IMPORT_TESTS
-import platform
-
-import pyarrow
-import pyarrow.parquet
-import pyarrow.plasma
-import pyarrow.fs
-import pyarrow._hdfs
-import pyarrow.dataset
-import pyarrow.flight
-
-if platform.system() == "Darwin":
- macos_version = tuple(map(int, platform.mac_ver()[0].split('.')))
- check_s3fs = macos_version >= (10, 13)
-else:
- check_s3fs = True
-
-if check_s3fs:
- import pyarrow._s3fs
-IMPORT_TESTS
-}
-
-test_linux_wheels() {
- local py_arches="3.6m 3.7m 3.8 3.9"
- local manylinuxes="2010 2014"
-
- for py_arch in ${py_arches}; do
- local env=_verify_wheel-${py_arch}
- conda create -yq -n ${env} python=${py_arch//[mu]/}
- conda activate ${env}
- pip install -U pip
-
- for ml_spec in ${manylinuxes}; do
- # check the mandatory and optional imports
- pip install python-rc/${VERSION}-rc${RC_NUMBER}/pyarrow-${VERSION}-cp${py_arch//[mu.]/}-cp${py_arch//./}-manylinux${ml_spec}_x86_64.whl
- check_python_imports
-
- # install test requirements and execute the tests
- pip install -r ${ARROW_DIR}/python/requirements-test.txt
- python -c 'import pyarrow; pyarrow.create_library_symlinks()'
- pytest --pyargs pyarrow
- done
-
- conda deactivate
- done
-}
-
-test_macos_wheels() {
- local py_arches="3.6m 3.7m 3.8 3.9"
-
- for py_arch in ${py_arches}; do
- local env=_verify_wheel-${py_arch}
- conda create -yq -n ${env} python=${py_arch//m/}
- conda activate ${env}
- pip install -U pip
-
- # check the mandatory and optional imports
- pip install --find-links python-rc/${VERSION}-rc${RC_NUMBER} pyarrow==${VERSION}
- check_python_imports
-
- # install test requirements and execute the tests
- pip install -r ${ARROW_DIR}/python/requirements-test.txt
- python -c 'import pyarrow; pyarrow.create_library_symlinks()'
- pytest --pyargs pyarrow
-
- conda deactivate
- done
-}
-
-test_wheels() {
- clone_testing_repositories
-
- local download_dir=binaries
- mkdir -p ${download_dir}
-
- if [ "$(uname)" == "Darwin" ]; then
- local filter_regex=.*macosx.*
- else
- local filter_regex=.*manylinux.*
- fi
-
- python $SOURCE_DIR/download_rc_binaries.py $VERSION $RC_NUMBER \
- --regex=${filter_regex} \
- --dest=${download_dir}
-
- verify_dir_artifact_signatures ${download_dir}
-
- pushd ${download_dir}
-
- if [ "$(uname)" == "Darwin" ]; then
- test_macos_wheels
- else
- test_linux_wheels
- fi
-
- popd
-}
-
-# By default test all functionalities.
-# To deactivate one test, deactivate the test and all of its dependents
-# To explicitly select one test, set TEST_DEFAULT=0 TEST_X=1
-
-# Install NodeJS locally for running the JavaScript tests rather than using the
-# system Node installation, which may be too old.
-: ${INSTALL_NODE:=1}
-
-if [ "${ARTIFACT}" == "source" ]; then
- : ${TEST_SOURCE:=1}
-elif [ "${ARTIFACT}" == "wheels" ]; then
- TEST_WHEELS=1
-else
- TEST_BINARY_DISTRIBUTIONS=1
-fi
-: ${TEST_SOURCE:=0}
-: ${TEST_WHEELS:=0}
-: ${TEST_BINARY_DISTRIBUTIONS:=0}
-
-: ${TEST_DEFAULT:=1}
-: ${TEST_JAVA:=${TEST_DEFAULT}}
-: ${TEST_CPP:=${TEST_DEFAULT}}
-: ${TEST_CSHARP:=${TEST_DEFAULT}}
-: ${TEST_GLIB:=${TEST_DEFAULT}}
-: ${TEST_RUBY:=${TEST_DEFAULT}}
-: ${TEST_PYTHON:=${TEST_DEFAULT}}
-: ${TEST_JS:=${TEST_DEFAULT}}
-: ${TEST_GO:=${TEST_DEFAULT}}
-: ${TEST_RUST:=${TEST_DEFAULT}}
-: ${TEST_INTEGRATION:=${TEST_DEFAULT}}
-if [ ${TEST_BINARY_DISTRIBUTIONS} -gt 0 ]; then
- TEST_BINARY_DISTRIBUTIONS_DEFAULT=${TEST_DEFAULT}
-else
- TEST_BINARY_DISTRIBUTIONS_DEFAULT=0
-fi
-: ${TEST_BINARY:=${TEST_BINARY_DISTRIBUTIONS_DEFAULT}}
-: ${TEST_APT:=${TEST_BINARY_DISTRIBUTIONS_DEFAULT}}
-: ${TEST_YUM:=${TEST_BINARY_DISTRIBUTIONS_DEFAULT}}
-
-# For selective Integration testing, set TEST_DEFAULT=0 TEST_INTEGRATION_X=1 TEST_INTEGRATION_Y=1
-: ${TEST_INTEGRATION_CPP:=${TEST_INTEGRATION}}
-: ${TEST_INTEGRATION_JAVA:=${TEST_INTEGRATION}}
-: ${TEST_INTEGRATION_JS:=${TEST_INTEGRATION}}
-: ${TEST_INTEGRATION_GO:=${TEST_INTEGRATION}}
-
-# Automatically test if its activated by a dependent
-TEST_GLIB=$((${TEST_GLIB} + ${TEST_RUBY}))
-TEST_CPP=$((${TEST_CPP} + ${TEST_GLIB} + ${TEST_PYTHON} + ${TEST_INTEGRATION_CPP}))
-TEST_JAVA=$((${TEST_JAVA} + ${TEST_INTEGRATION_JAVA}))
-TEST_JS=$((${TEST_JS} + ${TEST_INTEGRATION_JS}))
-TEST_GO=$((${TEST_GO} + ${TEST_INTEGRATION_GO}))
-TEST_INTEGRATION=$((${TEST_INTEGRATION} + ${TEST_INTEGRATION_CPP} + ${TEST_INTEGRATION_JAVA} + ${TEST_INTEGRATION_JS} + ${TEST_INTEGRATION_GO}))
-
-NEED_MINICONDA=$((${TEST_CPP} + ${TEST_WHEELS} + ${TEST_BINARY} + ${TEST_INTEGRATION}))
-
-: ${TEST_ARCHIVE:=apache-arrow-${VERSION}.tar.gz}
-case "${TEST_ARCHIVE}" in
- /*)
- ;;
- *)
- TEST_ARCHIVE=${PWD}/${TEST_ARCHIVE}
- ;;
-esac
-
-TEST_SUCCESS=no
-
-setup_tempdir "arrow-${VERSION}"
-echo "Working in sandbox ${ARROW_TMPDIR}"
-cd ${ARROW_TMPDIR}
-
-if [ ${NEED_MINICONDA} -gt 0 ]; then
- setup_miniconda
-fi
-
-if [ "${ARTIFACT}" == "source" ]; then
- dist_name="apache-arrow-${VERSION}"
- if [ ${TEST_SOURCE} -gt 0 ]; then
- import_gpg_keys
- if [ ! -d "${dist_name}" ]; then
- fetch_archive ${dist_name}
- tar xf ${dist_name}.tar.gz
- fi
- else
- mkdir -p ${dist_name}
- if [ ! -f ${TEST_ARCHIVE} ]; then
- echo "${TEST_ARCHIVE} not found"
- exit 1
- fi
- tar xf ${TEST_ARCHIVE} -C ${dist_name} --strip-components=1
- fi
- pushd ${dist_name}
- test_source_distribution
- popd
-elif [ "${ARTIFACT}" == "wheels" ]; then
- import_gpg_keys
- test_wheels
-else
- import_gpg_keys
- test_binary_distribution
-fi
-
-TEST_SUCCESS=yes
-echo 'Release candidate looks good!'
-exit 0
diff --git a/dev/release/verify-yum.sh b/dev/release/verify-yum.sh
deleted file mode 100755
index b9c46c4..0000000
--- a/dev/release/verify-yum.sh
+++ /dev/null
@@ -1,154 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -exu
-
-if [ $# -lt 2 ]; then
- echo "Usage: $0 VERSION rc"
- echo " $0 VERSION release"
- echo " $0 VERSION local"
- echo " e.g.: $0 0.13.0 rc # Verify 0.13.0 RC"
- echo " e.g.: $0 0.13.0 release # Verify 0.13.0"
- echo " e.g.: $0 0.13.0-dev20210203 local # Verify 0.13.0-dev20210203 on local"
- exit 1
-fi
-
-VERSION="$1"
-TYPE="$2"
-
-local_prefix="/arrow/dev/tasks/linux-packages"
-
-artifactory_base_url="https://apache.jfrog.io/artifactory/arrow/centos"
-if [ "${TYPE}" = "rc" ]; then
- artifactory_base_url+="-rc"
-fi
-
-distribution=$(. /etc/os-release && echo "${ID}")
-distribution_version=$(. /etc/os-release && echo "${VERSION_ID}")
-
-cmake_package=cmake
-cmake_command=cmake
-have_flight=yes
-have_gandiva=yes
-have_glib=yes
-have_parquet=yes
-install_command="dnf install -y --enablerepo=powertools"
-case "${distribution}-${distribution_version}" in
- centos-7)
- cmake_package=cmake3
- cmake_command=cmake3
- have_flight=no
- have_gandiva=no
- install_command="yum install -y"
- ;;
-esac
-if [ "$(arch)" = "aarch64" ]; then
- have_gandiva=no
-fi
-
-if [ "${TYPE}" = "local" ]; then
- case "${VERSION}" in
- *-dev*)
- package_version="$(echo "${VERSION}" | sed -e 's/-dev\(.*\)$/-0.dev\1/g')"
- ;;
- *-rc*)
- package_version="$(echo "${VERSION}" | sed -e 's/-rc.*$//g')"
- package_version+="-1"
- ;;
- *)
- package_version="${VERSION}-1"
- ;;
- esac
- package_version+=".el${distribution_version}"
- release_path="${local_prefix}/yum/repositories"
- release_path+="/centos/${distribution_version}/$(arch)/Packages"
- release_path+="/apache-arrow-release-${package_version}.noarch.rpm"
- ${install_command} "${release_path}"
-else
- package_version="${VERSION}"
- ${install_command} \
- ${artifactory_base_url}/${distribution_version}/apache-arrow-release-latest.rpm
-fi
-
-if [ "${TYPE}" = "local" ]; then
- sed \
- -i"" \
- -e "s,baseurl=https://apache\.jfrog\.io/artifactory/arrow/,baseurl=file://${local_prefix}/yum/repositories/,g" \
- /etc/yum.repos.d/Apache-Arrow.repo
- keys="${local_prefix}/KEYS"
- if [ -f "${keys}" ]; then
- cp "${keys}" /etc/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow
- fi
-else
- if [ "${TYPE}" = "rc" ]; then
- sed \
- -i"" \
- -e "s,/centos/,/centos-rc/,g" \
- /etc/yum.repos.d/Apache-Arrow.repo
- fi
-fi
-
-${install_command} --enablerepo=epel arrow-devel-${package_version}
-${install_command} \
- ${cmake_package} \
- gcc-c++ \
- git \
- make
-mkdir -p build
-cp -a /arrow/cpp/examples/minimal_build build
-pushd build/minimal_build
-${cmake_command} .
-make -j$(nproc)
-./arrow_example
-popd
-
-if [ "${have_glib}" = "yes" ]; then
- ${install_command} --enablerepo=epel arrow-glib-devel-${package_version}
- ${install_command} --enablerepo=epel arrow-glib-doc-${package_version}
-fi
-${install_command} --enablerepo=epel arrow-python-devel-${package_version}
-
-if [ "${have_glib}" = "yes" ]; then
- ${install_command} --enablerepo=epel plasma-glib-devel-${package_version}
- ${install_command} --enablerepo=epel plasma-glib-doc-${package_version}
-else
- ${install_command} --enablerepo=epel plasma-devel-${package_version}
-fi
-
-if [ "${have_flight}" = "yes" ]; then
- ${install_command} --enablerepo=epel arrow-flight-devel-${package_version}
-fi
-
-if [ "${have_gandiva}" = "yes" ]; then
- if [ "${have_glib}" = "yes" ]; then
- ${install_command} --enablerepo=epel gandiva-glib-devel-${package_version}
- ${install_command} --enablerepo=epel gandiva-glib-doc-${package_version}
- else
- ${install_command} --enablerepo=epel gandiva-devel-${package_version}
- fi
-fi
-
-if [ "${have_parquet}" = "yes" ]; then
- if [ "${have_glib}" = "yes" ]; then
- ${install_command} --enablerepo=epel parquet-glib-devel-${package_version}
- ${install_command} --enablerepo=epel parquet-glib-doc-${package_version}
- else
- ${install_command} --enablerepo=epel parquet-devel-${package_version}
- fi
-fi
diff --git a/dev/tasks/README.md b/dev/tasks/README.md
deleted file mode 100644
index 1af9739..0000000
--- a/dev/tasks/README.md
+++ /dev/null
@@ -1,19 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-
-See the usage guide under the [documentation page](../../docs/source/developers/crossbow.rst)
diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.6.____cpython.yaml
deleted file mode 100644
index dd4c041..0000000
--- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.6.____cpython.yaml
+++ /dev/null
@@ -1,70 +0,0 @@
-aws_sdk_cpp:
-- 1.8.151
-bzip2:
-- '1'
-c_compiler:
-- gcc
-c_compiler_version:
-- '9'
-cdt_name:
-- cos6
-channel_sources:
-- conda-forge,defaults
-channel_targets:
-- conda-forge main
-cuda_compiler:
-- nvcc
-cuda_compiler_version:
-- '10.2'
-cxx_compiler:
-- gxx
-cxx_compiler_version:
-- '9'
-docker_image:
-- quay.io/condaforge/linux-anvil-cuda:10.2
-gflags:
-- '2.2'
-glog:
-- 0.4.0
-grpc_cpp:
-- '1.36'
-libprotobuf:
-- '3.15'
-lz4_c:
-- 1.9.3
-numpy:
-- '1.17'
-orc:
-- 1.6.7
-pin_run_as_build:
- bzip2:
- max_pin: x
- lz4-c:
- max_pin: x.x.x
- python:
- min_pin: x.x
- max_pin: x.x
- zlib:
- max_pin: x.x
-python:
-- 3.6.* *_cpython
-re2:
-- 2021.04.01
-snappy:
-- '1'
-target_platform:
-- linux-64
-thrift_cpp:
-- 0.14.1
-zip_keys:
-- - c_compiler_version
- - cxx_compiler_version
-- - cuda_compiler_version
- - cdt_name
- - docker_image
-- - python
- - numpy
-zlib:
-- '1.2'
-zstd:
-- '1.4'
diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.7.____cpython.yaml
deleted file mode 100644
index f0c4392..0000000
--- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.7.____cpython.yaml
+++ /dev/null
@@ -1,70 +0,0 @@
-aws_sdk_cpp:
-- 1.8.151
-bzip2:
-- '1'
-c_compiler:
-- gcc
-c_compiler_version:
-- '9'
-cdt_name:
-- cos6
-channel_sources:
-- conda-forge,defaults
-channel_targets:
-- conda-forge main
-cuda_compiler:
-- nvcc
-cuda_compiler_version:
-- '10.2'
-cxx_compiler:
-- gxx
-cxx_compiler_version:
-- '9'
-docker_image:
-- quay.io/condaforge/linux-anvil-cuda:10.2
-gflags:
-- '2.2'
-glog:
-- 0.4.0
-grpc_cpp:
-- '1.36'
-libprotobuf:
-- '3.15'
-lz4_c:
-- 1.9.3
-numpy:
-- '1.17'
-orc:
-- 1.6.7
-pin_run_as_build:
- bzip2:
- max_pin: x
- lz4-c:
- max_pin: x.x.x
- python:
- min_pin: x.x
- max_pin: x.x
- zlib:
- max_pin: x.x
-python:
-- 3.7.* *_cpython
-re2:
-- 2021.04.01
-snappy:
-- '1'
-target_platform:
-- linux-64
-thrift_cpp:
-- 0.14.1
-zip_keys:
-- - c_compiler_version
- - cxx_compiler_version
-- - cuda_compiler_version
- - cdt_name
- - docker_image
-- - python
- - numpy
-zlib:
-- '1.2'
-zstd:
-- '1.4'
diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.8.____cpython.yaml
deleted file mode 100644
index 149e70f..0000000
--- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.8.____cpython.yaml
+++ /dev/null
@@ -1,70 +0,0 @@
-aws_sdk_cpp:
-- 1.8.151
-bzip2:
-- '1'
-c_compiler:
-- gcc
-c_compiler_version:
-- '9'
-cdt_name:
-- cos6
-channel_sources:
-- conda-forge,defaults
-channel_targets:
-- conda-forge main
-cuda_compiler:
-- nvcc
-cuda_compiler_version:
-- '10.2'
-cxx_compiler:
-- gxx
-cxx_compiler_version:
-- '9'
-docker_image:
-- quay.io/condaforge/linux-anvil-cuda:10.2
-gflags:
-- '2.2'
-glog:
-- 0.4.0
-grpc_cpp:
-- '1.36'
-libprotobuf:
-- '3.15'
-lz4_c:
-- 1.9.3
-numpy:
-- '1.17'
-orc:
-- 1.6.7
-pin_run_as_build:
- bzip2:
- max_pin: x
- lz4-c:
- max_pin: x.x.x
- python:
- min_pin: x.x
- max_pin: x.x
- zlib:
- max_pin: x.x
-python:
-- 3.8.* *_cpython
-re2:
-- 2021.04.01
-snappy:
-- '1'
-target_platform:
-- linux-64
-thrift_cpp:
-- 0.14.1
-zip_keys:
-- - c_compiler_version
- - cxx_compiler_version
-- - cuda_compiler_version
- - cdt_name
- - docker_image
-- - python
- - numpy
-zlib:
-- '1.2'
-zstd:
-- '1.4'
diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.19python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.19python3.9.____cpython.yaml
deleted file mode 100644
index fb15d4e..0000000
--- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.19python3.9.____cpython.yaml
+++ /dev/null
@@ -1,70 +0,0 @@
-aws_sdk_cpp:
-- 1.8.151
-bzip2:
-- '1'
-c_compiler:
-- gcc
-c_compiler_version:
-- '9'
-cdt_name:
-- cos6
-channel_sources:
-- conda-forge,defaults
-channel_targets:
-- conda-forge main
-cuda_compiler:
-- nvcc
-cuda_compiler_version:
-- '10.2'
-cxx_compiler:
-- gxx
-cxx_compiler_version:
-- '9'
-docker_image:
-- quay.io/condaforge/linux-anvil-cuda:10.2
-gflags:
-- '2.2'
-glog:
-- 0.4.0
-grpc_cpp:
-- '1.36'
-libprotobuf:
-- '3.15'
-lz4_c:
-- 1.9.3
-numpy:
-- '1.19'
-orc:
-- 1.6.7
-pin_run_as_build:
- bzip2:
- max_pin: x
- lz4-c:
- max_pin: x.x.x
- python:
- min_pin: x.x
- max_pin: x.x
- zlib:
- max_pin: x.x
-python:
-- 3.9.* *_cpython
-re2:
-- 2021.04.01
-snappy:
-- '1'
-target_platform:
-- linux-64
-thrift_cpp:
-- 0.14.1
-zip_keys:
-- - c_compiler_version
- - cxx_compiler_version
-- - cuda_compiler_version
- - cdt_name
- - docker_image
-- - python
- - numpy
-zlib:
-- '1.2'
-zstd:
-- '1.4'
diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml
deleted file mode 100644
index d977f9e..0000000
--- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml
+++ /dev/null
@@ -1,70 +0,0 @@
-aws_sdk_cpp:
-- 1.8.151
-bzip2:
-- '1'
-c_compiler:
-- gcc
-c_compiler_version:
-- '9'
-cdt_name:
-- cos6
-channel_sources:
-- conda-forge,defaults
-channel_targets:
-- conda-forge main
-cuda_compiler:
-- nvcc
-cuda_compiler_version:
-- None
-cxx_compiler:
-- gxx
-cxx_compiler_version:
-- '9'
-docker_image:
-- quay.io/condaforge/linux-anvil-comp7
-gflags:
-- '2.2'
-glog:
-- 0.4.0
-grpc_cpp:
-- '1.36'
-libprotobuf:
-- '3.15'
-lz4_c:
-- 1.9.3
-numpy:
-- '1.17'
-orc:
-- 1.6.7
-pin_run_as_build:
- bzip2:
- max_pin: x
- lz4-c:
- max_pin: x.x.x
- python:
- min_pin: x.x
- max_pin: x.x
- zlib:
- max_pin: x.x
-python:
-- 3.6.* *_cpython
-re2:
-- 2021.04.01
-snappy:
-- '1'
-target_platform:
-- linux-64
-thrift_cpp:
-- 0.14.1
-zip_keys:
-- - c_compiler_version
- - cxx_compiler_version
-- - cuda_compiler_version
- - cdt_name
- - docker_image
-- - python
- - numpy
-zlib:
-- '1.2'
-zstd:
-- '1.4'
diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml
deleted file mode 100644
index 6ffa87a..0000000
--- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml
+++ /dev/null
@@ -1,70 +0,0 @@
-aws_sdk_cpp:
-- 1.8.151
-bzip2:
-- '1'
-c_compiler:
-- gcc
-c_compiler_version:
-- '9'
-cdt_name:
-- cos6
-channel_sources:
-- conda-forge,defaults
-channel_targets:
-- conda-forge main
-cuda_compiler:
-- nvcc
-cuda_compiler_version:
-- None
-cxx_compiler:
-- gxx
-cxx_compiler_version:
-- '9'
-docker_image:
-- quay.io/condaforge/linux-anvil-comp7
-gflags:
-- '2.2'
-glog:
-- 0.4.0
-grpc_cpp:
-- '1.36'
-libprotobuf:
-- '3.15'
-lz4_c:
-- 1.9.3
-numpy:
-- '1.17'
-orc:
-- 1.6.7
-pin_run_as_build:
- bzip2:
- max_pin: x
- lz4-c:
- max_pin: x.x.x
- python:
- min_pin: x.x
- max_pin: x.x
- zlib:
- max_pin: x.x
-python:
-- 3.7.* *_cpython
-re2:
-- 2021.04.01
-snappy:
-- '1'
-target_platform:
-- linux-64
-thrift_cpp:
-- 0.14.1
-zip_keys:
-- - c_compiler_version
- - cxx_compiler_version
-- - cuda_compiler_version
- - cdt_name
- - docker_image
-- - python
- - numpy
-zlib:
-- '1.2'
-zstd:
-- '1.4'
diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml
deleted file mode 100644
index 7105f63..0000000
--- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml
+++ /dev/null
@@ -1,70 +0,0 @@
-aws_sdk_cpp:
-- 1.8.151
-bzip2:
-- '1'
-c_compiler:
-- gcc
-c_compiler_version:
-- '9'
-cdt_name:
-- cos6
-channel_sources:
-- conda-forge,defaults
-channel_targets:
-- conda-forge main
-cuda_compiler:
-- nvcc
-cuda_compiler_version:
-- None
-cxx_compiler:
-- gxx
-cxx_compiler_version:
-- '9'
-docker_image:
-- quay.io/condaforge/linux-anvil-comp7
-gflags:
-- '2.2'
-glog:
-- 0.4.0
-grpc_cpp:
-- '1.36'
-libprotobuf:
-- '3.15'
-lz4_c:
-- 1.9.3
-numpy:
-- '1.17'
-orc:
-- 1.6.7
-pin_run_as_build:
- bzip2:
- max_pin: x
- lz4-c:
- max_pin: x.x.x
- python:
- min_pin: x.x
- max_pin: x.x
- zlib:
- max_pin: x.x
-python:
-- 3.8.* *_cpython
-re2:
-- 2021.04.01
-snappy:
-- '1'
-target_platform:
-- linux-64
-thrift_cpp:
-- 0.14.1
-zip_keys:
-- - c_compiler_version
- - cxx_compiler_version
-- - cuda_compiler_version
- - cdt_name
- - docker_image
-- - python
- - numpy
-zlib:
-- '1.2'
-zstd:
-- '1.4'
diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml
deleted file mode 100644
index efe0148..0000000
--- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml
+++ /dev/null
@@ -1,70 +0,0 @@
-aws_sdk_cpp:
-- 1.8.151
-bzip2:
-- '1'
-c_compiler:
-- gcc
-c_compiler_version:
-- '9'
-cdt_name:
-- cos6
-channel_sources:
-- conda-forge,defaults
-channel_targets:
-- conda-forge main
-cuda_compiler:
-- nvcc
-cuda_compiler_version:
-- None
-cxx_compiler:
-- gxx
-cxx_compiler_version:
-- '9'
-docker_image:
-- quay.io/condaforge/linux-anvil-comp7
-gflags:
-- '2.2'
-glog:
-- 0.4.0
-grpc_cpp:
-- '1.36'
-libprotobuf:
-- '3.15'
-lz4_c:
-- 1.9.3
-numpy:
-- '1.19'
-orc:
-- 1.6.7
-pin_run_as_build:
- bzip2:
- max_pin: x
- lz4-c:
- max_pin: x.x.x
- python:
- min_pin: x.x
- max_pin: x.x
- zlib:
- max_pin: x.x
-python:
-- 3.9.* *_cpython
-re2:
-- 2021.04.01
-snappy:
-- '1'
-target_platform:
-- linux-64
-thrift_cpp:
-- 0.14.1
-zip_keys:
-- - c_compiler_version
- - cxx_compiler_version
-- - cuda_compiler_version
- - cdt_name
- - docker_image
-- - python
- - numpy
-zlib:
-- '1.2'
-zstd:
-- '1.4'
diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.6.____cpython.yaml
deleted file mode 100644
index f2d3cea..0000000
--- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.6.____cpython.yaml
+++ /dev/null
@@ -1,71 +0,0 @@
-BUILD:
-- aarch64-conda_cos7-linux-gnu
-aws_sdk_cpp:
-- 1.8.63
-boost_cpp:
-- 1.74.0
-bzip2:
-- '1'
-c_compiler:
-- gcc
-c_compiler_version:
-- '7'
-cdt_arch:
-- aarch64
-cdt_name:
-- cos7
-channel_sources:
-- conda-forge
-channel_targets:
-- conda-forge main
-cuda_compiler_version:
-- None
-cxx_compiler:
-- gxx
-cxx_compiler_version:
-- '7'
-docker_image:
-- condaforge/linux-anvil-aarch64
-gflags:
-- '2.2'
-glog:
-- 0.4.0
-grpc_cpp:
-- '1.32'
-libprotobuf:
-- '3.13'
-lz4_c:
-- 1.9.2
-numpy:
-- '1.16'
-orc:
-- 1.6.5
-pin_run_as_build:
- boost-cpp:
- max_pin: x.x.x
- bzip2:
- max_pin: x
- lz4-c:
- max_pin: x.x.x
- python:
- min_pin: x.x
- max_pin: x.x
- zlib:
- max_pin: x.x
-python:
-- 3.6.* *_cpython
-re2:
-- 2020.10.01
-snappy:
-- '1'
-target_platform:
-- linux-aarch64
-zip_keys:
-- - c_compiler_version
- - cxx_compiler_version
-- - numpy
- - python
-zlib:
-- '1.2'
-zstd:
-- '1.4'
diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.7.____cpython.yaml
deleted file mode 100644
index 611c39c..0000000
--- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.7.____cpython.yaml
+++ /dev/null
@@ -1,71 +0,0 @@
-BUILD:
-- aarch64-conda_cos7-linux-gnu
-aws_sdk_cpp:
-- 1.8.63
-boost_cpp:
-- 1.74.0
-bzip2:
-- '1'
-c_compiler:
-- gcc
-c_compiler_version:
-- '7'
-cdt_arch:
-- aarch64
-cdt_name:
-- cos7
-channel_sources:
-- conda-forge
-channel_targets:
-- conda-forge main
-cuda_compiler_version:
-- None
-cxx_compiler:
-- gxx
-cxx_compiler_version:
-- '7'
-docker_image:
-- condaforge/linux-anvil-aarch64
-gflags:
-- '2.2'
-glog:
-- 0.4.0
-grpc_cpp:
-- '1.32'
-libprotobuf:
-- '3.13'
-lz4_c:
-- 1.9.2
-numpy:
-- '1.16'
-orc:
-- 1.6.5
-pin_run_as_build:
- boost-cpp:
- max_pin: x.x.x
- bzip2:
- max_pin: x
- lz4-c:
- max_pin: x.x.x
- python:
- min_pin: x.x
- max_pin: x.x
- zlib:
- max_pin: x.x
-python:
-- 3.7.* *_cpython
-re2:
-- 2020.10.01
-snappy:
-- '1'
-target_platform:
-- linux-aarch64
-zip_keys:
-- - c_compiler_version
- - cxx_compiler_version
-- - numpy
- - python
-zlib:
-- '1.2'
-zstd:
-- '1.4'
diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.8.____cpython.yaml
deleted file mode 100644
index 2f0fc0e..0000000
--- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.8.____cpython.yaml
+++ /dev/null
@@ -1,71 +0,0 @@
-BUILD:
-- aarch64-conda_cos7-linux-gnu
-aws_sdk_cpp:
-- 1.8.63
-boost_cpp:
-- 1.74.0
-bzip2:
-- '1'
-c_compiler:
-- gcc
-c_compiler_version:
-- '7'
-cdt_arch:
-- aarch64
-cdt_name:
-- cos7
-channel_sources:
-- conda-forge
-channel_targets:
-- conda-forge main
-cuda_compiler_version:
-- None
-cxx_compiler:
-- gxx
-cxx_compiler_version:
-- '7'
-docker_image:
-- condaforge/linux-anvil-aarch64
-gflags:
-- '2.2'
-glog:
-- 0.4.0
-grpc_cpp:
-- '1.32'
-libprotobuf:
-- '3.13'
-lz4_c:
-- 1.9.2
-numpy:
-- '1.16'
-orc:
-- 1.6.5
-pin_run_as_build:
- boost-cpp:
- max_pin: x.x.x
- bzip2:
- max_pin: x
- lz4-c:
- max_pin: x.x.x
- python:
- min_pin: x.x
- max_pin: x.x
- zlib:
- max_pin: x.x
-python:
-- 3.8.* *_cpython
-re2:
-- 2020.10.01
-snappy:
-- '1'
-target_platform:
-- linux-aarch64
-zip_keys:
-- - c_compiler_version
- - cxx_compiler_version
-- - numpy
- - python
-zlib:
-- '1.2'
-zstd:
-- '1.4'
diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.9.____cpython.yaml
deleted file mode 100644
index 2ec8720..0000000
--- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.9.____cpython.yaml
+++ /dev/null
@@ -1,71 +0,0 @@
-BUILD:
-- aarch64-conda_cos7-linux-gnu
-aws_sdk_cpp:
-- 1.8.63
-boost_cpp:
-- 1.74.0
-bzip2:
-- '1'
-c_compiler:
-- gcc
-c_compiler_version:
-- '7'
-cdt_arch:
-- aarch64
-cdt_name:
-- cos7
-channel_sources:
-- conda-forge
-channel_targets:
-- conda-forge main
-cuda_compiler_version:
-- None
-cxx_compiler:
-- gxx
-cxx_compiler_version:
-- '7'
-docker_image:
-- condaforge/linux-anvil-aarch64
-gflags:
-- '2.2'
-glog:
-- 0.4.0
-grpc_cpp:
-- '1.32'
-libprotobuf:
-- '3.13'
-lz4_c:
-- 1.9.2
-numpy:
-- '1.19'
-orc:
-- 1.6.5
-pin_run_as_build:
- boost-cpp:
- max_pin: x.x.x
- bzip2:
- max_pin: x
- lz4-c:
- max_pin: x.x.x
- python:
- min_pin: x.x
- max_pin: x.x
- zlib:
- max_pin: x.x
-python:
-- 3.9.* *_cpython
-re2:
-- 2020.10.01
-snappy:
-- '1'
-target_platform:
-- linux-aarch64
-zip_keys:
-- - c_compiler_version
- - cxx_compiler_version
-- - numpy
- - python
-zlib:
-- '1.2'
-zstd:
-- '1.4'
diff --git a/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.6.____cpython.yaml
deleted file mode 100644
index 7b2dbb3..0000000
--- a/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.6.____cpython.yaml
+++ /dev/null
@@ -1,65 +0,0 @@
-MACOSX_DEPLOYMENT_TARGET:
-- '10.9'
-aws_sdk_cpp:
-- 1.8.151
-bzip2:
-- '1'
-c_compiler:
-- clang
-c_compiler_version:
-- '11'
-channel_sources:
-- conda-forge,defaults
-channel_targets:
-- conda-forge main
-cuda_compiler_version:
-- None
-cxx_compiler:
-- clangxx
-cxx_compiler_version:
-- '11'
-gflags:
-- '2.2'
-glog:
-- 0.4.0
-grpc_cpp:
-- '1.36'
-libprotobuf:
-- '3.15'
-lz4_c:
-- 1.9.3
-macos_machine:
-- x86_64-apple-darwin13.4.0
-numpy:
-- '1.17'
-orc:
-- 1.6.7
-pin_run_as_build:
- bzip2:
- max_pin: x
- lz4-c:
- max_pin: x.x.x
- python:
- min_pin: x.x
- max_pin: x.x
- zlib:
- max_pin: x.x
-python:
-- 3.6.* *_cpython
-re2:
-- 2021.04.01
-snappy:
-- '1'
-target_platform:
-- osx-64
-thrift_cpp:
-- 0.14.1
-zip_keys:
-- - c_compiler_version
- - cxx_compiler_version
-- - python
- - numpy
-zlib:
-- '1.2'
-zstd:
-- '1.4'
diff --git a/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.7.____cpython.yaml
deleted file mode 100644
index 8e3e828..0000000
--- a/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.7.____cpython.yaml
+++ /dev/null
@@ -1,65 +0,0 @@
-MACOSX_DEPLOYMENT_TARGET:
-- '10.9'
-aws_sdk_cpp:
-- 1.8.151
-bzip2:
-- '1'
-c_compiler:
-- clang
-c_compiler_version:
-- '11'
-channel_sources:
-- conda-forge,defaults
-channel_targets:
-- conda-forge main
-cuda_compiler_version:
-- None
-cxx_compiler:
-- clangxx
-cxx_compiler_version:
-- '11'
-gflags:
-- '2.2'
-glog:
-- 0.4.0
-grpc_cpp:
-- '1.36'
-libprotobuf:
-- '3.15'
-lz4_c:
-- 1.9.3
-macos_machine:
-- x86_64-apple-darwin13.4.0
-numpy:
-- '1.17'
-orc:
-- 1.6.7
-pin_run_as_build:
- bzip2:
- max_pin: x
- lz4-c:
- max_pin: x.x.x
- python:
- min_pin: x.x
- max_pin: x.x
- zlib:
- max_pin: x.x
-python:
-- 3.7.* *_cpython
-re2:
-- 2021.04.01
-snappy:
-- '1'
-target_platform:
-- osx-64
-thrift_cpp:
-- 0.14.1
-zip_keys:
-- - c_compiler_version
- - cxx_compiler_version
-- - python
- - numpy
-zlib:
-- '1.2'
-zstd:
-- '1.4'
diff --git a/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.8.____cpython.yaml
deleted file mode 100644
index cdd53c6..0000000
--- a/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.8.____cpython.yaml
+++ /dev/null
@@ -1,65 +0,0 @@
-MACOSX_DEPLOYMENT_TARGET:
-- '10.9'
-aws_sdk_cpp:
-- 1.8.151
-bzip2:
-- '1'
-c_compiler:
-- clang
-c_compiler_version:
-- '11'
-channel_sources:
-- conda-forge,defaults
-channel_targets:
-- conda-forge main
-cuda_compiler_version:
-- None
-cxx_compiler:
-- clangxx
-cxx_compiler_version:
-- '11'
-gflags:
-- '2.2'
-glog:
-- 0.4.0
-grpc_cpp:
-- '1.36'
-libprotobuf:
-- '3.15'
-lz4_c:
-- 1.9.3
-macos_machine:
-- x86_64-apple-darwin13.4.0
-numpy:
-- '1.17'
-orc:
-- 1.6.7
-pin_run_as_build:
- bzip2:
- max_pin: x
- lz4-c:
- max_pin: x.x.x
- python:
- min_pin: x.x
- max_pin: x.x
- zlib:
- max_pin: x.x
-python:
-- 3.8.* *_cpython
-re2:
-- 2021.04.01
-snappy:
-- '1'
-target_platform:
-- osx-64
-thrift_cpp:
-- 0.14.1
-zip_keys:
-- - c_compiler_version
- - cxx_compiler_version
-- - python
- - numpy
-zlib:
-- '1.2'
-zstd:
-- '1.4'
diff --git a/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.19python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.19python3.9.____cpython.yaml
deleted file mode 100644
index 37df6a9..0000000
--- a/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.19python3.9.____cpython.yaml
+++ /dev/null
@@ -1,65 +0,0 @@
-MACOSX_DEPLOYMENT_TARGET:
-- '10.9'
-aws_sdk_cpp:
-- 1.8.151
-bzip2:
-- '1'
-c_compiler:
-- clang
-c_compiler_version:
-- '11'
-channel_sources:
-- conda-forge,defaults
-channel_targets:
-- conda-forge main
-cuda_compiler_version:
-- None
-cxx_compiler:
-- clangxx
-cxx_compiler_version:
-- '11'
-gflags:
-- '2.2'
-glog:
-- 0.4.0
-grpc_cpp:
-- '1.36'
-libprotobuf:
-- '3.15'
-lz4_c:
-- 1.9.3
-macos_machine:
-- x86_64-apple-darwin13.4.0
-numpy:
-- '1.19'
-orc:
-- 1.6.7
-pin_run_as_build:
- bzip2:
- max_pin: x
- lz4-c:
- max_pin: x.x.x
- python:
- min_pin: x.x
- max_pin: x.x
- zlib:
- max_pin: x.x
-python:
-- 3.9.* *_cpython
-re2:
-- 2021.04.01
-snappy:
-- '1'
-target_platform:
-- osx-64
-thrift_cpp:
-- 0.14.1
-zip_keys:
-- - c_compiler_version
- - cxx_compiler_version
-- - python
- - numpy
-zlib:
-- '1.2'
-zstd:
-- '1.4'
diff --git a/dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.8.____cpython.yaml
deleted file mode 100644
index 5894b8e..0000000
--- a/dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.8.____cpython.yaml
+++ /dev/null
@@ -1,65 +0,0 @@
-MACOSX_DEPLOYMENT_TARGET:
-- '11.0'
-aws_sdk_cpp:
-- 1.8.151
-bzip2:
-- '1'
-c_compiler:
-- clang
-c_compiler_version:
-- '11'
-channel_sources:
-- conda-forge/label/rust_dev,conda-forge
-channel_targets:
-- conda-forge main
-cuda_compiler_version:
-- None
-cxx_compiler:
-- clangxx
-cxx_compiler_version:
-- '11'
-gflags:
-- '2.2'
-glog:
-- 0.4.0
-grpc_cpp:
-- '1.36'
-libprotobuf:
-- '3.15'
-lz4_c:
-- 1.9.3
-macos_machine:
-- arm64-apple-darwin20.0.0
-numpy:
-- '1.19'
-orc:
-- 1.6.7
-pin_run_as_build:
- bzip2:
- max_pin: x
- lz4-c:
- max_pin: x.x.x
- python:
- min_pin: x.x
- max_pin: x.x
- zlib:
- max_pin: x.x
-python:
-- 3.8.* *_cpython
-re2:
-- 2021.04.01
-snappy:
-- '1'
-target_platform:
-- osx-arm64
-thrift_cpp:
-- 0.14.1
-zip_keys:
-- - c_compiler_version
- - cxx_compiler_version
-- - python
- - numpy
-zlib:
-- '1.2'
-zstd:
-- '1.4'
diff --git a/dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.9.____cpython.yaml
deleted file mode 100644
index 4e6014c..0000000
--- a/dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.9.____cpython.yaml
+++ /dev/null
@@ -1,65 +0,0 @@
-MACOSX_DEPLOYMENT_TARGET:
-- '11.0'
-aws_sdk_cpp:
-- 1.8.151
-bzip2:
-- '1'
-c_compiler:
-- clang
-c_compiler_version:
-- '11'
-channel_sources:
-- conda-forge/label/rust_dev,conda-forge
-channel_targets:
-- conda-forge main
-cuda_compiler_version:
-- None
-cxx_compiler:
-- clangxx
-cxx_compiler_version:
-- '11'
-gflags:
-- '2.2'
-glog:
-- 0.4.0
-grpc_cpp:
-- '1.36'
-libprotobuf:
-- '3.15'
-lz4_c:
-- 1.9.3
-macos_machine:
-- arm64-apple-darwin20.0.0
-numpy:
-- '1.19'
-orc:
-- 1.6.7
-pin_run_as_build:
- bzip2:
- max_pin: x
- lz4-c:
- max_pin: x.x.x
- python:
- min_pin: x.x
- max_pin: x.x
- zlib:
- max_pin: x.x
-python:
-- 3.9.* *_cpython
-re2:
-- 2021.04.01
-snappy:
-- '1'
-target_platform:
-- osx-arm64
-thrift_cpp:
-- 0.14.1
-zip_keys:
-- - c_compiler_version
- - cxx_compiler_version
-- - python
- - numpy
-zlib:
-- '1.2'
-zstd:
-- '1.4'
diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base3.6.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base3.6.yaml
deleted file mode 100644
index ac945ce..0000000
--- a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base3.6.yaml
+++ /dev/null
@@ -1,22 +0,0 @@
-c_compiler:
-- gcc
-c_compiler_version:
-- '7'
-channel_sources:
-- conda-forge,defaults
-channel_targets:
-- conda-forge main
-cxx_compiler:
-- gxx
-cxx_compiler_version:
-- '7'
-docker_image:
-- condaforge/linux-anvil-comp7
-pin_run_as_build:
- r-base:
- min_pin: x.x
- max_pin: x.x
-r_base:
-- '3.6'
-target_platform:
-- linux-64
diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.0.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.0.yaml
deleted file mode 100644
index 51d26f8..0000000
--- a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.0.yaml
+++ /dev/null
@@ -1,22 +0,0 @@
-c_compiler:
-- gcc
-c_compiler_version:
-- '7'
-channel_sources:
-- conda-forge,defaults
-channel_targets:
-- conda-forge main
-cxx_compiler:
-- gxx
-cxx_compiler_version:
-- '7'
-docker_image:
-- condaforge/linux-anvil-comp7
-pin_run_as_build:
- r-base:
- min_pin: x.x
- max_pin: x.x
-r_base:
-- '4.0'
-target_platform:
-- linux-64
diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base3.6.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base3.6.yaml
deleted file mode 100644
index e3c5b89..0000000
--- a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base3.6.yaml
+++ /dev/null
@@ -1,26 +0,0 @@
-MACOSX_DEPLOYMENT_TARGET:
-- '10.9'
-c_compiler:
-- clang
-c_compiler_version:
-- '10'
-channel_sources:
-- conda-forge,defaults
-channel_targets:
-- conda-forge main
-cxx_compiler:
-- clangxx
-cxx_compiler_version:
-- '10'
-macos_machine:
-- x86_64-apple-darwin13.4.0
-macos_min_version:
-- '10.9'
-pin_run_as_build:
- r-base:
- min_pin: x.x
- max_pin: x.x
-r_base:
-- '3.6'
-target_platform:
-- osx-64
diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.0.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.0.yaml
deleted file mode 100644
index 8343a28..0000000
--- a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.0.yaml
+++ /dev/null
@@ -1,26 +0,0 @@
-MACOSX_DEPLOYMENT_TARGET:
-- '10.9'
-c_compiler:
-- clang
-c_compiler_version:
-- '10'
-channel_sources:
-- conda-forge,defaults
-channel_targets:
-- conda-forge main
-cxx_compiler:
-- clangxx
-cxx_compiler_version:
-- '10'
-macos_machine:
-- x86_64-apple-darwin13.4.0
-macos_min_version:
-- '10.9'
-pin_run_as_build:
- r-base:
- min_pin: x.x
- max_pin: x.x
-r_base:
-- '4.0'
-target_platform:
-- osx-64
diff --git a/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base3.6.yaml b/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base3.6.yaml
deleted file mode 100644
index 3fb7f88..0000000
--- a/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base3.6.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
-channel_sources:
-- conda-forge,defaults
-channel_targets:
-- conda-forge main
-pin_run_as_build:
- r-base:
- min_pin: x.x
- max_pin: x.x
-r_base:
-- '3.6'
-target_platform:
-- win-64
diff --git a/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base4.0.yaml b/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base4.0.yaml
deleted file mode 100644
index 02c2a70..0000000
--- a/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base4.0.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
-channel_sources:
-- conda-forge,defaults
-channel_targets:
-- conda-forge main
-pin_run_as_build:
- r-base:
- min_pin: x.x
- max_pin: x.x
-r_base:
-- '4.0'
-target_platform:
-- win-64
diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml
deleted file mode 100644
index 8fbbb64..0000000
--- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml
+++ /dev/null
@@ -1,55 +0,0 @@
-aws_sdk_cpp:
-- 1.8.151
-bzip2:
-- '1'
-c_compiler:
-- vs2017
-channel_sources:
-- conda-forge,defaults
-channel_targets:
-- conda-forge main
-cuda_compiler:
-- nvcc
-cuda_compiler_version:
-- None
-cxx_compiler:
-- vs2017
-gflags:
-- '2.2'
-glog:
-- 0.4.0
-grpc_cpp:
-- '1.36'
-libprotobuf:
-- '3.15'
-lz4_c:
-- 1.9.3
-numpy:
-- '1.17'
-pin_run_as_build:
- bzip2:
- max_pin: x
- lz4-c:
- max_pin: x.x.x
- python:
- min_pin: x.x
- max_pin: x.x
- zlib:
- max_pin: x.x
-python:
-- 3.6.* *_cpython
-re2:
-- 2021.04.01
-snappy:
-- '1'
-target_platform:
-- win-64
-thrift_cpp:
-- 0.14.1
-zip_keys:
-- - numpy
- - python
-zlib:
-- '1.2'
-zstd:
-- '1.4'
diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml
deleted file mode 100644
index 4b702a3..0000000
--- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml
+++ /dev/null
@@ -1,55 +0,0 @@
-aws_sdk_cpp:
-- 1.8.151
-bzip2:
-- '1'
-c_compiler:
-- vs2017
-channel_sources:
-- conda-forge,defaults
-channel_targets:
-- conda-forge main
-cuda_compiler:
-- nvcc
-cuda_compiler_version:
-- None
-cxx_compiler:
-- vs2017
-gflags:
-- '2.2'
-glog:
-- 0.4.0
-grpc_cpp:
-- '1.36'
-libprotobuf:
-- '3.15'
-lz4_c:
-- 1.9.3
-numpy:
-- '1.17'
-pin_run_as_build:
- bzip2:
- max_pin: x
- lz4-c:
- max_pin: x.x.x
- python:
- min_pin: x.x
- max_pin: x.x
- zlib:
- max_pin: x.x
-python:
-- 3.7.* *_cpython
-re2:
-- 2021.04.01
-snappy:
-- '1'
-target_platform:
-- win-64
-thrift_cpp:
-- 0.14.1
-zip_keys:
-- - numpy
- - python
-zlib:
-- '1.2'
-zstd:
-- '1.4'
diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml
deleted file mode 100644
index 6ae6c2f..0000000
--- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml
+++ /dev/null
@@ -1,55 +0,0 @@
-aws_sdk_cpp:
-- 1.8.151
-bzip2:
-- '1'
-c_compiler:
-- vs2017
-channel_sources:
-- conda-forge,defaults
-channel_targets:
-- conda-forge main
-cuda_compiler:
-- nvcc
-cuda_compiler_version:
-- None
-cxx_compiler:
-- vs2017
-gflags:
-- '2.2'
-glog:
-- 0.4.0
-grpc_cpp:
-- '1.36'
-libprotobuf:
-- '3.15'
-lz4_c:
-- 1.9.3
-numpy:
-- '1.17'
-pin_run_as_build:
- bzip2:
- max_pin: x
- lz4-c:
- max_pin: x.x.x
- python:
- min_pin: x.x
- max_pin: x.x
- zlib:
- max_pin: x.x
-python:
-- 3.8.* *_cpython
-re2:
-- 2021.04.01
-snappy:
-- '1'
-target_platform:
-- win-64
-thrift_cpp:
-- 0.14.1
-zip_keys:
-- - numpy
- - python
-zlib:
-- '1.2'
-zstd:
-- '1.4'
diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml
deleted file mode 100644
index 73a8b50..0000000
--- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml
+++ /dev/null
@@ -1,55 +0,0 @@
-aws_sdk_cpp:
-- 1.8.151
-bzip2:
-- '1'
-c_compiler:
-- vs2017
-channel_sources:
-- conda-forge,defaults
-channel_targets:
-- conda-forge main
-cuda_compiler:
-- nvcc
-cuda_compiler_version:
-- None
-cxx_compiler:
-- vs2017
-gflags:
-- '2.2'
-glog:
-- 0.4.0
-grpc_cpp:
-- '1.36'
-libprotobuf:
-- '3.15'
-lz4_c:
-- 1.9.3
-numpy:
-- '1.19'
-pin_run_as_build:
- bzip2:
- max_pin: x
- lz4-c:
- max_pin: x.x.x
- python:
- min_pin: x.x
- max_pin: x.x
- zlib:
- max_pin: x.x
-python:
-- 3.9.* *_cpython
-re2:
-- 2021.04.01
-snappy:
-- '1'
-target_platform:
-- win-64
-thrift_cpp:
-- 0.14.1
-zip_keys:
-- - numpy
- - python
-zlib:
-- '1.2'
-zstd:
-- '1.4'
diff --git a/dev/tasks/conda-recipes/.scripts/logging_utils.sh b/dev/tasks/conda-recipes/.scripts/logging_utils.sh
deleted file mode 100644
index a53ef3f..0000000
--- a/dev/tasks/conda-recipes/.scripts/logging_utils.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-# Provide a unified interface for the different logging
-# utilities CI providers offer. If unavailable, provide
-# a compatible fallback (e.g. bare `echo xxxxxx`).
-
-function startgroup {
- # Start a foldable group of log lines
- # Pass a single argument, quoted
- case ${CI:-} in
- azure )
- echo "##[group]$1";;
- travis )
- echo "$1"
- echo -en 'travis_fold:start:'"${1// /}"'\\r';;
- * )
- echo "$1";;
- esac
-}
-
-function endgroup {
- # End a foldable group of log lines
- # Pass a single argument, quoted
- case ${CI:-} in
- azure )
- echo "##[endgroup]";;
- travis )
- echo -en 'travis_fold:end:'"${1// /}"'\\r';;
- esac
-}
diff --git a/dev/tasks/conda-recipes/README.md b/dev/tasks/conda-recipes/README.md
deleted file mode 100644
index 39f82f1..0000000
--- a/dev/tasks/conda-recipes/README.md
+++ /dev/null
@@ -1,67 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-# Conda Forge recipes
-
-This directory must be migrated periodically with the upstrem updates of
-[arrow-cpp-feedstock][arrow-cpp-feedstock],
-[parquet-cpp-feedstock][parquet-cpp-feedstock].
-conda-forge repositories because of multiple vendored files.
-
-## Keeping the recipes synchronized
-
-The recipes here are tested on nightly basis, so they follow the development
-versions of arrow instead of the upstream recipes, which are suitable for the
-latest releases.
-
-### Backporting from the upstream feedstocks
-
-In most of the cases these recipes are more accurate, then the upstream
-feedstocks. Although the upstream feedstocks regularly receive automatic updates
-by the conda-forge team so we need to backport those changes to the crossbow
-recipes. Most of these updates are touching the version pinning files
-(under `.ci_support`) and other CI related configuration files.
-
-Because all three recipes must be built in the same continuous integration
-job prefer porting from the [arrpw-cpp feedstock][arrow-cpp-feedstock].
-
-#### Updating the variants:
-
-Copy the configuration files from `arrow-cpp-feedstock/.ci_support` to the
-`.ci_support` folder.
-
-#### Updating the CI configurations:
-
-The `.azure-pipelines/azure-pipelines-[linux|osx|win].yml` should be ported
-to the local counterparts under `.azure-pipelines` with keeping the crossbow
-related parts (the cloning of arrow and the jinja templated variables) and
-moving the matrix definitions like [this][matrix-definition] to the crossbow
-[tasks.yml][../tasks.yml] config file.
-
-
-### Porting recipes from crossbow to the upstream feedstocks
-
-Theoretically these recipes should be up to date with the actual version of
-Arrow, so during the release procedure the content of these recipes should be
-copied to the upstream feedstocks.
-
-
-[arrow-cpp-feedstock]: https://github.com/conda-forge/arrow-cpp-feedstock
-[parquet-cpp-feedstock]: https://github.com/conda-forge/parquet-cpp-feedstock
-[matrix-definition]: https://github.com/conda-forge/arrow-cpp-feedstock/blob/master/.azure-pipelines/azure-pipelines-linux.yml#L12
diff --git a/dev/tasks/conda-recipes/arrow-cpp/LLVM_LICENSE.txt b/dev/tasks/conda-recipes/arrow-cpp/LLVM_LICENSE.txt
deleted file mode 100644
index 461398b..0000000
--- a/dev/tasks/conda-recipes/arrow-cpp/LLVM_LICENSE.txt
+++ /dev/null
@@ -1,68 +0,0 @@
-==============================================================================
-LLVM Release License
-==============================================================================
-University of Illinois/NCSA
-Open Source License
-
-Copyright (c) 2003-2018 University of Illinois at Urbana-Champaign.
-All rights reserved.
-
-Developed by:
-
- LLVM Team
-
- University of Illinois at Urbana-Champaign
-
- http://llvm.org
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal with
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
- * Redistributions of source code must retain the above copyright notice,
- this list of conditions and the following disclaimers.
-
- * Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimers in the
- documentation and/or other materials provided with the distribution.
-
- * Neither the names of the LLVM Team, University of Illinois at
- Urbana-Champaign, nor the names of its contributors may be used to
- endorse or promote products derived from this Software without specific
- prior written permission.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
-SOFTWARE.
-
-==============================================================================
-Copyrights and Licenses for Third Party Software Distributed with LLVM:
-==============================================================================
-The LLVM software contains code written by third parties. Such software will
-have its own individual LICENSE.TXT file in the directory in which it appears.
-This file will describe the copyrights, license, and restrictions which apply
-to that code.
-
-The disclaimer of warranty in the University of Illinois Open Source License
-applies to all code in the LLVM Distribution, and nothing in any of the
-other licenses gives permission to use the names of the LLVM Team or the
-University of Illinois to endorse or promote products derived from this
-Software.
-
-The following pieces of software have additional or alternate copyrights,
-licenses, and/or restrictions:
-
-Program Directory
-------- ---------
-Google Test llvm/utils/unittest/googletest
-OpenBSD regex llvm/lib/Support/{reg*, COPYRIGHT.regex}
-pyyaml tests llvm/test/YAMLParser/{*.data, LICENSE.TXT}
-ARM contributions llvm/lib/Target/ARM/LICENSE.TXT
-md5 contributions llvm/lib/Support/MD5.cpp llvm/include/llvm/Support/MD5.h
diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat b/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat
deleted file mode 100644
index 2cc6ed1..0000000
--- a/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat
+++ /dev/null
@@ -1,54 +0,0 @@
-@echo on
-
-mkdir "%SRC_DIR%"\cpp\build
-pushd "%SRC_DIR%"\cpp\build
-
-:: Enable CUDA support
-if "%cuda_compiler_version%"=="None" (
- set "EXTRA_CMAKE_ARGS=-DARROW_CUDA=OFF"
-) else (
- REM this should move to nvcc-feedstock
- set "CUDA_PATH=%CUDA_PATH:\=/%"
- set "CUDA_HOME=%CUDA_HOME:\=/%"
-
- set "EXTRA_CMAKE_ARGS=-DARROW_CUDA=ON"
-)
-
-cmake -G "Ninja" ^
- -DBUILD_SHARED_LIBS=ON ^
- -DCMAKE_INSTALL_PREFIX="%LIBRARY_PREFIX%" ^
- -DARROW_DEPENDENCY_SOURCE=SYSTEM ^
- -DARROW_PACKAGE_PREFIX="%LIBRARY_PREFIX%" ^
- -DLLVM_TOOLS_BINARY_DIR="%LIBRARY_BIN%" ^
- -DPython3_EXECUTABLE="%PYTHON%" ^
- -DARROW_WITH_BZ2:BOOL=ON ^
- -DARROW_WITH_ZLIB:BOOL=ON ^
- -DARROW_WITH_ZSTD:BOOL=ON ^
- -DARROW_WITH_LZ4:BOOL=ON ^
- -DARROW_WITH_SNAPPY:BOOL=ON ^
- -DARROW_WITH_BROTLI:BOOL=ON ^
- -DARROW_BOOST_USE_SHARED:BOOL=ON ^
- -DARROW_BUILD_TESTS:BOOL=OFF ^
- -DARROW_BUILD_UTILITIES:BOOL=OFF ^
- -DARROW_BUILD_STATIC:BOOL=OFF ^
- -DCMAKE_BUILD_TYPE=release ^
- -DARROW_SSE42:BOOL=OFF ^
- -DARROW_PYTHON:BOOL=ON ^
- -DARROW_MIMALLOC:BOOL=ON ^
- -DARROW_DATASET:BOOL=ON ^
- -DARROW_FLIGHT:BOOL=ON ^
- -DARROW_HDFS:BOOL=ON ^
- -DARROW_PARQUET:BOOL=ON ^
- -DARROW_GANDIVA:BOOL=ON ^
- -DARROW_ORC:BOOL=ON ^
- -DARROW_S3:BOOL=ON ^
- -DBoost_NO_BOOST_CMAKE=ON ^
- -DCMAKE_UNITY_BUILD=ON ^
- %EXTRA_CMAKE_ARGS% ^
- ..
-if errorlevel 1 exit 1
-
-cmake --build . --target install --config Release
-if errorlevel 1 exit 1
-
-popd
diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat b/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat
deleted file mode 100644
index 89cec37..0000000
--- a/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat
+++ /dev/null
@@ -1,44 +0,0 @@
-@echo on
-pushd "%SRC_DIR%"\python
-
-@rem the symlinks for cmake modules don't work here
-@rem NOTE: In contrast to conda-forge, they work here as we clone from git.
-@rem del cmake_modules\BuildUtils.cmake
-@rem del cmake_modules\SetupCxxFlags.cmake
-@rem del cmake_modules\CompilerInfo.cmake
-@rem del cmake_modules\FindNumPy.cmake
-@rem del cmake_modules\FindPythonLibsNew.cmake
-@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\BuildUtils.cmake" cmake_modules\
-@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\SetupCxxFlags.cmake" cmake_modules\
-@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\CompilerInfo.cmake" cmake_modules\
-@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\FindNumPy.cmake" cmake_modules\
-@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\FindPythonLibsNew.cmake" cmake_modules\
-
-SET ARROW_HOME=%LIBRARY_PREFIX%
-SET SETUPTOOLS_SCM_PRETEND_VERSION=%PKG_VERSION%
-SET PYARROW_BUILD_TYPE=release
-SET PYARROW_WITH_S3=1
-SET PYARROW_WITH_HDFS=1
-SET PYARROW_WITH_DATASET=1
-SET PYARROW_WITH_FLIGHT=1
-SET PYARROW_WITH_GANDIVA=1
-SET PYARROW_WITH_PARQUET=1
-SET PYARROW_CMAKE_GENERATOR=Ninja
-
-:: Enable CUDA support
-if "%cuda_compiler_version%"=="None" (
- set "PYARROW_WITH_CUDA=0"
-) else (
- set "PYARROW_WITH_CUDA=1"
-)
-
-%PYTHON% setup.py ^
- build_ext ^
- install --single-version-externally-managed ^
- --record=record.txt
-if errorlevel 1 exit 1
-popd
-
-if [%PKG_NAME%] == [pyarrow] (
- rd /s /q %SP_DIR%\pyarrow\tests
-)
diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh
deleted file mode 100644
index f9c1d97..0000000
--- a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh
+++ /dev/null
@@ -1,95 +0,0 @@
-#!/bin/bash
-
-set -e
-set -x
-
-mkdir cpp/build
-pushd cpp/build
-
-EXTRA_CMAKE_ARGS=""
-
-# Include g++'s system headers
-if [ "$(uname)" == "Linux" ]; then
- SYSTEM_INCLUDES=$(echo | ${CXX} -E -Wp,-v -xc++ - 2>&1 | grep '^ ' | awk '{print "-isystem;" substr($1, 1)}' | tr '\n' ';')
- EXTRA_CMAKE_ARGS=" -DARROW_GANDIVA_PC_CXX_FLAGS=${SYSTEM_INCLUDES}"
-fi
-
-# Enable CUDA support
-if [[ ! -z "${cuda_compiler_version+x}" && "${cuda_compiler_version}" != "None" ]]
-then
- if [[ -z "${CUDA_HOME+x}" ]]
- then
- echo "cuda_compiler_version=${cuda_compiler_version} CUDA_HOME=$CUDA_HOME"
- CUDA_GDB_EXECUTABLE=$(which cuda-gdb || exit 0)
- if [[ -n "$CUDA_GDB_EXECUTABLE" ]]
- then
- CUDA_HOME=$(dirname $(dirname $CUDA_GDB_EXECUTABLE))
- else
- echo "Cannot determine CUDA_HOME: cuda-gdb not in PATH"
- return 1
- fi
- fi
- EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=ON -DCUDA_TOOLKIT_ROOT_DIR=${CUDA_HOME} -DCMAKE_LIBRARY_PATH=${CUDA_HOME}/lib64/stubs"
-else
- EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=OFF"
-fi
-
-if [[ "${target_platform}" == "osx-arm64" ]]; then
- # We need llvm 11+ support in Arrow for this
- EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_GANDIVA=OFF"
- sed -ie "s;protoc-gen-grpc.*$;protoc-gen-grpc=${BUILD_PREFIX}/bin/grpc_cpp_plugin\";g" ../src/arrow/flight/CMakeLists.txt
- sed -ie 's;"--with-jemalloc-prefix\=je_arrow_";"--with-jemalloc-prefix\=je_arrow_" "--with-lg-page\=14";g' ../cmake_modules/ThirdpartyToolchain.cmake
-else
- EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_GANDIVA=ON"
-fi
-
-cmake \
- -DARROW_BOOST_USE_SHARED=ON \
- -DARROW_BUILD_BENCHMARKS=OFF \
- -DARROW_BUILD_STATIC=OFF \
- -DARROW_BUILD_TESTS=OFF \
- -DARROW_BUILD_UTILITIES=OFF \
- -DBUILD_SHARED_LIBS=ON \
- -DARROW_DATASET=ON \
- -DARROW_DEPENDENCY_SOURCE=SYSTEM \
- -DARROW_FLIGHT=ON \
- -DARROW_FLIGHT_REQUIRE_TLSCREDENTIALSOPTIONS=ON \
- -DARROW_HDFS=ON \
- -DARROW_JEMALLOC=ON \
- -DARROW_MIMALLOC=ON \
- -DARROW_ORC=ON \
- -DARROW_PACKAGE_PREFIX=$PREFIX \
- -DARROW_PARQUET=ON \
- -DARROW_PLASMA=ON \
- -DARROW_PYTHON=ON \
- -DARROW_S3=ON \
- -DARROW_SIMD_LEVEL=NONE \
- -DARROW_USE_LD_GOLD=ON \
- -DARROW_WITH_BROTLI=ON \
- -DARROW_WITH_BZ2=ON \
- -DARROW_WITH_LZ4=ON \
- -DARROW_WITH_SNAPPY=ON \
- -DARROW_WITH_ZLIB=ON \
- -DARROW_WITH_ZSTD=ON \
- -DCMAKE_BUILD_TYPE=release \
- -DCMAKE_INSTALL_LIBDIR=lib \
- -DCMAKE_INSTALL_PREFIX=$PREFIX \
- -DLLVM_TOOLS_BINARY_DIR=$PREFIX/bin \
- -DPython3_EXECUTABLE=${PYTHON} \
- -DProtobuf_PROTOC_EXECUTABLE=$BUILD_PREFIX/bin/protoc \
- -GNinja \
- ${EXTRA_CMAKE_ARGS} \
- ..
-
-# Commented out until jemalloc and mimalloc are fixed upstream
-if [[ "${target_platform}" == "osx-arm64" ]]; then
- ninja jemalloc_ep-prefix/src/jemalloc_ep-stamp/jemalloc_ep-patch mimalloc_ep-prefix/src/mimalloc_ep-stamp/mimalloc_ep-patch
- cp $BUILD_PREFIX/share/gnuconfig/config.* jemalloc_ep-prefix/src/jemalloc_ep/build-aux/
- sed -ie 's/list(APPEND mi_cflags -march=native)//g' mimalloc_ep-prefix/src/mimalloc_ep/CMakeLists.txt
- # Use the correct register for thread-local storage
- sed -ie 's/tpidr_el0/tpidrro_el0/g' mimalloc_ep-prefix/src/mimalloc_ep/include/mimalloc-internal.h
-fi
-
-ninja install
-
-popd
diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh
deleted file mode 100644
index a394e99..0000000
--- a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/bin/sh
-
-set -e
-set -x
-
-# Build dependencies
-export ARROW_HOME=$PREFIX
-export PARQUET_HOME=$PREFIX
-export SETUPTOOLS_SCM_PRETEND_VERSION=$PKG_VERSION
-export PYARROW_BUILD_TYPE=release
-export PYARROW_BUNDLE_ARROW_CPP_HEADERS=0
-export PYARROW_WITH_DATASET=1
-export PYARROW_WITH_FLIGHT=1
-if [[ "${target_platform}" == "osx-arm64" ]]; then
- # We need llvm 11+ support in Arrow for this
- export PYARROW_WITH_GANDIVA=0
-else
- export PYARROW_WITH_GANDIVA=1
-fi
-export PYARROW_WITH_HDFS=1
-export PYARROW_WITH_ORC=1
-export PYARROW_WITH_PARQUET=1
-export PYARROW_WITH_PLASMA=1
-export PYARROW_WITH_S3=1
-export PYARROW_CMAKE_GENERATOR=Ninja
-BUILD_EXT_FLAGS=""
-
-# Enable CUDA support
-if [[ ! -z "${cuda_compiler_version+x}" && "${cuda_compiler_version}" != "None" ]]; then
- export PYARROW_WITH_CUDA=1
-else
- export PYARROW_WITH_CUDA=0
-fi
-
-# Resolve: Make Error at cmake_modules/SetupCxxFlags.cmake:338 (message): Unsupported arch flag: -march=.
-if [[ "${target_platform}" == "linux-aarch64" ]]; then
- export PYARROW_CMAKE_OPTIONS="-DARROW_ARMV8_ARCH=armv8-a"
-fi
-
-cd python
-
-$PYTHON setup.py \
- build_ext \
- install --single-version-externally-managed \
- --record=record.txt
-
-if [[ "$PKG_NAME" == "pyarrow" ]]; then
- rm -r ${SP_DIR}/pyarrow/tests
-fi
diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml
deleted file mode 100644
index 48a8629..0000000
--- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml
+++ /dev/null
@@ -1,302 +0,0 @@
-# NOTE: In constrast to the conda-forge recipe, ARROW_VERSION is a templated variable here.
-{% set version = ARROW_VERSION %}
-{% set cuda_enabled = cuda_compiler_version != "None" %}
-{% set build_ext_version = ARROW_VERSION %}
-{% set build_ext = "cuda" if cuda_enabled else "cpu" %}
-{% set proc_build_number = "0" %}
-
-package:
- name: arrow-cpp-ext
- version: {{ version }}
-
-source:
- path: ../../../../
-
-build:
- number: 0
- # for cuda on win/linux, building with 9.2 is enough to be compatible with all later versions,
- # since arrow is only using libcuda, and not libcudart.
- skip: true # [(win or linux) and cuda_compiler_version not in ("None", "10.2")]
- skip: true # [osx and cuda_compiler_version != "None"]
- run_exports:
- - {{ pin_subpackage("arrow-cpp", max_pin="x.x.x") }}
-
-outputs:
- - name: arrow-cpp-proc
- version: {{ build_ext_version }}
- build:
- number: {{ proc_build_number }}
- string: "{{ build_ext }}"
- test:
- commands:
- - exit 0
- about:
- home: http://github.com/apache/arrow
- license: Apache-2.0
- license_file:
- - LICENSE.txt
- summary: 'A meta-package to select Arrow build variant'
-
- - name: arrow-cpp
- script: build-arrow.sh # [not win]
- script: bld-arrow.bat # [win]
- version: {{ version }}
- build:
- string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }}
- run_exports:
- - {{ pin_subpackage("arrow-cpp", max_pin="x.x.x") }}
- ignore_run_exports:
- - cudatoolkit
- track_features:
- {{ "- arrow-cuda" if cuda_enabled else "" }}
- requirements:
- build:
- - python # [build_platform != target_platform]
- - cross-python_{{ target_platform }} # [build_platform != target_platform]
- - cython # [build_platform != target_platform]
- - numpy # [build_platform != target_platform]
- - gnuconfig # [osx and arm64]
- - libprotobuf
- - grpc-cpp
- - cmake
- - autoconf # [unix]
- - ninja
- - make # [unix]
- - {{ compiler('c') }}
- - {{ compiler('cxx') }}
- - {{ compiler("cuda") }} # [cuda_compiler_version != "None"]
- host:
- - aws-sdk-cpp
- - boost-cpp >=1.70
- - brotli
- - bzip2
- - c-ares
- - gflags
- - glog
- - grpc-cpp
- - libprotobuf
- - clangdev 10 # [not (osx and arm64)]
- - llvmdev 10 # [not (osx and arm64)]
- - libutf8proc
- - lz4-c
- - numpy
- - orc # [unix]
- - python
- - rapidjson
- - re2
- - snappy
- - thrift-cpp
- - zlib
- - zstd
- run:
- - {{ pin_compatible('numpy', lower_bound='1.16') }}
- - python
- run_constrained:
- - arrow-cpp-proc * {{ build_ext }}
- - cudatoolkit >=9.2 # [cuda_compiler_version != "None"]
-
- about:
- home: http://github.com/apache/arrow
- license: Apache-2.0
- license_file:
- - LICENSE.txt
- summary: C++ libraries for Apache Arrow
-
- test:
- commands:
- # headers
- - test -f $PREFIX/include/arrow/api.h # [unix]
- - test -f $PREFIX/include/arrow/flight/types.h # [unix]
- - test -f $PREFIX/include/plasma/client.h # [unix]
- - test -f $PREFIX/include/gandiva/engine.h # [unix and not (osx and arm64)]
- - test -f $PREFIX/include/parquet/api/reader.h # [unix]
- - if not exist %LIBRARY_INC%\\arrow\\api.h exit 1 # [win]
- - if not exist %LIBRARY_INC%\\gandiva\\engine.h exit 1 # [win]
- - if not exist %LIBRARY_INC%\\parquet\\api\\reader.h exit 1 # [win]
-
- # shared
- - test -f $PREFIX/lib/libarrow.so # [linux]
- - test -f $PREFIX/lib/libarrow_dataset.so # [linux]
- - test -f $PREFIX/lib/libarrow_flight.so # [linux]
- - test -f $PREFIX/lib/libarrow_python.so # [linux]
- - test -f $PREFIX/lib/libparquet.so # [linux]
- - test -f $PREFIX/lib/libgandiva.so # [linux]
- - test -f $PREFIX/lib/libplasma.so # [linux]
- - test -f $PREFIX/lib/libarrow_cuda${SHLIB_EXT} # [(cuda_compiler_version != "None") and unix]
- - test ! -f $PREFIX/lib/libarrow_cuda${SHLIB_EXT} # [(cuda_compiler_version == "None") and unix]
- - if not exist %PREFIX%\\Library\\bin\\arrow_cuda.dll exit 1 # [(cuda_compiler_version != "None") and win]
- - if exist %PREFIX%\\Library\\bin\\arrow_cuda.dll exit 1 # [(cuda_compiler_version == "None") and win]
- - test -f $PREFIX/lib/libarrow.dylib # [osx]
- - test -f $PREFIX/lib/libarrow_dataset.dylib # [osx]
- - test -f $PREFIX/lib/libarrow_python.dylib # [osx]
- - test -f $PREFIX/lib/libgandiva.dylib # [osx and not arm64]
- - test -f $PREFIX/lib/libparquet.dylib # [osx]
- - test -f $PREFIX/lib/libplasma.dylib # [osx]
- - if not exist %PREFIX%\\Library\\bin\\arrow.dll exit 1 # [win]
- - if not exist %PREFIX%\\Library\\bin\\arrow_dataset.dll exit 1 # [win]
- - if not exist %PREFIX%\\Library\\bin\\arrow_flight.dll exit 1 # [win]
- - if not exist %PREFIX%\\Library\\bin\\arrow_python.dll exit 1 # [win]
- - if not exist %PREFIX%\\Library\\bin\\parquet.dll exit 1 # [win]
- - if not exist %PREFIX%\\Library\\bin\\gandiva.dll exit 1 # [win]
-
- # absence of static libraries
- - test ! -f $PREFIX/lib/libarrow.a # [unix]
- - test ! -f $PREFIX/lib/libarrow_dataset.a # [unix]
- - test ! -f $PREFIX/lib/libarrow_flight.a # [unix]
- - test ! -f $PREFIX/lib/libarrow_python.a # [unix]
- - test ! -f $PREFIX/lib/libplasma.a # [unix]
- - test ! -f $PREFIX/lib/libparquet.a # [unix]
- - test ! -f $PREFIX/lib/libgandiva.a # [unix]
- - if exist %PREFIX%\\Library\\lib\\arrow_static.lib exit 1 # [win]
- - if exist %PREFIX%\\Library\\lib\\arrow_dataset_static.lib exit 1 # [win]
- - if exist %PREFIX%\\Library\\lib\\arrow_flight_static.lib exit 1 # [win]
- - if exist %PREFIX%\\Library\\lib\\arrow_python_static.lib exit 1 # [win]
- - if exist %PREFIX%\\Library\\lib\\parquet_static.lib exit 1 # [win]
- - if exist %PREFIX%\\Library\\lib\\gandiva_static.lib exit 1 # [win]
-
- - name: pyarrow
- script: build-pyarrow.sh # [not win]
- script: bld-pyarrow.bat # [win]
- version: {{ version }}
- build:
- string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }}
- ignore_run_exports:
- - cudatoolkit
- track_features:
- {{ "- arrow-cuda" if cuda_enabled else "" }}
- requirements:
- build:
- - python # [build_platform != target_platform]
- - cross-python_{{ target_platform }} # [build_platform != target_platform]
- - cython # [build_platform != target_platform]
- - numpy # [build_platform != target_platform]
- - cmake
- - ninja
- - make # [unix]
- - {{ compiler('c') }}
- - {{ compiler('cxx') }}
- # pyarrow does not require nvcc but it needs to link against libraries in arrow-cpp=*=*cuda
- - {{ compiler("cuda") }} # [cuda_compiler_version != "None"]
- host:
- - {{ pin_subpackage('arrow-cpp', exact=True) }}
- - cython
- - numpy
- - python
- - setuptools
- - setuptools_scm
- - six
- run:
- - {{ pin_subpackage('arrow-cpp', exact=True) }}
- - {{ pin_compatible('numpy', lower_bound='1.16') }}
- # empty parquet-cpp metapackage, force old versions to be uninstalled
- - parquet-cpp 1.5.1.*
- - python
- run_constrained:
- - arrow-cpp-proc * {{ build_ext }}
- - cudatoolkit >=9.2 # [cuda_compiler_version != "None"]
-
- about:
- home: http://github.com/apache/arrow
- license: Apache-2.0
- license_file:
- - LICENSE.txt
- summary: Python libraries for Apache Arrow
-
- test:
- imports:
- - pyarrow
- - pyarrow.dataset
- - pyarrow.flight
- - pyarrow.gandiva # [not (osx and arm64)]
- - pyarrow.orc # [unix]
- - pyarrow.parquet
- - pyarrow.plasma # [unix]
- - pyarrow.fs
- - pyarrow._s3fs
- - pyarrow._hdfs
- # We can only test importing cuda package but cannot run when a
- # CUDA device is not available, for instance, when building from CI.
- # On Windows, we cannot even do that due to `nvcuda.dll` not being found, see
- # https://conda-forge.org/docs/maintainer/knowledge_base.html#nvcuda-dll-cannot-be-found-on-windows
- # However, we check below for (at least) the presence of a correctly-compiled module
- - pyarrow.cuda # [cuda_compiler_version != "None" and not win]
- commands:
- - test ! -f ${SP_DIR}/pyarrow/tests/test_array.py # [unix]
- - if exist %SP_DIR%/pyarrow/tests/test_array.py exit 1 # [win]
- # Need to remove dot from PY_VER; %MYVAR:x=y% replaces "x" in %MYVAR% with "y"
- - if not exist %SP_DIR%/pyarrow/_cuda.cp%PY_VER:.=%-win_amd64.pyd exit 1 # [win and cuda_compiler_version != "None"]
-
- - name: pyarrow-tests
- script: build-pyarrow.sh # [not win]
- script: bld-pyarrow.bat # [win]
- version: {{ version }}
- build:
- string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }}
- ignore_run_exports:
- - cudatoolkit
- track_features:
- {{ "- arrow-cuda" if cuda_enabled else "" }}
- requirements:
- build:
- - python # [build_platform != target_platform]
- - cross-python_{{ target_platform }} # [build_platform != target_platform]
- - cython # [build_platform != target_platform]
- - numpy # [build_platform != target_platform]
- - cmake
- - ninja
- - make # [unix]
- - {{ compiler('c') }}
- - {{ compiler('cxx') }}
- # pyarrow does not require nvcc but it needs to link against libraries in arrow-cpp=*=*cuda
- - {{ compiler("cuda") }} # [cuda_compiler_version != "None"]
- host:
- - {{ pin_subpackage('arrow-cpp', exact=True) }}
- - {{ pin_subpackage('pyarrow', exact=True) }}
- - cython
- - numpy
- - python
- - setuptools
- - setuptools_scm
- - six
- run:
- - {{ pin_subpackage('pyarrow', exact=True) }}
- - python
- run_constrained:
- - arrow-cpp-proc * {{ build_ext }}
- - cudatoolkit >=9.2 # [cuda_compiler_version != "None"]
-
- about:
- home: http://github.com/apache/arrow
- license: Apache-2.0
- license_file:
- - LICENSE.txt
- summary: Python test files for Apache Arrow
-
- test:
- commands:
- - test -f ${SP_DIR}/pyarrow/tests/test_array.py # [unix]
- - if not exist %SP_DIR%/pyarrow/tests/test_array.py exit 1 # [win]
-
-about:
- home: http://github.com/apache/arrow
- license: Apache-2.0
- license_file:
- - LICENSE.txt
- summary: C++ and Python libraries for Apache Arrow
-
-extra:
- recipe-maintainers:
- - wesm
- - xhochy
- - leifwalsh
- - jreback
- - cpcloud
- - pcmoritz
- - robertnishihara
- - siddharthteotia
- - kou
- - kszucs
- - pitrou
- - pearu
- - nealrichardson
- - jakirkham
diff --git a/dev/tasks/conda-recipes/azure.clean.yml b/dev/tasks/conda-recipes/azure.clean.yml
deleted file mode 100644
index 55ac365..0000000
--- a/dev/tasks/conda-recipes/azure.clean.yml
+++ /dev/null
@@ -1,28 +0,0 @@
-jobs:
-- job: linux
- pool:
- vmImage: ubuntu-18.04
- timeoutInMinutes: 360
-
- steps:
- - script: |
- git clone --no-checkout {{ arrow.remote }} arrow
- git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
- git -C arrow checkout FETCH_HEAD
- git -C arrow submodule update --init --recursive
- displayName: Clone arrow
-
- - script: |
- conda install -y -c conda-forge pandas anaconda-client packaging
- displayName: Install requirements
-
- - script: |
- {% if arrow.branch == 'master' %}
- mkdir -p $HOME/.continuum/anaconda-client/tokens/
- echo $(CROSSBOW_ANACONDA_TOKEN) > $HOME/.continuum/anaconda-client/tokens/https%3A%2F%2Fapi.anaconda.org.token
- {% endif %}
- eval "$(conda shell.bash hook)"
- conda activate base
- python3 arrow/dev/tasks/conda-recipes/clean.py {% if arrow.branch == 'master' %}FORCE{% endif %}
- displayName: Delete outdated packages
-
diff --git a/dev/tasks/conda-recipes/azure.linux.yml b/dev/tasks/conda-recipes/azure.linux.yml
deleted file mode 100755
index 161fd14..0000000
--- a/dev/tasks/conda-recipes/azure.linux.yml
+++ /dev/null
@@ -1,38 +0,0 @@
-{% import 'macros.jinja' as macros with context %}
-
-jobs:
-- job: linux
- pool:
- vmImage: ubuntu-16.04
- timeoutInMinutes: 360
-
- variables:
- CONFIG: {{ config }}
- R_CONFIG: {{ r_config|default("") }}
- ARROW_VERSION: {{ arrow.no_rc_version }}
- UPLOAD_PACKAGES: False
-
- steps:
- # configure qemu binfmt-misc running. This allows us to run docker containers
- # embedded qemu-static
- - script: |
- docker run --rm --privileged multiarch/qemu-user-static:register --reset --credential yes
- ls /proc/sys/fs/binfmt_misc/
- displayName: Configure binfmt_misc
- condition: not(startsWith(variables['CONFIG'], 'linux_64'))
-
- {{ macros.azure_checkout_arrow() }}
-
- - task: CondaEnvironment@1
- inputs:
- packageSpecs: 'anaconda-client shyaml'
- installOptions: '-c conda-forge'
- updateConda: false
-
- - script: |
- mkdir build_artifacts
- CI=azure arrow/dev/tasks/conda-recipes/run_docker_build.sh $(pwd)/build_artifacts
- displayName: Run docker build
-
- {{ macros.azure_upload_releases("build_artifacts/linux-64/*.tar.bz2") }}
- {{ macros.azure_upload_anaconda("build_artifacts/linux-64/*.tar.bz2") }}
diff --git a/dev/tasks/conda-recipes/azure.osx.yml b/dev/tasks/conda-recipes/azure.osx.yml
deleted file mode 100755
index dbb1a68..0000000
--- a/dev/tasks/conda-recipes/azure.osx.yml
+++ /dev/null
@@ -1,80 +0,0 @@
-{% import 'macros.jinja' as macros with context %}
-
-jobs:
-- job: osx
- pool:
- vmImage: macOS-10.14
- timeoutInMinutes: 360
- variables:
- CONFIG: {{ config }}
- R_CONFIG: {{ r_config|default("") }}
- ARROW_VERSION: {{ arrow.no_rc_version }}
- UPLOAD_PACKAGES: False
- steps:
- - script: |
- echo "Removing homebrew from Azure to avoid conflicts."
- curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/uninstall > ~/uninstall_homebrew
- chmod +x ~/uninstall_homebrew
- ~/uninstall_homebrew -fq
- rm ~/uninstall_homebrew
- displayName: Remove homebrew
-
- - bash: |
- echo "##vso[task.prependpath]$CONDA/bin"
- sudo chown -R $USER $CONDA
- displayName: Add conda to PATH
-
- - script: |
- source activate base
- conda install -n base -c conda-forge --quiet --yes conda-forge-ci-setup=3 conda-build
- displayName: 'Add conda-forge-ci-setup=3'
-
- {{ macros.azure_checkout_arrow() }}
-
- - script: |
- source activate base
- echo "Configuring conda."
-
- setup_conda_rc ./ ./ ./.ci_support/${CONFIG}.yaml
- export CI=azure
- source run_conda_forge_build_setup
- conda update --yes --quiet --override-channels -c conda-forge -c defaults --all
- displayName: Configure conda and conda-build
- workingDirectory: arrow/dev/tasks/conda-recipes
- env:
- OSX_FORCE_SDK_DOWNLOAD: "1"
-
- - script: |
- source activate base
- mangle_compiler ./ ./ ./.ci_support/${CONFIG}.yaml
- workingDirectory: arrow/dev/tasks/conda-recipes
- displayName: Mangle compiler
-
- - script: |
- source activate base
- make_build_number ./ ./ ./.ci_support/${CONFIG}.yaml
- workingDirectory: arrow/dev/tasks/conda-recipes
- displayName: Generate build number clobber file
-
- - script: |
- source activate base
- set +x
- if [[ "${CONFIG}" == osx_arm* ]]; then
- EXTRA_CB_OPTIONS="${EXTRA_CB_OPTIONS:-} --no-test"
- fi
- conda build arrow-cpp \
- -m ./.ci_support/${CONFIG}.yaml \
- --clobber-file ./.ci_support/clobber_${CONFIG}.yaml \
- ${EXTRA_CB_OPTIONS:-} \
- --output-folder ./build_artifacts
-
- if [ ! -z "${R_CONFIG}" ]; then
- conda build r-arrow \
- -m ./.ci_support/r/${R_CONFIG}.yaml \
- --output-folder ./build_artifacts
- fi
- workingDirectory: arrow/dev/tasks/conda-recipes
- displayName: Build recipes
-
- {{ macros.azure_upload_releases("arrow/dev/tasks/conda-recipes/build_artifacts/osx-*/*.tar.bz2") }}
- {{ macros.azure_upload_anaconda("arrow/dev/tasks/conda-recipes/build_artifacts/osx-*/*.tar.bz2") }}
diff --git a/dev/tasks/conda-recipes/azure.win.yml b/dev/tasks/conda-recipes/azure.win.yml
deleted file mode 100755
index a3ec693..0000000
--- a/dev/tasks/conda-recipes/azure.win.yml
+++ /dev/null
@@ -1,77 +0,0 @@
-{% import 'macros.jinja' as macros with context %}
-
-jobs:
-- job: win
- pool:
- vmImage: vs2017-win2016
- timeoutInMinutes: 360
- variables:
- CONFIG: {{ config }}
- R_CONFIG: {{ r_config|default("") }}
- ARROW_VERSION: {{ arrow.no_rc_version }}
- CONDA_BLD_PATH: D:\\bld\\
- UPLOAD_PACKAGES: False
-
- steps:
- - script: |
- choco install vcpython27 -fdv -y --debug
- condition: contains(variables['CONFIG'], 'vs2008')
- displayName: Install vcpython27.msi (if needed)
-
- - powershell: |
- Set-PSDebug -Trace 1
- $batchcontent = @"
- ECHO ON
- SET vcpython=C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0
- DIR "%vcpython%"
- CALL "%vcpython%\vcvarsall.bat" %*
- "@
- $batchDir = "C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0\VC"
- $batchPath = "$batchDir" + "\vcvarsall.bat"
- New-Item -Path $batchPath -ItemType "file" -Force
- Set-Content -Value $batchcontent -Path $batchPath
- Get-ChildItem -Path $batchDir
- Get-ChildItem -Path ($batchDir + '\..')
- condition: contains(variables['CONFIG'], 'vs2008')
- displayName: Patch vs2008 (if needed)
-
- - task: CondaEnvironment@1
- inputs:
- packageSpecs: 'python=3.6 conda-build conda conda-forge::conda-forge-ci-setup=3 pip' # Optional
- installOptions: "-c conda-forge"
- updateConda: true
- displayName: Install conda-build and activate environment
- - script: set PYTHONUNBUFFERED=1
-
- {{ macros.azure_checkout_arrow()|indent(2) }}
-
- # Configure the VM
- - script: setup_conda_rc .\ .\ .\.ci_support\%CONFIG%.yaml
- workingDirectory: arrow\dev\tasks\conda-recipes
-
- # Configure the VM.
- - script: |
- set "CI=azure"
- call activate base
- run_conda_forge_build_setup
- displayName: conda-forge build setup
- workingDirectory: arrow\dev\tasks\conda-recipes
-
- - script: |
- conda.exe build arrow-cpp parquet-cpp -m .ci_support\%CONFIG%.yaml
- displayName: Build recipe
- workingDirectory: arrow\dev\tasks\conda-recipes
- env:
- PYTHONUNBUFFERED: 1
- condition: not(contains(variables['CONFIG'], 'vs2008'))
-
- - script: |
- conda.exe build r-arrow -m .ci_support\r\%R_CONFIG%.yaml
- displayName: Build recipe
- workingDirectory: arrow\dev\tasks\conda-recipes
- env:
- PYTHONUNBUFFERED: 1
- condition: contains(variables['R_CONFIG'], 'win')
-
- {{ macros.azure_upload_releases("D:\bld\win-64\*.tar.bz2")|indent(2) }}
- {{ macros.azure_upload_anaconda("D:\bld\win-64\*.tar.bz2")|indent(2) }}
diff --git a/dev/tasks/conda-recipes/build_steps.sh b/dev/tasks/conda-recipes/build_steps.sh
deleted file mode 100755
index 25864c0..0000000
--- a/dev/tasks/conda-recipes/build_steps.sh
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/usr/bin/env bash
-
-# NOTE: This script has been slightly adopted to suite the Apache Arrow / crossbow CI
-# setup. The next time this is updated to the current version on conda-forge,
-# you will also make this additions afterwards.
-
-# PLEASE NOTE: This script has been automatically generated by conda-smithy. Any changes here
-# will be lost next time ``conda smithy rerender`` is run. If you would like to make permanent
-# changes to this script, consider a proposal to conda-smithy so that other feedstocks can also
-# benefit from the improvement.
-
-set -xeuo pipefail
-
-output_dir=${1}
-
-export PYTHONUNBUFFERED=1
-export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/feedstock_root}"
-export CI_SUPPORT="${FEEDSTOCK_ROOT}/.ci_support"
-export CONFIG_FILE="${CI_SUPPORT}/${CONFIG}.yaml"
-
-cat >~/.condarc <<CONDARC
-
-conda-build:
- root-dir: ${output_dir}
-
-CONDARC
-
-conda install --yes --quiet conda-forge-ci-setup=3 conda-build pip -c conda-forge
-
-# set up the condarc
-setup_conda_rc "${FEEDSTOCK_ROOT}" "${FEEDSTOCK_ROOT}" "${CONFIG_FILE}"
-
-source run_conda_forge_build_setup
-
-# make the build number clobber
-make_build_number "${FEEDSTOCK_ROOT}" "${FEEDSTOCK_ROOT}" "${CONFIG_FILE}"
-
-export CONDA_BLD_PATH="${output_dir}"
-
-conda build \
- "${FEEDSTOCK_ROOT}/arrow-cpp" \
- "${FEEDSTOCK_ROOT}/parquet-cpp" \
- -m "${CI_SUPPORT}/${CONFIG}.yaml" \
- --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" \
- --output-folder "${output_dir}"
-
-if [ ! -z "${R_CONFIG:-}" ]; then
- conda build \
- "${FEEDSTOCK_ROOT}/r-arrow" \
- -m "${CI_SUPPORT}/r/${R_CONFIG}.yaml" \
- --output-folder "${output_dir}"
-fi
-
-
-touch "${output_dir}/conda-forge-build-done-${CONFIG}"
diff --git a/dev/tasks/conda-recipes/clean.py b/dev/tasks/conda-recipes/clean.py
deleted file mode 100644
index bd31c87..0000000
--- a/dev/tasks/conda-recipes/clean.py
+++ /dev/null
@@ -1,80 +0,0 @@
-from subprocess import check_output, check_call
-from typing import List
-
-import json
-import os
-import pandas as pd
-import sys
-
-from packaging.version import Version
-
-
-VERSIONS_TO_KEEP = 5
-PACKAGES = [
- "arrow-cpp",
- "arrow-cpp-proc",
- "parquet-cpp",
- "pyarrow",
- "pyarrow-tests",
- "r-arrow",
-]
-PLATFORMS = [
- "linux-64",
- "linux-aarch64",
- "osx-64",
- "win-64",
-]
-EXCLUDED_PATTERNS = [
- ["r-arrow", "linux-aarch64"],
-]
-
-
-def packages_to_delete(package_name: str, platform: str) -> List[str]:
- env = os.environ.copy()
- env["CONDA_SUBDIR"] = platform
- pkgs_json = check_output(
- [
- "conda",
- "search",
- "--json",
- "-c",
- "arrow-nightlies",
- "--override-channels",
- package_name,
- ],
- env=env,
- )
- pkgs = pd.DataFrame(json.loads(pkgs_json)[package_name])
- pkgs["version"] = pkgs["version"].map(Version)
- pkgs["py_version"] = pkgs["build"].str.slice(0, 4)
-
- to_delete = []
-
- for (subdir, python), group in pkgs.groupby(["subdir", "py_version"]):
- group = group.sort_values(by="version", ascending=False)
-
- if len(group) > VERSIONS_TO_KEEP:
- del_candidates = group[VERSIONS_TO_KEEP:]
- to_delete += (
- f"arrow-nightlies/{package_name}/"
- + del_candidates["version"].astype(str)
- + del_candidates["url"].str.replace(
- "https://conda.anaconda.org/arrow-nightlies", ""
- )
- ).to_list()
-
- return to_delete
-
-
-if __name__ == "__main__":
- to_delete = []
- for package in PACKAGES:
- for platform in PLATFORMS:
- if [package, platform] in EXCLUDED_PATTERNS:
- continue
- to_delete += packages_to_delete(package, platform)
-
- for name in to_delete:
- print(f"Deleting {name} …")
- if "FORCE" in sys.argv:
- check_call(["anaconda", "remove", "-f", name])
diff --git a/dev/tasks/conda-recipes/conda-forge.yml b/dev/tasks/conda-recipes/conda-forge.yml
deleted file mode 100644
index 4c07b5d..0000000
--- a/dev/tasks/conda-recipes/conda-forge.yml
+++ /dev/null
@@ -1 +0,0 @@
-channel_priority: strict
diff --git a/dev/tasks/conda-recipes/drone-steps.sh b/dev/tasks/conda-recipes/drone-steps.sh
deleted file mode 100755
index dffdb41..0000000
--- a/dev/tasks/conda-recipes/drone-steps.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-OUTPUT_DIR=$1
-QUEUE_REMOTE_URL=$2
-TASK_BRANCH=$3
-TASK_TAG=$4
-UPLOAD_TO_ANACONDA=$5
-
-conda install -y mamba
-$FEEDSTOCK_ROOT/build_steps.sh ${OUTPUT_DIR}
-
-# Upload as Github release
-mamba install -y anaconda-client shyaml -c conda-forge
-
-pushd $DRONE_WORKSPACE
-pip install -e arrow/dev/archery[crossbow]
-archery crossbow \
- --queue-path . \
- --queue-remote ${QUEUE_REMOTE_URL} \
- upload-artifacts \
- --sha ${TASK_BRANCH} \
- --tag ${TASK_TAG} \
- --pattern "${OUTPUT_DIR}/linux-aarch64/*.tar.bz2"
-
-if [[ "${UPLOAD_TO_ANACONDA}" == "1" ]]; then
- anaconda -t ${CROSSBOW_ANACONDA_TOKEN} upload --force build_artifacts/linux-aarch64/*.tar.bz2
-fi
diff --git a/dev/tasks/conda-recipes/drone.yml b/dev/tasks/conda-recipes/drone.yml
deleted file mode 100644
index a461c79..0000000
--- a/dev/tasks/conda-recipes/drone.yml
+++ /dev/null
@@ -1,43 +0,0 @@
----
-kind: pipeline
-name: {{ config }}
-
-platform:
- os: linux
- arch: arm64
-
-# Omit double builds with crossbow
-trigger:
- event:
- - push
-
-steps:
-- name: Install and build
- image: condaforge/linux-anvil-aarch64
- environment:
- CONFIG: {{ config }}
- UPLOAD_PACKAGES: False
- ARROW_VERSION: {{ arrow.no_rc_version }}
- PLATFORM: linux-aarch64
- BINSTAR_TOKEN:
- from_secret: BINSTAR_TOKEN
- FEEDSTOCK_TOKEN:
- from_secret: FEEDSTOCK_TOKEN
- STAGING_BINSTAR_TOKEN:
- from_secret: STAGING_BINSTAR_TOKEN
- CROSSBOW_GITHUB_TOKEN:
- from_secret: CROSSBOW_GITHUB_TOKEN
- CROSSBOW_ANACONDA_TOKEN:
- from_secret: CROSSBOW_ANACONDA_TOKEN
- commands:
- - export RECIPE_ROOT="$FEEDSTOCK_ROOT/arrow-cpp"
- - export CI=drone
- - export GIT_BRANCH="{{ arrow.branch }}"
- - export FEEDSTOCK_NAME=arrow-cpp
- - export FEEDSTOCK_ROOT="$DRONE_WORKSPACE/arrow/dev/tasks/conda-recipes"
- - sed -i '$ichown -R conda:conda "$FEEDSTOCK_ROOT"' /opt/docker/bin/entrypoint
- - yum install -y git
- - git clone --no-checkout {{ arrow.remote }} arrow
- - pushd arrow && git fetch -t {{ arrow.remote }} {{ arrow.branch }} && git checkout FETCH_HEAD && git submodule update --init --recursive && popd
- - mkdir -p $(pwd)/build_artifacts && chmod a+rwx $(pwd)/build_artifacts
- - /opt/docker/bin/entrypoint $FEEDSTOCK_ROOT/drone-steps.sh $(pwd)/build_artifacts {{ queue_remote_url }} {{ task.branch }} {{ task.tag }} {% if arrow.branch == 'master' %}1{% else %}0{% endif %}
diff --git a/dev/tasks/conda-recipes/parquet-cpp/meta.yaml b/dev/tasks/conda-recipes/parquet-cpp/meta.yaml
deleted file mode 100644
index 5de06c3..0000000
--- a/dev/tasks/conda-recipes/parquet-cpp/meta.yaml
+++ /dev/null
@@ -1,51 +0,0 @@
-# ARROW-3229: this is a meta-package to prevent conflicts in the future
-
-{% set parquet_version = "1.5.1" %}
-
-package:
- name: parquet-cpp
- version: {{ parquet_version }}
-
-build:
- number: 0
- skip: true # [win32]
- skip: true # [win and py<35]
-
-requirements:
- host:
- # NOTE: in the upstream feedstock use >= instead of =
- - arrow-cpp ={{ ARROW_VERSION }}
- run:
- - arrow-cpp ={{ ARROW_VERSION }}
-
-test:
- commands:
- # headers
- - test -f $PREFIX/include/parquet/api/reader.h # [unix]
- - if not exist %LIBRARY_INC%\\parquet\\api\\reader.h exit 1 # [win]
-
- # shared
- - test -f $PREFIX/lib/libparquet.so # [linux]
- - test -f $PREFIX/lib/libparquet.dylib # [osx]
- - if not exist %PREFIX%\\Library\\bin\\parquet.dll exit 1 # [win]
-
- # absence of static libraries
- - test ! -f $PREFIX/lib/libparquet.a # [unix]
- - if exist %PREFIX%\\Library\\lib\\parquet_static.lib exit 1 # [win]
-
-about:
- home: http://github.com/apache/arrow
- license: Apache 2.0
- summary: 'C++ libraries for the Apache Parquet file format'
-
-extra:
- recipe-maintainers:
- - wesm
- - xhochy
- - leifwalsh
- - jreback
- - cpcloud
- - siddharthteotia
- - kou
- - kszucs
- - pitrou
diff --git a/dev/tasks/conda-recipes/r-arrow/bld.bat b/dev/tasks/conda-recipes/r-arrow/bld.bat
deleted file mode 100644
index a193ddc..0000000
--- a/dev/tasks/conda-recipes/r-arrow/bld.bat
+++ /dev/null
@@ -1,9 +0,0 @@
-bash %RECIPE_DIR%/build_win.sh
-IF %ERRORLEVEL% NEQ 0 exit 1
-cp %RECIPE_DIR%/configure.win r
-IF %ERRORLEVEL% NEQ 0 exit 1
-cp %RECIPE_DIR%/install.libs.R r/src
-IF %ERRORLEVEL% NEQ 0 exit 1
-set "MAKEFLAGS=-j%CPU_COUNT%"
-"%R%" CMD INSTALL --build r
-IF %ERRORLEVEL% NEQ 0 exit 1
... 10223 lines suppressed ...