You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by an...@apache.org on 2023/10/23 16:28:03 UTC
[beam] 01/01: change extension of beam sdist from .zip to .tar.gz
This is an automated email from the ASF dual-hosted git repository.
anandinguva pushed a commit to branch zip_to_tar
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 81d74f6fedbd0340ac9585f196cb072d64d6b3d6
Author: Anand Inguva <an...@gmail.com>
AuthorDate: Mon Oct 23 12:25:33 2023 -0400
change extension of beam sdist from .zip to .tar.gz
---
.github/actions/common-rc-validation/action.yaml | 10 +--
.github/workflows/build_release_candidate.yml | 16 ++--
.github/workflows/run_rc_validation.yml | 93 +++++++++++-----------
contributor-docs/release-guide.md | 10 +--
.../python_release_automation_utils.sh | 6 +-
.../src/main/scripts/build_release_candidate.sh | 16 ++--
.../main/scripts/deploy_release_candidate_pypi.sh | 4 +-
release/src/main/scripts/run_rc_validation.sh | 48 +++++------
sdks/python/scripts/run_snapshot_publish.sh | 6 +-
9 files changed, 104 insertions(+), 105 deletions(-)
diff --git a/.github/actions/common-rc-validation/action.yaml b/.github/actions/common-rc-validation/action.yaml
index 23efa93d153..51738e13812 100644
--- a/.github/actions/common-rc-validation/action.yaml
+++ b/.github/actions/common-rc-validation/action.yaml
@@ -36,19 +36,19 @@ runs:
shell: bash
run: |
echo "---------------------Downloading Python Staging RC----------------------------"
- wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip
- wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512
- if [[ ! -f apache-beam-$RELEASE_VER.zip ]]; then
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.tar.gz
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.tar.gz.sha512
+ if [[ ! -f apache-beam-$RELEASE_VER.tar.gz ]]; then
{ echo "Fail to download Python Staging RC files." ;exit 1; }
fi
echo "--------------------------Verifying Hashes------------------------------------"
- sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512
+ sha512sum -c apache-beam-${RELEASE_VER}.tar.gz.sha512
`which pip` install --upgrade pip
`which pip` install --upgrade setuptools
- name: Installing python SDK
shell: bash
- run: pip install apache-beam-${RELEASE_VER}.zip[gcp]
+ run: pip install apache-beam-${RELEASE_VER}.tar.gz[gcp]
\ No newline at end of file
diff --git a/.github/workflows/build_release_candidate.yml b/.github/workflows/build_release_candidate.yml
index 12f1537dac1..7c3e7b98c4b 100644
--- a/.github/workflows/build_release_candidate.yml
+++ b/.github/workflows/build_release_candidate.yml
@@ -122,18 +122,18 @@ jobs:
mkdir -p beam/${{ github.event.inputs.RELEASE }}
cd beam/${{ github.event.inputs.RELEASE }}
RC_DIR="beam-${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}"
- RC_ZIP="${RC_DIR}.zip"
+ RC_ZIP="${RC_DIR}.tar.gz"
RELEASE_DIR="beam-${{ github.event.inputs.RELEASE }}"
RC_TAG="v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}"
- SOURCE_RELEASE_ZIP="apache-beam-${{ github.event.inputs.RELEASE }}-source-release.zip"
+ SOURCE_RELEASE_ZIP="apache-beam-${{ github.event.inputs.RELEASE }}-source-release.tar.gz"
# Check whether there is an existing dist dir
if (svn ls "${SOURCE_RELEASE_ZIP}"); then
echo "Removing existing ${SOURCE_RELEASE_ZIP}."
svn delete "${SOURCE_RELEASE_ZIP}"
fi
- echo "Downloading: https://github.com/apache/beam/archive/${RC_TAG}.zip"
- wget https://github.com/apache/beam/archive/${RC_TAG}.zip -O "${RC_ZIP}"
+ echo "Downloading: https://github.com/apache/beam/archive/${RC_TAG}.tar.gz"
+ wget https://github.com/apache/beam/archive/${RC_TAG}.tar.gz -O "${RC_ZIP}"
unzip "$RC_ZIP"
rm "$RC_ZIP"
@@ -197,11 +197,11 @@ jobs:
cd "${RELEASE_DIR}"/python
- echo "------Checking Hash Value for apache-beam-${RELEASE}.zip-----"
- sha512sum -c "apache-beam-${RELEASE}.zip.sha512"
+ echo "------Checking Hash Value for apache-beam-${RELEASE}.tar.gz-----"
+ sha512sum -c "apache-beam-${RELEASE}.tar.gz.sha512"
- echo "------Signing Source Release apache-beam-${RELEASE}.zip------"
- gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --detach-sig "apache-beam-${RELEASE}.zip"
+ echo "------Signing Source Release apache-beam-${RELEASE}.tar.gz------"
+ gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --detach-sig "apache-beam-${RELEASE}.tar.gz"
for artifact in *.whl; do
echo "----------Checking Hash Value for ${artifact} wheel-----------"
diff --git a/.github/workflows/run_rc_validation.yml b/.github/workflows/run_rc_validation.yml
index 4902fee8101..35fb3ce05ea 100644
--- a/.github/workflows/run_rc_validation.yml
+++ b/.github/workflows/run_rc_validation.yml
@@ -17,14 +17,14 @@
# To learn more about GitHub Actions in Apache Beam check the CI.m
-name: Run RC Validation
+name: Run RC Validation
on:
workflow_dispatch:
inputs:
RELEASE_VER:
description: Beam current Release Version
required: true
- default: 2.42.0
+ default: 2.42.0
USER_GCS_BUCKET:
description: Bucket to upload results
required: true
@@ -57,9 +57,9 @@ on:
type: boolean
required: true
default: true
-env:
+env:
RC_TAG: "v${{github.event.inputs.RELEASE_VER}}-RC${{github.event.inputs.RC_NUM}}"
- RELEASE_VER: ${{github.event.inputs.RELEASE_VER}}
+ RELEASE_VER: ${{github.event.inputs.RELEASE_VER}}
USER_GCP_PROJECT: apache-beam-testing
PYTHON_RC_DOWNLOAD_URL: https://dist.apache.org/repos/dist/dev/beam
USER_GCP_REGION: us-central1
@@ -100,19 +100,19 @@ jobs:
- name: Comment on PR to Trigger Python ReleaseCandidate Test
run: |
gh pr comment "$GITHUB_PR_URL" --body "Run Python ReleaseCandidate"
-
+
sql_taxi_with_dataflow:
runs-on: [self-hosted,ubuntu-20.04]
if: ${{github.event.inputs.RUN_SQL_TAXI_WITH_DATAFLOW == 'true'}}
strategy:
- matrix:
+ matrix:
py_version: [3.8]
steps:
- name: Checkout code
uses: actions/checkout@v4
- with:
- ref: ${{env.RC_TAG}}
-
+ with:
+ ref: ${{env.RC_TAG}}
+
- name: Install Python
uses: actions/setup-python@v4
with:
@@ -154,23 +154,23 @@ jobs:
--num_workers 5 \
--output_topic projects/${USER_GCP_PROJECT}/topics/${SQL_TAXI_TOPIC} \
--beam_services="{\":sdks:java:extensions:sql:expansion-service:shadowJar\": \"${SQL_EXPANSION_SERVICE_JAR}\"}" \
- --sdk_location apache-beam-${RELEASE_VER}.zip || true
+ --sdk_location apache-beam-${RELEASE_VER}.tar.gz || true
- name: Checking Results
run: |
gcloud pubsub subscriptions pull --project=${USER_GCP_PROJECT} --limit=5 ${SQL_TAXI_SUBSCRIPTION}
gcloud pubsub subscriptions pull --project=${USER_GCP_PROJECT} --limit=5 ${SQL_TAXI_SUBSCRIPTION}
- name: Removing Pub Sub Topic
if: always()
- run: |
+ run: |
gcloud pubsub topics delete --project=${USER_GCP_PROJECT} ${SQL_TAXI_TOPIC}
gcloud pubsub subscriptions delete --project=${USER_GCP_PROJECT} ${SQL_TAXI_SUBSCRIPTION}
python_cross_validation:
runs-on: [self-hosted,ubuntu-20.04]
- if: ${{github.event.inputs.RUN_PYTHON_CROSS_VALIDATION == 'true'}}
+ if: ${{github.event.inputs.RUN_PYTHON_CROSS_VALIDATION == 'true'}}
strategy:
- matrix:
+ matrix:
py_version: [3.8]
steps:
- name: Checkout code
@@ -183,7 +183,7 @@ jobs:
echo "====================Checking Environment & Variables================="
echo ""
echo "running validations on release ${{github.event.inputs.RELEASE_VER}} RC${{github.event.inputs.RC_NUM}}."
- - name: Install Kubectl
+ - name: Install Kubectl
uses: azure/setup-kubectl@v3
- name: Setup Java JDK
@@ -196,7 +196,7 @@ jobs:
uses: actions/setup-python@v4
with:
python-version: ${{matrix.py_version}}
-
+
- name: Setting python env
uses: ./.github/actions/common-rc-validation
@@ -206,19 +206,19 @@ jobs:
- name: Installing gcloud-auth-plugin
run: sudo apt-get install google-cloud-sdk-gke-gcloud-auth-plugin
- - name: Setting Kafka Cluster Name
+ - name: Setting Kafka Cluster Name
run: |
- echo "KAFKA_CLUSTER_NAME=xlang-kafka-cluster-$RANDOM">> $GITHUB_ENV
-
+ echo "KAFKA_CLUSTER_NAME=xlang-kafka-cluster-$RANDOM">> $GITHUB_ENV
+
- name: Creating Kafka Cluster
run: |
gcloud container clusters create --project=${USER_GCP_PROJECT} --region=${USER_GCP_REGION} --no-enable-ip-alias $KAFKA_CLUSTER_NAME
kubectl apply -R -f .test-infra/kubernetes/kafka-cluster
-
+
- name: Waiting for Kafka cluster to be ready
run: kubectl wait --for=condition=Ready pod/kafka-0 --timeout=1200s
- - name: Start xlang Kafka Taxi with Dataflow Runner
+ - name: Start xlang Kafka Taxi with Dataflow Runner
run: |
echo "BOOTSTRAP_SERVERS=$(kubectl get svc outside-0 -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):32400" >> $GITHUB_ENV
echo "KAFKA_TAXI_DF_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV
@@ -243,16 +243,16 @@ jobs:
--temp_location=${USER_GCS_BUCKET}/temp/ \
--with_metadata \
--beam_services="{\"sdks:java:io:expansion-service:shadowJar\": \"${KAFKA_EXPANSION_SERVICE_JAR}\"}" \
- --sdk_location apache-beam-${RELEASE_VER}.zip || true
+ --sdk_location apache-beam-${RELEASE_VER}.tar.gz || true
- name: Checking executions results
run: |
- bq head -n 10 ${KAFKA_TAXI_DF_DATASET}.xlang_kafka_taxi
+ bq head -n 10 ${KAFKA_TAXI_DF_DATASET}.xlang_kafka_taxi
- name: Remove BigQuery Dataset
if: always()
run: |
bq rm -f ${KAFKA_TAXI_DF_DATASET}.xlang_kafka_taxi
bq rm -f ${KAFKA_TAXI_DF_DATASET}
-
+
- name: Delete Kafka Cluster
if: always()
run: gcloud container clusters delete --project=${USER_GCP_PROJECT} --region=${USER_GCP_REGION} --async -q $KAFKA_CLUSTER_NAME
@@ -266,8 +266,8 @@ jobs:
steps:
- name: Sending PubSub name to env
run: |
- echo "SHARED_PUBSUB_TOPIC=leader_board-${GITHUB_ACTOR}-python-topic-$(date +%m%d)_$RANDOM" >> $GITHUB_ENV
- - id: generate_pubsub_name
+ echo "SHARED_PUBSUB_TOPIC=leader_board-${GITHUB_ACTOR}-python-topic-$(date +%m%d)_$RANDOM" >> $GITHUB_ENV
+ - id: generate_pubsub_name
run: |
echo "::set-output name=pubsub::$SHARED_PUBSUB_TOPIC"
- name: Creating Pub Sub Topics
@@ -287,7 +287,7 @@ jobs:
uses: actions/setup-python@v4
with:
python-version: '3.8'
-
+
- name: Setting python env
uses: ./.github/actions/common-rc-validation
with:
@@ -343,14 +343,14 @@ jobs:
run: |
ls
cd word-count-beam
- timeout --preserve-status 50m mvn compile exec:java -Dexec.mainClass=org.apache.beam.examples.complete.game.injector.Injector -Dexec.args="${USER_GCP_PROJECT} ${{needs.generate_shared_pubsub.outputs.name}} none" || true
-
+ timeout --preserve-status 50m mvn compile exec:java -Dexec.mainClass=org.apache.beam.examples.complete.game.injector.Injector -Dexec.args="${USER_GCP_PROJECT} ${{needs.generate_shared_pubsub.outputs.name}} none" || true
+
direct_runner_leaderboard:
runs-on: [self-hosted, ubuntu-20.04]
if: ${{github.event.inputs.RUN_DIRECT_RUNNER_TESTS == 'true' }}
strategy:
- matrix:
+ matrix:
py_version: [3.8]
needs: generate_shared_pubsub
steps:
@@ -363,13 +363,13 @@ jobs:
uses: actions/setup-python@v4
with:
python-version: ${{matrix.py_version}}
-
+
- name: Setting python env
uses: ./.github/actions/common-rc-validation
with:
RELEASE_VER: ${{env.RELEASE_VER}}
PYTHON_RC_DOWNLOAD_URL: ${{env.PYTHON_RC_DOWNLOAD_URL}}
-
+
- name: Exporting leaderboard Dataset Name
run: echo "LEADERBOARD_DIRECT_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV
- name: Creating Dataset
@@ -389,11 +389,11 @@ jobs:
bq head -n 10 ${LEADERBOARD_DIRECT_DATASET}.leader_board_teams
- name: Removing BigQuery Dataset
if: always()
- run: |
+ run: |
bq rm -f ${LEADERBOARD_DIRECT_DATASET}.leader_board_users
bq rm -f ${LEADERBOARD_DIRECT_DATASET}.leader_board_teams
bq rm -f $LEADERBOARD_DIRECT_DATASET
-
+
dataflow_runner_leaderboard:
runs-on: [self-hosted,ubuntu-20.04]
if: ${{github.event.inputs.RUN_DATAFLOW_RUNNER_TESTS=='true'}}
@@ -411,13 +411,13 @@ jobs:
uses: actions/setup-python@v4
with:
python-version: ${{matrix.py_version}}
-
+
- name: Setting python env
uses: ./.github/actions/common-rc-validation
with:
RELEASE_VER: ${{env.RELEASE_VER}}
PYTHON_RC_DOWNLOAD_URL: ${{env.PYTHON_RC_DOWNLOAD_URL}}
-
+
- name: Exporting Dataflow Dataset Name
run: echo "LEADERBOARD_DF_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV
- name: Creating Dataset
@@ -434,18 +434,18 @@ jobs:
--dataset ${LEADERBOARD_DF_DATASET} \
--runner DataflowRunner \
--temp_location=${USER_GCS_BUCKET}/temp/ \
- --sdk_location apache-beam-${RELEASE_VER}.zip || true
+ --sdk_location apache-beam-${RELEASE_VER}.tar.gz || true
- name: Checking results
run: |
bq head -n 10 ${LEADERBOARD_DF_DATASET}.leader_board_users
bq head -n 10 ${LEADERBOARD_DF_DATASET}.leader_board_teams
- name: Removing BigQuery Dataset
if: always()
- run: |
+ run: |
bq rm -f ${LEADERBOARD_DF_DATASET}.leader_board_users
bq rm -f ${LEADERBOARD_DF_DATASET}.leader_board_teams
bq rm -f $LEADERBOARD_DF_DATASET
-
+
direct_runner_gamestats:
runs-on: [self-hosted,ubuntu-20.04]
@@ -463,13 +463,13 @@ jobs:
uses: actions/setup-python@v4
with:
python-version: ${{matrix.py_version}}
-
+
- name: Setting python env
uses: ./.github/actions/common-rc-validation
with:
RELEASE_VER: ${{env.RELEASE_VER}}
PYTHON_RC_DOWNLOAD_URL: ${{env.PYTHON_RC_DOWNLOAD_URL}}
-
+
- name: Exporting Gamestates Direct Dataset Name
run: echo "GAMESTATS_DIRECT_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV
- name: Creating Dataset
@@ -490,7 +490,7 @@ jobs:
bq head -n 10 ${GAMESTATS_DIRECT_DATASET}.game_stats_sessions
- name: Removing BigQuery Dataset
if: always()
- run: |
+ run: |
bq rm -f ${GAMESTATS_DIRECT_DATASET}.game_stats_sessions
bq rm -f ${GAMESTATS_DIRECT_DATASET}.game_stats_teams
bq rm -f $GAMESTATS_DIRECT_DATASET
@@ -512,13 +512,13 @@ jobs:
uses: actions/setup-python@v4
with:
python-version: ${{matrix.py_version}}
-
+
- name: Setting python env
uses: ./.github/actions/common-rc-validation
with:
RELEASE_VER: ${{env.RELEASE_VER}}
PYTHON_RC_DOWNLOAD_URL: ${{env.PYTHON_RC_DOWNLOAD_URL}}
-
+
- name: Exporting Gamestates Direct Dataset Name
run: echo "GAMESTATS_DF_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV
- name: Creating Dataset
@@ -535,7 +535,7 @@ jobs:
--dataset ${GAMESTATS_DF_DATASET} \
--runner DataflowRunner \
--temp_location=${USER_GCS_BUCKET}/temp/ \
- --sdk_location apache-beam-${RELEASE_VER}.zip \
+ --sdk_location apache-beam-${RELEASE_VER}.tar.gz \
--fixed_window_duration ${FIXED_WINDOW_DURATION} || true
- name: Checking Results
run: |
@@ -545,9 +545,9 @@ jobs:
if: always()
run: |
bq rm -f ${GAMESTATS_DF_DATASET}.game_stats_teams
- bq rm -f ${GAMESTATS_DF_DATASET}.game_stats_sessions
+ bq rm -f ${GAMESTATS_DF_DATASET}.game_stats_sessions
bq rm -f $GAMESTATS_DF_DATASET
-
+
remove_shared_pubsub:
runs-on: [self-hosted,ubuntu-20.04]
needs: [java_injector, generate_shared_pubsub]
@@ -555,4 +555,3 @@ jobs:
steps:
- name: Deleting Shared Pub Sub
run: gcloud pubsub topics delete --project=${USER_GCP_PROJECT} ${{needs.generate_shared_pubsub.outputs.name}}
-
\ No newline at end of file
diff --git a/contributor-docs/release-guide.md b/contributor-docs/release-guide.md
index e8e8ead0902..712e77a25b1 100644
--- a/contributor-docs/release-guide.md
+++ b/contributor-docs/release-guide.md
@@ -801,8 +801,8 @@ You can (optionally) also do additional verification by:
signature/checksum files of Java artifacts may not contain filenames. Hence
you might need to compare checksums/signatures manually or modify the files by
appending the filenames.)
-- [ ] Check signatures (e.g. `gpg --verify apache-beam-1.2.3-python.zip.asc
- apache-beam-1.2.3-python.zip`)
+- [ ] Check signatures (e.g. `gpg --verify apache-beam-1.2.3-python.tar.gz.asc
+ apache-beam-1.2.3-python.tar.gz`)
- [ ] `grep` for legal headers in each file.
- [ ] Run all jenkins suites and include links to passing tests in the voting
email.
@@ -1018,14 +1018,14 @@ write to BigQuery, and create a cluster of machines for running containers (for
* **Verify the hashes**
```
- sha512sum -c apache-beam-2.5.0-python.zip.sha512
- sha512sum -c apache-beam-2.5.0-source-release.zip.sha512
+ sha512sum -c apache-beam-2.5.0-python.tar.gz.sha512
+ sha512sum -c apache-beam-2.5.0-source-release.tar.gz.sha512
```
* **Build SDK**
```
sudo apt-get install unzip
- unzip apache-beam-2.5.0-source-release.zip
+ unzip apache-beam-2.5.0-source-release.tar.gz
python setup.py sdist
```
* **Setup virtual environment**
diff --git a/release/src/main/python-release/python_release_automation_utils.sh b/release/src/main/python-release/python_release_automation_utils.sh
index 2f5a9ac0a5d..337ece8ba64 100644
--- a/release/src/main/python-release/python_release_automation_utils.sh
+++ b/release/src/main/python-release/python_release_automation_utils.sh
@@ -97,7 +97,7 @@ function download_files() {
wget -r -l2 --no-parent -nd -A "$BEAM_PYTHON_SDK_WHL*" $RC_STAGING_URL
else
- BEAM_PYTHON_SDK_ZIP="apache-beam-$VERSION.zip"
+ BEAM_PYTHON_SDK_ZIP="apache-beam-$VERSION.tar.gz"
wget -r -l2 --no-parent -nd -A "$BEAM_PYTHON_SDK_ZIP*" $RC_STAGING_URL
fi
}
@@ -128,7 +128,7 @@ function get_sha512_name() {
if [[ $1 = *"wheel"* ]]; then
echo $(ls | grep "/*.whl.sha512$")
else
- echo $(ls | grep "/*.zip.sha512$")
+ echo $(ls | grep "/*.tar.gz.sha512$")
fi
}
@@ -142,7 +142,7 @@ function get_asc_name() {
if [[ $1 = *"wheel"* ]]; then
echo $(ls | grep "/*.whl.asc$")
else
- echo $(ls | grep "/*.zip.asc$")
+ echo $(ls | grep "/*.tar.gz.asc$")
fi
}
diff --git a/release/src/main/scripts/build_release_candidate.sh b/release/src/main/scripts/build_release_candidate.sh
index d0e6310f50a..745d726d765 100755
--- a/release/src/main/scripts/build_release_candidate.sh
+++ b/release/src/main/scripts/build_release_candidate.sh
@@ -193,19 +193,19 @@ if [[ $confirmation = "y" ]]; then
echo "----------------Downloading Source Release-------------------"
# GitHub strips the "v" from "v2.29.0" in naming zip and the dir inside it
RC_DIR="beam-${RELEASE}-RC${RC_NUM}"
- RC_ZIP="${RC_DIR}.zip"
+ RC_ZIP="${RC_DIR}.tar.gz"
# We want to strip the -RC1 suffix from the directory name inside the zip
RELEASE_DIR="beam-${RELEASE}"
- SOURCE_RELEASE_ZIP="apache-beam-${RELEASE}-source-release.zip"
+ SOURCE_RELEASE_ZIP="apache-beam-${RELEASE}-source-release.tar.gz"
# Check whether there is an existing dist dir
if (svn ls "${SOURCE_RELEASE_ZIP}"); then
echo "Removing existing ${SOURCE_RELEASE_ZIP}."
svn delete "${SOURCE_RELEASE_ZIP}"
fi
- echo "Downloading: ${GIT_BEAM_ARCHIVE}/${RC_TAG}.zip"
- wget ${GIT_BEAM_ARCHIVE}/${RC_TAG}.zip -O "${RC_ZIP}"
+ echo "Downloading: ${GIT_BEAM_ARCHIVE}/${RC_TAG}.tar.gz"
+ wget ${GIT_BEAM_ARCHIVE}/${RC_TAG}.tar.gz -O "${RC_ZIP}"
unzip "$RC_ZIP"
rm "$RC_ZIP"
@@ -268,11 +268,11 @@ if [[ $confirmation = "y" ]]; then
cd "${SVN_ARTIFACTS_DIR}"
- echo "------Checking Hash Value for apache-beam-${RELEASE}.zip-----"
- sha512sum -c "apache-beam-${RELEASE}.zip.sha512"
+ echo "------Checking Hash Value for apache-beam-${RELEASE}.tar.gz-----"
+ sha512sum -c "apache-beam-${RELEASE}.tar.gz.sha512"
- echo "------Signing Source Release apache-beam-${RELEASE}.zip------"
- gpg --local-user "${SIGNING_KEY}" --armor --detach-sig "apache-beam-${RELEASE}.zip"
+ echo "------Signing Source Release apache-beam-${RELEASE}.tar.gz------"
+ gpg --local-user "${SIGNING_KEY}" --armor --detach-sig "apache-beam-${RELEASE}.tar.gz"
for artifact in *.whl; do
echo "----------Checking Hash Value for ${artifact} wheel-----------"
diff --git a/release/src/main/scripts/deploy_release_candidate_pypi.sh b/release/src/main/scripts/deploy_release_candidate_pypi.sh
index d1fef8c6078..9ee90212dda 100755
--- a/release/src/main/scripts/deploy_release_candidate_pypi.sh
+++ b/release/src/main/scripts/deploy_release_candidate_pypi.sh
@@ -141,8 +141,8 @@ python3 "${SCRIPT_DIR}/download_github_actions_artifacts.py" \
cd "${PYTHON_ARTIFACTS_DIR}"
-echo "------Checking Hash Value for apache-beam-${RELEASE}rc${RC_NUMBER}.zip-----"
-sha512sum -c "apache-beam-${RELEASE}rc${RC_NUMBER}.zip.sha512"
+echo "------Checking Hash Value for apache-beam-${RELEASE}rc${RC_NUMBER}.tar.gz-----"
+sha512sum -c "apache-beam-${RELEASE}rc${RC_NUMBER}.tar.gz.sha512"
for artifact in *.whl; do
echo "----------Checking Hash Value for ${artifact} wheel-----------"
diff --git a/release/src/main/scripts/run_rc_validation.sh b/release/src/main/scripts/run_rc_validation.sh
index 7f32c297966..0f2bfe4aaec 100755
--- a/release/src/main/scripts/run_rc_validation.sh
+++ b/release/src/main/scripts/run_rc_validation.sh
@@ -300,14 +300,14 @@ if [[ ("$python_leaderboard_direct" = true \
cd ${LOCAL_BEAM_DIR}
echo "---------------------Downloading Python Staging RC----------------------------"
- wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip
- wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512
- if [[ ! -f apache-beam-${RELEASE_VER}.zip ]]; then
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.tar.gz
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.tar.gz.sha512
+ if [[ ! -f apache-beam-${RELEASE_VER}.tar.gz ]]; then
{ echo "Fail to download Python Staging RC files." ;exit 1; }
fi
echo "--------------------------Verifying Hashes------------------------------------"
- sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512
+ sha512sum -c apache-beam-${RELEASE_VER}.tar.gz.sha512
echo "--------------------------Updating ~/.m2/settings.xml-------------------------"
cd ~
@@ -378,7 +378,7 @@ if [[ ("$python_leaderboard_direct" = true \
pip install --upgrade pip setuptools wheel
echo "--------------------------Installing Python SDK-------------------------------"
- pip install apache-beam-${RELEASE_VER}.zip[gcp]
+ pip install apache-beam-${RELEASE_VER}.tar.gz[gcp]
echo "----------------Starting Leaderboard with DirectRunner-----------------------"
if [[ "$python_leaderboard_direct" = true ]]; then
@@ -434,7 +434,7 @@ if [[ ("$python_leaderboard_direct" = true \
--dataset ${LEADERBOARD_DF_DATASET} \
--runner DataflowRunner \
--temp_location=${USER_GCS_BUCKET}/temp/ \
- --sdk_location apache-beam-${RELEASE_VER}.zip; \
+ --sdk_location apache-beam-${RELEASE_VER}.tar.gz; \
exec bash"
echo "***************************************************************"
@@ -509,7 +509,7 @@ if [[ ("$python_leaderboard_direct" = true \
--dataset ${GAMESTATS_DF_DATASET} \
--runner DataflowRunner \
--temp_location=${USER_GCS_BUCKET}/temp/ \
- --sdk_location apache-beam-${RELEASE_VER}.zip \
+ --sdk_location apache-beam-${RELEASE_VER}.tar.gz \
--fixed_window_duration ${FIXED_WINDOW_DURATION}; exec bash"
echo "***************************************************************"
@@ -566,14 +566,14 @@ if [[ ("$python_xlang_quickstart" = true) \
cd ${LOCAL_BEAM_DIR}
echo "---------------------Downloading Python Staging RC----------------------------"
- wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip
- wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512
- if [[ ! -f apache-beam-${RELEASE_VER}.zip ]]; then
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.tar.gz
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.tar.gz.sha512
+ if [[ ! -f apache-beam-${RELEASE_VER}.tar.gz ]]; then
{ echo "Failed to download Python Staging RC files." ;exit 1; }
fi
echo "--------------------------Verifying Hashes------------------------------------"
- sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512
+ sha512sum -c apache-beam-${RELEASE_VER}.tar.gz.sha512
`which pip` install --upgrade pip
`which pip` install --upgrade setuptools
@@ -593,7 +593,7 @@ if [[ ("$python_xlang_quickstart" = true) \
ln -s ${LOCAL_BEAM_DIR}/sdks beam_env_${py_version}/lib/sdks
echo "--------------------------Installing Python SDK-------------------------------"
- pip install apache-beam-${RELEASE_VER}.zip
+ pip install apache-beam-${RELEASE_VER}.tar.gz
echo '************************************************************';
echo '* Running Python Multi-language Quickstart with DirectRunner';
@@ -672,14 +672,14 @@ if [[ ("$java_xlang_quickstart" = true) \
cd ${LOCAL_BEAM_DIR}
echo "---------------------Downloading Python Staging RC----------------------------"
- wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip
- wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512
- if [[ ! -f apache-beam-${RELEASE_VER}.zip ]]; then
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.tar.gz
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.tar.gz.sha512
+ if [[ ! -f apache-beam-${RELEASE_VER}.tar.gz ]]; then
{ echo "Failed to download Python Staging RC files." ;exit 1; }
fi
echo "--------------------------Verifying Hashes------------------------------------"
- sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512
+ sha512sum -c apache-beam-${RELEASE_VER}.tar.gz.sha512
`which pip` install --upgrade pip
`which pip` install --upgrade setuptools
@@ -699,7 +699,7 @@ if [[ ("$java_xlang_quickstart" = true) \
ln -s ${LOCAL_BEAM_DIR}/sdks beam_env_${py_version}/lib/sdks
echo "--------------------------Installing Python SDK-------------------------------"
- pip install apache-beam-${RELEASE_VER}.zip[dataframe]
+ pip install apache-beam-${RELEASE_VER}.tar.gz[dataframe]
# Deacrivating in the main shell. We will reactivate the virtual environment new shells
# for the expansion service and the job server.
@@ -768,14 +768,14 @@ if [[ ("$python_xlang_kafka_taxi_dataflow" = true
cd ${LOCAL_BEAM_DIR}
echo "---------------------Downloading Python Staging RC----------------------------"
- wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip
- wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512
- if [[ ! -f apache-beam-${RELEASE_VER}.zip ]]; then
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.tar.gz
+ wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.tar.gz.sha512
+ if [[ ! -f apache-beam-${RELEASE_VER}.tar.gz ]]; then
{ echo "Fail to download Python Staging RC files." ;exit 1; }
fi
echo "--------------------------Verifying Hashes------------------------------------"
- sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512
+ sha512sum -c apache-beam-${RELEASE_VER}.tar.gz.sha512
`which pip` install --upgrade pip
`which pip` install --upgrade setuptools
@@ -807,7 +807,7 @@ if [[ ("$python_xlang_kafka_taxi_dataflow" = true
ln -s ${LOCAL_BEAM_DIR}/sdks beam_env_${py_version}/lib/sdks
echo "--------------------------Installing Python SDK-------------------------------"
- pip install apache-beam-${RELEASE_VER}.zip[gcp]
+ pip install apache-beam-${RELEASE_VER}.tar.gz[gcp]
echo "----------------Starting XLang Kafka Taxi with DataflowRunner---------------------"
if [[ "$python_xlang_kafka_taxi_dataflow" = true ]]; then
@@ -837,7 +837,7 @@ if [[ ("$python_xlang_kafka_taxi_dataflow" = true
--temp_location=${USER_GCS_BUCKET}/temp/ \
--with_metadata \
--beam_services=\"{\\\"sdks:java:io:expansion-service:shadowJar\\\": \\\"${KAFKA_EXPANSION_SERVICE_JAR}\\\"}\" \
- --sdk_location apache-beam-${RELEASE_VER}.zip; \
+ --sdk_location apache-beam-${RELEASE_VER}.tar.gz; \
exec bash"
echo "***************************************************************"
@@ -882,7 +882,7 @@ if [[ ("$python_xlang_kafka_taxi_dataflow" = true
--temp_location=${USER_GCS_BUCKET}/temp/ \
--output_topic projects/${USER_GCP_PROJECT}/topics/${SQL_TAXI_TOPIC} \
--beam_services=\"{\\\":sdks:java:extensions:sql:expansion-service:shadowJar\\\": \\\"${SQL_EXPANSION_SERVICE_JAR}\\\"}\" \
- --sdk_location apache-beam-${RELEASE_VER}.zip; \
+ --sdk_location apache-beam-${RELEASE_VER}.tar.gz; \
exec bash"
echo "***************************************************************"
diff --git a/sdks/python/scripts/run_snapshot_publish.sh b/sdks/python/scripts/run_snapshot_publish.sh
index 6379e6f2108..bc379077349 100755
--- a/sdks/python/scripts/run_snapshot_publish.sh
+++ b/sdks/python/scripts/run_snapshot_publish.sh
@@ -21,7 +21,7 @@ BUCKET=gs://beam-python-nightly-snapshots
VERSION=$(awk '/__version__/{print $3}' $WORKSPACE/sdks/python/apache_beam/version.py)
VERSION=$(echo $VERSION | cut -c 2- | rev | cut -c 2- | rev)
time=$(date +"%Y-%m-%dT%H:%M:%S")
-SNAPSHOT="apache-beam-$VERSION-$time.zip"
+SNAPSHOT="apache-beam-$VERSION-$time.tar.gz"
DEP_SNAPSHOT_ROOT="$BUCKET/dependency_requirements_snapshot"
DEP_SNAPSHOT_FILE_NAME="beam-py-requirements-$time.txt"
@@ -30,8 +30,8 @@ DEP_SNAPSHOT_FILE_NAME="beam-py-requirements-$time.txt"
# and located under Gradle build directory.
cd $WORKSPACE/sdks/python/build
-# Rename the file to be apache-beam-{VERSION}-{datetime}.zip
-for file in "apache-beam-$VERSION*.zip"; do
+# Rename the file to be apache-beam-{VERSION}-{datetime}.tar.gz
+for file in "apache-beam-$VERSION*.tar.gz"; do
mv $file $SNAPSHOT
done