You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/07/07 17:49:54 UTC
[airflow] branch master updated: generate go client from openapi
spec (#9502)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 6c15885 generate go client from openapi spec (#9502)
6c15885 is described below
commit 6c158853ae4ae47c65a6fe1d830f47b2ad08eb5d
Author: QP Hou <qp...@scribd.com>
AuthorDate: Tue Jul 7 10:48:28 2020 -0700
generate go client from openapi spec (#9502)
* generate go client from openapi spec
* move openapi codegen to seperate workflow
---
.github/workflows/cancel.yml | 2 +-
.github/workflows/{cancel.yml => openapi.yml} | 23 ++++---
.pre-commit-config.yaml | 2 +-
chart/values.yaml | 36 +++++-----
clients/README.md | 30 +++++++++
.../workflows/cancel.yml => clients/gen/common.sh | 31 +++++----
clients/gen/go.sh | 77 ++++++++++++++++++++++
scripts/ci/docker-compose/local.yml | 4 +-
scripts/ci/openapi/client_codegen_diff.sh | 43 ++++++++++++
9 files changed, 204 insertions(+), 44 deletions(-)
diff --git a/.github/workflows/cancel.yml b/.github/workflows/cancel.yml
index 5a26c0f..e831a28 100644
--- a/.github/workflows/cancel.yml
+++ b/.github/workflows/cancel.yml
@@ -14,7 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
+---
name: Cancel
on: [push, pull_request]
jobs:
diff --git a/.github/workflows/cancel.yml b/.github/workflows/openapi.yml
similarity index 70%
copy from .github/workflows/cancel.yml
copy to .github/workflows/openapi.yml
index 5a26c0f..915c40e 100644
--- a/.github/workflows/cancel.yml
+++ b/.github/workflows/openapi.yml
@@ -14,17 +14,22 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
+---
+name: OpenAPI Build
+on:
+ push:
+ branches: ['master']
+ pull_request:
+ branches: ['master']
-name: Cancel
-on: [push, pull_request]
jobs:
- cancel:
- name: 'Cancel Previous Runs'
+ test-openapi-client-generation:
+ name: "Test OpenAPI client generation"
runs-on: ubuntu-latest
- timeout-minutes: 3
steps:
- - uses: styfle/cancel-workflow-action@0.3.2
+ - uses: actions/checkout@master
with:
- # Fetch via https://api.github.com/repos/apache/airflow/actions/workflows/ci.yml
- workflow_id: 1029499
- access_token: ${{ github.token }}
+ fetch-depth: 0
+ - name: "Generate client codegen diff"
+ run: ./scripts/ci/openapi/client_codegen_diff.sh
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index e5c3658..e0bd201 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -25,7 +25,7 @@ repos:
rev: v1.1.7
hooks:
- id: forbid-tabs
- exclude: ^docs/Makefile$
+ exclude: ^docs/Makefile$|^clients/gen/go.sh
- id: insert-license
name: Add license for all SQL files
files: \.sql$
diff --git a/chart/values.yaml b/chart/values.yaml
index 8fa7ab6..f0d5fa4 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -223,12 +223,12 @@ webserver:
extraNetworkPolicies: []
resources: {}
- # limits:
- # cpu: 100m
- # memory: 128Mi
- # requests:
- # cpu: 100m
- # memory: 128Mi
+ # limits:
+ # cpu: 100m
+ # memory: 128Mi
+ # requests:
+ # cpu: 100m
+ # memory: 128Mi
# Create initial user.
defaultUser:
@@ -265,12 +265,12 @@ flower:
# Additional network policies as needed
extraNetworkPolicies: []
resources: {}
- # limits:
- # cpu: 100m
- # memory: 128Mi
- # requests:
- # cpu: 100m
- # memory: 128Mi
+ # limits:
+ # cpu: 100m
+ # memory: 128Mi
+ # requests:
+ # cpu: 100m
+ # memory: 128Mi
service:
type: ClusterIP
@@ -281,12 +281,12 @@ statsd:
# Additional network policies as needed
extraNetworkPolicies: []
resources: {}
- # limits:
- # cpu: 100m
- # memory: 128Mi
- # requests:
- # cpu: 100m
- # memory: 128Mi
+ # limits:
+ # cpu: 100m
+ # memory: 128Mi
+ # requests:
+ # cpu: 100m
+ # memory: 128Mi
service:
extraAnnotations: {}
diff --git a/clients/README.md b/clients/README.md
new file mode 100644
index 0000000..668a98a
--- /dev/null
+++ b/clients/README.md
@@ -0,0 +1,30 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+Airflow OpenAPI clients
+=======================
+
+Supported languages:
+
+* [Golang](https://github.com/apache/airflow-client-go) generated through `./gen/go.sh`.
+
+
+## Dependencies
+
+All client generation scripts use [pre-commit](https://pre-commit.com/#install)
+to prepend license header to generated code.
diff --git a/.github/workflows/cancel.yml b/clients/gen/common.sh
old mode 100644
new mode 100755
similarity index 63%
copy from .github/workflows/cancel.yml
copy to clients/gen/common.sh
index 5a26c0f..46fc645
--- a/.github/workflows/cancel.yml
+++ b/clients/gen/common.sh
@@ -1,3 +1,5 @@
+#!/bin/bash
+
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,16 +17,19 @@
# specific language governing permissions and limitations
# under the License.
-name: Cancel
-on: [push, pull_request]
-jobs:
- cancel:
- name: 'Cancel Previous Runs'
- runs-on: ubuntu-latest
- timeout-minutes: 3
- steps:
- - uses: styfle/cancel-workflow-action@0.3.2
- with:
- # Fetch via https://api.github.com/repos/apache/airflow/actions/workflows/ci.yml
- workflow_id: 1029499
- access_token: ${{ github.token }}
+OPENAPI_GENERATOR_CLI_VER=4.3.1
+GIT_USER=${GIT_USER:-apache}
+
+function gen_client {
+ lang=$1
+ shift
+ docker run --rm \
+ -v "${SPEC_PATH}:/spec" \
+ -v "${OUTPUT_DIR}:/output" \
+ openapitools/openapi-generator-cli:v${OPENAPI_GENERATOR_CLI_VER} \
+ generate \
+ --input-spec "/spec" \
+ --generator-name "${lang}" \
+ --git-user-id "${GIT_USER}" \
+ --output "/output" "$@"
+}
diff --git a/clients/gen/go.sh b/clients/gen/go.sh
new file mode 100755
index 0000000..8c58efa
--- /dev/null
+++ b/clients/gen/go.sh
@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if [ "$#" -ne 2 ]; then
+ echo "USAGE: $0 SPEC_PATH OUTPUT_DIR"
+ exit 1
+fi
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+# shellcheck source=./clients/gen/common.sh
+source "${SCRIPT_DIR}/common.sh"
+
+VERSION=1.0.0
+go_config=(
+ "packageVersion=${VERSION}"
+ "enumClassPrefix=true"
+)
+
+SPEC_PATH=$(realpath "$1")
+if [ ! -d "$2" ]; then
+ echo "$2 is not a valid directory or does not exist."
+ exit 1
+fi
+OUTPUT_DIR=$(realpath "$2")
+
+# create openapi ignore file to keep generated code clean
+cat <<EOF > "${OUTPUT_DIR}/.openapi-generator-ignore"
+.travis.yml
+git_push.sh
+EOF
+
+set -ex
+IFS=','
+
+SPEC_PATH="${SPEC_PATH}" \
+OUTPUT_DIR="${OUTPUT_DIR}" \
+ gen_client go \
+ --package-name airflow \
+ --git-repo-id airflow/clients/go/airflow \
+ --additional-properties "${go_config[*]}"
+
+# patch generated client to support problem HTTP API
+# this patch can be removed after following upstream patch gets merged:
+# https://github.com/OpenAPITools/openapi-generator/pull/6793
+cd "${OUTPUT_DIR}" && patch -b <<'EOF'
+--- client.go
++++ client.go
+@@ -37,7 +37,7 @@ import (
+ )
+
+ var (
+- jsonCheck = regexp.MustCompile(`(?i:(?:application|text)/(?:vnd\.[^;]+\+)?json)`)
++ jsonCheck = regexp.MustCompile(`(?i:(?:application|text)/(?:vnd\.[^;]+\+)?(?:problem\+)?json)`)
+ xmlCheck = regexp.MustCompile(`(?i:(?:application|text)/xml)`)
+ )
+EOF
+
+pushd "${OUTPUT_DIR}"
+ # prepend license headers
+ pre-commit run --all-files || true
+popd
diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml
index 0e24db7..7ba83a5 100644
--- a/scripts/ci/docker-compose/local.yml
+++ b/scripts/ci/docker-compose/local.yml
@@ -18,8 +18,8 @@
version: "2.2"
services:
airflow:
- stdin_open: true # docker run -i
- tty: true # docker run -t
+ stdin_open: true # docker run -i
+ tty: true # docker run -t
# We need to mount files an directories individually because some files
# such apache_airflow.egg-info should not be mounted from host
# we only mount those files that it makes sense to edit while developing
diff --git a/scripts/ci/openapi/client_codegen_diff.sh b/scripts/ci/openapi/client_codegen_diff.sh
new file mode 100755
index 0000000..6b846fe
--- /dev/null
+++ b/scripts/ci/openapi/client_codegen_diff.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+SCRIPTS_CI_DIR=$(dirname "${BASH_SOURCE[0]}")
+
+# shellcheck source=scripts/ci/libraries/_initialization.sh
+. "${SCRIPTS_CI_DIR}"/../libraries/_initialization.sh
+get_environment_for_builds_on_ci
+
+git remote add target "https://github.com/${CI_TARGET_REPO}"
+git fetch target "${CI_TARGET_BRANCH}:${CI_TARGET_BRANCH}" --depth=1
+
+echo "Diffing openapi spec against ${CI_TARGET_BRANCH}..."
+
+SPEC_FILE=airflow/api_connexion/openapi/v1.yaml
+if ! git diff --name-only "${CI_TARGET_BRANCH}" HEAD | grep "${SPEC_FILE}" ; then
+ echo "no openapi spec change detected, going to skip client code gen validation."
+ exit 0
+fi
+
+echo "openapi spec change detected. comparing codegen diff..."
+
+mkdir -p ./clients/go/airflow
+./clients/gen/go.sh ./airflow/api_connexion/openapi/v1.yaml ./clients/go/airflow
+mkdir -p ./clients/go_target_branch/airflow
+git checkout "${CI_TARGET_BRANCH}" ./airflow/api_connexion/openapi/v1.yaml
+./clients/gen/go.sh ./airflow/api_connexion/openapi/v1.yaml ./clients/go_target_branch/airflow
+diff ./clients/go_target_branch/airflow ./clients/go/airflow || true