You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by am...@apache.org on 2020/08/21 16:33:49 UTC
[beam] branch master updated: [BEAM-10049] Add licenses for go
dependencies in python, java, and go SDK containers
This is an automated email from the ASF dual-hosted git repository.
amyrvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 081ad8c [BEAM-10049] Add licenses for go dependencies in python, java, and go SDK containers
new 209bc21 Merge pull request #12650 from alanmyrvold/licensesGo
081ad8c is described below
commit 081ad8c33fb2bd53df6cff39f09eed7ce804ce05
Author: Alan Myrvold <am...@google.com>
AuthorDate: Thu Aug 20 22:07:08 2020 +0000
[BEAM-10049] Add licenses for go dependencies in python, java, and go SDK containers
---
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 3 ++
.../container => release/go-licenses}/Dockerfile | 21 +++-------
release/go-licenses/common.gradle | 45 ++++++++++++++++++++++
release/go-licenses/get-licenses.sh | 26 +++++++++++++
release/go-licenses/go/build.gradle | 25 ++++++++++++
release/go-licenses/java/build.gradle | 25 ++++++++++++
release/go-licenses/py/build.gradle | 25 ++++++++++++
sdks/go/container/Dockerfile | 12 ++++++
sdks/go/container/build.gradle | 13 ++++++-
sdks/java/container/Dockerfile | 13 +++++++
sdks/java/container/Dockerfile-java11 | 12 ++++++
sdks/java/container/build.gradle | 9 +++++
sdks/python/container/Dockerfile | 12 ++++++
sdks/python/container/common.gradle | 9 +++++
.../container/license_scripts/pull_licenses_py.py | 1 -
settings.gradle | 3 ++
16 files changed, 237 insertions(+), 17 deletions(-)
diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index ffe1e3c..3133c4f 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -1497,6 +1497,9 @@ class BeamModulePlugin implements Plugin<Project> {
project.tasks.dockerPrepare.dependsOn project.tasks.copyLicenses
}
+ project.ext.applyDockerRunNature = {
+ project.apply plugin: "com.palantir.docker-run"
+ }
/** ***********************************************************************************************/
project.ext.applyGroovyNature = {
diff --git a/sdks/go/container/Dockerfile b/release/go-licenses/Dockerfile
similarity index 71%
copy from sdks/go/container/Dockerfile
copy to release/go-licenses/Dockerfile
index 770d4eb..43e7870 100644
--- a/sdks/go/container/Dockerfile
+++ b/release/go-licenses/Dockerfile
@@ -16,18 +16,9 @@
# limitations under the License.
###############################################################################
-FROM debian:buster
-MAINTAINER "Apache Beam <de...@beam.apache.org>"
-
-RUN apt-get update && \
- DEBIAN_FRONTEND=noninteractive apt-get install -y \
- ca-certificates \
- && \
- rm -rf /var/lib/apt/lists/*
-
-ADD target/linux_amd64/boot /opt/apache/beam/
-
-COPY target/LICENSE /opt/apache/beam/
-COPY target/NOTICE /opt/apache/beam/
-
-ENTRYPOINT ["/opt/apache/beam/boot"]
+FROM golang:1.15.0-buster
+RUN go get github.com/google/go-licenses
+COPY get-licenses.sh /opt/apache/beam/
+ARG sdk_location
+ENV sdk_location=$sdk_location
+ENTRYPOINT /opt/apache/beam/get-licenses.sh
diff --git a/release/go-licenses/common.gradle b/release/go-licenses/common.gradle
new file mode 100644
index 0000000..751b8d9
--- /dev/null
+++ b/release/go-licenses/common.gradle
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+def licenseOutput = "${buildDir}/output"
+
+docker {
+ name "golicenses-${sdkName}"
+ dockerfile file('../Dockerfile')
+ files '../get-licenses.sh'
+ noCache false
+ buildArgs(['sdk_location': "github.com/apache/beam/sdks/${sdkName}/container"])
+}
+
+// The mkdir happens at configuration time to allow dockerRun volume to be declared.
+mkdir "${licenseOutput}"
+
+dockerRun {
+ name "license-container-${sdkName}"
+ image "golicenses-${sdkName}:latest"
+ volumes "${licenseOutput}": '/output'
+ daemonize false
+ clean true
+}
+tasks.dockerRun.dependsOn 'docker'
+
+task createLicenses {
+ outputs.cacheIf { true }
+ outputs.file("$buildDir/output/licenses/list.csv")
+ dependsOn 'dockerRun'
+}
diff --git a/release/go-licenses/get-licenses.sh b/release/go-licenses/get-licenses.sh
new file mode 100755
index 0000000..727fb9b
--- /dev/null
+++ b/release/go-licenses/get-licenses.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -ex
+rm -rf /output/*
+
+go get $sdk_location
+
+go-licenses save $sdk_location --save_path=/output/licenses
+go-licenses csv $sdk_location | tee /output/licenses/list.csv
+chmod -R a+w /output/*
diff --git a/release/go-licenses/go/build.gradle b/release/go-licenses/go/build.gradle
new file mode 100644
index 0000000..bb49353
--- /dev/null
+++ b/release/go-licenses/go/build.gradle
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+plugins { id 'org.apache.beam.module' }
+applyDockerNature()
+applyDockerRunNature()
+description = 'Apache Beam :: Release :: Go Licenses :: Go'
+
+project.ext.sdkName = 'go'
+apply from: '../common.gradle'
diff --git a/release/go-licenses/java/build.gradle b/release/go-licenses/java/build.gradle
new file mode 100644
index 0000000..96a328d
--- /dev/null
+++ b/release/go-licenses/java/build.gradle
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+plugins { id 'org.apache.beam.module' }
+applyDockerNature()
+applyDockerRunNature()
+description = 'Apache Beam :: Release :: Go Licenses :: Java'
+
+project.ext.sdkName = 'java'
+apply from: '../common.gradle'
diff --git a/release/go-licenses/py/build.gradle b/release/go-licenses/py/build.gradle
new file mode 100644
index 0000000..a8e5205
--- /dev/null
+++ b/release/go-licenses/py/build.gradle
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+plugins { id 'org.apache.beam.module' }
+applyDockerNature()
+applyDockerRunNature()
+description = 'Apache Beam :: Release :: Go Licenses :: Python'
+
+project.ext.sdkName = 'python'
+apply from: '../common.gradle'
diff --git a/sdks/go/container/Dockerfile b/sdks/go/container/Dockerfile
index 770d4eb..b9ed238 100644
--- a/sdks/go/container/Dockerfile
+++ b/sdks/go/container/Dockerfile
@@ -19,6 +19,8 @@
FROM debian:buster
MAINTAINER "Apache Beam <de...@beam.apache.org>"
+ARG pull_licenses
+
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y \
ca-certificates \
@@ -30,4 +32,14 @@ ADD target/linux_amd64/boot /opt/apache/beam/
COPY target/LICENSE /opt/apache/beam/
COPY target/NOTICE /opt/apache/beam/
+# Add golang licenses. Because the go-license directory may be empty if
+# pull_licenses is false, and COPY fails if there are no files,
+# copy an extra LICENSE file then remove it.
+COPY target/LICENSE target/go-licenses/* /opt/apache/beam/third_party_licenses/golang/
+RUN rm /opt/apache/beam/third_party_licenses/golang/LICENSE
+RUN if [ "$pull_licenses" = "false" ] ; then \
+ # Remove above golang license and dir if pull licenses false
+ rm -rf /opt/apache/beam/third_party_licenses ; \
+ fi
+
ENTRYPOINT ["/opt/apache/beam/boot"]
diff --git a/sdks/go/container/build.gradle b/sdks/go/container/build.gradle
index c22cbc6..7d54e56 100644
--- a/sdks/go/container/build.gradle
+++ b/sdks/go/container/build.gradle
@@ -52,6 +52,17 @@ docker {
project.rootProject["docker-repository-root"] :
project.docker_image_default_repo_root)
files "./build/"
+ buildArgs(['pull_licenses': project.rootProject.hasProperty(["docker-pull-licenses"]) ||
+ project.rootProject.hasProperty(["isRelease"])])
}
-// Ensure that making the docker image builds any required artifacts
dockerPrepare.dependsOn goBuild
+
+// Ensure that making the docker image builds any required artifacts
+if (project.rootProject.hasProperty(["docker-pull-licenses"])) {
+ task copyGolangLicenses(type: Copy) {
+ from "${project(':release:go-licenses:go').buildDir}/output"
+ into "build/target/go-licenses"
+ dependsOn ':release:go-licenses:go:createLicenses'
+ }
+ dockerPrepare.dependsOn 'copyGolangLicenses'
+}
diff --git a/sdks/java/container/Dockerfile b/sdks/java/container/Dockerfile
index f2b1c06..b637384 100644
--- a/sdks/java/container/Dockerfile
+++ b/sdks/java/container/Dockerfile
@@ -19,6 +19,8 @@
FROM openjdk:8
MAINTAINER "Apache Beam <de...@beam.apache.org>"
+ARG pull_licenses
+
ADD target/slf4j-api.jar /opt/apache/beam/jars/
ADD target/slf4j-jdk14.jar /opt/apache/beam/jars/
ADD target/beam-sdks-java-harness.jar /opt/apache/beam/jars/
@@ -35,4 +37,15 @@ COPY target/NOTICE /opt/apache/beam/
# copy third party licenses
ADD target/third_party_licenses /opt/apache/beam/third_party_licenses/
+
+# Add golang licenses. Because the go-license directory may be empty if
+# pull_licenses is false, and COPY fails if there are no files,
+# copy an extra LICENSE file then remove it.
+COPY target/LICENSE target/go-licenses/* /opt/apache/beam/third_party_licenses/golang/
+RUN rm /opt/apache/beam/third_party_licenses/golang/LICENSE
+RUN if [ "$pull_licenses" = "false" ] ; then \
+ # Remove above license dir if pull licenses false
+ rm -rf /opt/apache/beam/third_party_licenses ; \
+ fi
+
ENTRYPOINT ["/opt/apache/beam/boot"]
diff --git a/sdks/java/container/Dockerfile-java11 b/sdks/java/container/Dockerfile-java11
index 0ca4fa3..1d85e85 100644
--- a/sdks/java/container/Dockerfile-java11
+++ b/sdks/java/container/Dockerfile-java11
@@ -19,6 +19,8 @@
FROM openjdk:11
MAINTAINER "Apache Beam <de...@beam.apache.org>"
+ARG pull_licenses
+
ADD target/slf4j-api.jar /opt/apache/beam/jars/
ADD target/slf4j-jdk14.jar /opt/apache/beam/jars/
ADD target/beam-sdks-java-harness.jar /opt/apache/beam/jars/
@@ -30,4 +32,14 @@ ADD target/kafka-clients.jar /opt/apache/beam/jars/
ADD target/linux_amd64/boot /opt/apache/beam/
+# Add golang licenses. Because the go-license directory may be empty if
+# pull_licenses is false, and COPY fails if there are no files,
+# copy an extra LICENSE file then remove it.
+COPY target/LICENSE target/go-licenses/* /opt/apache/beam/third_party_licenses/golang/
+RUN rm /opt/apache/beam/third_party_licenses/golang/LICENSE
+RUN if [ "$pull_licenses" = "false" ] ; then \
+ # Remove above license dir if pull licenses false
+ rm -rf /opt/apache/beam/third_party_licenses ; \
+ fi
+
ENTRYPOINT ["/opt/apache/beam/boot"]
diff --git a/sdks/java/container/build.gradle b/sdks/java/container/build.gradle
index 92a46b0..2f21bae 100644
--- a/sdks/java/container/build.gradle
+++ b/sdks/java/container/build.gradle
@@ -96,6 +96,8 @@ docker {
project.rootProject["docker-tag"] : project.sdk_version)
dockerfile project.file("./${dockerfileName}")
files "./build/"
+ buildArgs(['pull_licenses': project.rootProject.hasProperty(["docker-pull-licenses"]) ||
+ project.rootProject.hasProperty(["isRelease"])])
}
if (project.rootProject.hasProperty(["docker-pull-licenses"]) ||
@@ -109,6 +111,13 @@ if (project.rootProject.hasProperty(["docker-pull-licenses"]) ||
}
}
dockerPrepare.dependsOn pullLicenses
+
+ task copyGolangLicenses(type: Copy) {
+ from "${project(':release:go-licenses:java').buildDir}/output"
+ into "build/target/go-licenses"
+ dependsOn ':release:go-licenses:java:createLicenses'
+ }
+ dockerPrepare.dependsOn 'copyGolangLicenses'
} else {
task createFile(type: Exec) {
executable "sh"
diff --git a/sdks/python/container/Dockerfile b/sdks/python/container/Dockerfile
index f28d6e3..7f87e1c 100644
--- a/sdks/python/container/Dockerfile
+++ b/sdks/python/container/Dockerfile
@@ -56,6 +56,14 @@ COPY target/apache-beam.tar.gz /opt/apache/beam/tars/
RUN pip install -v /opt/apache/beam/tars/apache-beam.tar.gz[gcp]
COPY target/license_scripts /tmp/license_scripts/
+
+# Add golang licenses. Because the go-license directory may be empty if
+# pull_licenses is false, and COPY fails if there are no files,
+# copy an extra LICENSE file then remove it.
+COPY target/LICENSE target/go-licenses/* /opt/apache/beam/third_party_licenses/golang/
+RUN rm /opt/apache/beam/third_party_licenses/golang/LICENSE
+
+COPY target/license_scripts /tmp/license_scripts/
RUN if [ "$pull_licenses" = "true" ] ; then \
pip install 'pip-licenses<3.0.0;python_version>="3.5"' && \
pip install 'pip-licenses==1.18.0;python_version<="2.7"' && \
@@ -63,11 +71,15 @@ RUN if [ "$pull_licenses" = "true" ] ; then \
pip uninstall -y pip-licenses && \
# Remove pip cache.
rm -rf /root/.cache/pip ; \
+ else \
+ # Remove above golang license and dir if pull licenses false
+ rm -rf /opt/apache/beam/third_party_licenses ; \
fi
# Log complete list of what exact packages and versions are installed.
RUN pip freeze --all
+
COPY target/LICENSE /opt/apache/beam/
COPY target/NOTICE /opt/apache/beam/
diff --git a/sdks/python/container/common.gradle b/sdks/python/container/common.gradle
index 2a6ec55..bb4eba9 100644
--- a/sdks/python/container/common.gradle
+++ b/sdks/python/container/common.gradle
@@ -69,3 +69,12 @@ docker {
dockerPrepare.dependsOn copyLauncherDependencies
dockerPrepare.dependsOn copyDockerfileDependencies
dockerPrepare.dependsOn copyLicenseScripts
+
+if (project.rootProject.hasProperty(["docker-pull-licenses"])) {
+ task copyGolangLicenses(type: Copy) {
+ from "${project(':release:go-licenses:py').buildDir}/output"
+ into "build/target/go-licenses"
+ dependsOn ':release:go-licenses:py:createLicenses'
+ }
+ dockerPrepare.dependsOn 'copyGolangLicenses'
+}
diff --git a/sdks/python/container/license_scripts/pull_licenses_py.py b/sdks/python/container/license_scripts/pull_licenses_py.py
index e85ff70..c987015 100644
--- a/sdks/python/container/license_scripts/pull_licenses_py.py
+++ b/sdks/python/container/license_scripts/pull_licenses_py.py
@@ -117,7 +117,6 @@ def pull_from_url(dep, configs):
if __name__ == "__main__":
- os.makedirs(LICENSE_DIR)
no_licenses = []
logging.getLogger().setLevel(logging.INFO)
diff --git a/settings.gradle b/settings.gradle
index 7119cf1..887c607 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -19,6 +19,9 @@
rootProject.name = "beam"
include ":release"
+include ":release:go-licenses:go"
+include ":release:go-licenses:java"
+include ":release:go-licenses:py"
include ":examples:java"
include ":examples:kotlin"