You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by dc...@apache.org on 2022/07/22 13:12:35 UTC
[datasketches-memory] 01/01: Add custom deploy script
This is an automated email from the ASF dual-hosted git repository.
dcromberge pushed a commit to branch deployment-changes
in repository https://gitbox.apache.org/repos/asf/datasketches-memory.git
commit 332485b07cba39df3098ccfb9f4c8d15873ab5dc
Author: David Cromberge <dc...@apache.org>
AuthorDate: Fri Jul 22 14:09:58 2022 +0100
Add custom deploy script
The maven project deployment is not compatible with the maven
multi-release configuration unless a dedicated maven module is
added solely for packaging and deployment. Instead, this approach
uses the maven plugin directly and should be compatible with future
additions such as memory-17.
---
.github/workflows/maven.yml | 2 +-
.gitignore | 2 +-
README.md | 13 ++--
datasketches-memory-java11/pom.xml | 16 +----
datasketches-memory-java8/pom.xml | 2 +-
docs/eclipse.md | 2 +-
docs/intellij.md | 2 +-
docs/maven.md | 2 +-
docs/module-system.md | 2 +-
pom.xml | 2 +-
tools/scripts/{ => assets}/CheckMemoryJar.java | 0
tools/scripts/{ => assets}/LoremIpsum.txt | 0
.../{getGitProperties.sh => get-git-properties.sh} | 0
...kage-mr-jar.sh => package-multi-release-jar.sh} | 6 +-
...ackage-jar.sh => package-single-release-jar.sh} | 31 ++-------
tools/scripts/sign-deploy-jar.sh | 77 ++++++++++++++++++++++
tools/scripts/{test-mr-jar.sh => test-jar.sh} | 35 ++++------
17 files changed, 120 insertions(+), 74 deletions(-)
diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
index cbc6f27..22fc03f 100644
--- a/.github/workflows/maven.yml
+++ b/.github/workflows/maven.yml
@@ -106,7 +106,7 @@ jobs:
# The GitTag for CI purposes is irrelevant
- name: Custom build script
run: |
- ./tools/scripts/compile-package-jar.sh $JAVA_HOME x.y.z .
+ ./tools/scripts/package-single-release-jar.sh $JAVA_HOME x.y.z .
shell: bash
- name: Test & Report
diff --git a/.gitignore b/.gitignore
index 10c850e..f36d4f4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -60,7 +60,7 @@ build.xml
*.properties
*.releaseBackup
*.next
-*.tag
+*.asc
# Jekyll
_site/
diff --git a/README.md b/README.md
index 2023943..ce9c5ee 100644
--- a/README.md
+++ b/README.md
@@ -151,14 +151,14 @@ This will create the following Jars:
#### Building for a specific java version
-A build script named **compile-package-jar.sh** has been provided to package a JAR for a specific java version. This is necessary in cases where a developer is unable to install all the required versions of the JDK that are required as part of the Maven build.
+A build script named **package-single-release-jar.sh** has been provided to package a JAR for a specific java version. This is necessary in cases where a developer is unable to install all the required versions of the JDK that are required as part of the Maven build.
The build script performs the following steps:
1. Sets up staging directories under **target/** for the package files
2. Uses git commands to gather information about the current Git commit and branch
3. Compiles java source tree
-4. Packages a JAR containing compiled sources together with the Manifest, Licence and Notice files
+4. Packages a JAR containing compiled sources together with the Manifest, License and Notice files
5. Checks and tests the assembled JAR by using the API to access four different resource types
The build script is located in the **tools/scripts/** directory and requires the following arguments:
@@ -171,15 +171,15 @@ For example, if the project base directory is `/src/datasketches-memory`;
To run the script for a release version:
- ./tools/scripts/compile-package-jar.sh $JAVA_HOME 2.1.0 /src/datasketches-memory
+ ./tools/scripts/package-single-release-jar.sh $JAVA_HOME 2.1.0 /src/datasketches-memory
To run the script for a snapshot version:
- ./tools/scripts/compile-package-jar.sh $JAVA_HOME 2.2.0-SNAPSHOT /src/datasketches-memory
+ ./tools/scripts/package-single-release-jar.sh $JAVA_HOME 2.2.0-SNAPSHOT /src/datasketches-memory
To run the script for an RC version:
- ./tools/scripts/compile-package-jar.sh $JAVA_HOME 2.1.0-RC1 /src/datasketches-memory
+ ./tools/scripts/package-single-release-jar.sh $JAVA_HOME 2.1.0-RC1 /src/datasketches-memory
Note that the script does **not** use the _Git Version Tag_ to adjust the working copy to a remote tag - it is expected that the user has a pristine copy of the desired branch/tag available **before** using the script.
@@ -200,3 +200,6 @@ In order to build and contribute to this project, please read the relevant IDE d
- [Eclipse IDE Setup](docs/eclipse.md)
- [IntelliJ IDE Setup](docs/intellij.md)
+
+For releasing to AppNexus, please use the `sign-deploy-jar.sh` script in the scripts directory.
+See the documentation within the script for usage instructions.
diff --git a/datasketches-memory-java11/pom.xml b/datasketches-memory-java11/pom.xml
index b37e4c7..ab4c738 100644
--- a/datasketches-memory-java11/pom.xml
+++ b/datasketches-memory-java11/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.datasketches</groupId>
- <artifactId>datasketches-memory-root</artifactId>
+ <artifactId>datasketches-memory</artifactId>
<version>2.2.0</version>
</parent>
@@ -66,8 +66,7 @@
</property>
</activation>
<properties>
- <assembly.script.path>${maven.multiModuleProjectDirectory}/tools/scripts/package-mr-jar.sh</assembly.script.path>
- <assembly.checks.script.path>${maven.multiModuleProjectDirectory}/tools/scripts/test-mr-jar.sh</assembly.checks.script.path>
+ <assembly.script.path>${maven.multiModuleProjectDirectory}/tools/scripts/package-multi-release-jar.sh</assembly.script.path>
<assembly.java.home>${env.JAVA_HOME}</assembly.java.home>
<assembly.git.tag>${project.version}</assembly.git.tag>
<assembly.base.dir>${maven.multiModuleProjectDirectory}</assembly.base.dir>
@@ -92,17 +91,6 @@
<commandlineArgs>${assembly.script.path} ${assembly.java.home} ${assembly.git.tag} ${assembly.base.dir}</commandlineArgs>
</configuration>
</execution>
- <execution>
- <id>check-mr-jar</id>
- <goals>
- <goal>exec</goal>
- </goals>
- <phase>package</phase>
- <configuration>
- <executable>bash</executable>
- <commandlineArgs>${assembly.checks.script.path} ${assembly.java.home} ${assembly.git.tag} ${assembly.base.dir}</commandlineArgs>
- </configuration>
- </execution>
</executions>
</plugin>
</plugins>
diff --git a/datasketches-memory-java8/pom.xml b/datasketches-memory-java8/pom.xml
index 06d62be..9e21d00 100644
--- a/datasketches-memory-java8/pom.xml
+++ b/datasketches-memory-java8/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.datasketches</groupId>
- <artifactId>datasketches-memory-root</artifactId>
+ <artifactId>datasketches-memory</artifactId>
<version>2.2.0</version>
</parent>
diff --git a/docs/eclipse.md b/docs/eclipse.md
index 8516fed..1ca35f2 100644
--- a/docs/eclipse.md
+++ b/docs/eclipse.md
@@ -34,7 +34,7 @@ for a Maven module should be set as follows:
| Maven submodule | JDK |
| -------------------------- | --- |
-| datasketches-memory-root | 8 |
+| datasketches-memory | 8 |
| datasketches-memory-java8 | 8 |
| datasketches-memory-java11 | 11 |
diff --git a/docs/intellij.md b/docs/intellij.md
index 69396b4..47e9654 100644
--- a/docs/intellij.md
+++ b/docs/intellij.md
@@ -35,7 +35,7 @@ for a maven module should be set as follows:
| Maven submodule | JDK |
| -------------------------- | --- |
-| datasketches-memory-root | 8 |
+| datasketches-memory | 8 |
| datasketches-memory-java8 | 8 |
| datasketches-memory-java11 | 11 |
diff --git a/docs/maven.md b/docs/maven.md
index 314e023..5fd9868 100644
--- a/docs/maven.md
+++ b/docs/maven.md
@@ -20,7 +20,7 @@
# Maven Configuration: Multi-Module Project
This project is a multi-module Maven project. A multi-module Maven project consists of an
-aggregator project (the `datasketches-memory-root` project), together with a set of submodules.
+aggregator project (the `datasketches-memory` project), together with a set of submodules.
The aggregator's configuration is inherited by each submodule, thus reducing duplication.
Datasketches Memory makes use of some features of the Java platform, for example, `Unsafe`,
diff --git a/docs/module-system.md b/docs/module-system.md
index bf16bfe..b5cf78a 100644
--- a/docs/module-system.md
+++ b/docs/module-system.md
@@ -49,7 +49,7 @@ dependency graph.
#### org.apache.datasketches.memory
-In the `datasketches-memory-java11` maven submodule root, the following module declaration has
+In the `datasketches-memory-java11` maven submodule source root, the following module declaration has
been added:
```java
diff --git a/pom.xml b/pom.xml
index d1529ab..14c1948 100644
--- a/pom.xml
+++ b/pom.xml
@@ -31,7 +31,7 @@ under the License.
</parent>
<groupId>org.apache.datasketches</groupId>
- <artifactId>datasketches-memory-root</artifactId>
+ <artifactId>datasketches-memory</artifactId>
<version>2.2.0</version>
<!-- Required for multi-module project -->
<packaging>pom</packaging>
diff --git a/tools/scripts/CheckMemoryJar.java b/tools/scripts/assets/CheckMemoryJar.java
similarity index 100%
rename from tools/scripts/CheckMemoryJar.java
rename to tools/scripts/assets/CheckMemoryJar.java
diff --git a/tools/scripts/LoremIpsum.txt b/tools/scripts/assets/LoremIpsum.txt
similarity index 100%
rename from tools/scripts/LoremIpsum.txt
rename to tools/scripts/assets/LoremIpsum.txt
diff --git a/tools/scripts/getGitProperties.sh b/tools/scripts/get-git-properties.sh
similarity index 100%
rename from tools/scripts/getGitProperties.sh
rename to tools/scripts/get-git-properties.sh
diff --git a/tools/scripts/package-mr-jar.sh b/tools/scripts/package-multi-release-jar.sh
similarity index 96%
rename from tools/scripts/package-mr-jar.sh
rename to tools/scripts/package-multi-release-jar.sh
index edb67a9..57b423e 100755
--- a/tools/scripts/package-mr-jar.sh
+++ b/tools/scripts/package-multi-release-jar.sh
@@ -20,7 +20,7 @@
# This is a general bash script to build a datasketches-memory-X.jar
# with multi-release functionality. The sources, test-sources, tests and
# javadoc jars are also included in the output.
-# It does use git and also uses the script getGitProperties.sh.
+# It does use git and also uses the script get-git-properties.sh.
#
# NOTE: This script assumes that `mvn package` has been run prior to invocation.
# By default, it is called from the maven exec-plugin.
@@ -129,7 +129,7 @@ Created-By: Apache Datasketches Memory package-mr-jar.sh
Multi-Release: true
EOF
#### Generate git.properties file ####
-echo "$($ScriptsDir/getGitProperties.sh $ProjectBaseDir $ProjectArtifactId $GitTag)" >> ${ArchiveDir}/MANIFEST.MF
+echo "$($ScriptsDir/get-git-properties.sh $ProjectBaseDir $ProjectArtifactId $GitTag)" >> ${ArchiveDir}/MANIFEST.MF
###########################
#### MULTI-RELEASE JAR ####
@@ -185,3 +185,5 @@ prepare_jar $PackageJavaDoc
rsync -q -a -I --filter="- .*" ${MemoryJava8Docs} $PackageJavaDoc
${Jar_} cfm $OutputJavaDoc ${ArchiveDir}/MANIFEST.MF -C $PackageJavaDoc .
echo "Created javadoc jar ${OutputJavaDoc}"
+
+echo "$($ScriptsDir/test-jar.sh $JDKHome $GitTag $OutputMrJar $ProjectBaseDir)"
diff --git a/tools/scripts/compile-package-jar.sh b/tools/scripts/package-single-release-jar.sh
similarity index 81%
rename from tools/scripts/compile-package-jar.sh
rename to tools/scripts/package-single-release-jar.sh
index 6eb3a81..e58ff57 100755
--- a/tools/scripts/compile-package-jar.sh
+++ b/tools/scripts/package-single-release-jar.sh
@@ -23,7 +23,8 @@
# who do not wish to install several versions of the JDK on their
# machine.
# The script does not assume a POM file and does not use Maven.
-# It does use git and also uses the script getGitProperties.sh.
+# It does use git and also uses the script get-git-properties.sh and
+# test-jar.sh scripts.
# Required Input Parameters:
# \$1 = absolute path of JDK home directory
@@ -46,7 +47,7 @@ ProjectBaseDir=$3 #this must be an absolute path
#### Setup absolute directory references ####
ProjectArtifactId="memory"
ScriptsDir=${ProjectBaseDir}/tools/scripts/
-MemoryMapFile=$ScriptsDir/LoremIpsum.txt
+MemoryMapFile=$ScriptsDir/assets/LoremIpsum.txt
#### Initialise path dependent variables ####
OutputDir=target
@@ -100,12 +101,12 @@ cp NOTICE $PackageMeta
#### Generate MANIFEST.MF ####
cat >> ${PackageManifest}<< EOF
Manifest-Version: 1.0
-Created-By: Apache Datasketches Memory compile-package-jar.sh
+Created-By: Apache Datasketches Memory package-single-release-jar.sh
Multi-Release: false
EOF
#### Generate git.properties file ####
-echo "$($ScriptsDir/getGitProperties.sh $ProjectBaseDir $ProjectArtifactId $GitTag)" >> $PackageManifest
+echo "$($ScriptsDir/get-git-properties.sh $ProjectBaseDir $ProjectArtifactId $GitTag)" >> $PackageManifest
#### Copy source tree to target/src
rsync -a -I $MemoryJava8Src $PackageSrc
@@ -134,6 +135,7 @@ echo
echo "Building JAR from ${PackageContents}..."
${Jar_} cfm $OutputJar ${PackageManifest} -C $PackageContents .
echo
+echo "Successfully built ${OutputJar}"
# Uncomment this section to display JAR contents
# echo "--- JAR CONTENTS ---"
@@ -141,23 +143,4 @@ echo
# ${Jar_} tf ${OutputJar}
# echo
-echo "--- RUN JAR CHECKS ---"
-echo
-if [[ $JavaVersion -eq 8 ]]; then
- ${Javac_} -cp $OutputJar -d $PackageTests $(find $ScriptsDir -name '*.java')
- ${Java_} -cp $PackageTests:$OutputJar org.apache.datasketches.memory.tools.scripts.CheckMemoryJar $MemoryMapFile
-else
- ${Javac_} \
- --add-modules org.apache.datasketches.memory \
- -p "$OutputJar" -d $PackageTests $(find $ScriptsDir -name '*.java')
-
- ${Java_} \
- --add-modules org.apache.datasketches.memory \
- --add-exports java.base/jdk.internal.misc=org.apache.datasketches.memory \
- --add-exports java.base/jdk.internal.ref=org.apache.datasketches.memory \
- --add-opens java.base/java.nio=org.apache.datasketches.memory \
- --add-opens java.base/sun.nio.ch=org.apache.datasketches.memory \
- -p $OutputJar -cp $PackageTests org.apache.datasketches.memory.tools.scripts.CheckMemoryJar $MemoryMapFile
-fi
-echo
-echo "Successfully built ${OutputJar}"
+echo "$($ScriptsDir/test-jar.sh $JDKHome $GitTag $OutputJar $ProjectBaseDir)"
diff --git a/tools/scripts/sign-deploy-jar.sh b/tools/scripts/sign-deploy-jar.sh
new file mode 100755
index 0000000..646e564
--- /dev/null
+++ b/tools/scripts/sign-deploy-jar.sh
@@ -0,0 +1,77 @@
+#!/bin/bash -e
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This is a general bash script to sign and deploy a datasketches-memory-X.jar.
+# This is intended to be used for releasing the Memory component to Maven central.
+
+# Required Input Parameters:
+# \$1 = Git Version Tag for this deployment
+# Example tag for SNAPSHOT : 1.0.0-SNAPSHOT
+# Example tag for Release Candidate: 1.0.0-RC1
+# Example tag for Release : 1.0.0
+# \$2 = absolute path of project.basedir
+# For example: $ <this script>.sh 2.1.0 .
+
+#### Extract GitTag, TestJar and ProjectBaseDir from input parameters ####
+GitTag=$1
+ProjectBaseDir=$2
+
+#### Setup absolute directory references ####
+OutputDir=${ProjectBaseDir}/target
+
+OutputMrJar=${OutputDir}/datasketches-memory-${GitTag}.jar
+OutputTests=${OutputDir}/datasketches-memory-${GitTag}-tests.jar
+OutputJavaDoc=${OutputDir}/datasketches-memory-${GitTag}-javadoc.jar
+OutputSources=${OutputDir}/datasketches-memory-${GitTag}-sources.jar
+OutputTestSources=${OutputDir}/datasketches-memory-${GitTag}-test-sources.jar
+OutputPom=${OutputDir}/datasketches-memory-${GitTag}-pom
+
+#### Use GNU-GPG to create signature
+sign_file () {
+ File=$1
+ gpg --verbose --personal-digest-preferences=SHA512 --detach-sign -a $File
+}
+
+### Deploy to nexus
+if [[ $GitTag == *SNAPSHOT ]]
+then
+ echo "Using SNAPSHOT repository."
+ DistributionsUrl=https://repository.apache.org/content/repositories/snapshots/
+ DistributionsId=apache.snapshots.https
+else
+ echo "Using RELEASES repository."
+ DistributionsUrl=https://repository.apache.org/service/local/staging/deploy/maven2/
+ DistributionsId=apache.releases.https
+fi;
+
+mvn org.apache.maven.plugins:maven-gpg-plugin:3.0.1:sign-and-deploy-file \
+ -Durl=$DistributionsUrl\
+ -DrepositoryId=$DistributionsId \
+ -Dfile=$OutputMrJar \
+ -Dsources=$OutputSources \
+ -Dfiles=$OutputTests,$OutputTestSources \
+ -Dtypes=jar,jar \
+ -Dclassifiers=tests,test-sources \
+ -Djavadoc=$OutputJavaDoc \
+ -Dpackaging=jar \
+ -Dversion=$GitTag \
+ -DupdateReleaseInfo=true \
+ -DpomFile=${ProjectBaseDir}/pom.xml
+
+echo "Successfully signed and deployed jars"
diff --git a/tools/scripts/test-mr-jar.sh b/tools/scripts/test-jar.sh
similarity index 76%
rename from tools/scripts/test-mr-jar.sh
rename to tools/scripts/test-jar.sh
index f846c38..a565073 100755
--- a/tools/scripts/test-mr-jar.sh
+++ b/tools/scripts/test-jar.sh
@@ -17,8 +17,7 @@
# specific language governing permissions and limitations
# under the License.
-# This is a general bash script to test a datasketches-memory-X.jar
-# with multi-release functionality.
+# This is a general bash script to test a datasketches-memory-X.jar.
# This is intended to be used for C/I matrix testing or for quick
# verification of the output from the assembly process.
@@ -28,29 +27,23 @@
# Example tag for SNAPSHOT : 1.0.0-SNAPSHOT
# Example tag for Release Candidate: 1.0.0-RC1
# Example tag for Release : 1.0.0
-# \$3 = absolute path of project.basedir
-# For example: $ <this script>.sh $JAVA_HOME 2.1.0 .
+# \$3 = absolute path of datasketches-memory-X.jar
+# \$4 = absolute path of project.basedir
+# For example: $ <this script>.sh $JAVA_HOME 2.1.0 target/datasketches-memory-X.jar .
-if [ -z "$1" ]; then echo "Missing JDK home"; exit 1; fi
-if [ -z "$2" ]; then echo "Missing Git Tag"; exit 1; fi
-if [ -z "$3" ]; then echo "Missing project.basedir"; exit 1; fi
-
-#### Extract JDKHome, Version and ProjectBaseDir from input parameters ####
+#### Extract JDKHome, Version, TestJar and ProjectBaseDir from input parameters ####
JDKHome=$1
GitTag=$2
-ProjectBaseDir=$3 #this must be an absolute path
+TestJar=$3
+ProjectBaseDir=$4
#### Setup absolute directory references ####
ProjectArtifactId="memory"
ScriptsDir=${ProjectBaseDir}/tools/scripts/
-MemoryMapFile=$ScriptsDir/LoremIpsum.txt
+MemoryMapFile=$ScriptsDir/assets/LoremIpsum.txt
#### Initialise path dependent variables ####
-OutputDir=target
-OutputJar=${OutputDir}/datasketches-memory-${GitTag}.jar
-
-PackageDir=${OutputDir}/archive-tmp
-PackageChecks=${PackageDir}/checks
+PackageChecks=target/archive-tmp/checks
#### Move to project directory ####
cd ${ProjectBaseDir}
@@ -84,12 +77,12 @@ mkdir -p $PackageChecks
echo "--- RUN JAR CHECKS ---"
echo
if [[ $JavaVersion -eq 8 ]]; then
- ${Javac_} -cp $OutputJar -d $PackageChecks $(find $ScriptsDir -name '*.java')
- ${Java_} -cp $PackageChecks:$OutputJar org.apache.datasketches.memory.tools.scripts.CheckMemoryJar $MemoryMapFile
+ ${Javac_} -cp $TestJar -d $PackageChecks $(find ${ScriptsDir/assets} -name '*.java')
+ ${Java_} -cp $PackageChecks:$TestJar org.apache.datasketches.memory.tools.scripts.CheckMemoryJar $MemoryMapFile
else
${Javac_} \
--add-modules org.apache.datasketches.memory \
- -p "$OutputJar" -d $PackageChecks $(find $ScriptsDir -name '*.java')
+ -p "$TestJar" -d $PackageChecks $(find ${ScriptsDir/assets} -name '*.java')
${Java_} \
--add-modules org.apache.datasketches.memory \
@@ -97,7 +90,7 @@ else
--add-exports java.base/jdk.internal.ref=org.apache.datasketches.memory \
--add-opens java.base/java.nio=org.apache.datasketches.memory \
--add-opens java.base/sun.nio.ch=org.apache.datasketches.memory \
- -p $OutputJar -cp $PackageChecks org.apache.datasketches.memory.tools.scripts.CheckMemoryJar $MemoryMapFile
+ -p $TestJar -cp $PackageChecks org.apache.datasketches.memory.tools.scripts.CheckMemoryJar $MemoryMapFile
fi
echo
-echo "Successfully checked ${OutputJar}"
+echo "Successfully checked ${TestJar}"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org