You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/11/23 15:24:44 UTC
[arrow] branch master updated: ARROW-1577: [JS] add ASF release
scripts
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new ea0fb37 ARROW-1577: [JS] add ASF release scripts
ea0fb37 is described below
commit ea0fb370e2ee0d11b2fbdb149247af3695fd394a
Author: Paul Taylor <pa...@me.com>
AuthorDate: Thu Nov 23 10:24:39 2017 -0500
ARROW-1577: [JS] add ASF release scripts
@wesm does [this](https://github.com/apache/arrow/commit/b270dbad4f4fd70e19613b93148ec2ae3596e1fd#diff-fc8acbd4f42fb5e6b0cad14928b68115R59) look good?
Author: Paul Taylor <pa...@me.com>
Author: Wes McKinney <we...@twosigma.com>
Closes #1346 from trxcllnt/js-asf-release-scripts and squashes the following commits:
ac246cd1 [Wes McKinney] Update npm-release.sh while integration testing from release tarball is hard
469beeac [Wes McKinney] Add to READMEs about how to release and verify JS release
160f5c30 [Paul Taylor] remove tag-version-prefix from npmrc
854020d2 [Wes McKinney] Run unit tests without integration tests for now
94859222 [Wes McKinney] Add JavaScript release verification script
dd06c468 [Wes McKinney] Create tag in source release script
2c6ce86c [Paul Taylor] number of npm-release.sh script arguments is 0
e1c3a96c [Paul Taylor] npm install before release, use npm version --git-tag-version with tag-version-prefix
58700435 [Wes McKinney] Number of arguments is 2
1133fb8c [Wes McKinney] Remove upstream Arrow version tag. Bump JS version to 0.2.0
30b602f1 [Paul Taylor] add dev/release/js-source-release.sh script
be82c087 [Paul Taylor] add npm-release.sh script to publish npm modules
f72fc833 [Paul Taylor] update test_js commands in verify-release-candidate.sh
5194951d [Paul Taylor] npm pack the js sources into the release tarball
d73c47a0 [Paul Taylor] prepare js lib for release
7d9c84a0 [Paul Taylor] remove tslib patches as it doesn't work on windows
---
ci/travis_script_integration.sh | 2 +-
dev/README.md | 8 +++
dev/release/README | 13 ++++
dev/release/js-source-release.sh | 103 +++++++++++++++++++++++++++++
dev/release/js-verify-release-candidate.sh | 100 ++++++++++++++++++++++++++++
dev/release/verify-release-candidate.sh | 11 ++-
js/.npmrc | 2 +-
js/README.md | 22 +++---
js/gulp/package-task.js | 8 +--
js/gulp/test-task.js | 3 +-
js/gulp/util.js | 16 ++++-
js/gulpfile.js | 4 +-
js/npm-release.sh | 12 +++-
js/package.json | 23 ++++---
14 files changed, 292 insertions(+), 35 deletions(-)
diff --git a/ci/travis_script_integration.sh b/ci/travis_script_integration.sh
index 105a0cc..9514a30 100755
--- a/ci/travis_script_integration.sh
+++ b/ci/travis_script_integration.sh
@@ -51,7 +51,7 @@ pushd $ARROW_JS_DIR
npm run lint
npm run build
# create initial test data
-npm run test:createTestData
+npm run create:testdata
# run once to write the snapshots
npm test -- -t ts -u --integration
# run again to test all builds against the snapshots
diff --git a/dev/README.md b/dev/README.md
index f3e963c..62ffb0a 100644
--- a/dev/README.md
+++ b/dev/README.md
@@ -113,3 +113,11 @@ Studio 2015):
```
dev/release/verify-release-candidate.bat apache-arrow-0.7.0.tar.gz
```
+
+### Verifying the JavaScript release
+
+For JavaScript-specific releases, use a different verification script:
+
+```shell
+bash dev/release/js-verify-release-candidate.sh 0.7.0 0
+```
\ No newline at end of file
diff --git a/dev/release/README b/dev/release/README
index dd2f512..8e0104e 100644
--- a/dev/release/README
+++ b/dev/release/README
@@ -49,3 +49,16 @@ Logon to the apache repository: https://repository.apache.org/#stagingRepositori
Select the arrow staging repository you just just created: orgapachearrow-100x
Click the "close" button
Once validation has passed, click the "release" button
+
+# Releasing JavaScript
+
+* Make release branch then tag the release
+
+git checkout -b release-js-X.Y.Z
+git tag -a apache-arrow-js-X.Y.Z
+
+* Build the source release (requires NodeJS)
+
+dev/release/js-source-release.sh X.Y.Z $RC_NUM
+
+* After release vote push tag, and rebase master on release branch
\ No newline at end of file
diff --git a/dev/release/js-source-release.sh b/dev/release/js-source-release.sh
new file mode 100755
index 0000000..bf32acd
--- /dev/null
+++ b/dev/release/js-source-release.sh
@@ -0,0 +1,103 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+set -e
+
+SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+if [ "$#" -ne 2 ]; then
+ echo "Usage: $0 <js-version> <rc-num>"
+ exit
+fi
+
+js_version=$1
+rc=$2
+
+tag=apache-arrow-js-${js_version}
+tagrc=${tag}-rc${rc}
+
+echo "Preparing source for tag ${tag}"
+
+tarball=${tag}.tar.gz
+
+# cd to $ARROW_HOME/js
+cd $SOURCE_DIR/../../js
+JS_SRC_DIR="$PWD"
+# npm pack the js source files
+npm install
+
+npm version --no-git-tag-version $js_version
+git add package.json
+git commit -m "[Release] Apache Arrow JavaScript $js_version"
+git tag -a ${tag}
+
+release_hash=`git rev-list $tag 2> /dev/null | head -n 1 `
+
+if [ -z "$release_hash" ]; then
+ echo "Cannot continue: unknown git tag: $tag"
+ exit
+fi
+
+echo "Using commit $release_hash"
+
+cd $SOURCE_DIR
+
+rm -rf js-tmp
+# `npm pack` writes the .tgz file to the current dir, so cd into js-tmp
+mkdir -p js-tmp
+cd js-tmp
+# run npm pack on `arrow/js`
+npm pack ${JS_SRC_DIR}
+# unzip and remove the npm pack tarball
+tar -xzf *.tgz && rm *.tgz
+# `npm pack` puts files in a dir called "package"
+cp $JS_SRC_DIR/../NOTICE.txt package
+cp $JS_SRC_DIR/../LICENSE.txt package
+# rename "package" to $tag
+mv package ${tag}
+tar czf ${tarball} ${tag}
+rm -rf ${tag}
+
+${SOURCE_DIR}/run-rat.sh ${tarball}
+
+# sign the archive
+gpg --armor --output ${tarball}.asc --detach-sig ${tarball}
+gpg --print-md MD5 ${tarball} > ${tarball}.md5
+sha1sum $tarball > ${tarball}.sha1
+sha256sum $tarball > ${tarball}.sha256
+sha512sum $tarball > ${tarball}.sha512
+
+# check out the arrow RC folder
+svn co --depth=empty https://dist.apache.org/repos/dist/dev/arrow js-rc-tmp
+
+# add the release candidate for the tag
+mkdir -p js-rc-tmp/${tagrc}
+cp ${tarball}* js-rc-tmp/${tagrc}
+svn add js-rc-tmp/${tagrc}
+svn ci -m 'Apache Arrow JavaScript ${version} RC${rc}' js-rc-tmp/${tagrc}
+
+cd -
+
+# clean up
+rm -rf js-tmp
+
+echo "Success! The release candidate is available here:"
+echo " https://dist.apache.org/repos/dist/dev/arrow/${tagrc}"
+echo ""
+echo "Commit SHA1: ${release_hash}"
diff --git a/dev/release/js-verify-release-candidate.sh b/dev/release/js-verify-release-candidate.sh
new file mode 100755
index 0000000..5a37e10
--- /dev/null
+++ b/dev/release/js-verify-release-candidate.sh
@@ -0,0 +1,100 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Requirements
+# - nodejs >= 6.0.0 (best way is to use nvm)
+
+case $# in
+ 2) VERSION="$1"
+ RC_NUMBER="$2"
+ ;;
+
+ *) echo "Usage: $0 X.Y.Z RC_NUMBER"
+ exit 1
+ ;;
+esac
+
+set -ex
+
+HERE=$(cd `dirname "${BASH_SOURCE[0]:-$0}"` && pwd)
+
+ARROW_DIST_URL='https://dist.apache.org/repos/dist/dev/arrow'
+
+download_dist_file() {
+ curl -f -O $ARROW_DIST_URL/$1
+}
+
+download_rc_file() {
+ download_dist_file apache-arrow-js-${VERSION}-rc${RC_NUMBER}/$1
+}
+
+import_gpg_keys() {
+ download_dist_file KEYS
+ gpg --import KEYS
+}
+
+fetch_archive() {
+ local dist_name=$1
+ download_rc_file ${dist_name}.tar.gz
+ download_rc_file ${dist_name}.tar.gz.asc
+ download_rc_file ${dist_name}.tar.gz.md5
+ download_rc_file ${dist_name}.tar.gz.sha512
+ gpg --verify ${dist_name}.tar.gz.asc ${dist_name}.tar.gz
+ gpg --print-md MD5 ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.md5
+ if [ "$(uname)" == "Darwin" ]; then
+ shasum -a 512 ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha512
+ else
+ sha512sum ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha512
+ fi
+}
+
+setup_tempdir() {
+ cleanup() {
+ rm -fr "$TMPDIR"
+ }
+ trap cleanup EXIT
+ TMPDIR=$(mktemp -d -t "$1.XXXXX")
+}
+
+setup_tempdir "arrow-js-$VERSION"
+echo "Working in sandbox $TMPDIR"
+cd $TMPDIR
+
+VERSION=$1
+RC_NUMBER=$2
+
+TARBALL=apache-arrow-js-$1.tar.gz
+
+import_gpg_keys
+
+DIST_NAME="apache-arrow-js-${VERSION}"
+fetch_archive $DIST_NAME
+tar xvzf ${DIST_NAME}.tar.gz
+cd ${DIST_NAME}
+
+npm install
+# npx run-s clean:all lint create:testdata build
+# npm run test -- -t ts -u --integration
+# npm run test -- --integration
+npx run-s clean:all lint build
+npm run test
+
+echo 'Release candidate looks good!'
+exit 0
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
index 8d1bed7..7c5e7fd 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -173,7 +173,16 @@ test_glib() {
test_js() {
pushd js
npm install
- npm run validate
+ # clean, lint, and build JS source
+ npm run clean:all
+ npm run lint
+ npm run build
+ # create initial integration test data
+ npm run create:testdata
+ # run once to write the snapshots
+ npm test -- -t ts -u --integration
+ # run again to test all builds against the snapshots
+ npm test -- --integration
popd
}
diff --git a/js/.npmrc b/js/.npmrc
index 71ffabd..b6b25d1 100644
--- a/js/.npmrc
+++ b/js/.npmrc
@@ -1,2 +1,2 @@
save-prefix=
-package-lock=false
+package-lock=false
\ No newline at end of file
diff --git a/js/README.md b/js/README.md
index e07be6b..9b1753f 100644
--- a/js/README.md
+++ b/js/README.md
@@ -183,22 +183,24 @@ The base `apache-arrow` package includes all the compilation targets for conveni
The targets are also published under the `@apache-arrow` namespace:
```sh
-npm install @apache-arrow/es5-cjs # ES5 CommonJS target
-npm install @apache-arrow/es5-esm # ES5 ESModules target
-npm install @apache-arrow/es5-umd # ES5 UMD target
-npm install @apache-arrow/es2015-cjs # ES2015 CommonJS target
-npm install @apache-arrow/es2015-esm # ES2015 ESModules target
-npm install @apache-arrow/es2015-umd # ES2015 UMD target
-npm install @apache-arrow/esnext-esm # ESNext CommonJS target
-npm install @apache-arrow/esnext-esm # ESNext ESModules target
-npm install @apache-arrow/esnext-umd # ESNext UMD target
+npm install apache-arrow # <-- combined es5/CommonJS + UMD, es2015/ESModules + UMD, and TypeScript package
+npm install @apache-arrow/ts # standalone TypeScript package
+npm install @apache-arrow/es5-cjs # standalone es5/CommonJS package
+npm install @apache-arrow/es5-esm # standalone es5/ESModules package
+npm install @apache-arrow/es5-umd # standalone es5/UMD package
+npm install @apache-arrow/es2015-cjs # standalone es2015/CommonJS package
+npm install @apache-arrow/es2015-esm # standalone es2015/ESModules package
+npm install @apache-arrow/es2015-umd # standalone es2015/UMD package
+npm install @apache-arrow/esnext-esm # standalone esNext/CommonJS package
+npm install @apache-arrow/esnext-esm # standalone esNext/ESModules package
+npm install @apache-arrow/esnext-umd # standalone esNext/UMD package
```
### Why we package like this
The JS community is a diverse group with a varied list of target environments and tool chains. Publishing multiple packages accommodates projects of all stripes.
-If you think we missed a compilation target and it's a blocker for adoption, please open an issue. We're here for you ❤️.
+If you think we missed a compilation target and it's a blocker for adoption, please open an issue.
# License
diff --git a/js/gulp/package-task.js b/js/gulp/package-task.js
index ad56d17..824284e 100644
--- a/js/gulp/package-task.js
+++ b/js/gulp/package-task.js
@@ -48,13 +48,7 @@ const createMainPackageJson = (target, format) => (orig) => ({
module: `${mainExport}.mjs`,
browser: `${mainExport}.es5.min.js`,
[`browser:es2015`]: `${mainExport}.es2015.min.js`,
- [`@std/esm`]: { esm: `mjs` },
- // Temporary workaround until https://github.com/Microsoft/tslib/pull/44 is merged
- scripts: {
- postinstall: `npm i shx && npm run tslib_mjs && npm run tslib_pkg && npm r shx`,
- tslib_mjs: `shx cp $(node -e \"console.log(require.resolve('tslib/tslib.es6.js'))\") $(node -e \"var r=require,p=r('path');console.log(p.join(p.dirname(r.resolve('tslib')),'tslib.mjs'))\")`,
- tslib_pkg: `node -e \"var r=require,p=r('path'),f=r('fs'),k=p.join(p.dirname(r.resolve('tslib')),'package.json'),x=JSON.parse(f.readFileSync(k));x.main='tslib';f.writeFileSync(k,JSON.stringify(x))\"`
- }
+ [`@std/esm`]: { esm: `mjs` }
});
const createTypeScriptPackageJson = (target, format) => (orig) => ({
diff --git a/js/gulp/test-task.js b/js/gulp/test-task.js
index ae6b807..ac9755e 100644
--- a/js/gulp/test-task.js
+++ b/js/gulp/test-task.js
@@ -73,12 +73,13 @@ const CPP_STREAM_TO_FILE = path.join(CPP_EXE_PATH, 'stream-to-file');
const CPP_FILE_TO_STREAM = path.join(CPP_EXE_PATH, 'file-to-stream');
const testFilesDir = path.join(ARROW_HOME, 'js/test/data');
+const snapshotsDir = path.join(ARROW_HOME, 'js/test/__snapshots__');
const cppFilesDir = path.join(testFilesDir, 'cpp');
const javaFilesDir = path.join(testFilesDir, 'java');
const jsonFilesDir = path.join(testFilesDir, 'json');
async function cleanTestData() {
- return await del(`${testFilesDir}/**`);
+ return await del([`${testFilesDir}/**`, `${snapshotsDir}/**`]);
}
async function createTestJSON() {
diff --git a/js/gulp/util.js b/js/gulp/util.js
index e23fc39..23476da 100644
--- a/js/gulp/util.js
+++ b/js/gulp/util.js
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+const fs = require('fs');
const path = require(`path`);
const pump = require(`pump`);
const { Observable, ReplaySubject } = require('rxjs');
@@ -27,13 +28,26 @@ const releasesRootDir = `targets`;
const knownTargets = [`es5`, `es2015`, `esnext`];
const knownModules = [`cjs`, `esm`, `cls`, `umd`];
const moduleFormatsToSkipCombosOf = { cls: true };
-const metadataFiles = [`../LICENSE.txt`, `../NOTICE.txt`, `README.md`];
const packageJSONFields = [
`version`, `license`, `description`,
`author`, `homepage`, `repository`,
`bugs`, `keywords`, `dependencies`
];
+const metadataFiles = [`LICENSE.txt`, `NOTICE.txt`, `README.md`].map((filename) => {
+ let err = false, prefixes = [`./`, `../`];
+ let p = prefixes.find((prefix) => {
+ try {
+ fs.statSync(path.resolve(path.join(prefix, filename)));
+ } catch (e) { return false; }
+ return true;
+ });
+ if (!p) {
+ throw new Error(`Couldn't find ${filename} in ./ or ../`);
+ }
+ return path.join(p, filename);
+});
+
// see: https://github.com/google/closure-compiler/blob/c1372b799d94582eaf4b507a4a22558ff26c403c/src/com/google/javascript/jscomp/CompilerOptions.java#L2988
const gCCLanguageNames = {
es5: `ECMASCRIPT5`,
diff --git a/js/gulpfile.js b/js/gulpfile.js
index cd0ac70..a603632 100644
--- a/js/gulpfile.js
+++ b/js/gulpfile.js
@@ -86,8 +86,8 @@ const buildConcurrent = (tasks) => () =>
.merge(...knownTargets.map((target) =>
del(`${targetDir(target, `cls`)}/**`)))));
-gulp.task(`test:cleanTestData`, cleanTestData);
-gulp.task(`test:createTestData`, createTestData);
+gulp.task(`clean:testdata`, cleanTestData);
+gulp.task(`create:testdata`, createTestData);
gulp.task( `test`, gulp.series(getTasks(`test`)));
gulp.task(`debug`, gulp.series(getTasks(`debug`)));
gulp.task(`clean`, gulp.parallel(getTasks(`clean`)));
diff --git a/js/npm-release.sh b/js/npm-release.sh
old mode 100644
new mode 100755
index 0bf70d9..42cd73c
--- a/js/npm-release.sh
+++ b/js/npm-release.sh
@@ -16,5 +16,15 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+set -e
-lerna publish --yes --skip-git --force-publish=*
\ No newline at end of file
+# validate the targets pass all tests before publishing
+npm install
+# npx run-s clean:all lint create:testdata build
+# npm run test -- -t ts -u --integration
+# npm run test -- --integration
+npx run-s clean:all lint build
+npm run test
+
+# publish the JS target modules to npm
+npx lerna publish --yes --skip-git --force-publish=*
diff --git a/js/package.json b/js/package.json
index 1a110b2..9933f75 100644
--- a/js/package.json
+++ b/js/package.json
@@ -17,16 +17,16 @@
"debug": "gulp debug",
"perf": "node ./perf/index.js",
"release": "./npm-release.sh",
- "validate": "run-s --silent lint build clean",
+ "clean:all": "run-p clean clean:testdata",
+ "clean:testdata": "gulp clean:testdata",
+ "create:testdata": "gulp create:testdata",
"test:coverage": "gulp test -t ts --coverage",
- "test:cleanTestData": "gulp test:cleanTestData",
- "test:createTestData": "gulp test:createTestData",
"doc": "shx rm -rf ./doc && esdoc",
- "lint": "npm-run-all -p lint:*",
+ "lint": "run-p lint:*",
"lint:src": "tslint --fix --project -p tsconfig.json -c tslint.json \"src/**/*.ts\"",
"lint:test": "tslint --fix --project -p test/tsconfig.json -c tslint.json \"test/**/*.ts\"",
"prepublishOnly": "echo \"Error: do 'npm run release' instead of 'npm publish'\" && exit 1",
- "postinstall": "shx cp node_modules/tslib/tslib.es6.js node_modules/tslib/tslib.mjs"
+ "version": "npm install && npm run clean:all"
},
"repository": {
"type": "git",
@@ -43,12 +43,15 @@
},
"homepage": "https://github.com/apache/arrow/blob/master/js/README.md",
"files": [
+ "bin",
"src",
- "dist",
- "types",
- "targets",
- "LICENSE",
- "README.md"
+ "gulp",
+ "test",
+ "*.json",
+ "tsconfig",
+ "README.md",
+ "gulpfile.js",
+ "npm-release.sh"
],
"peerDependencies": {
"command-line-usage": "4.0.1"
--
To stop receiving notification emails like this one, please contact
['"commits@arrow.apache.org" <co...@arrow.apache.org>'].