You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2021/10/10 12:56:47 UTC
[hbase] branch HBASE-26318 updated (1cf0c10 -> a57ce16)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a change to branch HBASE-26318
in repository https://gitbox.apache.org/repos/asf/hbase.git.
omit 1cf0c10 HBASE-26318 Publish test logs for flaky jobs to nightlies
add b94188f HBASE-26329 Upgrade commons-io to 2.11.0 (#3722)
add 42fe5e5 HBASE-26259 Fallback support to pure Java compression (#3691)
add 11b94d3 HBASE-26313 Publish the test logs for our nightly jobs to nightlies.apache.org (#3713)
add aad8935 HBASE-26317 Publish the test logs for pre commit jenkins job to nightlies (#3715)
add 7ae71e8 HBASE-26318 Publish test logs for flaky jobs to nightlies (#3717)
add 39a20c5 HBASE-26324 Reuse compressors and decompressors in WAL CompressionContext (#3728)
add a384c23 HBASE-26335 Minor improvements to IntegrationTestLoadCommonCrawl (#3731)
add c410bae HBASE-26339 SshPublisher will skip uploading artifacts if the build is failure (#3736)
add 7832518 HBASE-26312 Shell scan fails with timestamp (#3734)
add 82ccd33 HBASE-26186 jenkins script for caching artifacts should verify cached file before relying on it. (#3590)
new a57ce16 HBASE-26341 Upload dashboard html for flaky find job to nightlies
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (1cf0c10)
\
N -- N -- N refs/heads/HBASE-26318 (a57ce16)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
dev-support/Jenkinsfile | 262 ++++++++++++++-----
dev-support/Jenkinsfile_GitHub | 108 ++++++--
.../flaky-tests/flaky-reporting.Jenkinsfile | 28 +-
dev-support/gen_redirect_html.py | 37 +++
.../cache-apache-project-artifact.sh | 18 +-
hbase-assembly/pom.xml | 20 ++
.../hadoop/hbase/io/compress/Compression.java | 282 +++++++++++++++------
.../hadoop/hbase/io/compress/CompressionUtil.java | 36 +++
.../org/apache/hadoop/hbase/util/UnsafeAccess.java | 31 +--
.../hbase/io/compress/CompressionTestBase.java | 141 +++++++++++
.../hadoop/hbase/util}/RandomDistribution.java | 2 +-
.../hbase-compression-aircompressor/pom.xml | 182 +++++++++++++
.../compress/aircompressor/HadoopCompressor.java | 212 ++++++++++++++++
.../compress/aircompressor/HadoopDecompressor.java | 148 +++++++++++
.../hbase/io/compress/aircompressor/Lz4Codec.java | 156 ++++++++++++
.../hbase/io/compress/aircompressor/LzoCodec.java | 156 ++++++++++++
.../io/compress/aircompressor/SnappyCodec.java | 156 ++++++++++++
.../hbase/io/compress/aircompressor/ZstdCodec.java | 164 ++++++++++++
.../aircompressor/TestHFileCompressionLz4.java | 33 ++-
.../aircompressor/TestHFileCompressionLzo.java | 33 ++-
.../aircompressor/TestHFileCompressionSnappy.java | 33 ++-
.../aircompressor/TestHFileCompressionZstd.java | 33 ++-
.../io/compress/aircompressor/TestLz4Codec.java | 49 ++++
.../io/compress/aircompressor/TestLzoCodec.java | 50 ++++
.../io/compress/aircompressor/TestSnappyCodec.java | 50 ++++
.../aircompressor/TestWALCompressionLz4.java | 69 +++++
.../aircompressor/TestWALCompressionLzo.java | 69 +++++
.../aircompressor/TestWALCompressionSnappy.java | 69 +++++
.../aircompressor/TestWALCompressionZstd.java | 69 +++++
.../io/compress/aircompressor/TestZstdCodec.java | 50 ++++
hbase-compression/hbase-compression-lz4/pom.xml | 166 ++++++++++++
.../hadoop/hbase/io/compress/lz4/Lz4Codec.java | 118 +++++++++
.../hbase/io/compress/lz4/Lz4Compressor.java | 203 +++++++++++++++
.../hbase/io/compress/lz4/Lz4Decompressor.java | 142 +++++++++++
.../io/compress/lz4/TestHFileCompressionLz4.java | 33 ++-
.../hadoop/hbase/io/compress/lz4/TestLz4Codec.java | 50 ++++
.../io/compress/lz4/TestWALCompressionLz4.java | 69 +++++
hbase-compression/hbase-compression-snappy/pom.xml | 166 ++++++++++++
.../hbase/io/compress/xerial/SnappyCodec.java | 118 +++++++++
.../hbase/io/compress/xerial/SnappyCompressor.java | 182 +++++++++++++
.../io/compress/xerial/SnappyDecompressor.java | 135 ++++++++++
.../xerial/TestHFileCompressionSnappy.java | 33 ++-
.../hbase/io/compress/xerial/TestSnappyCodec.java | 50 ++++
.../compress/xerial/TestWALCompressionSnappy.java | 69 +++++
hbase-compression/hbase-compression-xz/pom.xml | 166 ++++++++++++
.../hadoop/hbase/io/compress/xz/LzmaCodec.java | 119 +++++++++
.../hbase/io/compress/xz/LzmaCompressor.java | 241 ++++++++++++++++++
.../hbase/io/compress/xz/LzmaDecompressor.java | 166 ++++++++++++
.../io/compress/xz/TestHFileCompressionLzma.java | 33 ++-
.../hadoop/hbase/io/compress/xz/TestLzmaCodec.java | 57 +++++
.../io/compress/xz/TestWALCompressionLzma.java | 69 +++++
hbase-compression/hbase-compression-zstd/pom.xml | 155 +++++++++++
.../hadoop/hbase/io/compress/zstd/ZstdCodec.java | 126 +++++++++
.../hbase/io/compress/zstd/ZstdCompressor.java | 186 ++++++++++++++
.../hbase/io/compress/zstd/ZstdDecompressor.java | 136 ++++++++++
.../io/compress/zstd/TestHFileCompressionZstd.java | 33 ++-
.../io/compress/zstd/TestWALCompressionZstd.java | 69 +++++
.../hbase/io/compress/zstd/TestZstdCodec.java | 58 +++++
hbase-compression/pom.xml | 110 ++++++++
.../hbase/test/IntegrationTestLoadCommonCrawl.java | 162 +++++++-----
.../apache/hadoop/hbase/PerformanceEvaluation.java | 2 +-
.../src/main/resources/supplemental-models.xml | 15 ++
.../hbase/regionserver/wal/CompressionContext.java | 20 +-
.../hadoop/hbase/io/compress/HFileTestBase.java | 131 ++++++++++
.../apache/hadoop/hbase/io/hfile/KVGenerator.java | 1 +
.../apache/hadoop/hbase/io/hfile/KeySampler.java | 2 +-
.../hadoop/hbase/io/hfile/TestHFileEncryption.java | 10 +-
.../hadoop/hbase/io/hfile/TestHFileSeek.java | 1 +
.../wal/TestWALCellCodecWithCompression.java | 19 +-
...mpressedWAL.java => CompressedWALTestBase.java} | 109 +++-----
.../apache/hadoop/hbase/wal/TestCompressedWAL.java | 104 +-------
.../wal/TestCompressedWALValueCompression.java | 88 +++++++
hbase-shell/src/main/ruby/hbase/table.rb | 2 +-
.../apache/hadoop/hbase/client/TestTableShell.java | 2 +-
pom.xml | 62 ++++-
75 files changed, 6201 insertions(+), 533 deletions(-)
create mode 100755 dev-support/gen_redirect_html.py
create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/CompressionUtil.java
create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/io/compress/CompressionTestBase.java
rename {hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile => hbase-common/src/test/java/org/apache/hadoop/hbase/util}/RandomDistribution.java (99%)
create mode 100644 hbase-compression/hbase-compression-aircompressor/pom.xml
create mode 100644 hbase-compression/hbase-compression-aircompressor/src/main/java/org/apache/hadoop/hbase/io/compress/aircompressor/HadoopCompressor.java
create mode 100644 hbase-compression/hbase-compression-aircompressor/src/main/java/org/apache/hadoop/hbase/io/compress/aircompressor/HadoopDecompressor.java
create mode 100644 hbase-compression/hbase-compression-aircompressor/src/main/java/org/apache/hadoop/hbase/io/compress/aircompressor/Lz4Codec.java
create mode 100644 hbase-compression/hbase-compression-aircompressor/src/main/java/org/apache/hadoop/hbase/io/compress/aircompressor/LzoCodec.java
create mode 100644 hbase-compression/hbase-compression-aircompressor/src/main/java/org/apache/hadoop/hbase/io/compress/aircompressor/SnappyCodec.java
create mode 100644 hbase-compression/hbase-compression-aircompressor/src/main/java/org/apache/hadoop/hbase/io/compress/aircompressor/ZstdCodec.java
copy hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestTableShell.java => hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestHFileCompressionLz4.java (52%)
copy hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestTableShell.java => hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestHFileCompressionLzo.java (52%)
copy hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestTableShell.java => hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestHFileCompressionSnappy.java (52%)
copy hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestTableShell.java => hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestHFileCompressionZstd.java (52%)
create mode 100644 hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestLz4Codec.java
create mode 100644 hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestLzoCodec.java
create mode 100644 hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestSnappyCodec.java
create mode 100644 hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
create mode 100644 hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
create mode 100644 hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionSnappy.java
create mode 100644 hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionZstd.java
create mode 100644 hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestZstdCodec.java
create mode 100644 hbase-compression/hbase-compression-lz4/pom.xml
create mode 100644 hbase-compression/hbase-compression-lz4/src/main/java/org/apache/hadoop/hbase/io/compress/lz4/Lz4Codec.java
create mode 100644 hbase-compression/hbase-compression-lz4/src/main/java/org/apache/hadoop/hbase/io/compress/lz4/Lz4Compressor.java
create mode 100644 hbase-compression/hbase-compression-lz4/src/main/java/org/apache/hadoop/hbase/io/compress/lz4/Lz4Decompressor.java
copy hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestTableShell.java => hbase-compression/hbase-compression-lz4/src/test/java/org/apache/hadoop/hbase/io/compress/lz4/TestHFileCompressionLz4.java (53%)
create mode 100644 hbase-compression/hbase-compression-lz4/src/test/java/org/apache/hadoop/hbase/io/compress/lz4/TestLz4Codec.java
create mode 100644 hbase-compression/hbase-compression-lz4/src/test/java/org/apache/hadoop/hbase/io/compress/lz4/TestWALCompressionLz4.java
create mode 100644 hbase-compression/hbase-compression-snappy/pom.xml
create mode 100644 hbase-compression/hbase-compression-snappy/src/main/java/org/apache/hadoop/hbase/io/compress/xerial/SnappyCodec.java
create mode 100644 hbase-compression/hbase-compression-snappy/src/main/java/org/apache/hadoop/hbase/io/compress/xerial/SnappyCompressor.java
create mode 100644 hbase-compression/hbase-compression-snappy/src/main/java/org/apache/hadoop/hbase/io/compress/xerial/SnappyDecompressor.java
copy hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestTableShell.java => hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestHFileCompressionSnappy.java (52%)
create mode 100644 hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestSnappyCodec.java
create mode 100644 hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestWALCompressionSnappy.java
create mode 100644 hbase-compression/hbase-compression-xz/pom.xml
create mode 100644 hbase-compression/hbase-compression-xz/src/main/java/org/apache/hadoop/hbase/io/compress/xz/LzmaCodec.java
create mode 100644 hbase-compression/hbase-compression-xz/src/main/java/org/apache/hadoop/hbase/io/compress/xz/LzmaCompressor.java
create mode 100644 hbase-compression/hbase-compression-xz/src/main/java/org/apache/hadoop/hbase/io/compress/xz/LzmaDecompressor.java
copy hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestTableShell.java => hbase-compression/hbase-compression-xz/src/test/java/org/apache/hadoop/hbase/io/compress/xz/TestHFileCompressionLzma.java (52%)
create mode 100644 hbase-compression/hbase-compression-xz/src/test/java/org/apache/hadoop/hbase/io/compress/xz/TestLzmaCodec.java
create mode 100644 hbase-compression/hbase-compression-xz/src/test/java/org/apache/hadoop/hbase/io/compress/xz/TestWALCompressionLzma.java
create mode 100644 hbase-compression/hbase-compression-zstd/pom.xml
create mode 100644 hbase-compression/hbase-compression-zstd/src/main/java/org/apache/hadoop/hbase/io/compress/zstd/ZstdCodec.java
create mode 100644 hbase-compression/hbase-compression-zstd/src/main/java/org/apache/hadoop/hbase/io/compress/zstd/ZstdCompressor.java
create mode 100644 hbase-compression/hbase-compression-zstd/src/main/java/org/apache/hadoop/hbase/io/compress/zstd/ZstdDecompressor.java
copy hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestTableShell.java => hbase-compression/hbase-compression-zstd/src/test/java/org/apache/hadoop/hbase/io/compress/zstd/TestHFileCompressionZstd.java (52%)
create mode 100644 hbase-compression/hbase-compression-zstd/src/test/java/org/apache/hadoop/hbase/io/compress/zstd/TestWALCompressionZstd.java
create mode 100644 hbase-compression/hbase-compression-zstd/src/test/java/org/apache/hadoop/hbase/io/compress/zstd/TestZstdCodec.java
create mode 100644 hbase-compression/pom.xml
create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/io/compress/HFileTestBase.java
copy hbase-server/src/test/java/org/apache/hadoop/hbase/wal/{TestCompressedWAL.java => CompressedWALTestBase.java} (54%)
create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestCompressedWALValueCompression.java
[hbase] 01/01: HBASE-26341 Upload dashboard html for flaky find job
to nightlies
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-26318
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit a57ce163e9b59a06c8b55aa5067bd157c0ec25d5
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Sun Oct 10 20:49:28 2021 +0800
HBASE-26341 Upload dashboard html for flaky find job to nightlies
---
.../flaky-tests/flaky-reporting.Jenkinsfile | 28 ++++++++++------
dev-support/gen_redirect_html.py | 37 ++++++++++++++++++++++
2 files changed, 55 insertions(+), 10 deletions(-)
diff --git a/dev-support/flaky-tests/flaky-reporting.Jenkinsfile b/dev-support/flaky-tests/flaky-reporting.Jenkinsfile
index 25e3fde..61894a6 100644
--- a/dev-support/flaky-tests/flaky-reporting.Jenkinsfile
+++ b/dev-support/flaky-tests/flaky-reporting.Jenkinsfile
@@ -28,6 +28,9 @@ pipeline {
timeout (time: 15, unit: 'MINUTES')
timestamps()
}
+ environment {
+ ASF_NIGHTLIES = 'https://nightlies.apache.org'
+ }
parameters {
booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a lot more meta-information.')
}
@@ -45,7 +48,21 @@ pipeline {
docker build -t hbase-dev-support dev-support
docker run --ulimit nproc=12500 -v "${WORKSPACE}":/hbase -u `id -u`:`id -g` --workdir=/hbase hbase-dev-support \
python dev-support/flaky-tests/report-flakies.py --mvn -v -o output "${flaky_args[@]}"
-'''
+ '''
+ sshPublisher(publishers: [
+ sshPublisherDesc(configName: 'Nightlies',
+ transfers: [
+ sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
+ sourceFiles: "output/dashboard.html"
+ )
+ ]
+ )
+ ])
+ sh '''
+ if [ -f "output/dashboard.html" ]; then
+ ./dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/output/dashboard.html" > output/dashboard.html
+ fi
+ '''
}
}
}
@@ -53,15 +70,6 @@ pipeline {
always {
// Has to be relative to WORKSPACE.
archiveArtifacts artifacts: "output/*"
- publishHTML target: [
- allowMissing: true,
- keepAll: true,
- alwaysLinkToLastBuild: true,
- // Has to be relative to WORKSPACE
- reportDir: "output",
- reportFiles: 'dashboard.html',
- reportName: 'Flaky Test Report'
- ]
}
}
}
diff --git a/dev-support/gen_redirect_html.py b/dev-support/gen_redirect_html.py
new file mode 100755
index 0000000..4c47ba9
--- /dev/null
+++ b/dev-support/gen_redirect_html.py
@@ -0,0 +1,37 @@
+#!/usr/bin/python
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from string import Template
+
+if len(sys.argv) != 2 :
+ print "usage: %s <redirect url>" % sys.argv[0]
+ exit(1)
+
+url = sys.argv[1].replace(" ", "%20")
+template = Template("""<html>
+ <head>
+ <meta http-equiv="refresh" content="3; url='$url'" />
+ </head>
+ <body>
+ <p>Should be redirect to <a href="$url">this link</a> soon.</p>
+ </body>
+</html>""")
+
+output = template.substitute(url = url)
+print output