You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2018/06/15 01:57:19 UTC
[01/12] hbase git commit: HBASE-19377 Update Java API CC version
[Forced Update!]
Repository: hbase
Updated Branches:
refs/heads/HBASE-20331 ca0b54884 -> acaa4a5f8 (forced update)
HBASE-19377 Update Java API CC version
Compatibility checker complaining about hash collisions, newer versions
use longer id strings.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8648af07
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8648af07
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8648af07
Branch: refs/heads/HBASE-20331
Commit: 8648af07d4acc6cd31e0a55b6958aa739c7d0551
Parents: edf60b9
Author: Mike Drob <md...@apache.org>
Authored: Tue Jun 12 13:23:13 2018 -0500
Committer: Mike Drob <md...@apache.org>
Committed: Tue Jun 12 14:19:13 2018 -0500
----------------------------------------------------------------------
dev-support/checkcompatibility.py | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/8648af07/dev-support/checkcompatibility.py
----------------------------------------------------------------------
diff --git a/dev-support/checkcompatibility.py b/dev-support/checkcompatibility.py
index ea9c229..d132c35 100755
--- a/dev-support/checkcompatibility.py
+++ b/dev-support/checkcompatibility.py
@@ -156,7 +156,7 @@ def checkout_java_acc(force):
logging.info("Downloading Java ACC...")
- url = "https://github.com/lvc/japi-compliance-checker/archive/2.1.tar.gz"
+ url = "https://github.com/lvc/japi-compliance-checker/archive/2.4.tar.gz"
scratch_dir = get_scratch_dir()
path = os.path.join(scratch_dir, os.path.basename(url))
jacc = urllib2.urlopen(url)
@@ -166,7 +166,7 @@ def checkout_java_acc(force):
subprocess.check_call(["tar", "xzf", path],
cwd=scratch_dir)
- shutil.move(os.path.join(scratch_dir, "japi-compliance-checker-2.1"),
+ shutil.move(os.path.join(scratch_dir, "japi-compliance-checker-2.4"),
os.path.join(acc_dir))
@@ -266,6 +266,12 @@ def process_java_acc_output(output):
return_value[line[:6]] = values
return return_value
+def log_java_acc_version():
+ java_acc_path = os.path.join(
+ get_java_acc_dir(), "japi-compliance-checker.pl")
+
+ args = ["perl", java_acc_path, "-dumpversion"]
+ logging.info("Java ACC version: " + check_output(args))
def run_java_acc(src_name, src_jars, dst_name, dst_jars, annotations, skip_annotations, name):
""" Run the compliance checker to compare 'src' and 'dst'. """
@@ -479,6 +485,7 @@ def main():
# Download deps.
checkout_java_acc(args.force_download)
+ log_java_acc_version()
# Set up the build.
scratch_dir = get_scratch_dir()
[08/12] hbase git commit: HBASE-19735 Create a client-tarball assembly
Posted by bu...@apache.org.
HBASE-19735 Create a client-tarball assembly
Provides an extra client descriptor to build a second
tarball with a reduced set of dependencies. Not of great
impact now, but will build the way for better in the future.
Signed-off-by: Sean Busbey <bu...@apache.org>
Conflicts:
hbase-assembly/pom.xml
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b5017ead
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b5017ead
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b5017ead
Branch: refs/heads/HBASE-20331
Commit: b5017ead7472b621841bf54ec99a516ecf487af5
Parents: 4a1408f
Author: Josh Elser <el...@apache.org>
Authored: Wed Feb 7 18:37:39 2018 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Thu Jun 14 20:56:46 2018 -0500
----------------------------------------------------------------------
hbase-assembly/pom.xml | 33 +++--
.../src/main/assembly/client-components.xml | 92 +++++++++++++
hbase-assembly/src/main/assembly/client.xml | 137 +++++++++++++++++++
hbase-procedure/pom.xml | 4 +-
hbase-spark/pom.xml | 6 +
pom.xml | 10 ++
6 files changed, 268 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b5017ead/hbase-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml
index 5da105b..4fa859a 100644
--- a/hbase-assembly/pom.xml
+++ b/hbase-assembly/pom.xml
@@ -95,6 +95,7 @@
<tarLongFileMode>gnu</tarLongFileMode>
<descriptors>
<descriptor>${assembly.file}</descriptor>
+ <descriptor>src/main/assembly/client.xml</descriptor>
</descriptors>
</configuration>
</plugin>
@@ -220,22 +221,22 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-hadoop-compat</artifactId>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop-compat</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>${compat.module}</artifactId>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>${compat.module}</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-shell</artifactId>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-shell</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
@@ -315,6 +316,14 @@
<groupId>jline</groupId>
<artifactId>jline</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-shaded-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-shaded-mapreduce</artifactId>
+ </dependency>
</dependencies>
<profiles>
<profile>
http://git-wip-us.apache.org/repos/asf/hbase/blob/b5017ead/hbase-assembly/src/main/assembly/client-components.xml
----------------------------------------------------------------------
diff --git a/hbase-assembly/src/main/assembly/client-components.xml b/hbase-assembly/src/main/assembly/client-components.xml
new file mode 100644
index 0000000..2fd1b57
--- /dev/null
+++ b/hbase-assembly/src/main/assembly/client-components.xml
@@ -0,0 +1,92 @@
+<?xml version="1.0"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<!-- Filesets shared by different binary tars. -->
+<component>
+ <fileSets>
+ <!--Copy over the site if built as docs dir -->
+ <fileSet>
+ <directory>${project.basedir}/../target/site</directory>
+ <outputDirectory>docs</outputDirectory>
+ </fileSet>
+ <!-- Include top level text files-->
+ <fileSet>
+ <directory>${project.basedir}/..</directory>
+ <outputDirectory>.</outputDirectory>
+ <includes>
+ <include>CHANGES.txt</include>
+ <include>README.txt</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
+ <!-- Include the top level conf directory -->
+ <fileSet>
+ <directory>${project.basedir}/../conf</directory>
+ <outputDirectory>conf</outputDirectory>
+ <fileMode>0644</fileMode>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+ <!-- Include top level bin directory -->
+ <!-- First copy all but the *.cmd files-->
+ <fileSet>
+ <directory>${project.basedir}/../bin</directory>
+ <outputDirectory>bin</outputDirectory>
+ <includes>
+ <include>get-active-master.rb</include>
+ <include>hbase</include>
+ <include>hbase-common.sh</include>
+ <include>hbase-config.sh</include>
+ <include>hbase-jruby</include>
+ <include>hirb.rb</include>
+ <include></include>
+ </includes>
+ <fileMode>0755</fileMode>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+ <!--Now do the cmd files; we do not want these executable.-->
+ <fileSet>
+ <directory>${project.basedir}/../bin</directory>
+ <outputDirectory>bin</outputDirectory>
+ <includes>
+ <include>hbase.cmd</include>
+ <include>hbase-config.cmd</include>
+ </includes>
+ </fileSet>
+ <!-- Move the ruby code over -->
+ <fileSet>
+ <directory>${project.basedir}/../hbase-shell/src/main/ruby</directory>
+ <outputDirectory>lib/ruby</outputDirectory>
+ <fileMode>0644</fileMode>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+ <!-- Include native libraries -->
+ <fileSet>
+ <directory>${project.basedir}/../hbase-server/target/native</directory>
+ <outputDirectory>lib/native</outputDirectory>
+ <fileMode>0755</fileMode>
+ <directoryMode>0755</directoryMode>
+ <includes>
+ <include>*.so</include>
+ <include>*.dylib</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+</component>
http://git-wip-us.apache.org/repos/asf/hbase/blob/b5017ead/hbase-assembly/src/main/assembly/client.xml
----------------------------------------------------------------------
diff --git a/hbase-assembly/src/main/assembly/client.xml b/hbase-assembly/src/main/assembly/client.xml
new file mode 100644
index 0000000..7951961
--- /dev/null
+++ b/hbase-assembly/src/main/assembly/client.xml
@@ -0,0 +1,137 @@
+<?xml version="1.0"?>
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1 http://maven.apache.org/xsd/assembly-1.1.1.xsd">
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+ <!--This 'all' id is not appended to the produced bundle because we do this: http://maven.apache.org/plugins/maven-assembly-plugin/faq.html#required-classifiers -->
+ <id>client-bin</id>
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+ <!-- Override the root directory in the tarball -->
+ <baseDirectory>hbase-${project.version}-client</baseDirectory>
+ <componentDescriptors>
+ <componentDescriptor>src/main/assembly/client-components.xml</componentDescriptor>
+ </componentDescriptors>
+ <moduleSets>
+ <moduleSet>
+ <useAllReactorProjects>true</useAllReactorProjects>
+ <includes>
+ <!-- Keep this list sorted by name -->
+ <include>org.apache.hbase:hbase-annotations</include>
+ <include>org.apache.hbase:hbase-client</include>
+ <include>org.apache.hbase:hbase-common</include>
+ <include>org.apache.hbase:hbase-hadoop-compat</include>
+ <include>org.apache.hbase:hbase-hadoop2-compat</include>
+ <include>org.apache.hbase:hbase-mapreduce</include>
+ <include>org.apache.hbase:hbase-metrics</include>
+ <include>org.apache.hbase:hbase-metrics-api</include>
+ <include>org.apache.hbase:hbase-procedure</include>
+ <include>org.apache.hbase:hbase-protocol</include>
+ <include>org.apache.hbase:hbase-protocol-shaded</include>
+ <include>org.apache.hbase:hbase-server</include>
+ <include>org.apache.hbase:hbase-zookeeper</include>
+ </includes>
+ <!-- Binaries for the dependencies also go in the hbase-jars directory -->
+ <binaries>
+ <outputDirectory>lib</outputDirectory>
+ <unpack>false</unpack>
+ <dependencySets>
+ <!-- Exclude jruby-complete from hbase_home/lib -->
+ <dependencySet>
+ <excludes>
+ <exclude>com.sun.jersey:*</exclude>
+ <exclude>com.sun.jersey.contribs:*</exclude>
+ <!-- Exclude jars which typical clients don't need -->
+ <exclude>org.apache.hbase:hbase-external-blockcache</exclude>
+ <exclude>org.apache.hbase:hbase-http</exclude>
+ <exclude>org.apache.hbase:hbase-replication</exclude>
+ <exclude>org.apache.hbase:hbase-rest</exclude>
+ <exclude>org.apache.hbase:hbase-rsgroup</exclude>
+ <exclude>org.apache.hbase:hbase-shaded-client</exclude>
+ <exclude>org.apache.hbase:hbase-shaded-mapreduce</exclude>
+ <!-- At present, hbase-shell doesn't actually contain
+ any Java code we need to include. Ruby files are
+ copied elsewhere in this descriptor. -->
+ <exclude>org.apache.hbase:hbase-shell</exclude>
+ <exclude>org.apache.hbase:hbase-thrift</exclude>
+ <exclude>org.jruby:jruby-complete</exclude>
+ </excludes>
+ </dependencySet>
+ </dependencySets>
+ </binaries>
+ </moduleSet>
+ <moduleSet>
+ <useAllReactorProjects>true</useAllReactorProjects>
+ <includes>
+ <include>org.apache.hbase:hbase-shaded-client</include>
+ <include>org.apache.hbase:hbase-shaded-mapreduce</include>
+ </includes>
+ <!-- Binaries for the dependencies also go in the hbase-jars directory -->
+ <binaries>
+ <outputDirectory>shaded-lib</outputDirectory>
+ <unpack>false</unpack>
+ <dependencySets>
+ <dependencySet>
+ <includes>
+ <!-- Keep this list sorted by name -->
+ <include>org.apache.hbase:hbase-shaded-client</include>
+ <include>org.apache.hbase:hbase-shaded-mapreduce</include>
+ </includes>
+ <useTransitiveDependencies>false</useTransitiveDependencies>
+ </dependencySet>
+ </dependencySets>
+ </binaries>
+ </moduleSet>
+ </moduleSets>
+ <!-- Include the generated LICENSE and NOTICE files -->
+ <files>
+ <file>
+ <source>${project.build.directory}/maven-shared-archive-resources/META-INF/LICENSE</source>
+ <outputDirectory>.</outputDirectory>
+ <destName>LICENSE.txt</destName>
+ <lineEnding>unix</lineEnding>
+ </file>
+ <file>
+ <source>${project.build.directory}/NOTICE.aggregate</source>
+ <outputDirectory>.</outputDirectory>
+ <destName>NOTICE.txt</destName>
+ <lineEnding>unix</lineEnding>
+ </file>
+ <file>
+ <source>${basedir}/src/main/resources/META-INF/LEGAL</source>
+ <outputDirectory>.</outputDirectory>
+ <destName>LEGAL</destName>
+ <lineEnding>unix</lineEnding>
+ </file>
+ </files>
+
+ <!-- Add jruby-complete to hbase_home/lib/ruby.
+ Update JRUBY_PACKAGED_WITH_HBASE in bin/hbase and hbase.cmd if you would like to update outputDirectory below -->
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>lib/ruby</outputDirectory>
+ <includes>
+ <include>org.jruby:jruby-complete</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+
+</assembly>
http://git-wip-us.apache.org/repos/asf/hbase/blob/b5017ead/hbase-procedure/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-procedure/pom.xml b/hbase-procedure/pom.xml
index 3fedda8..b89acdb 100644
--- a/hbase-procedure/pom.xml
+++ b/hbase-procedure/pom.xml
@@ -57,8 +57,8 @@
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-common</artifactId>
- <version>${project.version}</version>
- <classifier>tests</classifier>
+ <type>test-jar</type>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
http://git-wip-us.apache.org/repos/asf/hbase/blob/b5017ead/hbase-spark/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-spark/pom.xml b/hbase-spark/pom.xml
index 7654be4..a5f96b4 100644
--- a/hbase-spark/pom.xml
+++ b/hbase-spark/pom.xml
@@ -238,6 +238,12 @@
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
<artifactId>hbase-annotations</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
http://git-wip-us.apache.org/repos/asf/hbase/blob/b5017ead/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a943125..b9a1b77 100755
--- a/pom.xml
+++ b/pom.xml
@@ -1853,6 +1853,16 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-shaded-client</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-shaded-mapreduce</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<!-- General dependencies -->
<dependency>
<groupId>com.github.stephenc.findbugs</groupId>
[05/12] hbase git commit: HBASE-20625 refactor some WALCellCodec
related code
Posted by bu...@apache.org.
HBASE-20625 refactor some WALCellCodec related code
Signed-off-by: Guanghao Zhang <zg...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0b28155d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0b28155d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0b28155d
Branch: refs/heads/HBASE-20331
Commit: 0b28155d274910b4e667b949d51f78809a1eff0b
Parents: 9e9db32
Author: jingyuntian <ti...@gmail.com>
Authored: Thu Jun 14 10:25:24 2018 +0800
Committer: Guanghao Zhang <zg...@apache.org>
Committed: Thu Jun 14 19:37:01 2018 +0800
----------------------------------------------------------------------
.../hbase/protobuf/ReplicationProtbufUtil.java | 61 ++++----------
.../wal/AbstractProtobufLogWriter.java | 3 +
.../wal/AsyncProtobufLogWriter.java | 1 -
.../regionserver/wal/CompressionContext.java | 54 ++++++------
.../regionserver/wal/ProtobufLogReader.java | 2 +
.../regionserver/wal/ProtobufLogWriter.java | 1 -
.../hbase/regionserver/wal/ReaderBase.java | 3 -
.../wal/SecureProtobufLogReader.java | 1 +
.../hbase/regionserver/wal/WALCellCodec.java | 87 ++++++++++++++------
.../replication/ClusterMarkingEntryFilter.java | 2 -
.../java/org/apache/hadoop/hbase/wal/WAL.java | 11 ---
.../org/apache/hadoop/hbase/wal/WALKeyImpl.java | 60 +++++---------
.../wal/FaultyProtobufLogReader.java | 3 -
13 files changed, 134 insertions(+), 155 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0b28155d/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java
index 81dd59e..157ad1b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java
@@ -24,29 +24,25 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.UUID;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner;
-import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.PrivateCellUtil;
-import org.apache.hadoop.hbase.wal.WALKeyImpl;
+import org.apache.hadoop.hbase.regionserver.wal.WALCellCodec;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.io.SizedCellScanner;
import org.apache.hadoop.hbase.ipc.HBaseRpcController;
import org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl;
import org.apache.hadoop.hbase.wal.WALEdit;
-import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.wal.WAL.Entry;
+import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
+
@InterfaceAudience.Private
public class ReplicationProtbufUtil {
/**
@@ -81,7 +77,7 @@ public class ReplicationProtbufUtil {
* found.
*/
public static Pair<AdminProtos.ReplicateWALEntryRequest, CellScanner>
- buildReplicateWALEntryRequest(final Entry[] entries) {
+ buildReplicateWALEntryRequest(final Entry[] entries) throws IOException {
return buildReplicateWALEntryRequest(entries, null, null, null, null);
}
@@ -97,53 +93,30 @@ public class ReplicationProtbufUtil {
*/
public static Pair<AdminProtos.ReplicateWALEntryRequest, CellScanner>
buildReplicateWALEntryRequest(final Entry[] entries, byte[] encodedRegionName,
- String replicationClusterId, Path sourceBaseNamespaceDir, Path sourceHFileArchiveDir) {
+ String replicationClusterId, Path sourceBaseNamespaceDir, Path sourceHFileArchiveDir)
+ throws IOException {
// Accumulate all the Cells seen in here.
List<List<? extends Cell>> allCells = new ArrayList<>(entries.length);
int size = 0;
- WALProtos.FamilyScope.Builder scopeBuilder = WALProtos.FamilyScope.newBuilder();
AdminProtos.WALEntry.Builder entryBuilder = AdminProtos.WALEntry.newBuilder();
AdminProtos.ReplicateWALEntryRequest.Builder builder =
AdminProtos.ReplicateWALEntryRequest.newBuilder();
- HBaseProtos.UUID.Builder uuidBuilder = HBaseProtos.UUID.newBuilder();
+
for (Entry entry: entries) {
entryBuilder.clear();
- // TODO: this duplicates a lot in WALKeyImpl#getBuilder
- WALProtos.WALKey.Builder keyBuilder = entryBuilder.getKeyBuilder();
- WALKeyImpl key = entry.getKey();
- keyBuilder.setEncodedRegionName(
- UnsafeByteOperations.unsafeWrap(encodedRegionName == null
- ? key.getEncodedRegionName()
- : encodedRegionName));
- keyBuilder.setTableName(UnsafeByteOperations.unsafeWrap(key.getTableName().getName()));
- long sequenceId = key.getSequenceId();
- keyBuilder.setLogSequenceNumber(sequenceId);
- keyBuilder.setWriteTime(key.getWriteTime());
- if (key.getNonce() != HConstants.NO_NONCE) {
- keyBuilder.setNonce(key.getNonce());
+ WALProtos.WALKey.Builder keyBuilder;
+ try {
+ keyBuilder = entry.getKey().getBuilder(WALCellCodec.getNoneCompressor());
+ } catch (IOException e) {
+ throw new IOException(
+ "There should not throw exception since NoneCompressor do not throw any exceptions", e);
}
- if (key.getNonceGroup() != HConstants.NO_NONCE) {
- keyBuilder.setNonceGroup(key.getNonceGroup());
- }
- for(UUID clusterId : key.getClusterIds()) {
- uuidBuilder.setLeastSigBits(clusterId.getLeastSignificantBits());
- uuidBuilder.setMostSigBits(clusterId.getMostSignificantBits());
- keyBuilder.addClusterIds(uuidBuilder.build());
- }
- if (key.getOrigLogSeqNum() > 0) {
- keyBuilder.setOrigSequenceNumber(key.getOrigLogSeqNum());
+ if(encodedRegionName != null){
+ keyBuilder.setEncodedRegionName(
+ UnsafeByteOperations.unsafeWrap(encodedRegionName));
}
+ entryBuilder.setKey(keyBuilder.build());
WALEdit edit = entry.getEdit();
- NavigableMap<byte[], Integer> scopes = key.getReplicationScopes();
- if (scopes != null && !scopes.isEmpty()) {
- for (Map.Entry<byte[], Integer> scope: scopes.entrySet()) {
- scopeBuilder.setFamily(UnsafeByteOperations.unsafeWrap(scope.getKey()));
- WALProtos.ScopeType scopeType =
- WALProtos.ScopeType.valueOf(scope.getValue().intValue());
- scopeBuilder.setScopeType(scopeType);
- keyBuilder.addScopes(scopeBuilder.build());
- }
- }
List<Cell> cells = edit.getCells();
// Add up the size. It is used later serializing out the kvs.
for (Cell cell: cells) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/0b28155d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractProtobufLogWriter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractProtobufLogWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractProtobufLogWriter.java
index 50ac101..ae084a4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractProtobufLogWriter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractProtobufLogWriter.java
@@ -183,6 +183,8 @@ public abstract class AbstractProtobufLogWriter {
this.cellEncoder = codec.getEncoder(getOutputStreamForCellEncoder());
if (doCompress) {
this.compressor = codec.getByteStringCompressor();
+ } else {
+ this.compressor = WALCellCodec.getNoneCompressor();
}
}
@@ -198,6 +200,7 @@ public abstract class AbstractProtobufLogWriter {
this.cellEncoder = codec.getEncoder(getOutputStreamForCellEncoder());
// We do not support compression
this.compressionContext = null;
+ this.compressor = WALCellCodec.getNoneCompressor();
} else {
initAfterHeader0(doCompress);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0b28155d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncProtobufLogWriter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncProtobufLogWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncProtobufLogWriter.java
index abdc24e..6368fb7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncProtobufLogWriter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncProtobufLogWriter.java
@@ -120,7 +120,6 @@ public class AsyncProtobufLogWriter extends AbstractProtobufLogWriter
@Override
public void append(Entry entry) {
int buffered = output.buffered();
- entry.setCompressionContext(compressionContext);
try {
entry.getKey().
getBuilder(compressor).setFollowingKvCount(entry.getEdit().size()).build()
http://git-wip-us.apache.org/repos/asf/hbase/blob/0b28155d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java
index 22ac499..16866e1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.hbase.regionserver.wal;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
+import java.util.EnumMap;
+import java.util.Map;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.yetus.audience.InterfaceAudience;
@@ -35,12 +37,12 @@ public class CompressionContext {
static final String ENABLE_WAL_TAGS_COMPRESSION =
"hbase.regionserver.wal.tags.enablecompression";
- // visible only for WALKey, until we move everything into o.a.h.h.wal
- public final Dictionary regionDict;
- public final Dictionary tableDict;
- public final Dictionary familyDict;
- final Dictionary qualifierDict;
- final Dictionary rowDict;
+ public enum DictionaryIndex {
+ REGION, TABLE, FAMILY, QUALIFIER, ROW
+ }
+
+ private final Map<DictionaryIndex, Dictionary> dictionaries =
+ new EnumMap<>(DictionaryIndex.class);
// Context used for compressing tags
TagCompressionContext tagCompressionContext = null;
@@ -49,33 +51,35 @@ public class CompressionContext {
InstantiationException, IllegalAccessException, InvocationTargetException {
Constructor<? extends Dictionary> dictConstructor =
dictType.getConstructor();
- regionDict = dictConstructor.newInstance();
- tableDict = dictConstructor.newInstance();
- familyDict = dictConstructor.newInstance();
- qualifierDict = dictConstructor.newInstance();
- rowDict = dictConstructor.newInstance();
- if (recoveredEdits) {
- // This will never change
- regionDict.init(1);
- tableDict.init(1);
+ for (DictionaryIndex dictionaryIndex : DictionaryIndex.values()) {
+ Dictionary newDictionary = dictConstructor.newInstance();
+ dictionaries.put(dictionaryIndex, newDictionary);
+ }
+ if(recoveredEdits) {
+ getDictionary(DictionaryIndex.REGION).init(1);
+ getDictionary(DictionaryIndex.TABLE).init(1);
} else {
- regionDict.init(Short.MAX_VALUE);
- tableDict.init(Short.MAX_VALUE);
+ getDictionary(DictionaryIndex.REGION).init(Short.MAX_VALUE);
+ getDictionary(DictionaryIndex.TABLE).init(Short.MAX_VALUE);
}
- rowDict.init(Short.MAX_VALUE);
- familyDict.init(Byte.MAX_VALUE);
- qualifierDict.init(Byte.MAX_VALUE);
+
+ getDictionary(DictionaryIndex.ROW).init(Short.MAX_VALUE);
+ getDictionary(DictionaryIndex.FAMILY).init(Byte.MAX_VALUE);
+ getDictionary(DictionaryIndex.QUALIFIER).init(Byte.MAX_VALUE);
+
if (hasTagCompression) {
tagCompressionContext = new TagCompressionContext(dictType, Short.MAX_VALUE);
}
}
+ public Dictionary getDictionary(Enum dictIndex) {
+ return dictionaries.get(dictIndex);
+ }
+
void clear() {
- regionDict.clear();
- tableDict.clear();
- familyDict.clear();
- qualifierDict.clear();
- rowDict.clear();
+ for(Dictionary dictionary : dictionaries.values()){
+ dictionary.clear();
+ }
if (tagCompressionContext != null) {
tagCompressionContext.clear();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0b28155d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
index 5d8d8c0..83398bd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
@@ -312,6 +312,8 @@ public class ProtobufLogReader extends ReaderBase {
this.cellDecoder = codec.getDecoder(this.inputStream);
if (this.hasCompression) {
this.byteStringUncompressor = codec.getByteStringUncompressor();
+ } else {
+ this.byteStringUncompressor = WALCellCodec.getNoneUncompressor();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0b28155d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java
index 2852047..b4e2cbf 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java
@@ -47,7 +47,6 @@ public class ProtobufLogWriter extends AbstractProtobufLogWriter
@Override
public void append(Entry entry) throws IOException {
- entry.setCompressionContext(compressionContext);
entry.getKey().getBuilder(compressor).
setFollowingKvCount(entry.getEdit().size()).build().writeDelimitedTo(output);
for (Cell cell : entry.getEdit().getCells()) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/0b28155d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReaderBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReaderBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReaderBase.java
index 4338f6d..27d40b2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReaderBase.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ReaderBase.java
@@ -92,9 +92,6 @@ public abstract class ReaderBase implements AbstractFSWALProvider.Reader {
if (e == null) {
e = new Entry();
}
- if (compressionContext != null) {
- e.setCompressionContext(compressionContext);
- }
boolean hasEntry = false;
try {
http://git-wip-us.apache.org/repos/asf/hbase/blob/0b28155d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogReader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogReader.java
index b1f17ad..e43d140 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureProtobufLogReader.java
@@ -141,6 +141,7 @@ public class SecureProtobufLogReader extends ProtobufLogReader {
this.cellDecoder = codec.getDecoder(this.inputStream);
// We do not support compression with WAL encryption
this.compressionContext = null;
+ this.byteStringUncompressor = WALCellCodec.getNoneUncompressor();
this.hasCompression = false;
} else {
super.initAfterCompression(cellCodecClsName);
http://git-wip-us.apache.org/repos/asf/hbase/blob/0b28155d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java
index 2922a10..34d83f7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.util.ReflectionUtils;
import org.apache.hadoop.io.IOUtils;
import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
+import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
/**
@@ -62,12 +63,6 @@ public class WALCellCodec implements Codec {
public static final String WAL_CELL_CODEC_CLASS_KEY = "hbase.regionserver.wal.codec";
protected final CompressionContext compression;
- protected final ByteStringUncompressor statelessUncompressor = new ByteStringUncompressor() {
- @Override
- public byte[] uncompress(ByteString data, Dictionary dict) throws IOException {
- return WALCellCodec.uncompressByteString(data, dict);
- }
- };
/**
* <b>All subclasses must implement a no argument constructor</b>
@@ -132,17 +127,32 @@ public class WALCellCodec implements Codec {
}
public interface ByteStringCompressor {
- ByteString compress(byte[] data, Dictionary dict) throws IOException;
+ ByteString compress(byte[] data, Enum dictIndex) throws IOException;
}
public interface ByteStringUncompressor {
- byte[] uncompress(ByteString data, Dictionary dict) throws IOException;
+ byte[] uncompress(ByteString data, Enum dictIndex) throws IOException;
+ }
+
+ static class StatelessUncompressor implements ByteStringUncompressor {
+ CompressionContext compressionContext;
+
+ public StatelessUncompressor(CompressionContext compressionContext) {
+ this.compressionContext = compressionContext;
+ }
+
+ @Override
+ public byte[] uncompress(ByteString data, Enum dictIndex) throws IOException {
+ return WALCellCodec.uncompressByteString(data, compressionContext.getDictionary(dictIndex));
+ }
}
- // TODO: it sucks that compression context is in WAL.Entry. It'd be nice if it was here.
- // Dictionary could be gotten by enum; initially, based on enum, context would create
- // an array of dictionaries.
static class BaosAndCompressor extends ByteArrayOutputStream implements ByteStringCompressor {
+ private CompressionContext compressionContext;
+
+ public BaosAndCompressor(CompressionContext compressionContext) {
+ this.compressionContext = compressionContext;
+ }
public ByteString toByteString() {
// We need this copy to create the ByteString as the byte[] 'buf' is not immutable. We reuse
// them.
@@ -150,8 +160,8 @@ public class WALCellCodec implements Codec {
}
@Override
- public ByteString compress(byte[] data, Dictionary dict) throws IOException {
- writeCompressed(data, dict);
+ public ByteString compress(byte[] data, Enum dictIndex) throws IOException {
+ writeCompressed(data, dictIndex);
// We need this copy to create the ByteString as the byte[] 'buf' is not immutable. We reuse
// them.
ByteString result = ByteString.copyFrom(this.buf, 0, this.count);
@@ -159,7 +169,8 @@ public class WALCellCodec implements Codec {
return result;
}
- private void writeCompressed(byte[] data, Dictionary dict) throws IOException {
+ private void writeCompressed(byte[] data, Enum dictIndex) throws IOException {
+ Dictionary dict = compressionContext.getDictionary(dictIndex);
assert dict != null;
short dictIdx = dict.findEntry(data, 0, data.length);
if (dictIdx == Dictionary.NOT_IN_DICTIONARY) {
@@ -172,6 +183,22 @@ public class WALCellCodec implements Codec {
}
}
+ static class NoneCompressor implements ByteStringCompressor {
+
+ @Override
+ public ByteString compress(byte[] data, Enum dictIndex) {
+ return UnsafeByteOperations.unsafeWrap(data);
+ }
+ }
+
+ static class NoneUncompressor implements ByteStringUncompressor {
+
+ @Override
+ public byte[] uncompress(ByteString data, Enum dictIndex) {
+ return data.toByteArray();
+ }
+ }
+
private static byte[] uncompressByteString(ByteString bs, Dictionary dict) throws IOException {
InputStream in = bs.newInput();
byte status = (byte)in.read();
@@ -209,9 +236,12 @@ public class WALCellCodec implements Codec {
// To support tags
int tagsLength = cell.getTagsLength();
StreamUtils.writeRawVInt32(out, tagsLength);
- PrivateCellUtil.compressRow(out, cell, compression.rowDict);
- PrivateCellUtil.compressFamily(out, cell, compression.familyDict);
- PrivateCellUtil.compressQualifier(out, cell, compression.qualifierDict);
+ PrivateCellUtil.compressRow(out, cell,
+ compression.getDictionary(CompressionContext.DictionaryIndex.ROW));
+ PrivateCellUtil.compressFamily(out, cell,
+ compression.getDictionary(CompressionContext.DictionaryIndex.FAMILY));
+ PrivateCellUtil.compressQualifier(out, cell,
+ compression.getDictionary(CompressionContext.DictionaryIndex.QUALIFIER));
// Write timestamp, type and value as uncompressed.
StreamUtils.writeLong(out, cell.getTimestamp());
out.write(cell.getTypeByte());
@@ -255,19 +285,22 @@ public class WALCellCodec implements Codec {
pos = Bytes.putInt(backingArray, pos, vlength);
// the row
- int elemLen = readIntoArray(backingArray, pos + Bytes.SIZEOF_SHORT, compression.rowDict);
+ int elemLen = readIntoArray(backingArray, pos + Bytes.SIZEOF_SHORT,
+ compression.getDictionary(CompressionContext.DictionaryIndex.ROW));
checkLength(elemLen, Short.MAX_VALUE);
pos = Bytes.putShort(backingArray, pos, (short)elemLen);
pos += elemLen;
// family
- elemLen = readIntoArray(backingArray, pos + Bytes.SIZEOF_BYTE, compression.familyDict);
+ elemLen = readIntoArray(backingArray, pos + Bytes.SIZEOF_BYTE,
+ compression.getDictionary(CompressionContext.DictionaryIndex.FAMILY));
checkLength(elemLen, Byte.MAX_VALUE);
pos = Bytes.putByte(backingArray, pos, (byte)elemLen);
pos += elemLen;
// qualifier
- elemLen = readIntoArray(backingArray, pos, compression.qualifierDict);
+ elemLen = readIntoArray(backingArray, pos,
+ compression.getDictionary(CompressionContext.DictionaryIndex.QUALIFIER));
pos += elemLen;
// timestamp, type and value
@@ -354,12 +387,18 @@ public class WALCellCodec implements Codec {
}
public ByteStringCompressor getByteStringCompressor() {
- // TODO: ideally this should also encapsulate compressionContext
- return new BaosAndCompressor();
+ return new BaosAndCompressor(compression);
}
public ByteStringUncompressor getByteStringUncompressor() {
- // TODO: ideally this should also encapsulate compressionContext
- return this.statelessUncompressor;
+ return new StatelessUncompressor(compression);
+ }
+
+ public static ByteStringCompressor getNoneCompressor() {
+ return new NoneCompressor();
+ }
+
+ public static ByteStringUncompressor getNoneUncompressor() {
+ return new NoneUncompressor();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0b28155d/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ClusterMarkingEntryFilter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ClusterMarkingEntryFilter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ClusterMarkingEntryFilter.java
index 6dc5001..5f92bbf 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ClusterMarkingEntryFilter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ClusterMarkingEntryFilter.java
@@ -60,8 +60,6 @@ public class ClusterMarkingEntryFilter implements WALEntryFilter {
if (edit != null && !edit.isEmpty()) {
// Mark that the current cluster has the change
logKey.addClusterId(clusterId);
- // We need to set the CC to null else it will be compressed when sent to the sink
- entry.setCompressionContext(null);
return entry;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0b28155d/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
index 3c85737..cf367cd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
@@ -23,7 +23,6 @@ import java.util.Map;
import java.util.Set;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.RegionInfo;
-import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
import org.apache.hadoop.hbase.regionserver.wal.WALCoprocessorHost;
@@ -278,16 +277,6 @@ public interface WAL extends Closeable, WALFileLengthProvider {
return key;
}
- /**
- * Set compression context for this entry.
- *
- * @param compressionContext
- * Compression context
- */
- public void setCompressionContext(CompressionContext compressionContext) {
- key.setCompressionContext(compressionContext);
- }
-
@Override
public String toString() {
return this.key + "=" + this.edit;
http://git-wip-us.apache.org/repos/asf/hbase/blob/0b28155d/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java
index 8828239..71cbaf4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java
@@ -37,7 +37,6 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
-import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;
@@ -117,8 +116,6 @@ public class WALKeyImpl implements WALKey {
*/
private MultiVersionConcurrencyControl.WriteEntry writeEntry;
- private CompressionContext compressionContext;
-
public WALKeyImpl() {
init(null, null, 0L, HConstants.LATEST_TIMESTAMP,
new ArrayList<>(), HConstants.NO_NONCE, HConstants.NO_NONCE, null, null);
@@ -332,13 +329,6 @@ public class WALKeyImpl implements WALKey {
this.sequenceId = sequenceId;
}
- /**
- * @param compressionContext Compression context to use
- */
- public void setCompressionContext(CompressionContext compressionContext) {
- this.compressionContext = compressionContext;
- }
-
/** @return encoded region name */
@Override
public byte [] getEncodedRegionName() {
@@ -517,18 +507,13 @@ public class WALKeyImpl implements WALKey {
this.encodedRegionName = encodedRegionName;
}
- public WALProtos.WALKey.Builder getBuilder(
- WALCellCodec.ByteStringCompressor compressor) throws IOException {
+ public WALProtos.WALKey.Builder getBuilder(WALCellCodec.ByteStringCompressor compressor)
+ throws IOException {
WALProtos.WALKey.Builder builder = WALProtos.WALKey.newBuilder();
- if (compressionContext == null) {
- builder.setEncodedRegionName(UnsafeByteOperations.unsafeWrap(this.encodedRegionName));
- builder.setTableName(UnsafeByteOperations.unsafeWrap(this.tablename.getName()));
- } else {
- builder.setEncodedRegionName(compressor.compress(this.encodedRegionName,
- compressionContext.regionDict));
- builder.setTableName(compressor.compress(this.tablename.getName(),
- compressionContext.tableDict));
- }
+ builder.setEncodedRegionName(
+ compressor.compress(this.encodedRegionName, CompressionContext.DictionaryIndex.REGION));
+ builder.setTableName(
+ compressor.compress(this.tablename.getName(), CompressionContext.DictionaryIndex.TABLE));
builder.setLogSequenceNumber(getSequenceId());
builder.setWriteTime(writeTime);
if (this.origLogSeqNum > 0) {
@@ -548,29 +533,22 @@ public class WALKeyImpl implements WALKey {
}
if (replicationScope != null) {
for (Map.Entry<byte[], Integer> e : replicationScope.entrySet()) {
- ByteString family = (compressionContext == null)
- ? UnsafeByteOperations.unsafeWrap(e.getKey())
- : compressor.compress(e.getKey(), compressionContext.familyDict);
- builder.addScopes(FamilyScope.newBuilder()
- .setFamily(family).setScopeType(ScopeType.forNumber(e.getValue())));
+ ByteString family =
+ compressor.compress(e.getKey(), CompressionContext.DictionaryIndex.FAMILY);
+ builder.addScopes(FamilyScope.newBuilder().setFamily(family)
+ .setScopeType(ScopeType.forNumber(e.getValue())));
}
}
return builder;
}
public void readFieldsFromPb(WALProtos.WALKey walKey,
- WALCellCodec.ByteStringUncompressor uncompressor)
- throws IOException {
- if (this.compressionContext != null) {
- this.encodedRegionName = uncompressor.uncompress(
- walKey.getEncodedRegionName(), compressionContext.regionDict);
- byte[] tablenameBytes = uncompressor.uncompress(
- walKey.getTableName(), compressionContext.tableDict);
- this.tablename = TableName.valueOf(tablenameBytes);
- } else {
- this.encodedRegionName = walKey.getEncodedRegionName().toByteArray();
- this.tablename = TableName.valueOf(walKey.getTableName().toByteArray());
- }
+ WALCellCodec.ByteStringUncompressor uncompressor) throws IOException {
+ this.encodedRegionName = uncompressor.uncompress(walKey.getEncodedRegionName(),
+ CompressionContext.DictionaryIndex.REGION);
+ byte[] tablenameBytes =
+ uncompressor.uncompress(walKey.getTableName(), CompressionContext.DictionaryIndex.TABLE);
+ this.tablename = TableName.valueOf(tablenameBytes);
clusterIds.clear();
for (HBaseProtos.UUID clusterId : walKey.getClusterIdsList()) {
clusterIds.add(new UUID(clusterId.getMostSigBits(), clusterId.getLeastSigBits()));
@@ -585,14 +563,14 @@ public class WALKeyImpl implements WALKey {
if (walKey.getScopesCount() > 0) {
this.replicationScope = new TreeMap<>(Bytes.BYTES_COMPARATOR);
for (FamilyScope scope : walKey.getScopesList()) {
- byte[] family = (compressionContext == null) ? scope.getFamily().toByteArray() :
- uncompressor.uncompress(scope.getFamily(), compressionContext.familyDict);
+ byte[] family =
+ uncompressor.uncompress(scope.getFamily(), CompressionContext.DictionaryIndex.FAMILY);
this.replicationScope.put(family, scope.getScopeType().getNumber());
}
}
setSequenceId(walKey.getLogSequenceNumber());
this.writeTime = walKey.getWriteTime();
- if(walKey.hasOrigSequenceNumber()) {
+ if (walKey.hasOrigSequenceNumber()) {
this.origLogSeqNum = walKey.getOrigSequenceNumber();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0b28155d/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultyProtobufLogReader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultyProtobufLogReader.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultyProtobufLogReader.java
index f1508e5..2c74f80 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultyProtobufLogReader.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultyProtobufLogReader.java
@@ -45,9 +45,6 @@ public class FaultyProtobufLogReader extends ProtobufLogReader {
boolean b;
do {
Entry e = new Entry();
- if (compressionContext != null) {
- e.setCompressionContext(compressionContext);
- }
b = readNext(e);
nextQueue.offer(e);
numberOfFileEntries++;
[03/12] hbase git commit: HBASE-20722 Make RegionServerTracker only
depend on children changed event
Posted by bu...@apache.org.
HBASE-20722 Make RegionServerTracker only depend on children changed event
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/423a0ab7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/423a0ab7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/423a0ab7
Branch: refs/heads/HBASE-20331
Commit: 423a0ab71a4e588f3efd2d2ab96a36c16e8b37b1
Parents: ec66434
Author: zhangduo <zh...@apache.org>
Authored: Wed Jun 13 21:11:23 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Thu Jun 14 08:36:37 2018 +0800
----------------------------------------------------------------------
.../hadoop/hbase/client/VersionInfoUtil.java | 2 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 64 +++---
.../hbase/master/RegionServerTracker.java | 225 ++++++++++---------
.../hadoop/hbase/master/ServerManager.java | 17 +-
.../hbase/master/TestAssignmentListener.java | 98 +-------
.../hbase/master/TestClockSkewDetection.java | 2 +-
.../hbase/master/TestMasterNoCluster.java | 18 +-
.../hbase/master/TestShutdownBackupMaster.java | 2 +-
8 files changed, 167 insertions(+), 261 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/423a0ab7/hbase-server/src/main/java/org/apache/hadoop/hbase/client/VersionInfoUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/VersionInfoUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/VersionInfoUtil.java
index 95984de..cde59eb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/VersionInfoUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/VersionInfoUtil.java
@@ -102,7 +102,7 @@ public final class VersionInfoUtil {
* @param versionInfo the VersionInfo object to pack
* @return the version number as int. (e.g. 0x0103004 is 1.3.4)
*/
- private static int getVersionNumber(final HBaseProtos.VersionInfo versionInfo) {
+ public static int getVersionNumber(final HBaseProtos.VersionInfo versionInfo) {
if (versionInfo != null) {
try {
final String[] components = getVersionComponents(versionInfo);
http://git-wip-us.apache.org/repos/asf/hbase/blob/423a0ab7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index f20cc61..883bb4f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -72,7 +72,6 @@ import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.PleaseHoldException;
import org.apache.hadoop.hbase.ReplicationPeerNotFoundException;
import org.apache.hadoop.hbase.ScheduledChore;
-import org.apache.hadoop.hbase.ServerMetricsBuilder;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableDescriptors;
import org.apache.hadoop.hbase.TableName;
@@ -87,6 +86,7 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.client.TableState;
+import org.apache.hadoop.hbase.client.VersionInfoUtil;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.exceptions.MergeRegionException;
@@ -212,7 +212,6 @@ import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionServerInfo;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceViolationPolicy;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
@@ -297,7 +296,7 @@ public class HMaster extends HRegionServer implements MasterServices {
// Manager and zk listener for master election
private final ActiveMasterManager activeMasterManager;
// Region server tracker
- RegionServerTracker regionServerTracker;
+ private RegionServerTracker regionServerTracker;
// Draining region server tracker
private DrainingServerTracker drainingServerTracker;
// Tracker for load balancer state
@@ -725,10 +724,16 @@ public class HMaster extends HRegionServer implements MasterServices {
}
/**
+ * <p>
* Initialize all ZK based system trackers.
+ * </p>
+ * <p>
+ * Will be overridden in tests.
+ * </p>
*/
- void initializeZKBasedSystemTrackers() throws IOException, InterruptedException, KeeperException,
- ReplicationException {
+ @VisibleForTesting
+ protected void initializeZKBasedSystemTrackers()
+ throws IOException, InterruptedException, KeeperException, ReplicationException {
this.balancer = LoadBalancerFactory.getLoadBalancer(conf);
this.normalizer = RegionNormalizerFactory.getRegionNormalizer(conf);
this.normalizer.setMasterServices(this);
@@ -1070,9 +1075,15 @@ public class HMaster extends HRegionServer implements MasterServices {
}
/**
+ * <p>
* Create a {@link ServerManager} instance.
+ * </p>
+ * <p>
+ * Will be overridden in tests.
+ * </p>
*/
- ServerManager createServerManager(final MasterServices master) throws IOException {
+ @VisibleForTesting
+ protected ServerManager createServerManager(final MasterServices master) throws IOException {
// We put this out here in a method so can do a Mockito.spy and stub it out
// w/ a mocked up ServerManager.
setupClusterConnection();
@@ -1082,17 +1093,11 @@ public class HMaster extends HRegionServer implements MasterServices {
private void waitForRegionServers(final MonitoredTask status)
throws IOException, InterruptedException {
this.serverManager.waitForRegionServers(status);
- // Check zk for region servers that are up but didn't register
- for (ServerName sn: this.regionServerTracker.getOnlineServers()) {
- // The isServerOnline check is opportunistic, correctness is handled inside
- if (!this.serverManager.isServerOnline(sn) &&
- serverManager.checkAndRecordNewServer(sn, ServerMetricsBuilder.of(sn))) {
- LOG.info("Registered server found up in zk but who has not yet reported in: " + sn);
- }
- }
}
- void initClusterSchemaService() throws IOException, InterruptedException {
+ // Will be overridden in tests
+ @VisibleForTesting
+ protected void initClusterSchemaService() throws IOException, InterruptedException {
this.clusterSchemaService = new ClusterSchemaServiceImpl(this);
this.clusterSchemaService.startAsync();
try {
@@ -1104,14 +1109,14 @@ public class HMaster extends HRegionServer implements MasterServices {
}
}
- void initQuotaManager() throws IOException {
+ private void initQuotaManager() throws IOException {
MasterQuotaManager quotaManager = new MasterQuotaManager(this);
this.assignmentManager.setRegionStateListener(quotaManager);
quotaManager.start();
this.quotaManager = quotaManager;
}
- SpaceQuotaSnapshotNotifier createQuotaSnapshotNotifier() {
+ private SpaceQuotaSnapshotNotifier createQuotaSnapshotNotifier() {
SpaceQuotaSnapshotNotifier notifier =
SpaceQuotaSnapshotNotifierFactory.getInstance().create(getConfiguration());
return notifier;
@@ -1207,8 +1212,8 @@ public class HMaster extends HRegionServer implements MasterServices {
.getFileSystem(), archiveDir, params);
getChoreService().scheduleChore(hfileCleaner);
- replicationBarrierCleaner =
- new ReplicationBarrierCleaner(conf, this, getConnection(), replicationPeerManager);
+ replicationBarrierCleaner = new ReplicationBarrierCleaner(conf, this, getConnection(),
+ replicationPeerManager);
getChoreService().scheduleChore(replicationBarrierCleaner);
serviceStarted = true;
@@ -1263,6 +1268,9 @@ public class HMaster extends HRegionServer implements MasterServices {
if (this.mpmHost != null) {
this.mpmHost.stop("server shutting down.");
}
+ if (this.regionServerTracker != null) {
+ this.regionServerTracker.stop();
+ }
}
private void startProcedureExecutor() throws IOException {
@@ -2626,21 +2634,17 @@ public class HMaster extends HRegionServer implements MasterServices {
}
public int getRegionServerInfoPort(final ServerName sn) {
- RegionServerInfo info = this.regionServerTracker.getRegionServerInfo(sn);
- if (info == null || info.getInfoPort() == 0) {
- return conf.getInt(HConstants.REGIONSERVER_INFO_PORT,
- HConstants.DEFAULT_REGIONSERVER_INFOPORT);
- }
- return info.getInfoPort();
+ int port = this.serverManager.getInfoPort(sn);
+ return port == 0 ? conf.getInt(HConstants.REGIONSERVER_INFO_PORT,
+ HConstants.DEFAULT_REGIONSERVER_INFOPORT) : port;
}
@Override
public String getRegionServerVersion(final ServerName sn) {
- RegionServerInfo info = this.regionServerTracker.getRegionServerInfo(sn);
- if (info != null && info.hasVersionInfo()) {
- return info.getVersionInfo().getVersion();
- }
- return "0.0.0"; //Lowest version to prevent move system region to unknown version RS.
+ // Will return 0 if the server is not online to prevent move system region to unknown version
+ // RS.
+ int versionNumber = this.serverManager.getServerVersion(sn);
+ return VersionInfoUtil.versionNumberToString(versionNumber);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/423a0ab7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
index 100476b..12c8e57 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
@@ -1,5 +1,4 @@
/**
- *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -20,155 +19,167 @@ package org.apache.hadoop.hbase.master;
import java.io.IOException;
import java.io.InterruptedIOException;
-import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
-import java.util.NavigableMap;
-import java.util.TreeMap;
-
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.stream.Collectors;
+import org.apache.hadoop.hbase.ServerMetrics;
+import org.apache.hadoop.hbase.ServerMetricsBuilder;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.VersionInfoUtil;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.zookeeper.ZKListener;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionServerInfo;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionServerInfo;
+
/**
+ * <p>
* Tracks the online region servers via ZK.
- *
- * <p>Handling of new RSs checking in is done via RPC. This class
- * is only responsible for watching for expired nodes. It handles
- * listening for changes in the RS node list and watching each node.
- *
- * <p>If an RS node gets deleted, this automatically handles calling of
+ * </p>
+ * <p>
+ * Handling of new RSs checking in is done via RPC. This class is only responsible for watching for
+ * expired nodes. It handles listening for changes in the RS node list. The only exception is when
+ * master restart, we will use the list fetched from zk to construct the initial set of live region
+ * servers.
+ * </p>
+ * <p>
+ * If an RS node gets deleted, this automatically handles calling of
* {@link ServerManager#expireServer(ServerName)}
+ * </p>
*/
@InterfaceAudience.Private
public class RegionServerTracker extends ZKListener {
private static final Logger LOG = LoggerFactory.getLogger(RegionServerTracker.class);
- private final NavigableMap<ServerName, RegionServerInfo> regionServers = new TreeMap<>();
- private ServerManager serverManager;
- private MasterServices server;
+ private final Set<ServerName> regionServers = new HashSet<>();
+ private final ServerManager serverManager;
+ private final MasterServices server;
+ // As we need to send request to zk when processing the nodeChildrenChanged event, we'd better
+ // move the operation to a single threaded thread pool in order to not block the zk event
+ // processing since all the zk listener across HMaster will be called in one thread sequentially.
+ private final ExecutorService executor;
- public RegionServerTracker(ZKWatcher watcher,
- MasterServices server, ServerManager serverManager) {
+ public RegionServerTracker(ZKWatcher watcher, MasterServices server,
+ ServerManager serverManager) {
super(watcher);
this.server = server;
this.serverManager = serverManager;
+ executor = Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setDaemon(true).setNameFormat("RegionServerTracker-%d").build());
+ }
+
+ private Pair<ServerName, RegionServerInfo> getServerInfo(String name)
+ throws KeeperException, IOException {
+ ServerName serverName = ServerName.parseServerName(name);
+ String nodePath = ZNodePaths.joinZNode(watcher.getZNodePaths().rsZNode, name);
+ byte[] data;
+ try {
+ data = ZKUtil.getData(watcher, nodePath);
+ } catch (InterruptedException e) {
+ throw (InterruptedIOException) new InterruptedIOException().initCause(e);
+ }
+ if (data == null) {
+ // we should receive a children changed event later and then we will expire it, so we still
+ // need to add it to the region server set.
+ LOG.warn("Server node {} does not exist, already dead?", name);
+ return Pair.newPair(serverName, null);
+ }
+ if (data.length == 0 || !ProtobufUtil.isPBMagicPrefix(data)) {
+ // this should not happen actually, unless we have bugs or someone has messed zk up.
+ LOG.warn("Invalid data for region server node {} on zookeeper, data length = {}", name,
+ data.length);
+ return Pair.newPair(serverName, null);
+ }
+ RegionServerInfo.Builder builder = RegionServerInfo.newBuilder();
+ int magicLen = ProtobufUtil.lengthOfPBMagic();
+ ProtobufUtil.mergeFrom(builder, data, magicLen, data.length - magicLen);
+ return Pair.newPair(serverName, builder.build());
}
/**
+ * <p>
* Starts the tracking of online RegionServers.
- *
- * <p>All RSs will be tracked after this method is called.
- *
- * @throws KeeperException
- * @throws IOException
+ * </p>
+ * <p>
+ * All RSs will be tracked after this method is called.
+ * </p>
*/
public void start() throws KeeperException, IOException {
watcher.registerListener(this);
- List<String> servers =
- ZKUtil.listChildrenAndWatchThem(watcher, watcher.getZNodePaths().rsZNode);
- refresh(servers);
- }
-
- private void refresh(final List<String> servers) throws IOException {
- synchronized(this.regionServers) {
- this.regionServers.clear();
- for (String n: servers) {
- ServerName sn = ServerName.parseServerName(ZKUtil.getNodeName(n));
- if (regionServers.get(sn) == null) {
- RegionServerInfo.Builder rsInfoBuilder = RegionServerInfo.newBuilder();
- try {
- String nodePath = ZNodePaths.joinZNode(watcher.getZNodePaths().rsZNode, n);
- byte[] data = ZKUtil.getData(watcher, nodePath);
- if (data != null && data.length > 0 && ProtobufUtil.isPBMagicPrefix(data)) {
- int magicLen = ProtobufUtil.lengthOfPBMagic();
- ProtobufUtil.mergeFrom(rsInfoBuilder, data, magicLen, data.length - magicLen);
- }
- if (LOG.isTraceEnabled()) {
- LOG.trace("Added tracking of RS " + nodePath);
- }
- } catch (KeeperException e) {
- LOG.warn("Get Rs info port from ephemeral node", e);
- } catch (IOException e) {
- LOG.warn("Illegal data from ephemeral node", e);
- } catch (InterruptedException e) {
- throw new InterruptedIOException();
- }
- this.regionServers.put(sn, rsInfoBuilder.build());
- }
+ synchronized (this) {
+ List<String> servers =
+ ZKUtil.listChildrenAndWatchForNewChildren(watcher, watcher.getZNodePaths().rsZNode);
+ for (String n : servers) {
+ Pair<ServerName, RegionServerInfo> pair = getServerInfo(n);
+ ServerName serverName = pair.getFirst();
+ RegionServerInfo info = pair.getSecond();
+ regionServers.add(serverName);
+ ServerMetrics serverMetrics = info != null
+ ? ServerMetricsBuilder.of(serverName,
+ VersionInfoUtil.getVersionNumber(info.getVersionInfo()))
+ : ServerMetricsBuilder.of(serverName);
+ serverManager.checkAndRecordNewServer(serverName, serverMetrics);
}
}
}
- private void remove(final ServerName sn) {
- synchronized(this.regionServers) {
- this.regionServers.remove(sn);
- }
+ public void stop() {
+ executor.shutdownNow();
}
- @Override
- public void nodeCreated(String path) {
- if (path.startsWith(watcher.getZNodePaths().rsZNode)) {
- String serverName = ZKUtil.getNodeName(path);
- LOG.info("RegionServer ephemeral node created, adding [" + serverName + "]");
- if (server.isInitialized()) {
- // Only call the check to move servers if a RegionServer was added to the cluster; in this
- // case it could be a server with a new version so it makes sense to run the check.
- server.checkIfShouldMoveSystemRegionAsync();
+ private synchronized void refresh() {
+ List<String> names;
+ try {
+ names = ZKUtil.listChildrenAndWatchForNewChildren(watcher, watcher.getZNodePaths().rsZNode);
+ } catch (KeeperException e) {
+ // here we need to abort as we failed to set watcher on the rs node which means that we can
+ // not track the node deleted evetnt any more.
+ server.abort("Unexpected zk exception getting RS nodes", e);
+ return;
+ }
+ Set<ServerName> servers =
+ names.stream().map(ServerName::parseServerName).collect(Collectors.toSet());
+ for (Iterator<ServerName> iter = regionServers.iterator(); iter.hasNext();) {
+ ServerName sn = iter.next();
+ if (!servers.contains(sn)) {
+ LOG.info("RegionServer ephemeral node deleted, processing expiration [{}]", sn);
+ serverManager.expireServer(sn);
+ iter.remove();
}
}
- }
-
- @Override
- public void nodeDeleted(String path) {
- if (path.startsWith(watcher.getZNodePaths().rsZNode)) {
- String serverName = ZKUtil.getNodeName(path);
- LOG.info("RegionServer ephemeral node deleted, processing expiration [" +
- serverName + "]");
- ServerName sn = ServerName.parseServerName(serverName);
- if (!serverManager.isServerOnline(sn)) {
- LOG.warn(serverName.toString() + " is not online or isn't known to the master."+
- "The latter could be caused by a DNS misconfiguration.");
- return;
+ // here we do not need to parse the region server info as it is useless now, we only need the
+ // server name.
+ boolean newServerAdded = false;
+ for (ServerName sn : servers) {
+ if (regionServers.add(sn)) {
+ newServerAdded = true;
+ LOG.info("RegionServer ephemeral node created, adding [" + sn + "]");
}
- remove(sn);
- this.serverManager.expireServer(sn);
+ }
+ if (newServerAdded && server.isInitialized()) {
+ // Only call the check to move servers if a RegionServer was added to the cluster; in this
+ // case it could be a server with a new version so it makes sense to run the check.
+ server.checkIfShouldMoveSystemRegionAsync();
}
}
@Override
public void nodeChildrenChanged(String path) {
- if (path.equals(watcher.getZNodePaths().rsZNode)
- && !server.isAborted() && !server.isStopped()) {
- try {
- List<String> servers =
- ZKUtil.listChildrenAndWatchThem(watcher, watcher.getZNodePaths().rsZNode);
- refresh(servers);
- } catch (IOException e) {
- server.abort("Unexpected zk exception getting RS nodes", e);
- } catch (KeeperException e) {
- server.abort("Unexpected zk exception getting RS nodes", e);
- }
- }
- }
-
- public RegionServerInfo getRegionServerInfo(final ServerName sn) {
- return regionServers.get(sn);
- }
-
- /**
- * Gets the online servers.
- * @return list of online servers
- */
- public List<ServerName> getOnlineServers() {
- synchronized (this.regionServers) {
- return new ArrayList<>(this.regionServers.keySet());
+ if (path.equals(watcher.getZNodePaths().rsZNode) && !server.isAborted() &&
+ !server.isStopped()) {
+ executor.execute(this::refresh);
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/423a0ab7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 86e5530..c746502 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -46,7 +46,6 @@ import org.apache.hadoop.hbase.ServerMetrics;
import org.apache.hadoop.hbase.ServerMetricsBuilder;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.YouAreDeadException;
-import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RetriesExhaustedException;
@@ -187,19 +186,13 @@ public class ServerManager {
/**
* Constructor.
- * @param master
- * @throws ZooKeeperConnectionException
*/
public ServerManager(final MasterServices master) {
- this(master, true);
- }
-
- ServerManager(final MasterServices master, final boolean connect) {
this.master = master;
Configuration c = master.getConfiguration();
maxSkew = c.getLong("hbase.master.maxclockskew", 30000);
warningSkew = c.getLong("hbase.master.warningclockskew", 10000);
- this.connection = connect? master.getClusterConnection(): null;
+ this.connection = master.getClusterConnection();
this.rpcControllerFactory = this.connection == null? null: connection.getRpcControllerFactory();
}
@@ -228,8 +221,7 @@ public class ServerManager {
* @throws IOException
*/
ServerName regionServerStartup(RegionServerStartupRequest request, int versionNumber,
- InetAddress ia)
- throws IOException {
+ InetAddress ia) throws IOException {
// Test for case where we get a region startup message from a regionserver
// that has been quickly restarted but whose znode expiration handler has
// not yet run, or from a server whose fail we are currently processing.
@@ -1068,4 +1060,9 @@ public class ServerManager {
ServerMetrics serverMetrics = onlineServers.get(serverName);
return serverMetrics != null ? serverMetrics.getVersionNumber() : 0;
}
+
+ public int getInfoPort(ServerName serverName) {
+ ServerMetrics serverMetrics = onlineServers.get(serverName);
+ return serverMetrics != null ? serverMetrics.getInfoServerPort() : 0;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/423a0ab7/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java
index e8f739b..1f22830 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java
@@ -20,18 +20,11 @@ package org.apache.hadoop.hbase.master;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.ServerLoad;
-import org.apache.hadoop.hbase.ServerMetricsBuilder;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
@@ -45,18 +38,13 @@ import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.JVMClusterUtil;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
import org.junit.AfterClass;
-import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
-import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -74,16 +62,6 @@ public class TestAssignmentListener {
@Rule
public TestName name = new TestName();
- private static final Abortable abortable = new Abortable() {
- @Override
- public boolean isAborted() {
- return false;
- }
-
- @Override
- public void abort(String why, Throwable e) {
- }
- };
static class DummyListener {
protected AtomicInteger modified = new AtomicInteger(0);
@@ -284,7 +262,7 @@ public class TestAssignmentListener {
admin.mergeRegionsAsync(regions.get(0).getEncodedNameAsBytes(),
regions.get(1).getEncodedNameAsBytes(), true);
listener.awaitModifications(expectedModifications);
- assertEquals(1, admin.getTableRegions(tableName).size());
+ assertEquals(1, admin.getRegions(tableName).size());
assertEquals(expectedLoadCount, listener.getLoadCount()); // new merged region added
assertEquals(expectedCloseCount, listener.getCloseCount()); // daughters removed
@@ -313,78 +291,4 @@ public class TestAssignmentListener {
}
return serverCount == 1;
}
-
- @Test
- public void testAddNewServerThatExistsInDraining() throws Exception {
- // Under certain circumstances, such as when we failover to the Backup
- // HMaster, the DrainingServerTracker is started with existing servers in
- // draining before all of the Region Servers register with the
- // ServerManager as "online". This test is to ensure that Region Servers
- // are properly added to the ServerManager.drainingServers when they
- // register with the ServerManager under these circumstances.
- Configuration conf = TEST_UTIL.getConfiguration();
- ZKWatcher zooKeeper = new ZKWatcher(conf,
- "zkWatcher-NewServerDrainTest", abortable, true);
- String baseZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
- HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
- String drainingZNode = ZNodePaths.joinZNode(baseZNode,
- conf.get("zookeeper.znode.draining.rs", "draining"));
-
- HMaster master = Mockito.mock(HMaster.class);
- Mockito.when(master.getConfiguration()).thenReturn(conf);
-
- ServerName SERVERNAME_A = ServerName.valueOf("mockserverbulk_a.org", 1000, 8000);
- ServerName SERVERNAME_B = ServerName.valueOf("mockserverbulk_b.org", 1001, 8000);
- ServerName SERVERNAME_C = ServerName.valueOf("mockserverbulk_c.org", 1002, 8000);
-
- // We'll start with 2 servers in draining that existed before the
- // HMaster started.
- ArrayList<ServerName> drainingServers = new ArrayList<>();
- drainingServers.add(SERVERNAME_A);
- drainingServers.add(SERVERNAME_B);
-
- // We'll have 2 servers that come online AFTER the DrainingServerTracker
- // is started (just as we see when we failover to the Backup HMaster).
- // One of these will already be a draining server.
- HashMap<ServerName, ServerLoad> onlineServers = new HashMap<>();
- onlineServers.put(SERVERNAME_A, new ServerLoad(ServerMetricsBuilder.of(SERVERNAME_A)));
- onlineServers.put(SERVERNAME_C, new ServerLoad(ServerMetricsBuilder.of(SERVERNAME_C)));
-
- // Create draining znodes for the draining servers, which would have been
- // performed when the previous HMaster was running.
- for (ServerName sn : drainingServers) {
- String znode = ZNodePaths.joinZNode(drainingZNode, sn.getServerName());
- ZKUtil.createAndFailSilent(zooKeeper, znode);
- }
-
- // Now, we follow the same order of steps that the HMaster does to setup
- // the ServerManager, RegionServerTracker, and DrainingServerTracker.
- ServerManager serverManager = new ServerManager(master);
-
- RegionServerTracker regionServerTracker = new RegionServerTracker(
- zooKeeper, master, serverManager);
- regionServerTracker.start();
-
- DrainingServerTracker drainingServerTracker = new DrainingServerTracker(
- zooKeeper, master, serverManager);
- drainingServerTracker.start();
-
- // Confirm our ServerManager lists are empty.
- Assert.assertEquals(new HashMap<ServerName, ServerLoad>(), serverManager.getOnlineServers());
- Assert.assertEquals(new ArrayList<ServerName>(), serverManager.getDrainingServersList());
-
- // checkAndRecordNewServer() is how servers are added to the ServerManager.
- ArrayList<ServerName> onlineDrainingServers = new ArrayList<>();
- for (ServerName sn : onlineServers.keySet()){
- // Here's the actual test.
- serverManager.checkAndRecordNewServer(sn, onlineServers.get(sn));
- if (drainingServers.contains(sn)){
- onlineDrainingServers.add(sn); // keeping track for later verification
- }
- }
-
- // Verify the ServerManager lists are correctly updated.
- Assert.assertEquals(onlineServers, serverManager.getOnlineServers());
- Assert.assertEquals(onlineDrainingServers, serverManager.getDrainingServersList());
- }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/423a0ab7/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java
index c68f94d..877c382 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java
@@ -58,7 +58,7 @@ public class TestClockSkewDetection {
when(conn.getRpcControllerFactory()).thenReturn(mock(RpcControllerFactory.class));
return conn;
}
- }, true);
+ });
LOG.debug("regionServerStartup 1");
InetAddress ia1 = InetAddress.getLocalHost();
http://git-wip-us.apache.org/repos/asf/hbase/blob/423a0ab7/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
index c50ae0b..e73ba75 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
@@ -23,8 +23,6 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HBaseClassTestRule;
@@ -203,10 +201,10 @@ public class TestMasterNoCluster {
}
@Override
- void initClusterSchemaService() throws IOException, InterruptedException {}
+ protected void initClusterSchemaService() throws IOException, InterruptedException {}
@Override
- ServerManager createServerManager(MasterServices master) throws IOException {
+ protected ServerManager createServerManager(MasterServices master) throws IOException {
ServerManager sm = super.createServerManager(master);
// Spy on the created servermanager
ServerManager spy = Mockito.spy(sm);
@@ -271,23 +269,15 @@ public class TestMasterNoCluster {
}
@Override
- void initClusterSchemaService() throws IOException, InterruptedException {}
+ protected void initClusterSchemaService() throws IOException, InterruptedException {}
@Override
- void initializeZKBasedSystemTrackers() throws IOException, InterruptedException,
+ protected void initializeZKBasedSystemTrackers() throws IOException, InterruptedException,
KeeperException, ReplicationException {
super.initializeZKBasedSystemTrackers();
// Record a newer server in server manager at first
getServerManager().recordNewServerWithLock(newServer,
new ServerLoad(ServerMetricsBuilder.of(newServer)));
-
- List<ServerName> onlineServers = new ArrayList<>();
- onlineServers.add(deadServer);
- onlineServers.add(newServer);
- // Mock the region server tracker to pull the dead server from zk
- regionServerTracker = Mockito.spy(regionServerTracker);
- Mockito.doReturn(onlineServers).when(
- regionServerTracker).getOnlineServers();
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/423a0ab7/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestShutdownBackupMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestShutdownBackupMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestShutdownBackupMaster.java
index 91108ae..9475d60 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestShutdownBackupMaster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestShutdownBackupMaster.java
@@ -60,7 +60,7 @@ public class TestShutdownBackupMaster {
}
@Override
- void initClusterSchemaService() throws IOException, InterruptedException {
+ protected void initClusterSchemaService() throws IOException, InterruptedException {
if (ARRIVE != null) {
ARRIVE.countDown();
CONTINUE.await();
[09/12] hbase git commit: HBASE-20333 Provide a shaded client that
allows downstream to provide Hadoop needs.
Posted by bu...@apache.org.
HBASE-20333 Provide a shaded client that allows downstream to provide Hadoop needs.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4a1408f9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4a1408f9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4a1408f9
Branch: refs/heads/HBASE-20331
Commit: 4a1408f9f6d4d8649cb1eec424fad26e46a88384
Parents: 2541b3b
Author: Sean Busbey <bu...@apache.org>
Authored: Tue Apr 24 14:51:12 2018 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Thu Jun 14 20:56:46 2018 -0500
----------------------------------------------------------------------
.../hbase-shaded-check-invariants/pom.xml | 5 ++
.../hbase-shaded-client-byo-hadoop/pom.xml | 70 ++++++++++++++++++++
hbase-shaded/hbase-shaded-client/pom.xml | 35 ++++++++--
hbase-shaded/hbase-shaded-mapreduce/pom.xml | 30 ++-------
hbase-shaded/pom.xml | 6 ++
5 files changed, 115 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/4a1408f9/hbase-shaded/hbase-shaded-check-invariants/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shaded/hbase-shaded-check-invariants/pom.xml b/hbase-shaded/hbase-shaded-check-invariants/pom.xml
index 7ba4a41..287a986 100644
--- a/hbase-shaded/hbase-shaded-check-invariants/pom.xml
+++ b/hbase-shaded/hbase-shaded-check-invariants/pom.xml
@@ -48,6 +48,11 @@
<artifactId>hbase-shaded-mapreduce</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-shaded-client-byo-hadoop</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<!-- parent pom defines these for children. :( :( :( -->
<dependency>
<groupId>com.github.stephenc.findbugs</groupId>
http://git-wip-us.apache.org/repos/asf/hbase/blob/4a1408f9/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml b/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml
new file mode 100644
index 0000000..c51a1af
--- /dev/null
+++ b/hbase-shaded/hbase-shaded-client-byo-hadoop/pom.xml
@@ -0,0 +1,70 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <!--
+ /**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ -->
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>hbase-shaded</artifactId>
+ <groupId>org.apache.hbase</groupId>
+ <version>3.0.0-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+ <artifactId>hbase-shaded-client-byo-hadoop</artifactId>
+ <name>Apache HBase - Shaded - Client</name>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ <plugin>
+ <!--Make it so assembly:single does nothing in here-->
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <skipAssembly>true</skipAssembly>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ </dependency>
+ </dependencies>
+
+ <profiles>
+ <profile>
+ <id>release</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+</project>
http://git-wip-us.apache.org/repos/asf/hbase/blob/4a1408f9/hbase-shaded/hbase-shaded-client/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shaded/hbase-shaded-client/pom.xml b/hbase-shaded/hbase-shaded-client/pom.xml
index 72a5b60..6152aad 100644
--- a/hbase-shaded/hbase-shaded-client/pom.xml
+++ b/hbase-shaded/hbase-shaded-client/pom.xml
@@ -28,7 +28,7 @@
<relativePath>..</relativePath>
</parent>
<artifactId>hbase-shaded-client</artifactId>
- <name>Apache HBase - Shaded - Client</name>
+ <name>Apache HBase - Shaded - Client (with Hadoop bundled)</name>
<build>
<plugins>
<plugin>
@@ -51,6 +51,7 @@
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
+ <version>${project.version}</version>
</dependency>
</dependencies>
@@ -59,10 +60,34 @@
<id>release</id>
<build>
<plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>aggregate-into-a-jar-with-relocated-third-parties</id>
+ <configuration>
+ <artifactSet>
+ <excludes>
+ <!--
+ Tell the shade plugin that in this case we want to include hadoop
+ by leaving out the exclude.
+ -->
+ <!-- The rest of these should be kept in sync with the parent pom -->
+ <exclude>org.apache.hbase:hbase-resource-bundle</exclude>
+ <exclude>org.slf4j:*</exclude>
+ <exclude>com.google.code.findbugs:*</exclude>
+ <exclude>com.github.stephenc.findbugs:*</exclude>
+ <exclude>org.apache.htrace:*</exclude>
+ <exclude>org.apache.yetus:*</exclude>
+ <exclude>log4j:*</exclude>
+ <exclude>commons-logging:*</exclude>
+ </excludes>
+ </artifactSet>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</profile>
http://git-wip-us.apache.org/repos/asf/hbase/blob/4a1408f9/hbase-shaded/hbase-shaded-mapreduce/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shaded/hbase-shaded-mapreduce/pom.xml b/hbase-shaded/hbase-shaded-mapreduce/pom.xml
index edc80cd..598f3af 100644
--- a/hbase-shaded/hbase-shaded-mapreduce/pom.xml
+++ b/hbase-shaded/hbase-shaded-mapreduce/pom.xml
@@ -169,32 +169,10 @@
<id>release</id>
<build>
<plugins>
- <!-- Tell the shade plugin we want to leave Hadoop as a dependency -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <id>aggregate-into-a-jar-with-relocated-third-parties</id>
- <configuration>
- <artifactSet>
- <excludes>
- <exclude>org.apache.hadoop:*</exclude>
- <!-- The rest of these should be kept in sync with the parent pom -->
- <exclude>org.apache.hbase:hbase-resource-bundle</exclude>
- <exclude>org.slf4j:*</exclude>
- <exclude>com.google.code.findbugs:*</exclude>
- <exclude>com.github.stephenc.findbugs:*</exclude>
- <exclude>org.apache.htrace:*</exclude>
- <exclude>org.apache.yetus:*</exclude>
- <exclude>log4j:*</exclude>
- <exclude>commons-logging:*</exclude>
- </excludes>
- </artifactSet>
- </configuration>
- </execution>
- </executions>
- </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ </plugin>
</plugins>
</build>
</profile>
http://git-wip-us.apache.org/repos/asf/hbase/blob/4a1408f9/hbase-shaded/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shaded/pom.xml b/hbase-shaded/pom.xml
index 93b122f..9eb30e0 100644
--- a/hbase-shaded/pom.xml
+++ b/hbase-shaded/pom.xml
@@ -39,6 +39,7 @@
<shaded.prefix>org.apache.hadoop.hbase.shaded</shaded.prefix>
</properties>
<modules>
+ <module>hbase-shaded-client-byo-hadoop</module>
<module>hbase-shaded-client</module>
<module>hbase-shaded-mapreduce</module>
<module>hbase-shaded-check-invariants</module>
@@ -131,6 +132,11 @@
<shadeTestJar>false</shadeTestJar>
<artifactSet>
<excludes>
+ <!-- default to excluding Hadoop, have module that want
+ to include it redefine the exclude list -->
+ <exclude>org.apache.hadoop:*</exclude>
+ <!-- the rest of this needs to be kept in sync with any
+ hadoop-including module -->
<exclude>org.apache.hbase:hbase-resource-bundle</exclude>
<exclude>org.slf4j:*</exclude>
<exclude>com.google.code.findbugs:*</exclude>
[04/12] hbase git commit: HBASE-20630 B&R: Delete command enhancements
Posted by bu...@apache.org.
HBASE-20630 B&R: Delete command enhancements
Signed-off-by: tedyu <yu...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9e9db324
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9e9db324
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9e9db324
Branch: refs/heads/HBASE-20331
Commit: 9e9db3245f2d825837f231c004eac99f83829a95
Parents: 423a0ab
Author: Vladimir Rodionov <vr...@hortonworks.com>
Authored: Wed Jun 6 16:49:15 2018 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Wed Jun 13 18:46:48 2018 -0700
----------------------------------------------------------------------
.../hadoop/hbase/backup/BackupDriver.java | 9 ++-
.../hbase/backup/BackupRestoreConstants.java | 7 +-
.../hbase/backup/impl/BackupCommands.java | 85 +++++++++++++++++---
.../hadoop/hbase/backup/TestBackupDelete.java | 58 ++++++++++++-
4 files changed, 146 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/9e9db324/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java
----------------------------------------------------------------------
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java
index 8baf2f0..6644d89 100644
--- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java
+++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java
@@ -17,10 +17,14 @@
*/
package org.apache.hadoop.hbase.backup;
+import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BACKUP_LIST_DESC;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH_DESC;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG_DESC;
+import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP;
+import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP_DESC;
+import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_LIST;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH_DESC;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER;
@@ -46,14 +50,13 @@ import org.apache.hadoop.hbase.backup.impl.BackupManager;
import org.apache.hadoop.hbase.util.AbstractHBaseTool;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
-
/**
*
* Command-line entry point for backup operation
@@ -152,10 +155,12 @@ public class BackupDriver extends AbstractHBaseTool {
addOptNoArg(OPTION_DEBUG, OPTION_DEBUG_DESC);
addOptWithArg(OPTION_TABLE, OPTION_TABLE_DESC);
addOptWithArg(OPTION_BANDWIDTH, OPTION_BANDWIDTH_DESC);
+ addOptWithArg(OPTION_LIST, OPTION_BACKUP_LIST_DESC);
addOptWithArg(OPTION_WORKERS, OPTION_WORKERS_DESC);
addOptWithArg(OPTION_RECORD_NUMBER, OPTION_RECORD_NUMBER_DESC);
addOptWithArg(OPTION_SET, OPTION_SET_DESC);
addOptWithArg(OPTION_PATH, OPTION_PATH_DESC);
+ addOptWithArg(OPTION_KEEP, OPTION_KEEP_DESC);
addOptWithArg(OPTION_YARN_QUEUE_NAME, OPTION_YARN_QUEUE_NAME_DESC);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/9e9db324/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
----------------------------------------------------------------------
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
index eaeef22..16ec3d2 100644
--- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
+++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
@@ -65,8 +65,9 @@ public interface BackupRestoreConstants {
String OPTION_TABLE_DESC = "Table name. If specified, only backup images,"
+ " which contain this table will be listed.";
- String OPTION_TABLE_LIST = "l";
+ String OPTION_LIST = "l";
String OPTION_TABLE_LIST_DESC = "Table name list, comma-separated.";
+ String OPTION_BACKUP_LIST_DESC = "Backup ids list, comma-separated.";
String OPTION_BANDWIDTH = "b";
String OPTION_BANDWIDTH_DESC = "Bandwidth per task (MapReduce task) in MB/s";
@@ -80,6 +81,10 @@ public interface BackupRestoreConstants {
String OPTION_PATH = "p";
String OPTION_PATH_DESC = "Backup destination root directory path";
+ String OPTION_KEEP = "k";
+ String OPTION_KEEP_DESC = "Specifies maximum age of backup (in days) to keep during bulk delete";
+
+
String OPTION_TABLE_MAPPING = "m";
String OPTION_TABLE_MAPPING_DESC =
"A comma separated list of target tables. "
http://git-wip-us.apache.org/repos/asf/hbase/blob/9e9db324/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
----------------------------------------------------------------------
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
index 3e28f51..fc2f1a2 100644
--- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
+++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
@@ -18,10 +18,14 @@
package org.apache.hadoop.hbase.backup.impl;
+import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BACKUP_LIST_DESC;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH_DESC;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG_DESC;
+import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP;
+import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP_DESC;
+import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_LIST;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH_DESC;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER;
@@ -61,11 +65,11 @@ import org.apache.hadoop.hbase.backup.util.BackupUtils;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
import org.apache.hbase.thirdparty.org.apache.commons.cli.HelpFormatter;
import org.apache.hbase.thirdparty.org.apache.commons.cli.Options;
+import org.apache.yetus.audience.InterfaceAudience;
/**
* General backup commands, options and usage messages
@@ -105,8 +109,7 @@ public final class BackupCommands {
public static final String HISTORY_CMD_USAGE = "Usage: hbase backup history [options]";
- public static final String DELETE_CMD_USAGE = "Usage: hbase backup delete <backup_id>\n"
- + " backup_id Backup image id\n";
+ public static final String DELETE_CMD_USAGE = "Usage: hbase backup delete [options]";
public static final String REPAIR_CMD_USAGE = "Usage: hbase backup repair\n";
@@ -560,29 +563,93 @@ public final class BackupCommands {
@Override
public void execute() throws IOException {
- if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) {
+
+ if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 1) {
printUsage();
throw new IOException(INCORRECT_USAGE);
}
+ if (!cmdline.hasOption(OPTION_KEEP) && !cmdline.hasOption(OPTION_LIST)) {
+ printUsage();
+ throw new IOException(INCORRECT_USAGE);
+ }
super.execute();
+ if (cmdline.hasOption(OPTION_KEEP)) {
+ executeDeleteOlderThan(cmdline);
+ } else if (cmdline.hasOption(OPTION_LIST)) {
+ executeDeleteListOfBackups(cmdline);
+ }
+ }
+
+ private void executeDeleteOlderThan(CommandLine cmdline) throws IOException {
+ String value = cmdline.getOptionValue(OPTION_KEEP);
+ int days = 0;
+ try {
+ days = Integer.parseInt(value);
+ } catch (NumberFormatException e) {
+ throw new IOException(value + " is not an integer number");
+ }
+ final long fdays = days;
+ BackupInfo.Filter dateFilter = new BackupInfo.Filter() {
+ @Override
+ public boolean apply(BackupInfo info) {
+ long currentTime = EnvironmentEdgeManager.currentTime();
+ long maxTsToDelete = currentTime - fdays * 24 * 3600 * 1000;
+ return info.getCompleteTs() <= maxTsToDelete;
+ }
+ };
+ List<BackupInfo> history = null;
+ try (final BackupSystemTable sysTable = new BackupSystemTable(conn);
+ BackupAdminImpl admin = new BackupAdminImpl(conn)) {
+ history = sysTable.getBackupHistory(-1, dateFilter);
+ String[] backupIds = convertToBackupIds(history);
+ int deleted = admin.deleteBackups(backupIds);
+ System.out.println("Deleted " + deleted + " backups. Total older than " + days + " days: "
+ + backupIds.length);
+ } catch (IOException e) {
+ System.err.println("Delete command FAILED. Please run backup repair tool to restore backup "
+ + "system integrity");
+ throw e;
+ }
+ }
+
+ private String[] convertToBackupIds(List<BackupInfo> history) {
+ String[] ids = new String[history.size()];
+ for (int i = 0; i < ids.length; i++) {
+ ids[i] = history.get(i).getBackupId();
+ }
+ return ids;
+ }
+
+ private void executeDeleteListOfBackups(CommandLine cmdline) throws IOException {
+ String value = cmdline.getOptionValue(OPTION_LIST);
+ String[] backupIds = value.split(",");
- String[] args = cmdline.getArgs();
- String[] backupIds = new String[args.length - 1];
- System.arraycopy(args, 1, backupIds, 0, backupIds.length);
try (BackupAdminImpl admin = new BackupAdminImpl(conn)) {
int deleted = admin.deleteBackups(backupIds);
- System.out.println("Deleted " + deleted + " backups. Total requested: " + (args.length -1));
+ System.out.println("Deleted " + deleted + " backups. Total requested: " + backupIds.length);
} catch (IOException e) {
System.err.println("Delete command FAILED. Please run backup repair tool to restore backup "
- + "system integrity");
+ + "system integrity");
throw e;
}
+
}
@Override
protected void printUsage() {
System.out.println(DELETE_CMD_USAGE);
+ Options options = new Options();
+ options.addOption(OPTION_KEEP, true, OPTION_KEEP_DESC);
+ options.addOption(OPTION_LIST, true, OPTION_BACKUP_LIST_DESC);
+
+ HelpFormatter helpFormatter = new HelpFormatter();
+ helpFormatter.setLeftPadding(2);
+ helpFormatter.setDescPadding(8);
+ helpFormatter.setWidth(100);
+ helpFormatter.setSyntaxPrefix("Options:");
+ helpFormatter.printHelp(" ", null, options, USAGE_FOOTER);
+
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/9e9db324/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java
----------------------------------------------------------------------
diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java
index 9e67ff7..bc8b346 100644
--- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java
+++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java
@@ -28,7 +28,10 @@ import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.EnvironmentEdge;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.util.ToolRunner;
+import org.junit.Assert;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -90,7 +93,7 @@ public class TestBackupDelete extends TestBackupBase {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
System.setOut(new PrintStream(baos));
- String[] args = new String[] { "delete", backupId };
+ String[] args = new String[] { "delete", "-l", backupId };
// Run backup
try {
@@ -104,4 +107,57 @@ public class TestBackupDelete extends TestBackupBase {
LOG.info(baos.toString());
assertTrue(output.indexOf("Deleted 1 backups") >= 0);
}
+
+ @Test
+ public void testBackupPurgeOldBackupsCommand() throws Exception {
+ LOG.info("test backup delete (purge old backups) on a single table with data: command-line");
+ List<TableName> tableList = Lists.newArrayList(table1);
+ EnvironmentEdgeManager.injectEdge(new EnvironmentEdge() {
+ // time - 2 days
+ @Override
+ public long currentTime() {
+ return System.currentTimeMillis() - 2 * 24 * 3600 * 1000 ;
+ }
+ });
+ String backupId = fullTableBackup(tableList);
+ assertTrue(checkSucceeded(backupId));
+
+ EnvironmentEdgeManager.reset();
+
+ LOG.info("backup complete");
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ System.setOut(new PrintStream(baos));
+
+ // Purge all backups which are older than 3 days
+ // Must return 0 (no backups were purged)
+ String[] args = new String[] { "delete", "-k", "3" };
+ // Run backup
+
+ try {
+ int ret = ToolRunner.run(conf1, new BackupDriver(), args);
+ assertTrue(ret == 0);
+ } catch (Exception e) {
+ LOG.error("failed", e);
+ Assert.fail(e.getMessage());
+ }
+ String output = baos.toString();
+ LOG.info(baos.toString());
+ assertTrue(output.indexOf("Deleted 0 backups") >= 0);
+
+ // Purge all backups which are older than 1 days
+ // Must return 1 deleted backup
+ args = new String[] { "delete", "-k", "1" };
+ // Run backup
+ baos.reset();
+ try {
+ int ret = ToolRunner.run(conf1, new BackupDriver(), args);
+ assertTrue(ret == 0);
+ } catch (Exception e) {
+ LOG.error("failed", e);
+ Assert.fail(e.getMessage());
+ }
+ output = baos.toString();
+ LOG.info(baos.toString());
+ assertTrue(output.indexOf("Deleted 1 backups") >= 0);
+ }
}
[02/12] hbase git commit: HBASE-20561 The way we stop a
ReplicationSource may cause the RS down
Posted by bu...@apache.org.
HBASE-20561 The way we stop a ReplicationSource may cause the RS down
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ec664343
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ec664343
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ec664343
Branch: refs/heads/HBASE-20331
Commit: ec66434380aee62289ccf7b173d765bbe7083718
Parents: 8648af0
Author: Guanghao Zhang <zg...@apache.org>
Authored: Tue Jun 12 22:19:39 2018 +0800
Committer: Guanghao Zhang <zg...@apache.org>
Committed: Wed Jun 13 17:58:59 2018 +0800
----------------------------------------------------------------------
.../regionserver/ReplicationSource.java | 24 +++++++++++++++--
.../regionserver/ReplicationSourceManager.java | 28 +++++++++++++++++---
.../hadoop/hbase/zookeeper/ZKWatcher.java | 4 ++-
3 files changed, 49 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/ec664343/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index d21d83c..b63712b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -499,9 +499,29 @@ public class ReplicationSource implements ReplicationSourceInterface {
Collection<ReplicationSourceShipper> workers = workerThreads.values();
for (ReplicationSourceShipper worker : workers) {
worker.stopWorker();
- worker.entryReader.interrupt();
- worker.interrupt();
+ worker.entryReader.setReaderRunning(false);
}
+
+ for (ReplicationSourceShipper worker : workers) {
+ if (worker.isAlive() || worker.entryReader.isAlive()) {
+ try {
+ // Wait worker to stop
+ Thread.sleep(this.sleepForRetries);
+ } catch (InterruptedException e) {
+ LOG.info("Interrupted while waiting " + worker.getName() + " to stop");
+ Thread.currentThread().interrupt();
+ }
+ // If worker still is alive after waiting, interrupt it
+ if (worker.isAlive()) {
+ worker.interrupt();
+ }
+ // If entry reader is alive after waiting, interrupt it
+ if (worker.entryReader.isAlive()) {
+ worker.entryReader.interrupt();
+ }
+ }
+ }
+
if (this.replicationEndpoint != null) {
this.replicationEndpoint.stop();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ec664343/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 9b4a22c..a370867 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.hbase.replication.ReplicationTracker;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -449,6 +450,24 @@ public class ReplicationSourceManager implements ReplicationListener {
void exec() throws ReplicationException;
}
+ /**
+ * Refresh replication source will terminate the old source first, then the source thread will be
+ * interrupted. Need to handle it instead of abort the region server.
+ */
+ private void interruptOrAbortWhenFail(ReplicationQueueOperation op) {
+ try {
+ op.exec();
+ } catch (ReplicationException e) {
+ if (e.getCause() != null && e.getCause() instanceof KeeperException.SystemErrorException
+ && e.getCause().getCause() != null && e.getCause()
+ .getCause() instanceof InterruptedException) {
+ throw new RuntimeException(
+ "Thread is interrupted, the replication source may be terminated");
+ }
+ server.abort("Failed to operate on replication queue", e);
+ }
+ }
+
private void abortWhenFail(ReplicationQueueOperation op) {
try {
op.exec();
@@ -484,8 +503,9 @@ public class ReplicationSourceManager implements ReplicationListener {
public void logPositionAndCleanOldLogs(String queueId, boolean queueRecovered,
WALEntryBatch entryBatch) {
String fileName = entryBatch.getLastWalPath().getName();
- abortWhenFail(() -> this.queueStorage.setWALPosition(server.getServerName(), queueId, fileName,
- entryBatch.getLastWalPosition(), entryBatch.getLastSeqIds()));
+ interruptOrAbortWhenFail(() -> this.queueStorage
+ .setWALPosition(server.getServerName(), queueId, fileName, entryBatch.getLastWalPosition(),
+ entryBatch.getLastSeqIds()));
cleanOldLogs(fileName, entryBatch.isEndOfFile(), queueId, queueRecovered);
}
@@ -523,7 +543,7 @@ public class ReplicationSourceManager implements ReplicationListener {
}
LOG.debug("Removing {} logs in the list: {}", walSet.size(), walSet);
for (String wal : walSet) {
- abortWhenFail(() -> this.queueStorage.removeWAL(server.getServerName(), id, wal));
+ interruptOrAbortWhenFail(() -> this.queueStorage.removeWAL(server.getServerName(), id, wal));
}
walSet.clear();
}
@@ -886,7 +906,7 @@ public class ReplicationSourceManager implements ReplicationListener {
}
public void cleanUpHFileRefs(String peerId, List<String> files) {
- abortWhenFail(() -> this.queueStorage.removeHFileRefs(peerId, files));
+ interruptOrAbortWhenFail(() -> this.queueStorage.removeHFileRefs(peerId, files));
}
int activeFailoverTaskCount() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/ec664343/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKWatcher.java
----------------------------------------------------------------------
diff --git a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKWatcher.java b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKWatcher.java
index cceedfd..ce00af4 100644
--- a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKWatcher.java
+++ b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKWatcher.java
@@ -607,7 +607,9 @@ public class ZKWatcher implements Watcher, Abortable, Closeable {
public void interruptedException(InterruptedException ie) throws KeeperException {
interruptedExceptionNoThrow(ie, true);
// Throw a system error exception to let upper level handle it
- throw new KeeperException.SystemErrorException();
+ KeeperException keeperException = new KeeperException.SystemErrorException();
+ keeperException.initCause(ie);
+ throw keeperException;
}
/**
[10/12] hbase git commit: HBASE-20615 emphasize shaded artifacts in
client tarball.
Posted by bu...@apache.org.
HBASE-20615 emphasize shaded artifacts in client tarball.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5c65a9f9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5c65a9f9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5c65a9f9
Branch: refs/heads/HBASE-20331
Commit: 5c65a9f94003000daac6619ef9e84ed8ee3b9487
Parents: b5017ea
Author: Sean Busbey <bu...@apache.org>
Authored: Fri May 18 11:11:42 2018 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Thu Jun 14 20:56:47 2018 -0500
----------------------------------------------------------------------
bin/hbase | 226 ++++++++++++++++---
bin/hbase-config.sh | 10 +
hbase-assembly/pom.xml | 21 +-
.../src/main/assembly/client-components.xml | 43 +++-
hbase-assembly/src/main/assembly/client.xml | 131 ++++++-----
hbase-assembly/src/main/assembly/components.xml | 3 +-
.../src/main/assembly/hadoop-two-compat.xml | 80 ++++++-
pom.xml | 6 +
8 files changed, 417 insertions(+), 103 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/5c65a9f9/bin/hbase
----------------------------------------------------------------------
diff --git a/bin/hbase b/bin/hbase
index 4f1c854..559a02e 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -71,11 +71,18 @@ if [ -d "${HBASE_HOME}/target" ]; then
in_dev_env=true
fi
+# Detect if we are in the omnibus tarball
+in_omnibus_tarball="false"
+if [ -f "${HBASE_HOME}/bin/hbase-daemons.sh" ]; then
+ in_omnibus_tarball="true"
+fi
+
read -d '' options_string << EOF
Options:
- --config DIR Configuration direction to use. Default: ./conf
- --hosts HOSTS Override the list in 'regionservers' file
- --auth-as-server Authenticate to ZooKeeper using servers configuration
+ --config DIR Configuration direction to use. Default: ./conf
+ --hosts HOSTS Override the list in 'regionservers' file
+ --auth-as-server Authenticate to ZooKeeper using servers configuration
+ --internal-classpath Skip attempting to use client facing jars (WARNING: unstable results between versions)
EOF
# if no args specified, show usage
if [ $# = 0 ]; then
@@ -87,16 +94,18 @@ if [ $# = 0 ]; then
echo " shell Run the HBase shell"
echo " hbck Run the hbase 'fsck' tool"
echo " snapshot Tool for managing snapshots"
- echo " wal Write-ahead-log analyzer"
- echo " hfile Store file analyzer"
- echo " zkcli Run the ZooKeeper shell"
- echo " master Run an HBase HMaster node"
- echo " regionserver Run an HBase HRegionServer node"
- echo " zookeeper Run a ZooKeeper server"
- echo " rest Run an HBase REST server"
- echo " thrift Run the HBase Thrift server"
- echo " thrift2 Run the HBase Thrift2 server"
- echo " clean Run the HBase clean up script"
+ if [ "${in_omnibus_tarball}" = "true" ]; then
+ echo " wal Write-ahead-log analyzer"
+ echo " hfile Store file analyzer"
+ echo " zkcli Run the ZooKeeper shell"
+ echo " master Run an HBase HMaster node"
+ echo " regionserver Run an HBase HRegionServer node"
+ echo " zookeeper Run a ZooKeeper server"
+ echo " rest Run an HBase REST server"
+ echo " thrift Run the HBase Thrift server"
+ echo " thrift2 Run the HBase Thrift2 server"
+ echo " clean Run the HBase clean up script"
+ fi
echo " classpath Dump hbase CLASSPATH"
echo " mapredcp Dump CLASSPATH entries required by mapreduce"
echo " pe Run PerformanceEvaluation"
@@ -186,9 +195,99 @@ for f in $HBASE_HOME/hbase-jars/hbase*.jar; do
fi
done
+#If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH
+# Allow this functionality to be disabled
+if [ "$HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP" != "true" ] ; then
+ HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which hadoop 2>/dev/null)
+fi
+
# Add libs to CLASSPATH
-for f in $HBASE_HOME/lib/*.jar; do
- CLASSPATH=${CLASSPATH}:$f;
+declare shaded_jar
+
+if [ "${INTERNAL_CLASSPATH}" != "true" ]; then
+ # find our shaded jars
+ declare shaded_client
+ declare shaded_client_byo_hadoop
+ declare shaded_mapreduce
+ for f in "${HBASE_HOME}"/lib/shaded-clients/hbase-shaded-client*.jar; do
+ if [[ "${f}" =~ byo-hadoop ]]; then
+ shaded_client_byo_hadoop="${f}"
+ else
+ shaded_client="${f}"
+ fi
+ done
+ for f in "${HBASE_HOME}"/lib/shaded-clients/hbase-shaded-mapreduce*.jar; do
+ shaded_mapreduce="${f}"
+ done
+
+ # If command can use our shaded client, use it
+ declare -a commands_in_client_jar=("classpath" "version")
+ for c in "${commands_in_client_jar[@]}"; do
+ if [ "${COMMAND}" = "${c}" ]; then
+ if [ -n "${HADOOP_IN_PATH}" ] && [ -f "${HADOOP_IN_PATH}" ]; then
+ # If we didn't find a jar above, this will just be blank and the
+ # check below will then default back to the internal classpath.
+ shaded_jar="${shaded_client_byo_hadoop}"
+ else
+ # If we didn't find a jar above, this will just be blank and the
+ # check below will then default back to the internal classpath.
+ shaded_jar="${shaded_client}"
+ fi
+ break
+ fi
+ done
+
+ # If command needs our shaded mapreduce, use it
+ # N.B "mapredcp" is not included here because in the shaded case it skips our built classpath
+ declare -a commands_in_mr_jar=("hbck" "snapshot" "canary" "regionsplitter" "pre-upgrade")
+ for c in "${commands_in_mr_jar[@]}"; do
+ if [ "${COMMAND}" = "${c}" ]; then
+ # If we didn't find a jar above, this will just be blank and the
+ # check below will then default back to the internal classpath.
+ shaded_jar="${shaded_mapreduce}"
+ break
+ fi
+ done
+
+ # Some commands specifically only can use shaded mapreduce when we'll get a full hadoop classpath at runtime
+ if [ -n "${HADOOP_IN_PATH}" ] && [ -f "${HADOOP_IN_PATH}" ]; then
+ declare -a commands_in_mr_need_hadoop=("backup" "restore" "rowcounter" "cellcounter")
+ for c in "${commands_in_mr_need_hadoop[@]}"; do
+ if [ "${COMMAND}" = "${c}" ]; then
+ # If we didn't find a jar above, this will just be blank and the
+ # check below will then default back to the internal classpath.
+ shaded_jar="${shaded_mapreduce}"
+ break
+ fi
+ done
+ fi
+fi
+
+
+if [ -n "${shaded_jar}" ] && [ -f "${shaded_jar}" ]; then
+ CLASSPATH="${CLASSPATH}:${shaded_jar}"
+# fall through to grabbing all the lib jars and hope we're in the omnibus tarball
+#
+# N.B. shell specifically can't rely on the shaded artifacts because RSGroups is only
+# available as non-shaded
+#
+# N.B. pe and ltt can't easily rely on shaded artifacts because they live in hbase-mapreduce:test-jar
+# and need some other jars that haven't been relocated. Currently enumerating that list
+# is too hard to be worth it.
+#
+else
+ for f in $HBASE_HOME/lib/*.jar; do
+ CLASSPATH=${CLASSPATH}:$f;
+ done
+ # make it easier to check for shaded/not later on.
+ shaded_jar=""
+fi
+for f in "${HBASE_HOME}"/lib/client-facing-thirdparty/*.jar; do
+ if [[ ! "${f}" =~ ^.*/htrace-core-3.*\.jar$ ]] && \
+ [ "${f}" != "htrace-core.jar$" ] && \
+ [[ ! "${f}" =~ ^.*/slf4j-log4j.*$ ]]; then
+ CLASSPATH="${CLASSPATH}:${f}"
+ fi
done
# default log directory & file
@@ -201,9 +300,9 @@ fi
function append_path() {
if [ -z "$1" ]; then
- echo $2
+ echo "$2"
else
- echo $1:$2
+ echo "$1:$2"
fi
}
@@ -214,18 +313,34 @@ if [ "$HBASE_LIBRARY_PATH" != "" ]; then
JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" "$HBASE_LIBRARY_PATH")
fi
-#If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH
-# Allow this functionality to be disabled
-if [ "$HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP" != "true" ] ; then
- HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which hadoop 2>/dev/null)
- if [ -f ${HADOOP_IN_PATH} ]; then
- HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$CLASSPATH" ${HADOOP_IN_PATH} \
- org.apache.hadoop.hbase.util.GetJavaProperty java.library.path 2>/dev/null)
- if [ -n "$HADOOP_JAVA_LIBRARY_PATH" ]; then
- JAVA_LIBRARY_PATH=$(append_path "${JAVA_LIBRARY_PATH}" "$HADOOP_JAVA_LIBRARY_PATH")
- fi
- CLASSPATH=$(append_path "${CLASSPATH}" `${HADOOP_IN_PATH} classpath 2>/dev/null`)
+#If configured and available, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH
+if [ -n "${HADOOP_IN_PATH}" ] && [ -f "${HADOOP_IN_PATH}" ]; then
+ HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$CLASSPATH" "${HADOOP_IN_PATH}" \
+ org.apache.hadoop.hbase.util.GetJavaProperty java.library.path)
+ if [ -n "$HADOOP_JAVA_LIBRARY_PATH" ]; then
+ JAVA_LIBRARY_PATH=$(append_path "${JAVA_LIBRARY_PATH}" "$HADOOP_JAVA_LIBRARY_PATH")
fi
+ CLASSPATH=$(append_path "${CLASSPATH}" "$(${HADOOP_IN_PATH} classpath 2>/dev/null)")
+else
+ # Otherwise, if we're providing Hadoop we should include htrace 3 if we were built with a version that needs it.
+ for f in "${HBASE_HOME}"/lib/client-facing-thirdparty/htrace-core-3*.jar "${HBASE_HOME}"/lib/client-facing-thirdparty/htrace-core.jar; do
+ if [ -f "${f}" ]; then
+ CLASSPATH="${CLASSPATH}:${f}"
+ break
+ fi
+ done
+ # Some commands require special handling when using shaded jars. For these cases, we rely on hbase-shaded-mapreduce
+ # instead of hbase-shaded-client* because we make use of some IA.Private classes that aren't in the latter. However,
+ # we don't invoke them using the "hadoop jar" command so we need to ensure there are some Hadoop classes available
+ # when we're not doing runtime hadoop classpath lookup.
+ #
+ # luckily the set of classes we need are those packaged in the shaded-client.
+ for c in "${commands_in_mr_jar[@]}"; do
+ if [ "${COMMAND}" = "${c}" ] && [ -n "${shaded_jar}" ]; then
+ CLASSPATH="${CLASSPATH}:${shaded_client:?We couldn\'t find the shaded client jar even though we did find the shaded MR jar. for command ${COMMAND} we need both. please use --internal-classpath as a workaround.}"
+ break
+ fi
+ done
fi
# Add user-specified CLASSPATH last
@@ -250,11 +365,11 @@ if [ -d "${HBASE_HOME}/build/native" -o -d "${HBASE_HOME}/lib/native" ]; then
JAVA_PLATFORM=`CLASSPATH=${CLASSPATH} ${JAVA} org.apache.hadoop.util.PlatformName | sed -e "s/ /_/g"`
fi
if [ -d "$HBASE_HOME/build/native" ]; then
- JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/build/native/${JAVA_PLATFORM}/lib)
+ JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" "${HBASE_HOME}/build/native/${JAVA_PLATFORM}/lib")
fi
if [ -d "${HBASE_HOME}/lib/native" ]; then
- JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/lib/native/${JAVA_PLATFORM})
+ JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" "${HBASE_HOME}/lib/native/${JAVA_PLATFORM}")
fi
fi
@@ -379,8 +494,30 @@ elif [ "$COMMAND" = "zkcli" ] ; then
done
elif [ "$COMMAND" = "backup" ] ; then
CLASS='org.apache.hadoop.hbase.backup.BackupDriver'
+ if [ -n "${shaded_jar}" ] ; then
+ for f in "${HBASE_HOME}"/lib/hbase-backup*.jar; do
+ if [ -f "${f}" ]; then
+ CLASSPATH="${CLASSPATH}:${f}"
+ break
+ fi
+ done
+ fi
elif [ "$COMMAND" = "restore" ] ; then
CLASS='org.apache.hadoop.hbase.backup.RestoreDriver'
+ if [ -n "${shaded_jar}" ] ; then
+ for f in "${HBASE_HOME}"/lib/hbase-backup*.jar; do
+ if [ -f "${f}" ]; then
+ CLASSPATH="${CLASSPATH}:${f}"
+ break
+ fi
+ done
+ for f in "${HBASE_HOME}"/lib/commons-lang3*.jar; do
+ if [ -f "${f}" ]; then
+ CLASSPATH="${CLASSPATH}:${f}"
+ break
+ fi
+ done
+ fi
elif [ "$COMMAND" = "upgrade" ] ; then
echo "This command was used to upgrade to HBase 0.96, it was removed in HBase 2.0.0."
echo "Please follow the documentation at http://hbase.apache.org/book.html#upgrading."
@@ -451,9 +588,24 @@ elif [ "$COMMAND" = "clean" ] ; then
"$bin"/hbase-cleanup.sh --config ${HBASE_CONF_DIR} $@
exit $?
elif [ "$COMMAND" = "mapredcp" ] ; then
+ # If we didn't find a jar above, this will just be blank and the
+ # check below will then default back to the internal classpath.
+ shaded_jar="${shaded_mapreduce}"
+ if [ "${INTERNAL_CLASSPATH}" != "true" ] && [ -f "${shaded_jar}" ]; then
+ echo -n "${shaded_jar}"
+ for f in "${HBASE_HOME}"/lib/client-facing-thirdparty/*.jar; do
+ if [[ ! "${f}" =~ ^.*/htrace-core-3.*\.jar$ ]] && \
+ [ "${f}" != "htrace-core.jar$" ] && \
+ [[ ! "${f}" =~ ^.*/slf4j-log4j.*$ ]]; then
+ echo -n ":${f}"
+ fi
+ done
+ echo ""
+ exit 0
+ fi
CLASS='org.apache.hadoop.hbase.util.MapreduceDependencyClasspathTool'
elif [ "$COMMAND" = "classpath" ] ; then
- echo $CLASSPATH
+ echo "$CLASSPATH"
exit 0
elif [ "$COMMAND" = "pe" ] ; then
CLASS='org.apache.hadoop.hbase.PerformanceEvaluation'
@@ -500,8 +652,20 @@ else
fi
HEAP_SETTINGS="$JAVA_HEAP_MAX $JAVA_OFFHEAP_MAX"
+# by now if we're running a command it means we need logging
+for f in ${HBASE_HOME}/lib/client-facing-thirdparty/slf4j-log4j*.jar; do
+ if [ -f "${f}" ]; then
+ CLASSPATH="${CLASSPATH}:${f}"
+ break
+ fi
+done
+
# Exec unless HBASE_NOEXEC is set.
export CLASSPATH
+if [ "${DEBUG}" = "true" ]; then
+ echo "classpath=${CLASSPATH}" >&2
+ HBASE_OPTS="${HBASE_OPTS} -Xdiag"
+fi
if [ "${HBASE_NOEXEC}" != "" ]; then
"$JAVA" -Dproc_$COMMAND -XX:OnOutOfMemoryError="kill -9 %p" $HEAP_SETTINGS $HBASE_OPTS $CLASS "$@"
http://git-wip-us.apache.org/repos/asf/hbase/blob/5c65a9f9/bin/hbase-config.sh
----------------------------------------------------------------------
diff --git a/bin/hbase-config.sh b/bin/hbase-config.sh
index 2e95ae7..1054751 100644
--- a/bin/hbase-config.sh
+++ b/bin/hbase-config.sh
@@ -84,6 +84,16 @@ do
exit 1
fi
shift
+ elif [ "--internal-classpath" = "$1" ]
+ then
+ shift
+ # shellcheck disable=SC2034
+ INTERNAL_CLASSPATH="true"
+ elif [ "--debug" = "$1" ]
+ then
+ shift
+ # shellcheck disable=SC2034
+ DEBUG="true"
else
# Presume we are at end of options and break
break
http://git-wip-us.apache.org/repos/asf/hbase/blob/5c65a9f9/hbase-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml
index 4fa859a..fd1415f 100644
--- a/hbase-assembly/pom.xml
+++ b/hbase-assembly/pom.xml
@@ -190,6 +190,19 @@
</plugins>
</build>
<dependencies>
+ <!-- client artifacts for downstream use -->
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-shaded-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-shaded-client-byo-hadoop</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-shaded-mapreduce</artifactId>
+ </dependency>
<!-- Intra-project dependencies -->
<dependency>
<groupId>org.apache.hbase</groupId>
@@ -316,14 +329,6 @@
<groupId>jline</groupId>
<artifactId>jline</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-shaded-client</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-shaded-mapreduce</artifactId>
- </dependency>
</dependencies>
<profiles>
<profile>
http://git-wip-us.apache.org/repos/asf/hbase/blob/5c65a9f9/hbase-assembly/src/main/assembly/client-components.xml
----------------------------------------------------------------------
diff --git a/hbase-assembly/src/main/assembly/client-components.xml b/hbase-assembly/src/main/assembly/client-components.xml
index 2fd1b57..2369f28 100644
--- a/hbase-assembly/src/main/assembly/client-components.xml
+++ b/hbase-assembly/src/main/assembly/client-components.xml
@@ -32,7 +32,8 @@
<directory>${project.basedir}/..</directory>
<outputDirectory>.</outputDirectory>
<includes>
- <include>CHANGES.txt</include>
+ <include>CHANGES.md</include>
+ <include>RELEASENOTES.md</include>
<include>README.txt</include>
</includes>
<fileMode>0644</fileMode>
@@ -56,7 +57,6 @@
<include>hbase-config.sh</include>
<include>hbase-jruby</include>
<include>hirb.rb</include>
- <include></include>
</includes>
<fileMode>0755</fileMode>
<directoryMode>0755</directoryMode>
@@ -88,5 +88,44 @@
<include>*.dylib</include>
</includes>
</fileSet>
+ <!-- This is only necessary until maven fixes the intra-project dependency bug
+ in maven 3.0. Until then, we have to include the test jars for sub-projects. When
+ fixed, the below dependencySet stuff is sufficient for pulling in the test jars as
+ well, as long as they are added as dependencies in this project. Right now, we only
+ have 1 submodule to accumulate, but we can copy/paste as necessary until maven is
+ fixed. -->
+ <!-- Used by PE and ltt -->
+ <fileSet>
+ <directory>${project.basedir}/../hbase-server/target/</directory>
+ <outputDirectory>lib</outputDirectory>
+ <includes>
+ <include>${server.test.jar}</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/../hbase-mapreduce/target/</directory>
+ <outputDirectory>lib</outputDirectory>
+ <includes>
+ <include>${mapreduce.test.jar}</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/../hbase-common/target/</directory>
+ <outputDirectory>lib</outputDirectory>
+ <includes>
+ <include>${common.test.jar}</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/../hbase-zookeeper/target/</directory>
+ <outputDirectory>lib</outputDirectory>
+ <includes>
+ <include>${zookeeper.test.jar}</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
</fileSets>
</component>
http://git-wip-us.apache.org/repos/asf/hbase/blob/5c65a9f9/hbase-assembly/src/main/assembly/client.xml
----------------------------------------------------------------------
diff --git a/hbase-assembly/src/main/assembly/client.xml b/hbase-assembly/src/main/assembly/client.xml
index 7951961..c875b95 100644
--- a/hbase-assembly/src/main/assembly/client.xml
+++ b/hbase-assembly/src/main/assembly/client.xml
@@ -31,73 +31,37 @@
<componentDescriptor>src/main/assembly/client-components.xml</componentDescriptor>
</componentDescriptors>
<moduleSets>
+ <!-- include regular jars so the shell can use them -->
<moduleSet>
<useAllReactorProjects>true</useAllReactorProjects>
<includes>
- <!-- Keep this list sorted by name -->
- <include>org.apache.hbase:hbase-annotations</include>
- <include>org.apache.hbase:hbase-client</include>
- <include>org.apache.hbase:hbase-common</include>
- <include>org.apache.hbase:hbase-hadoop-compat</include>
- <include>org.apache.hbase:hbase-hadoop2-compat</include>
- <include>org.apache.hbase:hbase-mapreduce</include>
- <include>org.apache.hbase:hbase-metrics</include>
- <include>org.apache.hbase:hbase-metrics-api</include>
- <include>org.apache.hbase:hbase-procedure</include>
- <include>org.apache.hbase:hbase-protocol</include>
- <include>org.apache.hbase:hbase-protocol-shaded</include>
- <include>org.apache.hbase:hbase-server</include>
- <include>org.apache.hbase:hbase-zookeeper</include>
+ <include>org.apache.hbase:hbase-shell</include>
</includes>
- <!-- Binaries for the dependencies also go in the hbase-jars directory -->
<binaries>
- <outputDirectory>lib</outputDirectory>
<unpack>false</unpack>
- <dependencySets>
- <!-- Exclude jruby-complete from hbase_home/lib -->
- <dependencySet>
+ <outputDirectory>lib</outputDirectory>
+ <dependencySets>
+ <dependencySet>
+ <!-- Exclude libraries that we put in their own dirs under lib/ -->
<excludes>
+ <exclude>org.jruby:jruby-complete</exclude>
<exclude>com.sun.jersey:*</exclude>
<exclude>com.sun.jersey.contribs:*</exclude>
- <!-- Exclude jars which typical clients don't need -->
- <exclude>org.apache.hbase:hbase-external-blockcache</exclude>
- <exclude>org.apache.hbase:hbase-http</exclude>
- <exclude>org.apache.hbase:hbase-replication</exclude>
- <exclude>org.apache.hbase:hbase-rest</exclude>
- <exclude>org.apache.hbase:hbase-rsgroup</exclude>
- <exclude>org.apache.hbase:hbase-shaded-client</exclude>
- <exclude>org.apache.hbase:hbase-shaded-mapreduce</exclude>
- <!-- At present, hbase-shell doesn't actually contain
- any Java code we need to include. Ruby files are
- copied elsewhere in this descriptor. -->
- <exclude>org.apache.hbase:hbase-shell</exclude>
- <exclude>org.apache.hbase:hbase-thrift</exclude>
- <exclude>org.jruby:jruby-complete</exclude>
+ <exclude>jline:jline</exclude>
+ <exclude>com.github.stephenc.findbugs:findbugs-annotations</exclude>
+ <exclude>commons-logging:commons-logging</exclude>
+ <exclude>log4j:log4j</exclude>
+ <exclude>org.apache.hbase:hbase-shaded-client</exclude>
+ <exclude>org.apache.hbase:hbase-shaded-client-byo-hadoop</exclude>
+ <exclude>org.apache.hbase:hbase-shaded-mapreduce</exclude>
+ <exclude>org.apache.htrace:htrace-core4</exclude>
+ <exclude>org.apache.htrace:htrace-core</exclude>
+ <exclude>org.apache.yetus:audience-annotations</exclude>
+ <exclude>org.slf4j:slf4j-api</exclude>
+ <exclude>org.slf4j:slf4j-log4j12</exclude>
</excludes>
- </dependencySet>
- </dependencySets>
- </binaries>
- </moduleSet>
- <moduleSet>
- <useAllReactorProjects>true</useAllReactorProjects>
- <includes>
- <include>org.apache.hbase:hbase-shaded-client</include>
- <include>org.apache.hbase:hbase-shaded-mapreduce</include>
- </includes>
- <!-- Binaries for the dependencies also go in the hbase-jars directory -->
- <binaries>
- <outputDirectory>shaded-lib</outputDirectory>
- <unpack>false</unpack>
- <dependencySets>
- <dependencySet>
- <includes>
- <!-- Keep this list sorted by name -->
- <include>org.apache.hbase:hbase-shaded-client</include>
- <include>org.apache.hbase:hbase-shaded-mapreduce</include>
- </includes>
- <useTransitiveDependencies>false</useTransitiveDependencies>
- </dependencySet>
- </dependencySets>
+ </dependencySet>
+ </dependencySets>
</binaries>
</moduleSet>
</moduleSets>
@@ -123,15 +87,66 @@
</file>
</files>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>lib/shaded-clients</outputDirectory>
+ <includes>
+ <include>org.apache.hbase:hbase-shaded-client</include>
+ <include>org.apache.hbase:hbase-shaded-mapreduce</include>
+ <include>org.apache.hbase:hbase-shaded-client-byo-hadoop</include>
+ </includes>
+ </dependencySet>
<!-- Add jruby-complete to hbase_home/lib/ruby.
Update JRUBY_PACKAGED_WITH_HBASE in bin/hbase and hbase.cmd if you would like to update outputDirectory below -->
- <dependencySets>
<dependencySet>
<outputDirectory>lib/ruby</outputDirectory>
<includes>
<include>org.jruby:jruby-complete</include>
</includes>
</dependencySet>
+ <!-- Include third party dependencies the shaded clients expose in the lib directory
+ N.B. this will conflict with the omnibus tarball but these should be precisely
+ the same artifacts so blind overwrite of either should be fine.
+ -->
+ <dependencySet>
+ <outputDirectory>lib/client-facing-thirdparty</outputDirectory>
+ <useTransitiveDependencies>true</useTransitiveDependencies>
+ <!-- Unfortunately, we have to whitelist these because Maven
+ currently doesn't use the dependency-reduced-pom after
+ the shaded module has done its thing. That means if we
+ did this as "transitives of the shaded modules" we'd
+ get a duplication of all the jars we already have in our
+ shaded artifacts. See MNG-5899.
+
+ Check that things match by listing files and making
+ sure the runtime scoped things are all present in the
+ tarball. e.g.:
+
+ for module in hbase-shaded-mapreduce hbase-shaded-client; do
+ mvn dependency:list -f hbase-shaded/${module}/dependency-reduced-pom.xml
+ done | \
+ grep -E "runtime|compile" | \
+ grep -v -E "junit|(optional)" | \
+ cut -d ' ' -f 3- | \
+ sort -u
+
+ TODO we should check this in nightly
+
+ Alternatively, we could
+ stop waiting for Maven to fix this and build the client
+ tarball in a different build.
+ -->
+ <includes>
+ <include>com.github.stephenc.findbugs:findbugs-annotations</include>
+ <include>commons-logging:commons-logging</include>
+ <include>log4j:log4j</include>
+ <include>org.apache.htrace:htrace-core4</include>
+ <include>org.apache.htrace:htrace-core</include>
+ <include>org.apache.yetus:audience-annotations</include>
+ <include>org.slf4j:slf4j-api</include>
+ <include>org.slf4j:slf4j-log4j12</include>
+ </includes>
+ </dependencySet>
</dependencySets>
</assembly>
http://git-wip-us.apache.org/repos/asf/hbase/blob/5c65a9f9/hbase-assembly/src/main/assembly/components.xml
----------------------------------------------------------------------
diff --git a/hbase-assembly/src/main/assembly/components.xml b/hbase-assembly/src/main/assembly/components.xml
index 59502d6..2eb16e7 100644
--- a/hbase-assembly/src/main/assembly/components.xml
+++ b/hbase-assembly/src/main/assembly/components.xml
@@ -37,7 +37,8 @@
<directory>${project.basedir}/..</directory>
<outputDirectory>.</outputDirectory>
<includes>
- <include>CHANGES.txt</include>
+ <include>CHANGES.md</include>
+ <include>RELEASENOTES.md</include>
<include>README.txt</include>
</includes>
<fileMode>0644</fileMode>
http://git-wip-us.apache.org/repos/asf/hbase/blob/5c65a9f9/hbase-assembly/src/main/assembly/hadoop-two-compat.xml
----------------------------------------------------------------------
diff --git a/hbase-assembly/src/main/assembly/hadoop-two-compat.xml b/hbase-assembly/src/main/assembly/hadoop-two-compat.xml
index 69a800b..7243783 100644
--- a/hbase-assembly/src/main/assembly/hadoop-two-compat.xml
+++ b/hbase-assembly/src/main/assembly/hadoop-two-compat.xml
@@ -66,18 +66,51 @@
<outputDirectory>lib</outputDirectory>
<unpack>false</unpack>
<dependencySets>
- <!-- Exclude jruby-complete from hbase_home/lib -->
<dependencySet>
+ <!-- Exclude libraries that we put in their own dirs under lib/ -->
<excludes>
<exclude>org.jruby:jruby-complete</exclude>
<exclude>com.sun.jersey:*</exclude>
<exclude>com.sun.jersey.contribs:*</exclude>
<exclude>jline:jline</exclude>
+ <exclude>org.apache.hbase:hbase-shaded-client</exclude>
+ <exclude>org.apache.hbase:hbase-shaded-client-byo-hadoop</exclude>
+ <exclude>org.apache.hbase:hbase-shaded-mapreduce</exclude>
+ <exclude>com.github.stephenc.findbugs:findbugs-annotations</exclude>
+ <exclude>commons-logging:commons-logging</exclude>
+ <exclude>log4j:log4j</exclude>
+ <exclude>org.apache.htrace:htrace-core4</exclude>
+ <exclude>org.apache.htrace:htrace-core</exclude>
+ <exclude>org.apache.yetus:audience-annotations</exclude>
+ <exclude>org.slf4j:slf4j-api</exclude>
+ <exclude>org.slf4j:slf4j-log4j12</exclude>
</excludes>
</dependencySet>
</dependencySets>
</binaries>
</moduleSet>
+ <!-- Include shaded clients in their own directory -->
+ <moduleSet>
+ <useAllReactorProjects>true</useAllReactorProjects>
+ <includes>
+ <include>org.apache.hbase:hbase-shaded-client</include>
+ <include>org.apache.hbase:hbase-shaded-mapreduce</include>
+ <include>org.apache.hbase:hbase-shaded-client-byo-hadoop</include>
+ </includes>
+ <binaries>
+ <outputDirectory>lib/shaded-clients</outputDirectory>
+ <unpack>false</unpack>
+ <dependencySets>
+ <dependencySet>
+ <includes>
+ <include>org.apache.hbase:hbase-shaded-client</include>
+ <include>org.apache.hbase:hbase-shaded-mapreduce</include>
+ <include>org.apache.hbase:hbase-shaded-client-byo-hadoop</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+ </binaries>
+ </moduleSet>
</moduleSets>
<!-- Include the generated LICENSE and NOTICE files -->
<files>
@@ -101,15 +134,56 @@
</file>
</files>
- <!-- Add jruby-complete to hbase_home/lib/ruby.
- Update JRUBY_PACKAGED_WITH_HBASE in bin/hbase and hbase.cmd if you would like to update outputDirectory below -->
<dependencySets>
+ <!-- Add jruby-complete to hbase_home/lib/ruby.
+ Update JRUBY_PACKAGED_WITH_HBASE in bin/hbase and hbase.cmd if you would like to update outputDirectory below -->
<dependencySet>
<outputDirectory>lib/ruby</outputDirectory>
<includes>
<include>org.jruby:jruby-complete</include>
</includes>
</dependencySet>
+ <!-- Include third party dependencies the shaded clients expose in the lib directory
+ -->
+ <dependencySet>
+ <outputDirectory>lib/client-facing-thirdparty</outputDirectory>
+ <useTransitiveDependencies>true</useTransitiveDependencies>
+ <!-- Unfortunately, we have to whitelist these because Maven
+ currently doesn't use the dependency-reduced-pom after
+ the shaded module has done its thing. That means if we
+ did this as "transitives of the shaded modules" we'd
+ get a duplication of all the jars we already have in our
+ shaded artifacts. See MNG-5899.
+
+ Check that things match by listing files and making
+ sure the runtime scoped things are all present in the
+ tarball. e.g.:
+
+ for module in hbase-shaded-mapreduce hbase-shaded-client; do
+ mvn dependency:list -f hbase-shaded/${module}/dependency-reduced-pom.xml
+ done | \
+ grep -E "runtime|compile" | \
+ grep -v -E "junit|(optional)" | \
+ cut -d ' ' -f 3- | \
+ sort -u
+
+ TODO we should check this in nightly
+
+ Alternatively, we could
+ stop waiting for Maven to fix this and build the client
+ tarball in a different build.
+ -->
+ <includes>
+ <include>com.github.stephenc.findbugs:findbugs-annotations</include>
+ <include>commons-logging:commons-logging</include>
+ <include>log4j:log4j</include>
+ <include>org.apache.htrace:htrace-core4</include>
+ <include>org.apache.htrace:htrace-core</include>
+ <include>org.apache.yetus:audience-annotations</include>
+ <include>org.slf4j:slf4j-api</include>
+ <include>org.slf4j:slf4j-log4j12</include>
+ </includes>
+ </dependencySet>
<dependencySet>
<outputDirectory>lib/zkcli</outputDirectory>
<includes>
http://git-wip-us.apache.org/repos/asf/hbase/blob/5c65a9f9/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b9a1b77..ea02fe8 100755
--- a/pom.xml
+++ b/pom.xml
@@ -1555,6 +1555,7 @@
<annotations.test.jar>hbase-annotations-${project.version}-tests.jar</annotations.test.jar>
<rsgroup.test.jar>hbase-rsgroup-${project.version}-tests.jar</rsgroup.test.jar>
<mapreduce.test.jar>hbase-mapreduce-${project.version}-tests.jar</mapreduce.test.jar>
+ <zookeeper.test.jar>hbase-zookeeper-${project.version}-tests.jar</zookeeper.test.jar>
<shell-executable>bash</shell-executable>
<surefire.version>2.21.0</surefire.version>
<surefire.provider>surefire-junit47</surefire.provider>
@@ -1860,6 +1861,11 @@
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-shaded-client-byo-hadoop</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-mapreduce</artifactId>
<version>${project.version}</version>
</dependency>
[12/12] hbase git commit: WIP tune down build retention while
troubleshooting.
Posted by bu...@apache.org.
WIP tune down build retention while troubleshooting.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/acaa4a5f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/acaa4a5f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/acaa4a5f
Branch: refs/heads/HBASE-20331
Commit: acaa4a5f8fe567d8a164a82c1519709404e4498a
Parents: e6f8941
Author: Sean Busbey <bu...@apache.org>
Authored: Wed May 23 14:14:45 2018 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Thu Jun 14 20:56:47 2018 -0500
----------------------------------------------------------------------
dev-support/Jenkinsfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/acaa4a5f/dev-support/Jenkinsfile
----------------------------------------------------------------------
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 59d3227..64144e9 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -24,7 +24,7 @@ pipeline {
cron('@daily')
}
options {
- buildDiscarder(logRotator(numToKeepStr: '30'))
+ buildDiscarder(logRotator(numToKeepStr: '1'))
timeout (time: 9, unit: 'HOURS')
timestamps()
skipDefaultCheckout()
[07/12] hbase git commit: HBASE-20332 shaded mapreduce module
shouldn't include hadoop
Posted by bu...@apache.org.
HBASE-20332 shaded mapreduce module shouldn't include hadoop
* modify the jar checking script to take args; make hadoop stuff optional
* separate out checking the artifacts that have hadoop vs those that don't.
* * Unfortunately means we need two modules for checking things
* * put in a safety check that the support script for checking jar contents is maintained in both modules
* * have to carve out an exception for o.a.hadoop.metrics2. :(
* fix duplicated class warning
* clean up dependencies in hbase-server and some modules that depend on it.
* allow Hadoop to have its own htrace where it needs it
* add a precommit check to make sure we're not using old htrace imports
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2541b3bb
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2541b3bb
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2541b3bb
Branch: refs/heads/HBASE-20331
Commit: 2541b3bbe8c5e0884575d748266035eb708adbe8
Parents: 04db900
Author: Sean Busbey <bu...@apache.org>
Authored: Mon Apr 9 13:37:44 2018 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Thu Jun 14 20:56:46 2018 -0500
----------------------------------------------------------------------
hbase-backup/pom.xml | 12 +-
.../src/main/resources/hbase/checkstyle.xml | 4 +-
hbase-client/pom.xml | 4 -
hbase-common/pom.xml | 12 --
hbase-endpoint/pom.xml | 14 --
hbase-examples/pom.xml | 12 --
hbase-external-blockcache/pom.xml | 4 -
hbase-hadoop2-compat/pom.xml | 6 -
hbase-it/pom.xml | 6 -
hbase-mapreduce/pom.xml | 30 +--
hbase-replication/pom.xml | 4 -
hbase-rest/pom.xml | 19 +-
hbase-rsgroup/pom.xml | 4 -
hbase-server/pom.xml | 103 ++++-----
.../hbase-shaded-check-invariants/pom.xml | 54 +++--
.../ensure-jars-have-correct-contents.sh | 92 ++++++--
hbase-shaded/hbase-shaded-mapreduce/pom.xml | 190 +++++++++++++++-
.../pom.xml | 215 +++++++++++++++++++
.../ensure-jars-have-correct-contents.sh | 129 +++++++++++
hbase-shaded/pom.xml | 13 ++
hbase-shell/pom.xml | 14 --
hbase-testing-util/pom.xml | 16 --
hbase-thrift/pom.xml | 16 --
pom.xml | 65 +++---
24 files changed, 773 insertions(+), 265 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/2541b3bb/hbase-backup/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-backup/pom.xml b/hbase-backup/pom.xml
index 7afd51e..00a996f 100644
--- a/hbase-backup/pom.xml
+++ b/hbase-backup/pom.xml
@@ -155,10 +155,6 @@
<artifactId>hadoop-common</artifactId>
<exclusions>
<exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- <exclusion>
<groupId>net.java.dev.jets3t</groupId>
<artifactId>jets3t</artifactId>
</exclusion>
@@ -264,9 +260,6 @@
<value>3.0</value>
</property>
</activation>
- <properties>
- <hadoop.version>3.0-SNAPSHOT</hadoop.version>
- </properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -276,6 +269,11 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-distcp</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
</dependencies>
</profile>
</profiles>
http://git-wip-us.apache.org/repos/asf/hbase/blob/2541b3bb/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml
----------------------------------------------------------------------
diff --git a/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml b/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml
index 7ad797c..148e256 100644
--- a/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml
+++ b/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml
@@ -86,8 +86,10 @@
org.apache.commons.lang,
org.apache.curator.shaded,
org.apache.hadoop.classification,
+ org.apache.htrace,
org.apache.htrace.shaded,
- org.codehaus.jackson"/>
+ org.codehaus.jackson,
+ org.htrace"/>
<property name="illegalClasses" value="
org.apache.commons.logging.Log,
org.apache.commons.logging.LogFactory"/>
http://git-wip-us.apache.org/repos/asf/hbase/blob/2541b3bb/hbase-client/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-client/pom.xml b/hbase-client/pom.xml
index f6247e3..bb99eec 100644
--- a/hbase-client/pom.xml
+++ b/hbase-client/pom.xml
@@ -235,10 +235,6 @@
<artifactId>hadoop-common</artifactId>
<exclusions>
<exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- <exclusion>
<groupId>net.java.dev.jets3t</groupId>
<artifactId>jets3t</artifactId>
</exclusion>
http://git-wip-us.apache.org/repos/asf/hbase/blob/2541b3bb/hbase-common/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-common/pom.xml b/hbase-common/pom.xml
index 5ae8e0b..0aaccb8 100644
--- a/hbase-common/pom.xml
+++ b/hbase-common/pom.xml
@@ -314,12 +314,6 @@
<artifactId>hadoop-common</artifactId>
<!--FYI This pulls in hadoop's guava. Its needed for Configuration
at least-->
- <exclusions>
- <exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- </exclusions>
</dependency>
</dependencies>
<build>
@@ -366,12 +360,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- </exclusions>
</dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hbase/blob/2541b3bb/hbase-endpoint/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-endpoint/pom.xml b/hbase-endpoint/pom.xml
index e9a8cf7..a831d3a 100644
--- a/hbase-endpoint/pom.xml
+++ b/hbase-endpoint/pom.xml
@@ -260,12 +260,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -297,10 +291,6 @@
<scope>test</scope>
<exclusions>
<exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- <exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
@@ -344,10 +334,6 @@
<artifactId>hadoop-minicluster</artifactId>
<exclusions>
<exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- <exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
http://git-wip-us.apache.org/repos/asf/hbase/blob/2541b3bb/hbase-examples/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-examples/pom.xml b/hbase-examples/pom.xml
index c74c1ba..e417b2d 100644
--- a/hbase-examples/pom.xml
+++ b/hbase-examples/pom.xml
@@ -232,12 +232,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- </exclusions>
</dependency>
</dependencies>
<build>
@@ -287,12 +281,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- </exclusions>
</dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hbase/blob/2541b3bb/hbase-external-blockcache/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-external-blockcache/pom.xml b/hbase-external-blockcache/pom.xml
index 24c33eb..2479b46 100644
--- a/hbase-external-blockcache/pom.xml
+++ b/hbase-external-blockcache/pom.xml
@@ -225,10 +225,6 @@
<artifactId>hadoop-common</artifactId>
<exclusions>
<exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- <exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
http://git-wip-us.apache.org/repos/asf/hbase/blob/2541b3bb/hbase-hadoop2-compat/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/pom.xml b/hbase-hadoop2-compat/pom.xml
index ab33c72..c1e4212 100644
--- a/hbase-hadoop2-compat/pom.xml
+++ b/hbase-hadoop2-compat/pom.xml
@@ -167,12 +167,6 @@ limitations under the License.
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
http://git-wip-us.apache.org/repos/asf/hbase/blob/2541b3bb/hbase-it/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-it/pom.xml b/hbase-it/pom.xml
index 9b1a167..4c17245 100644
--- a/hbase-it/pom.xml
+++ b/hbase-it/pom.xml
@@ -335,12 +335,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/hbase/blob/2541b3bb/hbase-mapreduce/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/pom.xml b/hbase-mapreduce/pom.xml
index af80737..2bf693b 100644
--- a/hbase-mapreduce/pom.xml
+++ b/hbase-mapreduce/pom.xml
@@ -196,6 +196,15 @@
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
+ <exclusions>
+ <!-- commons-logging is only used by hbase-http's HttpRequestLog and hbase-server's
+ HBaseTestingUtil. We don't need either of those here, so execlude it.
+ -->
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
@@ -246,10 +255,19 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
+ <!-- jackson(s) used by PerformanceEvaluation and it looks like TableMapReduceUtil -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
</dependencies>
<profiles>
<!-- Skip the tests in this module -->
@@ -285,10 +303,6 @@
<artifactId>hadoop-common</artifactId>
<exclusions>
<exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- <exclusion>
<groupId>net.java.dev.jets3t</groupId>
<artifactId>jets3t</artifactId>
</exclusion>
@@ -335,10 +349,6 @@
<artifactId>hadoop-hdfs</artifactId>
<exclusions>
<exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- <exclusion>
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
@@ -378,10 +388,6 @@
<scope>test</scope>
<exclusions>
<exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- <exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
http://git-wip-us.apache.org/repos/asf/hbase/blob/2541b3bb/hbase-replication/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-replication/pom.xml b/hbase-replication/pom.xml
index d05c60e..b999c1d 100644
--- a/hbase-replication/pom.xml
+++ b/hbase-replication/pom.xml
@@ -156,10 +156,6 @@
<artifactId>hadoop-common</artifactId>
<exclusions>
<exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- <exclusion>
<groupId>net.java.dev.jets3t</groupId>
<artifactId>jets3t</artifactId>
</exclusion>
http://git-wip-us.apache.org/repos/asf/hbase/blob/2541b3bb/hbase-rest/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-rest/pom.xml b/hbase-rest/pom.xml
index 617f254..d06feec 100644
--- a/hbase-rest/pom.xml
+++ b/hbase-rest/pom.xml
@@ -300,6 +300,19 @@
<artifactId>jackson-jaxrs-json-provider</artifactId>
</dependency>
<dependency>
+ <!-- We *might* need this for XMLStreamReader use in RemoteAdmin
+ TODO figure out if we can remove it.
+ -->
+ <groupId>org.codehaus.jettison</groupId>
+ <artifactId>jettison</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>stax</groupId>
+ <artifactId>stax-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
<!--For JspC used in ant task-->
<groupId>org.glassfish.web</groupId>
<artifactId>javax.servlet.jsp</artifactId>
@@ -374,12 +387,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/hbase/blob/2541b3bb/hbase-rsgroup/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-rsgroup/pom.xml b/hbase-rsgroup/pom.xml
index 2d9a10d..1cc3854 100644
--- a/hbase-rsgroup/pom.xml
+++ b/hbase-rsgroup/pom.xml
@@ -199,10 +199,6 @@
<artifactId>hadoop-common</artifactId>
<exclusions>
<exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- <exclusion>
<groupId>net.java.dev.jets3t</groupId>
<artifactId>jets3t</artifactId>
</exclusion>
http://git-wip-us.apache.org/repos/asf/hbase/blob/2541b3bb/hbase-server/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml
index 11361d8..e9daf93 100644
--- a/hbase-server/pom.xml
+++ b/hbase-server/pom.xml
@@ -71,6 +71,14 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-remote-resources-plugin</artifactId>
<version>1.5</version>
+ <dependencies>
+ <!-- resource bundle only needed at build time -->
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-resource-bundle</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
<executions>
<execution>
<id>default</id>
@@ -392,12 +400,6 @@
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-metrics</artifactId>
</dependency>
- <!-- resource bundle only needed at build time -->
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-resource-bundle</artifactId>
- <optional>true</optional>
- </dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
@@ -437,19 +439,16 @@
<artifactId>jetty-webapp</artifactId>
</dependency>
<dependency>
- <!--For JspC used in ant task-->
+ <!-- For JspC used in ant task, then needed at compile /runtime
+ because the source code made from the JSP refers to its runtime
+ -->
<groupId>org.glassfish.web</groupId>
<artifactId>javax.servlet.jsp</artifactId>
</dependency>
+ <!-- Also used by generated sources from our JSP -->
<dependency>
- <groupId>org.codehaus.jettison</groupId>
- <artifactId>jettison</artifactId>
- <exclusions>
- <exclusion>
- <groupId>stax</groupId>
- <artifactId>stax-api</artifactId>
- </exclusion>
- </exclusions>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>javax.servlet.jsp-api</artifactId>
</dependency>
<!-- General dependencies -->
<dependency>
@@ -501,9 +500,20 @@
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
</dependency>
+ <!-- Jackson only used in compile/runtime scope by BlockCacheUtil class
+ also used by some tests
+ -->
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
<dependency>
- <groupId>javax.ws.rs</groupId>
- <artifactId>javax.ws.rs-api</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
</dependency>
<!-- tracing Dependencies -->
@@ -512,11 +522,6 @@
<artifactId>htrace-core4</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- <version>${htrace-hadoop.version}</version>
- </dependency>
- <dependency>
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
</dependency>
@@ -556,6 +561,15 @@
<artifactId>httpcore</artifactId>
<scope>test</scope>
</dependency>
+ <!-- commons-logging is used by HBTU to monkey with log levels
+ have to put it at compile scope because Hadoop's IOUtils uses it
+ both for hadoop 2.7 and 3.0, so we'll fail at compile if it's at test scope.
+ -->
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ <scope>compile</scope>
+ </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-crypto</artifactId>
@@ -676,34 +690,10 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-distcp</artifactId>
- <version>${hadoop-two.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-auth</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-annotations</artifactId>
- <version>${hadoop-two.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<exclusions>
<exclusion>
@@ -796,21 +786,32 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-distcp</artifactId>
- <version>${hadoop-three.version}</version>
+ <artifactId>hadoop-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-annotations</artifactId>
- <version>${hadoop-three.version}</version>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/hbase/blob/2541b3bb/hbase-shaded/hbase-shaded-check-invariants/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shaded/hbase-shaded-check-invariants/pom.xml b/hbase-shaded/hbase-shaded-check-invariants/pom.xml
index 7322769..7ba4a41 100644
--- a/hbase-shaded/hbase-shaded-check-invariants/pom.xml
+++ b/hbase-shaded/hbase-shaded-check-invariants/pom.xml
@@ -26,7 +26,7 @@
Enforces our invariants for our shaded artifacts. e.g. shaded clients have
a specific set of transitive dependencies and shaded clients only contain
classes that are in particular packages. Does the enforcement through
- the maven-enforcer-plugin and and integration test.
+ the maven-enforcer-plugin and integration test.
</description>
<name>Apache HBase Shaded Packaging Invariants</name>
@@ -34,11 +34,15 @@
</properties>
<dependencies>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-shaded-client</artifactId>
- <version>${project.version}</version>
- </dependency>
+ <!-- Include here any client facing artifacts that presume
+ the runtime environment will have hadoop.
+
+ If our checks fail for the shaded mapreduce artifact,
+ then probably a dependency from hadoop has shown up
+ in the hbase-mapreduce module without being flagged
+ as 'provided' scope. See the note by the relevant
+ hadoop profile in that module.
+ -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-mapreduce</artifactId>
@@ -113,6 +117,8 @@
<exclude>com.github.stephenc.findbugs:*</exclude>
<!-- We leave HTrace as an unshaded dependnecy on purpose so that tracing within a JVM will work -->
<exclude>org.apache.htrace:*</exclude>
+ <!-- Our public API requires Hadoop at runtime to work -->
+ <exclude>org.apache.hadoop:*</exclude>
</excludes>
</banTransitiveDependencies>
<banDuplicateClasses>
@@ -158,18 +164,37 @@
</execution>
</executions>
</plugin>
- <!--
- Check that we actually relocated everything we included.
- It's critical that we don't ship third party dependencies that haven't
- been relocated under our pacakge space, since this will lead to
- difficult to debug classpath errors for downstream. Unfortunately, that
- means inspecting all the jars.
- -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>1.6.0</version>
<executions>
+ <!-- It's easier to have two copies of our validation
+ script than to copy it via remote-resources-plugin, but
+ we need to make sure they stay the same.
+ -->
+ <execution>
+ <id>make-sure-validation-files-are-in-sync</id>
+ <phase>validate</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <executable>diff</executable>
+ <requiresOnline>false</requiresOnline>
+ <arguments>
+ <argument>../hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh</argument>
+ <argument>../hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ <!--
+ Check that we actually relocated everything we included.
+ It's critical that we don't ship third party dependencies that haven't
+ been relocated under our package space, since this will lead to
+ difficult to debug classpath errors for downstream. Unfortunately, that
+ means inspecting all the jars.
+ -->
<execution>
<id>check-jar-contents</id>
<phase>integration-test</phase>
@@ -180,6 +205,9 @@
<executable>${shell-executable}</executable>
<workingDirectory>${project.build.testOutputDirectory}</workingDirectory>
<requiresOnline>false</requiresOnline>
+ <!-- Important that we don't pass the 'allow-hadoop' flag here, because
+ we allowed it as a provided dependency above.
+ -->
<arguments>
<argument>ensure-jars-have-correct-contents.sh</argument>
<argument>${hbase-client-artifacts}</argument>
http://git-wip-us.apache.org/repos/asf/hbase/blob/2541b3bb/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh
----------------------------------------------------------------------
diff --git a/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh
index 8bda8ce..eff1d20 100644
--- a/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh
+++ b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh
@@ -15,33 +15,67 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# Usage: $0 [/path/to/some/example.jar:/path/to/another/example/created.jar]
-#
-# accepts a single command line argument with a colon separated list of
-# paths to jars to check. Iterates through each such passed jar and checks
-# all the contained paths to make sure they follow the below constructed
-# safe list.
+set -e
+function usage {
+ echo "Usage: ${0} [options] [/path/to/some/example.jar:/path/to/another/example/created.jar]"
+ echo ""
+ echo " accepts a single command line argument with a colon separated list of"
+ echo " paths to jars to check. Iterates through each such passed jar and checks"
+ echo " all the contained paths to make sure they follow the below constructed"
+ echo " safe list."
+ echo ""
+ echo " --allow-hadoop Include stuff from the Apache Hadoop project in the list"
+ echo " of allowed jar contents. default: false"
+ echo " --debug print more info to stderr"
+ exit 1
+}
+# if no args specified, show usage
+if [ $# -lt 1 ]; then
+ usage
+fi
+
+# Get arguments
+declare allow_hadoop
+declare debug
+while [ $# -gt 0 ]
+do
+ case "$1" in
+ --allow-hadoop) shift; allow_hadoop="true";;
+ --debug) shift; debug="true";;
+ --) shift; break;;
+ -*) usage ;;
+ *) break;; # terminate while loop
+ esac
+done
+
+# should still have jars to check.
+if [ $# -lt 1 ]; then
+ usage
+fi
+if [ -n "${debug}" ]; then
+ echo "[DEBUG] Checking on jars: $*" >&2
+ echo "jar command is: $(which jar)" >&2
+ echo "grep command is: $(which grep)" >&2
+ grep -V >&2 || true
+fi
+
+IFS=: read -r -d '' -a artifact_list < <(printf '%s\0' "$1")
-# we have to allow the directories that lead to the org/apache/hadoop dir
-allowed_expr="(^org/$|^org/apache/$"
+# we have to allow the directories that lead to the hbase dirs
+allowed_expr="(^org/$|^org/apache/$|^org/apache/hadoop/$"
# We allow the following things to exist in our client artifacts:
-# * classes in packages that start with org.apache.hadoop, which by
-# convention should be in a path that looks like org/apache/hadoop
-allowed_expr+="|^org/apache/hadoop/"
+# * classes in packages that start with org.apache.hadoop.hbase, which by
+# convention should be in a path that looks like org/apache/hadoop/hbase
+allowed_expr+="|^org/apache/hadoop/hbase"
# * classes in packages that start with org.apache.hbase
allowed_expr+="|^org/apache/hbase/"
# * whatever in the "META-INF" directory
allowed_expr+="|^META-INF/"
# * the folding tables from jcodings
allowed_expr+="|^tables/"
-# * Hadoop's and HBase's default configuration files, which have the form
+# * HBase's default configuration files, which have the form
# "_module_-default.xml"
-allowed_expr+="|^[^-]*-default.xml$"
-# * Hadoop's versioning properties files, which have the form
-# "_module_-version-info.properties"
-allowed_expr+="|^[^-]*-version-info.properties$"
-# * Hadoop's application classloader properties file.
-allowed_expr+="|^org.apache.hadoop.application-classloader.properties$"
+allowed_expr+="|^hbase-default.xml$"
# public suffix list used by httpcomponents
allowed_expr+="|^mozilla/$"
allowed_expr+="|^mozilla/public-suffix-list.txt$"
@@ -51,12 +85,30 @@ allowed_expr+="|^properties.dtd$"
allowed_expr+="|^PropertyList-1.0.dtd$"
+if [ -n "${allow_hadoop}" ]; then
+ # * classes in packages that start with org.apache.hadoop, which by
+ # convention should be in a path that looks like org/apache/hadoop
+ allowed_expr+="|^org/apache/hadoop/"
+ # * Hadoop's default configuration files, which have the form
+ # "_module_-default.xml"
+ allowed_expr+="|^[^-]*-default.xml$"
+ # * Hadoop's versioning properties files, which have the form
+ # "_module_-version-info.properties"
+ allowed_expr+="|^[^-]*-version-info.properties$"
+ # * Hadoop's application classloader properties file.
+ allowed_expr+="|^org.apache.hadoop.application-classloader.properties$"
+else
+ # We have some classes for integrating with the Hadoop Metrics2 system
+ # that have to be in a particular package space due to access rules.
+ allowed_expr+="|^org/apache/hadoop/metrics2"
+fi
+
+
allowed_expr+=")"
declare -i bad_artifacts=0
declare -a bad_contents
-IFS=: read -r -d '' -a artifact_list < <(printf '%s\0' "$1")
for artifact in "${artifact_list[@]}"; do
- bad_contents=($(jar tf "${artifact}" | grep -v -E "${allowed_expr}"))
+ bad_contents=($(jar tf "${artifact}" | grep -v -E "${allowed_expr}" || true))
if [ ${#bad_contents[@]} -gt 0 ]; then
echo "[ERROR] Found artifact with unexpected contents: '${artifact}'"
echo " Please check the following and either correct the build or update"
http://git-wip-us.apache.org/repos/asf/hbase/blob/2541b3bb/hbase-shaded/hbase-shaded-mapreduce/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shaded/hbase-shaded-mapreduce/pom.xml b/hbase-shaded/hbase-shaded-mapreduce/pom.xml
index cfcc357..edc80cd 100644
--- a/hbase-shaded/hbase-shaded-mapreduce/pom.xml
+++ b/hbase-shaded/hbase-shaded-mapreduce/pom.xml
@@ -62,6 +62,10 @@
</plugins>
</build>
<dependencies>
+ <!--
+ We want to ensure needed hadoop bits are at provided scope for our shaded
+ artifact, so we list them below in hadoop specific profiles.
+ -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-mapreduce</artifactId>
@@ -138,10 +142,6 @@
<artifactId>jetty-webapp</artifactId>
</exclusion>
<exclusion>
- <groupId>org.glassfish.web</groupId>
- <artifactId>javax.servlet.jsp</artifactId>
- </exclusion>
- <exclusion>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>jersey-server</artifactId>
</exclusion>
@@ -149,6 +149,17 @@
<groupId>org.glassfish.jersey.containers</groupId>
<artifactId>jersey-container-servlet-core</artifactId>
</exclusion>
+ <!-- We excluded the server-side generated classes for JSP, so exclude
+ their runtime support libraries too
+ -->
+ <exclusion>
+ <groupId>org.glassfish.web</groupId>
+ <artifactId>javax.servlet.jsp</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>javax.servlet.jsp-api</artifactId>
+ </exclusion>
</exclusions>
</dependency>
</dependencies>
@@ -158,12 +169,175 @@
<id>release</id>
<build>
<plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- </plugin>
+ <!-- Tell the shade plugin we want to leave Hadoop as a dependency -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>aggregate-into-a-jar-with-relocated-third-parties</id>
+ <configuration>
+ <artifactSet>
+ <excludes>
+ <exclude>org.apache.hadoop:*</exclude>
+ <!-- The rest of these should be kept in sync with the parent pom -->
+ <exclude>org.apache.hbase:hbase-resource-bundle</exclude>
+ <exclude>org.slf4j:*</exclude>
+ <exclude>com.google.code.findbugs:*</exclude>
+ <exclude>com.github.stephenc.findbugs:*</exclude>
+ <exclude>org.apache.htrace:*</exclude>
+ <exclude>org.apache.yetus:*</exclude>
+ <exclude>log4j:*</exclude>
+ <exclude>commons-logging:*</exclude>
+ </excludes>
+ </artifactSet>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</profile>
+ <!-- These hadoop profiles should be derived from those in the hbase-mapreduce
+ module. Essentially, you must list the same hadoop-* dependencies
+ since provided dependencies are not transitively included.
+ -->
+ <!-- profile against Hadoop 2.x: This is the default. -->
+ <profile>
+ <id>hadoop-2.0</id>
+ <activation>
+ <property>
+ <!--Below formatting for dev-support/generate-hadoopX-poms.sh-->
+ <!--h2--><name>!hadoop.profile</name>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>net.java.dev.jets3t</groupId>
+ <artifactId>jets3t</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>jsp-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-compiler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>jsp-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>stax</groupId>
+ <artifactId>stax-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>xerces</groupId>
+ <artifactId>xercesImpl</artifactId>
+ </exclusion>
+ </exclusions>
+ <version>${hadoop-two.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+
+ <!--
+ profile for building against Hadoop 3.0.x. Activate using:
+ mvn -Dhadoop.profile=3.0
+ -->
+ <profile>
+ <id>hadoop-3.0</id>
+ <activation>
+ <property>
+ <name>hadoop.profile</name>
+ <value>3.0</value>
+ </property>
+ </activation>
+ <properties>
+ <hadoop.version>${hadoop-three.version}</hadoop.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ </profile>
</profiles>
</project>
http://git-wip-us.apache.org/repos/asf/hbase/blob/2541b3bb/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/pom.xml b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/pom.xml
new file mode 100644
index 0000000..07789f4
--- /dev/null
+++ b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/pom.xml
@@ -0,0 +1,215 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>hbase</artifactId>
+ <groupId>org.apache.hbase</groupId>
+ <version>3.0.0-SNAPSHOT</version>
+ <relativePath>../..</relativePath>
+ </parent>
+ <artifactId>hbase-shaded-with-hadoop-check-invariants</artifactId>
+ <packaging>pom</packaging>
+
+ <description>
+ Enforces our invariants for our shaded artifacts. e.g. shaded clients have
+ a specific set of transitive dependencies and shaded clients only contain
+ classes that are in particular packages. Does the enforcement through
+ the maven-enforcer-plugin and integration test.
+ </description>
+ <name>Apache HBase Shaded Packaging Invariants (with Hadoop bundled)</name>
+
+ <properties>
+ </properties>
+
+ <dependencies>
+ <!-- This should only be client facing artifacts that bundle
+ Apache Hadoop related artifacts.
+ -->
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-shaded-client</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <!-- parent pom defines these for children. :( :( :( -->
+ <dependency>
+ <groupId>com.github.stephenc.findbugs</groupId>
+ <artifactId>findbugs-annotations</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <!-- Test dependencies -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>extra-enforcer-rules</artifactId>
+ <version>1.0-beta-6</version>
+ </dependency>
+ </dependencies>
+ <executions>
+ <execution>
+ <id>enforce-banned-dependencies</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <skip>true</skip>
+ <rules>
+ <banTransitiveDependencies>
+<!--
+ <message>
+ Our client-facing artifacts are not supposed to have additional dependencies
+ and one or more of them do. The output from the enforcer plugin should give
+ specifics.
+ </message>
+-->
+ <excludes>
+ <!-- We leave logging stuff alone -->
+ <exclude>org.slf4j:*</exclude>
+ <exclude>log4j:*</exclude>
+ <exclude>commons-logging:*</exclude>
+ <!-- annotations that never change -->
+ <exclude>com.google.code.findbugs:*</exclude>
+ <exclude>com.github.stephenc.findbugs:*</exclude>
+ <!-- We leave HTrace as an unshaded dependnecy on purpose so that tracing within a JVM will work -->
+ <exclude>org.apache.htrace:*</exclude>
+ <!-- NB we don't exclude Hadoop from this check here, because the assumption is any needed classes
+ are contained in our artifacts.
+ -->
+ </excludes>
+ </banTransitiveDependencies>
+ <banDuplicateClasses>
+ <findAllDuplicates>true</findAllDuplicates>
+ </banDuplicateClasses>
+ </rules>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>test-resources</id>
+ <phase>pre-integration-test</phase>
+ <goals>
+ <goal>testResources</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <!-- create a maven pom property that has all of our dependencies.
+ below in the integration-test phase we'll pass this list
+ of paths to our jar checker script.
+ -->
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>put-client-artifacts-in-a-property</id>
+ <phase>pre-integration-test</phase>
+ <goals>
+ <goal>build-classpath</goal>
+ </goals>
+ <configuration>
+ <excludeScope>provided</excludeScope>
+ <excludeTransitive>true</excludeTransitive>
+ <outputProperty>hbase-client-artifacts</outputProperty>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.6.0</version>
+ <executions>
+ <!-- It's easier to have two copies of our validation
+ script than to copy it via remote-resources-plugin, but
+ we need to make sure they stay the same.
+ -->
+ <execution>
+ <id>make-sure-validation-files-are-in-sync</id>
+ <phase>validate</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <executable>diff</executable>
+ <requiresOnline>false</requiresOnline>
+ <arguments>
+ <argument>../hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh</argument>
+ <argument>../hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ <!--
+ Check that we actually relocated everything we included.
+ It's critical that we don't ship third party dependencies that haven't
+ been relocated under our package space, since this will lead to
+ difficult to debug classpath errors for downstream. Unfortunately, that
+ means inspecting all the jars.
+ -->
+ <execution>
+ <id>check-jar-contents-for-stuff-with-hadoop</id>
+ <phase>integration-test</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <executable>${shell-executable}</executable>
+ <workingDirectory>${project.build.testOutputDirectory}</workingDirectory>
+ <requiresOnline>false</requiresOnline>
+ <arguments>
+ <argument>ensure-jars-have-correct-contents.sh</argument>
+ <argument>--allow-hadoop</argument>
+ <argument>${hbase-client-artifacts}</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/hbase/blob/2541b3bb/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh
----------------------------------------------------------------------
diff --git a/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh
new file mode 100644
index 0000000..eff1d20
--- /dev/null
+++ b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh
@@ -0,0 +1,129 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+function usage {
+ echo "Usage: ${0} [options] [/path/to/some/example.jar:/path/to/another/example/created.jar]"
+ echo ""
+ echo " accepts a single command line argument with a colon separated list of"
+ echo " paths to jars to check. Iterates through each such passed jar and checks"
+ echo " all the contained paths to make sure they follow the below constructed"
+ echo " safe list."
+ echo ""
+ echo " --allow-hadoop Include stuff from the Apache Hadoop project in the list"
+ echo " of allowed jar contents. default: false"
+ echo " --debug print more info to stderr"
+ exit 1
+}
+# if no args specified, show usage
+if [ $# -lt 1 ]; then
+ usage
+fi
+
+# Get arguments
+declare allow_hadoop
+declare debug
+while [ $# -gt 0 ]
+do
+ case "$1" in
+ --allow-hadoop) shift; allow_hadoop="true";;
+ --debug) shift; debug="true";;
+ --) shift; break;;
+ -*) usage ;;
+ *) break;; # terminate while loop
+ esac
+done
+
+# should still have jars to check.
+if [ $# -lt 1 ]; then
+ usage
+fi
+if [ -n "${debug}" ]; then
+ echo "[DEBUG] Checking on jars: $*" >&2
+ echo "jar command is: $(which jar)" >&2
+ echo "grep command is: $(which grep)" >&2
+ grep -V >&2 || true
+fi
+
+IFS=: read -r -d '' -a artifact_list < <(printf '%s\0' "$1")
+
+# we have to allow the directories that lead to the hbase dirs
+allowed_expr="(^org/$|^org/apache/$|^org/apache/hadoop/$"
+# We allow the following things to exist in our client artifacts:
+# * classes in packages that start with org.apache.hadoop.hbase, which by
+# convention should be in a path that looks like org/apache/hadoop/hbase
+allowed_expr+="|^org/apache/hadoop/hbase"
+# * classes in packages that start with org.apache.hbase
+allowed_expr+="|^org/apache/hbase/"
+# * whatever in the "META-INF" directory
+allowed_expr+="|^META-INF/"
+# * the folding tables from jcodings
+allowed_expr+="|^tables/"
+# * HBase's default configuration files, which have the form
+# "_module_-default.xml"
+allowed_expr+="|^hbase-default.xml$"
+# public suffix list used by httpcomponents
+allowed_expr+="|^mozilla/$"
+allowed_expr+="|^mozilla/public-suffix-list.txt$"
+# Comes from commons-configuration, not sure if relocatable.
+allowed_expr+="|^digesterRules.xml$"
+allowed_expr+="|^properties.dtd$"
+allowed_expr+="|^PropertyList-1.0.dtd$"
+
+
+if [ -n "${allow_hadoop}" ]; then
+ # * classes in packages that start with org.apache.hadoop, which by
+ # convention should be in a path that looks like org/apache/hadoop
+ allowed_expr+="|^org/apache/hadoop/"
+ # * Hadoop's default configuration files, which have the form
+ # "_module_-default.xml"
+ allowed_expr+="|^[^-]*-default.xml$"
+ # * Hadoop's versioning properties files, which have the form
+ # "_module_-version-info.properties"
+ allowed_expr+="|^[^-]*-version-info.properties$"
+ # * Hadoop's application classloader properties file.
+ allowed_expr+="|^org.apache.hadoop.application-classloader.properties$"
+else
+ # We have some classes for integrating with the Hadoop Metrics2 system
+ # that have to be in a particular package space due to access rules.
+ allowed_expr+="|^org/apache/hadoop/metrics2"
+fi
+
+
+allowed_expr+=")"
+declare -i bad_artifacts=0
+declare -a bad_contents
+for artifact in "${artifact_list[@]}"; do
+ bad_contents=($(jar tf "${artifact}" | grep -v -E "${allowed_expr}" || true))
+ if [ ${#bad_contents[@]} -gt 0 ]; then
+ echo "[ERROR] Found artifact with unexpected contents: '${artifact}'"
+ echo " Please check the following and either correct the build or update"
+ echo " the allowed list with reasoning."
+ echo ""
+ for bad_line in "${bad_contents[@]}"; do
+ echo " ${bad_line}"
+ done
+ bad_artifacts=${bad_artifacts}+1
+ else
+ echo "[INFO] Artifact looks correct: '$(basename "${artifact}")'"
+ fi
+done
+
+# if there was atleast one bad artifact, exit with failure
+if [ "${bad_artifacts}" -gt 0 ]; then
+ exit 1
+fi
http://git-wip-us.apache.org/repos/asf/hbase/blob/2541b3bb/hbase-shaded/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shaded/pom.xml b/hbase-shaded/pom.xml
index 24c5158..93b122f 100644
--- a/hbase-shaded/pom.xml
+++ b/hbase-shaded/pom.xml
@@ -42,6 +42,7 @@
<module>hbase-shaded-client</module>
<module>hbase-shaded-mapreduce</module>
<module>hbase-shaded-check-invariants</module>
+ <module>hbase-shaded-with-hadoop-check-invariants</module>
</modules>
<dependencies>
<dependency>
@@ -118,6 +119,7 @@
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
+ <id>aggregate-into-a-jar-with-relocated-third-parties</id>
<phase>package</phase>
<goals>
<goal>shade</goal>
@@ -450,11 +452,22 @@
</excludes>
</filter>
<filter>
+ <!-- Duplication of classes that ship in commons-collections 2.x and 3.x
+ If we stop bundling a relevant commons-collections artifact we'll
+ need to revisit. See: https://s.apache.org/e09o
+ -->
+ <artifact>commons-beanutils:commons-beanutils-core</artifact>
+ <excludes>
+ <exclude>org/apache/commons/collections/*.class</exclude>
+ </excludes>
+ </filter>
+ <filter>
<!-- server side webapps that we don't need -->
<artifact>org.apache.hbase:hbase-server</artifact>
<excludes>
<exclude>hbase-webapps/*</exclude>
<exclude>hbase-webapps/**/*</exclude>
+ <exclude>**/*_jsp.class</exclude>
</excludes>
</filter>
<filter>
http://git-wip-us.apache.org/repos/asf/hbase/blob/2541b3bb/hbase-shell/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shell/pom.xml b/hbase-shell/pom.xml
index 8eaefaa..85f0415 100644
--- a/hbase-shell/pom.xml
+++ b/hbase-shell/pom.xml
@@ -277,12 +277,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -342,10 +336,6 @@
<scope>test</scope>
<exclusions>
<exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- <exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
@@ -409,10 +399,6 @@
<artifactId>hadoop-minicluster</artifactId>
<exclusions>
<exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- <exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
http://git-wip-us.apache.org/repos/asf/hbase/blob/2541b3bb/hbase-testing-util/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-testing-util/pom.xml b/hbase-testing-util/pom.xml
index 0f1b86c..032de61 100644
--- a/hbase-testing-util/pom.xml
+++ b/hbase-testing-util/pom.xml
@@ -140,12 +140,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -202,10 +196,6 @@
<scope>compile</scope>
<exclusions>
<exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- <exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
@@ -242,12 +232,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
<scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/hbase/blob/2541b3bb/hbase-thrift/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-thrift/pom.xml b/hbase-thrift/pom.xml
index 814650d..990e41c 100644
--- a/hbase-thrift/pom.xml
+++ b/hbase-thrift/pom.xml
@@ -503,12 +503,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -516,10 +510,6 @@
<scope>test</scope>
<exclusions>
<exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- <exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
@@ -576,12 +566,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- </exclusions>
</dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hbase/blob/2541b3bb/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ed7a172..a943125 100755
--- a/pom.xml
+++ b/pom.xml
@@ -1448,8 +1448,11 @@
<hadoop.guava.version>11.0.2</hadoop.guava.version>
<compat.module>hbase-hadoop2-compat</compat.module>
<assembly.file>src/main/assembly/hadoop-two-compat.xml</assembly.file>
- <audience-annotations.version>0.5.0</audience-annotations.version>
+ <!--This property is for hadoops netty. HBase netty
+ comes in via hbase-thirdparty hbase-shaded-netty-->
+ <netty.hadoop.version>3.6.2.Final</netty.hadoop.version>
<!-- end HBASE-15925 default hadoop compatibility values -->
+ <audience-annotations.version>0.5.0</audience-annotations.version>
<avro.version>1.7.7</avro.version>
<commons-codec.version>1.10</commons-codec.version>
<!-- pretty outdated -->
@@ -1477,7 +1480,6 @@
<junit.version>4.12</junit.version>
<hamcrest.version>1.3</hamcrest.version>
<htrace.version>4.2.0-incubating</htrace.version>
- <htrace-hadoop.version>3.2.0-incubating</htrace-hadoop.version>
<log4j.version>1.2.17</log4j.version>
<mockito-core.version>2.1.0</mockito-core.version>
<!--Internally we use a different version of protobuf. See hbase-protocol-shaded-->
@@ -1605,7 +1607,8 @@
org.mortbay.jetty:servlet-api, javax.servlet:servlet-api: These are excluded because they are
the same implementations. I chose org.mortbay.jetty:servlet-api-2.5 instead, which is a third
implementation of the same, because Hadoop also uses this version
- javax.servlet:jsp-api in favour of org.mortbay.jetty:jsp-api-2.1
+ javax.servlet:jsp-api in favour of javax.servlet.jsp:javax.servlet.jsp-api:2.3.1 since it
+ is what glassfish's jspC jar uses and that's where we get our own need for a jsp-api.
-->
<!-- Intra-module dependencies -->
<dependency>
@@ -1921,6 +1924,14 @@
<version>${commons-math.version}</version>
</dependency>
<dependency>
+ <!-- commons-logging is only used by hbase-http's HttpRequestLog and hbase-server's
+ HBaseTestingUtil.
+ -->
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ <version>1.2</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>${zookeeper.version}</version>
@@ -1985,6 +1996,16 @@
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.version}</version>
</dependency>
@@ -2087,6 +2108,12 @@
<version>${glassfish.jsp.version}</version>
</dependency>
<dependency>
+ <!-- this lib is used by the compiled Jsp from the above JspC -->
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>javax.servlet.jsp-api</artifactId>
+ <version>2.3.1</version>
+ </dependency>
+ <dependency>
<groupId>org.glassfish</groupId>
<artifactId>javax.el</artifactId>
<version>${glassfish.el.version}</version>
@@ -2551,10 +2578,6 @@
<version>${hadoop-two.version}</version>
<exclusions>
<exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- <exclusion>
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
@@ -2596,10 +2619,6 @@
<scope>test</scope>
<exclusions>
<exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- <exclusion>
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
@@ -2644,10 +2663,6 @@
<version>${hadoop-two.version}</version>
<exclusions>
<exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- <exclusion>
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
</exclusion>
@@ -2698,10 +2713,6 @@
<version>${hadoop-two.version}</version>
<exclusions>
<exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- <exclusion>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
</exclusion>
@@ -2820,10 +2831,6 @@
<version>${hadoop-three.version}</version>
<exclusions>
<exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- <exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</exclusion>
@@ -2869,10 +2876,6 @@
<scope>test</scope>
<exclusions>
<exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- <exclusion>
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
@@ -2953,10 +2956,6 @@
<artifactId>jersey-server</artifactId>
</exclusion>
<exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- <exclusion>
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
@@ -3021,10 +3020,6 @@
<version>${hadoop-three.version}</version>
<exclusions>
<exclusion>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- </exclusion>
- <exclusion>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
</exclusion>
[06/12] hbase git commit: HBASE-20733 QABot should run checkstyle
tests if the checkstyle configs change
Posted by bu...@apache.org.
HBASE-20733 QABot should run checkstyle tests if the checkstyle configs change
Signed-off-by: Mike Drob <md...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/04db9007
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/04db9007
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/04db9007
Branch: refs/heads/HBASE-20331
Commit: 04db900772889d70836dbd733f844782fb7adecd
Parents: 0b28155
Author: Sean Busbey <bu...@apache.org>
Authored: Thu Jun 14 13:00:08 2018 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Thu Jun 14 20:24:29 2018 -0500
----------------------------------------------------------------------
dev-support/hbase-personality.sh | 51 +++++++++++++++++++----------------
1 file changed, 28 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/04db9007/dev-support/hbase-personality.sh
----------------------------------------------------------------------
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
index 2c6e4a8..8004167 100755
--- a/dev-support/hbase-personality.sh
+++ b/dev-support/hbase-personality.sh
@@ -67,10 +67,12 @@ function personality_globals
# Yetus 0.7.0 enforces limits. Default proclimit is 1000.
# Up it. See HBASE-19902 for how we arrived at this number.
+ #shellcheck disable=SC2034
PROCLIMIT=10000
# Set docker container to run with 20g. Default is 4g in yetus.
# See HBASE-19902 for how we arrived at 20g.
+ #shellcheck disable=SC2034
DOCKERMEMLIMIT=20g
}
@@ -106,7 +108,7 @@ function personality_modules
local repostatus=$1
local testtype=$2
local extra=""
- local MODULES=(${CHANGED_MODULES[@]})
+ local MODULES=("${CHANGED_MODULES[@]}")
yetus_info "Personality: ${repostatus} ${testtype}"
@@ -130,6 +132,11 @@ function personality_modules
MODULES=(.)
fi
+ # If the checkstyle configs change, check everything.
+ if [[ "${testtype}" == checkstyle ]] && [[ "${MODULES[*]}" =~ hbase-checkstyle ]]; then
+ MODULES=(.)
+ fi
+
if [[ ${testtype} == mvninstall ]]; then
# shellcheck disable=SC2086
personality_enqueue_module . ${extra}
@@ -188,16 +195,31 @@ function personality_modules
function personality_file_tests
{
local filename=$1
+ yetus_debug "HBase specific personality_file_tests"
# If the change is to the refguide, then we don't need any builtin yetus tests
# the refguide test (below) will suffice for coverage.
if [[ ${filename} =~ src/main/asciidoc ]] ||
[[ ${filename} =~ src/main/xslt ]]; then
yetus_debug "Skipping builtin yetus checks for ${filename}. refguide test should pick it up."
- # fallback to checking which tests based on what yetus would do by default
- elif declare -f "${BUILDTOOL}_builtin_personality_file_tests" >/dev/null; then
- "${BUILDTOOL}_builtin_personality_file_tests" "${filename}"
- elif declare -f builtin_personality_file_tests >/dev/null; then
- builtin_personality_file_tests "${filename}"
+ else
+ # If we change our asciidoc, rebuild mvnsite
+ if [[ ${BUILDTOOL} = maven ]]; then
+ if [[ ${filename} =~ src/site || ${filename} =~ src/main/asciidoc ]]; then
+ yetus_debug "tests/mvnsite: ${filename}"
+ add_test mvnsite
+ fi
+ fi
+ # If we change checkstyle configs, run checkstyle
+ if [[ ${filename} =~ checkstyle.*\.xml ]]; then
+ yetus_debug "tests/checkstyle: ${filename}"
+ add_test checkstyle
+ fi
+ # fallback to checking which tests based on what yetus would do by default
+ if declare -f "${BUILDTOOL}_builtin_personality_file_tests" >/dev/null; then
+ "${BUILDTOOL}_builtin_personality_file_tests" "${filename}"
+ elif declare -f builtin_personality_file_tests >/dev/null; then
+ builtin_personality_file_tests "${filename}"
+ fi
fi
}
@@ -648,23 +670,6 @@ function hbaseanti_patchfile
return 0
}
-
-## @description hbase custom mvnsite file filter. See HBASE-15042
-## @audience private
-## @stability evolving
-## @param filename
-function mvnsite_filefilter
-{
- local filename=$1
-
- if [[ ${BUILDTOOL} = maven ]]; then
- if [[ ${filename} =~ src/site || ${filename} =~ src/main/asciidoc ]]; then
- yetus_debug "tests/mvnsite: ${filename}"
- add_test mvnsite
- fi
- fi
-}
-
## This is named so that yetus will check us right after running tests.
## Essentially, we check for normal failures and then we look for zombies.
#function hbase_unit_logfilter
[11/12] hbase git commit: HBASE-20334 add a test that verifies basic
client and MR integration
Posted by bu...@apache.org.
HBASE-20334 add a test that verifies basic client and MR integration
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e6f89414
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e6f89414
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e6f89414
Branch: refs/heads/HBASE-20331
Commit: e6f89414ffd52e4052f916adc30082b6547aae83
Parents: 5c65a9f
Author: Sean Busbey <bu...@apache.org>
Authored: Tue May 1 14:28:52 2018 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Thu Jun 14 20:56:47 2018 -0500
----------------------------------------------------------------------
dev-support/Jenkinsfile | 233 +++++++--
.../hbase_nightly_pseudo-distributed-test.sh | 516 +++++++++++++++++++
dev-support/hbase_nightly_source-artifact.sh | 14 +-
.../cache-apache-project-artifact.sh | 131 +++++
4 files changed, 836 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e6f89414/dev-support/Jenkinsfile
----------------------------------------------------------------------
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 2311e35..59d3227 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -60,54 +60,109 @@ pipeline {
booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a lot more meta-information.')
}
stages {
- stage ('yetus install') {
+ stage ('scm-checkout') {
steps {
- sh '''#!/usr/bin/env bash
-set -e
-echo "Ensure we have a copy of Apache Yetus."
-if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then
- YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}"
- echo "Checking for Yetus ${YETUS_RELEASE} in '${YETUS_DIR}'"
- if [ ! -d "${YETUS_DIR}" ]; then
- echo "New download of Apache Yetus version ${YETUS_RELEASE}."
- rm -rf "${WORKSPACE}/.gpg"
- mkdir -p "${WORKSPACE}/.gpg"
- chmod -R 700 "${WORKSPACE}/.gpg"
-
- echo "install yetus project KEYS"
- curl -L --fail -o "${WORKSPACE}/KEYS_YETUS" https://dist.apache.org/repos/dist/release/yetus/KEYS
- gpg --homedir "${WORKSPACE}/.gpg" --import "${WORKSPACE}/KEYS_YETUS"
-
- echo "download yetus release ${YETUS_RELEASE}"
- curl -L --fail -O "https://dist.apache.org/repos/dist/release/yetus/${YETUS_RELEASE}/yetus-${YETUS_RELEASE}-bin.tar.gz"
- curl -L --fail -O "https://dist.apache.org/repos/dist/release/yetus/${YETUS_RELEASE}/yetus-${YETUS_RELEASE}-bin.tar.gz.asc"
- echo "verifying yetus release"
- gpg --homedir "${WORKSPACE}/.gpg" --verify "yetus-${YETUS_RELEASE}-bin.tar.gz.asc"
- mv "yetus-${YETUS_RELEASE}-bin.tar.gz" yetus.tar.gz
- else
- echo "Reusing cached download of Apache Yetus version ${YETUS_RELEASE}."
- fi
-else
- YETUS_DIR="${WORKSPACE}/yetus-git"
- rm -rf "${YETUS_DIR}"
- echo "downloading from github"
- curl -L --fail https://api.github.com/repos/apache/yetus/tarball/HEAD -o yetus.tar.gz
-fi
-if [ ! -d "${YETUS_DIR}" ]; then
- echo "unpacking yetus into '${YETUS_DIR}'"
- mkdir -p "${YETUS_DIR}"
- gunzip -c yetus.tar.gz | tar xpf - -C "${YETUS_DIR}" --strip-components 1
-fi
- '''
- // Set up the file we need at PERSONALITY_FILE location
- dir ("tools") {
- sh """#!/usr/bin/env bash
-set -e
-echo "Downloading Project personality."
-curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
- """
+ dir('component') {
+ checkout scm
+ }
+ }
+ }
+ stage ('thirdparty installs') {
+ parallel {
+ stage ('yetus install') {
+ steps {
+ // directory must be unique for each parallel stage, because jenkins runs them in the same workspace :(
+ dir('downloads-yetus') {
+ // can't just do a simple echo or the directory won't be created. :(
+ sh '''#!/usr/bin/env bash
+ echo "Make sure we have a directory for downloading dependencies: $(pwd)"
+'''
+ }
+ sh '''#!/usr/bin/env bash
+ set -e
+ echo "Ensure we have a copy of Apache Yetus."
+ if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then
+ YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}"
+ echo "Checking for Yetus ${YETUS_RELEASE} in '${YETUS_DIR}'"
+ if [ ! -d "${YETUS_DIR}" ]; then
+ "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \
+ --working-dir "${WORKSPACE}/downloads-yetus" \
+ --keys 'https://www.apache.org/dist/yetus/KEYS' \
+ "${WORKSPACE}/yetus-${YETUS_RELEASE}-bin.tar.gz" \
+ "yetus/${YETUS_RELEASE}/yetus-${YETUS_RELEASE}-bin.tar.gz"
+ mv "yetus-${YETUS_RELEASE}-bin.tar.gz" yetus.tar.gz
+ else
+ echo "Reusing cached install of Apache Yetus version ${YETUS_RELEASE}."
+ fi
+ else
+ YETUS_DIR="${WORKSPACE}/yetus-git"
+ rm -rf "${YETUS_DIR}"
+ echo "downloading from github"
+ curl -L --fail https://api.github.com/repos/apache/yetus/tarball/HEAD -o yetus.tar.gz
+ fi
+ if [ ! -d "${YETUS_DIR}" ]; then
+ echo "unpacking yetus into '${YETUS_DIR}'"
+ mkdir -p "${YETUS_DIR}"
+ gunzip -c yetus.tar.gz | tar xpf - -C "${YETUS_DIR}" --strip-components 1
+ fi
+ '''
+ // Set up the file we need at PERSONALITY_FILE location
+ dir ("tools") {
+ sh """#!/usr/bin/env bash
+ set -e
+ echo "Downloading Project personality."
+ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
+ """
+ }
+ stash name: 'yetus', includes: "yetus-*/*,yetus-*/**/*,tools/personality.sh"
+ }
+ }
+ stage ('hadoop 2 cache') {
+ environment {
+ HADOOP2_VERSION="2.7.1"
+ }
+ steps {
+ // directory must be unique for each parallel stage, because jenkins runs them in the same workspace :(
+ dir('downloads-hadoop-2') {
+ sh '''#!/usr/bin/env bash
+ echo "Make sure we have a directory for downloading dependencies: $(pwd)"
+'''
+ }
+ sh '''#!/usr/bin/env bash
+ set -e
+ echo "Ensure we have a copy of Hadoop ${HADOOP2_VERSION}"
+ "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \
+ --working-dir "${WORKSPACE}/downloads-hadoop-2" \
+ --keys 'http://www.apache.org/dist/hadoop/common/KEYS' \
+ "${WORKSPACE}/hadoop-${HADOOP2_VERSION}-bin.tar.gz" \
+ "hadoop/common/hadoop-${HADOOP2_VERSION}/hadoop-${HADOOP2_VERSION}.tar.gz"
+ '''
+ stash name: 'hadoop-2', includes: "hadoop-${HADOOP2_VERSION}-bin.tar.gz"
+ }
+ }
+ stage ('hadoop 3 cache') {
+ environment {
+ HADOOP3_VERSION="3.0.0"
+ }
+ steps {
+ // directory must be unique for each parallel stage, because jenkins runs them in the same workspace :(
+ dir('downloads-hadoop-3') {
+ sh '''#!/usr/bin/env bash
+ echo "Make sure we have a directory for downloading dependencies: $(pwd)"
+'''
+ }
+ sh '''#!/usr/bin/env bash
+ set -e
+ echo "Ensure we have a copy of Hadoop ${HADOOP3_VERSION}"
+ "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \
+ --working-dir "${WORKSPACE}/downloads-hadoop-3" \
+ --keys 'http://www.apache.org/dist/hadoop/common/KEYS' \
+ "${WORKSPACE}/hadoop-${HADOOP3_VERSION}-bin.tar.gz" \
+ "hadoop/common/hadoop-${HADOOP3_VERSION}/hadoop-${HADOOP3_VERSION}.tar.gz"
+ '''
+ stash name: 'hadoop-3', includes: "hadoop-${HADOOP3_VERSION}-bin.tar.gz"
+ }
}
- stash name: 'yetus', includes: "yetus-*/*,yetus-*/**/*,tools/personality.sh"
}
}
stage ('init health results') {
@@ -149,6 +204,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
echo "-- Something went wrong running this stage, please [check relevant console output|${BUILD_URL}/console]." >> "${OUTPUT_DIR}/commentfile"
'''
unstash 'yetus'
+ // since we have a new node definition we need to re-do the scm checkout
dir('component') {
checkout scm
}
@@ -441,7 +497,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
// This is meant to mimic what a release manager will do to create RCs.
// See http://hbase.apache.org/book.html#maven.release
- stage ('create source tarball') {
+ stage ('packaging and integration') {
tools {
maven 'Maven (latest)'
// this needs to be set to the jdk that ought to be used to build releases on the branch the Jenkinsfile is stored in.
@@ -454,14 +510,17 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
sh '''#!/bin/bash -e
echo "Setting up directories"
rm -rf "output-srctarball" && mkdir "output-srctarball"
+ rm -rf "output-integration" && mkdir "output-integration" "output-integration/hadoop-2" "output-integration/hadoop-3" "output-integration/hadoop-3-shaded"
rm -rf "unpacked_src_tarball" && mkdir "unpacked_src_tarball"
+ rm -rf "hbase-install" && mkdir "hbase-install"
+ rm -rf "hbase-client" && mkdir "hbase-client"
+ rm -rf "hadoop-2" && mkdir "hadoop-2"
+ rm -rf "hadoop-3" && mkdir "hadoop-3"
rm -rf ".m2-for-repo" && mkdir ".m2-for-repo"
rm -rf ".m2-for-src" && mkdir ".m2-for-src"
- echo '(x) {color:red}-1 source release artifact{color}\n-- Something went wrong with this stage, [check relevant console output|${BUILD_URL}/console].' >output-srctarball/commentfile
+ echo "(x) {color:red}-1 source release artifact{color}\n-- Something went wrong with this stage, [check relevant console output|${BUILD_URL}/console]." >output-srctarball/commentfile
+ echo "(x) {color:red}-1 client integration test{color}\n-- Something went wrong with this stage, [check relevant console output|${BUILD_URL}/console]." >output-integration/commentfile
'''
- dir('component') {
- checkout scm
- }
sh '''#!/usr/bin/env bash
set -e
rm -rf "output-srctarball/machine" && mkdir "output-srctarball/machine"
@@ -470,6 +529,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
ls -lh "output-srctarball/machine"
'''
sh """#!/bin/bash -e
+ echo "Checking the steps for an RM to make a source artifact, then a binary artifact."
if "${env.BASEDIR}/dev-support/hbase_nightly_source-artifact.sh" \
--intermediate-file-dir output-srctarball \
--unpack-temp-dir unpacked_src_tarball \
@@ -483,12 +543,78 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
exit 1
fi
"""
+ echo "unpacking the hbase bin tarball into 'hbase-install' and the client tarball into 'hbase-client'"
+ sh '''#!/bin/bash -e
+ if [ 2 -ne $(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | wc -l) ]; then
+ echo '(x) {color:red}-1 testing binary artifact{color}\n-- source tarball did not produce the expected binaries.' >>output-srctarball/commentfile
+ exit 1
+ fi
+ install_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | sort | head -n 1)
+ tar --strip-component=1 -xzf "${install_artifact}" -C "hbase-install"
+ client_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | sort | tail -n 1)
+ tar --strip-component=1 -xzf "${client_artifact}" -C "hbase-client"
+'''
+ unstash 'hadoop-2'
+ echo "Attempting to use run an instance on top of Hadoop 2."
+ sh '''#!/bin/bash -xe
+ artifact=$(ls -1 "${WORKSPACE}"/hadoop-2*.tar.gz | head -n 1)
+ tar --strip-components=1 -xzf "${artifact}" -C "hadoop-2"
+ if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
+ --single-process \
+ --working-dir output-integration/hadoop-2 \
+ --hbase-client-install "hbase-client" \
+ "hbase-install" \
+ "hadoop-2/bin/hadoop" \
+ hadoop-2/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
+ hadoop-2/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \
+ >output-integration/hadoop-2.log 2>&1 ; then
+ echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 2. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-2.log]. (note that this means we didn't run on Hadoop 3)" >output-integration/commentfile
+ exit 2
+ fi
+'''
+ unstash 'hadoop-3'
+ echo "Attempting to use run an instance on top of Hadoop 3."
+ sh '''#!/bin/bash -e
+ artifact=$(ls -1 "${WORKSPACE}"/hadoop-3*.tar.gz | head -n 1)
+ tar --strip-components=1 -xzf "${artifact}" -C "hadoop-3"
+ if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
+ --single-process \
+ --working-dir output-integration/hadoop-3 \
+ --hbase-client-install hbase-client \
+ hbase-install \
+ hadoop-3/bin/hadoop \
+ hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
+ hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \
+ >output-integration/hadoop-3.log 2>&1 ; then
+ echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3.log]. (note that this means we didn't check the Hadoop 3 shaded client)" >output-integration/commentfile
+ exit 2
+ fi
+ echo "Attempting to use run an instance on top of Hadoop 3, relying on the Hadoop client artifacts for the example client program."
+ if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
+ --single-process \
+ --hadoop-client-classpath hadoop-3/share/hadoop/client/hadoop-client-api-*.jar:hadoop-3/share/hadoop/client/hadoop-client-runtime-*.jar \
+ --working-dir output-integration/hadoop-3-shaded \
+ --hbase-client-install hbase-client \
+ hbase-install \
+ hadoop-3/bin/hadoop \
+ hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
+ hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \
+ >output-integration/hadoop-3-shaded.log 2>&1 ; then
+ echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3 using Hadoop's shaded client. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3-shaded.log]." >output-integration/commentfile
+ exit 2
+ fi
+ echo "(/) {color:green}+1 client integration test{color}" >output-integration/commentfile
+'''
+
+
}
post {
always {
- stash name: 'srctarball-result', includes: "output-srctarball/commentfile"
+ stash name: 'srctarball-result', includes: "output-srctarball/commentfile,output-integration/commentfile"
archive 'output-srctarball/*'
archive 'output-srctarball/**/*'
+ archive 'output-integration/*'
+ archive 'output-integration/**/*'
}
}
}
@@ -509,7 +635,8 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
"${env.OUTPUT_DIR_RELATIVE_JDK7}/commentfile",
"${env.OUTPUT_DIR_RELATIVE_HADOOP2}/commentfile",
"${env.OUTPUT_DIR_RELATIVE_HADOOP3}/commentfile",
- 'output-srctarball/commentfile']
+ 'output-srctarball/commentfile',
+ 'output-integration/commentfile']
echo env.BRANCH_NAME
echo env.BUILD_URL
echo currentBuild.result
http://git-wip-us.apache.org/repos/asf/hbase/blob/e6f89414/dev-support/hbase_nightly_pseudo-distributed-test.sh
----------------------------------------------------------------------
diff --git a/dev-support/hbase_nightly_pseudo-distributed-test.sh b/dev-support/hbase_nightly_pseudo-distributed-test.sh
new file mode 100755
index 0000000..cc2dd5e
--- /dev/null
+++ b/dev-support/hbase_nightly_pseudo-distributed-test.sh
@@ -0,0 +1,516 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+function usage {
+ echo "Usage: ${0} [options] /path/to/component/bin-install /path/to/hadoop/executable /path/to/hadoop/hadoop-yarn-server-tests-tests.jar /path/to/hadoop/hadoop-mapreduce-client-jobclient-tests.jar"
+ echo ""
+ echo " --zookeeper-data /path/to/use Where the embedded zookeeper instance should write its data."
+ echo " defaults to 'zk-data' in the working-dir."
+ echo " --working-dir /path/to/use Path for writing configs and logs. must exist."
+ echo " defaults to making a directory via mktemp."
+ echo " --hadoop-client-classpath /path/to/some.jar:/path/to/another.jar classpath for hadoop jars."
+ echo " defaults to 'hadoop classpath'"
+ echo " --hbase-client-install /path/to/unpacked/client/tarball if given we'll look here for hbase client jars instead of the bin-install"
+ echo " --force-data-clean Delete all data in HDFS and ZK prior to starting up hbase"
+ echo " --single-process Run as single process instead of pseudo-distributed"
+ echo ""
+ exit 1
+}
+# if no args specified, show usage
+if [ $# -lt 4 ]; then
+ usage
+fi
+
+# Get arguments
+declare component_install
+declare hadoop_exec
+declare working_dir
+declare zk_data_dir
+declare clean
+declare distributed="true"
+declare hadoop_jars
+declare hbase_client
+while [ $# -gt 0 ]
+do
+ case "$1" in
+ --working-dir) shift; working_dir=$1; shift;;
+ --force-data-clean) shift; clean="true";;
+ --zookeeper-data) shift; zk_data_dir=$1; shift;;
+ --single-process) shift; distributed="false";;
+ --hadoop-client-classpath) shift; hadoop_jars="$1"; shift;;
+ --hbase-client-install) shift; hbase_client="$1"; shift;;
+ --) shift; break;;
+ -*) usage ;;
+ *) break;; # terminate while loop
+ esac
+done
+
+# should still have where component checkout is.
+if [ $# -lt 4 ]; then
+ usage
+fi
+component_install="$(cd "$(dirname "$1")"; pwd)/$(basename "$1")"
+hadoop_exec="$(cd "$(dirname "$2")"; pwd)/$(basename "$2")"
+yarn_server_tests_test_jar="$(cd "$(dirname "$3")"; pwd)/$(basename "$3")"
+mapred_jobclient_test_jar="$(cd "$(dirname "$4")"; pwd)/$(basename "$4")"
+
+if [ ! -x "${hadoop_exec}" ]; then
+ echo "hadoop cli does not appear to be executable." >&2
+ exit 1
+fi
+
+if [ ! -d "${component_install}" ]; then
+ echo "Path to HBase binary install should be a directory." >&2
+ exit 1
+fi
+
+if [ ! -f "${yarn_server_tests_test_jar}" ]; then
+ echo "Specified YARN server tests test jar is not a file." >&2
+ exit 1
+fi
+
+if [ ! -f "${mapred_jobclient_test_jar}" ]; then
+ echo "Specified MapReduce jobclient test jar is not a file." >&2
+ exit 1
+fi
+
+if [ -z "${working_dir}" ]; then
+ if ! working_dir="$(mktemp -d -t hbase-pseudo-dist-test)" ; then
+ echo "Failed to create temporary working directory. Please specify via --working-dir" >&2
+ exit 1
+ fi
+else
+ # absolutes please
+ working_dir="$(cd "$(dirname "${working_dir}")"; pwd)/$(basename "${working_dir}")"
+ if [ ! -d "${working_dir}" ]; then
+ echo "passed working directory '${working_dir}' must already exist." >&2
+ exit 1
+ fi
+fi
+
+if [ -z "${zk_data_dir}" ]; then
+ zk_data_dir="${working_dir}/zk-data"
+ mkdir "${zk_data_dir}"
+else
+ # absolutes please
+ zk_data_dir="$(cd "$(dirname "${zk_data_dir}")"; pwd)/$(basename "${zk_data_dir}")"
+ if [ ! -d "${zk_data_dir}" ]; then
+ echo "passed directory for unpacking the source tarball '${zk_data_dir}' must already exist."
+ exit 1
+ fi
+fi
+
+if [ -z "${hbase_client}" ]; then
+ hbase_client="${component_install}"
+else
+ echo "Using HBase client-side artifact"
+ # absolutes please
+ hbase_client="$(cd "$(dirname "${hbase_client}")"; pwd)/$(basename "${hbase_client}")"
+ if [ ! -d "${hbase_client}" ]; then
+ echo "If given hbase client install should be a directory with contents of the client tarball." >&2
+ exit 1
+ fi
+fi
+
+if [ -n "${hadoop_jars}" ]; then
+ declare -a tmp_jars
+ for entry in $(echo "${hadoop_jars}" | tr ':' '\n'); do
+ tmp_jars=("${tmp_jars[@]}" "$(cd "$(dirname "${entry}")"; pwd)/$(basename "${entry}")")
+ done
+ hadoop_jars="$(IFS=:; echo "${tmp_jars[*]}")"
+fi
+
+
+echo "You'll find logs and temp files in ${working_dir}"
+
+function redirect_and_run {
+ log_base=$1
+ shift
+ echo "$*" >"${log_base}.err"
+ "$@" >"${log_base}.out" 2>>"${log_base}.err"
+}
+
+(cd "${working_dir}"
+
+echo "Hadoop version information:"
+"${hadoop_exec}" version
+hadoop_version=$("${hadoop_exec}" version | head -n 1)
+hadoop_version="${hadoop_version#Hadoop }"
+if [ "${hadoop_version%.*.*}" -gt 2 ]; then
+ "${hadoop_exec}" envvars
+else
+ echo "JAVA_HOME: ${JAVA_HOME}"
+fi
+
+# Ensure that if some other Hadoop install happens to be present in the environment we ignore it.
+HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP="true"
+export HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP
+
+if [ -n "${clean}" ]; then
+ echo "Cleaning out ZooKeeper..."
+ rm -rf "${zk_data_dir:?}/*"
+fi
+
+echo "HBase version information:"
+"${component_install}/bin/hbase" version 2>/dev/null
+hbase_version=$("${component_install}/bin/hbase" version | head -n 1 2>/dev/null)
+hbase_version="${hbase_version#HBase }"
+
+if [ ! -s "${hbase_client}/lib/shaded-clients/hbase-shaded-mapreduce-${hbase_version}.jar" ]; then
+ echo "HBase binary install doesn't appear to include a shaded mapreduce artifact." >&2
+ exit 1
+fi
+
+if [ ! -s "${hbase_client}/lib/shaded-clients/hbase-shaded-client-${hbase_version}.jar" ]; then
+ echo "HBase binary install doesn't appear to include a shaded client artifact." >&2
+ exit 1
+fi
+
+if [ ! -s "${hbase_client}/lib/shaded-clients/hbase-shaded-client-byo-hadoop-${hbase_version}.jar" ]; then
+ echo "HBase binary install doesn't appear to include a shaded client artifact." >&2
+ exit 1
+fi
+
+echo "Writing out configuration for HBase."
+rm -rf "${working_dir}/hbase-conf"
+mkdir "${working_dir}/hbase-conf"
+
+if [ -f "${component_install}/conf/log4j.properties" ]; then
+ cp "${component_install}/conf/log4j.properties" "${working_dir}/hbase-conf/log4j.properties"
+else
+ cat >"${working_dir}/hbase-conf/log4j.properties" <<EOF
+# Define some default values that can be overridden by system properties
+hbase.root.logger=INFO,console
+
+# Define the root logger to the system property "hbase.root.logger".
+log4j.rootLogger=${hbase.root.logger}
+
+# Logging Threshold
+log4j.threshold=ALL
+# console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %.1000m%n
+EOF
+fi
+
+cat >"${working_dir}/hbase-conf/hbase-site.xml" <<EOF
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>hbase.rootdir</name>
+ <!-- We rely on the defaultFS being set in our hadoop confs -->
+ <value>/hbase</value>
+ </property>
+ <property>
+ <name>hbase.zookeeper.property.dataDir</name>
+ <value>${zk_data_dir}</value>
+ </property>
+ <property>
+ <name>hbase.cluster.distributed</name>
+ <value>${distributed}</value>
+ </property>
+</configuration>
+EOF
+
+if [ "true" = "${distributed}" ]; then
+ cat >"${working_dir}/hbase-conf/regionservers" <<EOF
+localhost
+EOF
+fi
+
+function cleanup {
+
+ echo "Shutting down HBase"
+ HBASE_CONF_DIR="${working_dir}/hbase-conf/" "${component_install}/bin/stop-hbase.sh"
+
+ if [ -f "${working_dir}/hadoop.pid" ]; then
+ echo "Shutdown: listing HDFS contents"
+ redirect_and_run "${working_dir}/hadoop_listing_at_end" \
+ "${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -ls -R /
+
+ echo "Shutting down Hadoop"
+ kill -6 "$(cat "${working_dir}/hadoop.pid")"
+ fi
+}
+
+trap cleanup EXIT SIGQUIT
+
+echo "Starting up Hadoop"
+
+HADOOP_CLASSPATH="${yarn_server_tests_test_jar}" "${hadoop_exec}" jar "${mapred_jobclient_test_jar}" minicluster -writeConfig "${working_dir}/hbase-conf/core-site.xml" -writeDetails "${working_dir}/hadoop_cluster_info.json" >"${working_dir}/hadoop_cluster_command.out" 2>"${working_dir}/hadoop_cluster_command.err" &
+echo "$!" > "${working_dir}/hadoop.pid"
+
+sleep_time=2
+until [ -s "${working_dir}/hbase-conf/core-site.xml" ]; do
+ printf '\twaiting for Hadoop to finish starting up.\n'
+ sleep "${sleep_time}"
+ sleep_time="$((sleep_time*2))"
+done
+
+if [ "${hadoop_version%.*.*}" -gt 2 ]; then
+ echo "Verifying configs"
+ "${hadoop_exec}" --config "${working_dir}/hbase-conf/" conftest
+fi
+
+if [ -n "${clean}" ]; then
+ echo "Cleaning out HDFS..."
+ "${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -rm -r /hbase
+ "${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -rm -r example/
+ "${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -rm -r example-region-listing.data
+fi
+
+echo "Listing HDFS contents"
+redirect_and_run "${working_dir}/hadoop_cluster_smoke" \
+ "${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -ls -R /
+
+echo "Starting up HBase"
+HBASE_CONF_DIR="${working_dir}/hbase-conf/" "${component_install}/bin/start-hbase.sh"
+
+sleep_time=2
+until "${component_install}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive >"${working_dir}/waiting_hbase_startup.log" 2>&1 <<EOF
+ count 'hbase:meta'
+EOF
+do
+ printf '\tretry waiting for hbase to come up.\n'
+ sleep "${sleep_time}"
+ sleep_time="$((sleep_time*2))"
+done
+
+echo "Setting up table 'test:example' with 1,000 regions"
+"${hbase_client}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive >"${working_dir}/table_create.log" 2>&1 <<EOF
+ create_namespace 'test'
+ create 'test:example', 'family1', 'family2', {NUMREGIONS => 1000, SPLITALGO => 'UniformSplit'}
+EOF
+
+echo "writing out example TSV to example.tsv"
+cat >"${working_dir}/example.tsv" <<EOF
+row1 value8 value8
+row3 value2
+row2 value9
+row10 value1
+pow1 value8 value8
+pow3 value2
+pow2 value9
+pow10 value1
+paw1 value8 value8
+paw3 value2
+paw2 value9
+paw10 value1
+raw1 value8 value8
+raw3 value2
+raw2 value9
+raw10 value1
+aow1 value8 value8
+aow3 value2
+aow2 value9
+aow10 value1
+aaw1 value8 value8
+aaw3 value2
+aaw2 value9
+aaw10 value1
+how1 value8 value8
+how3 value2
+how2 value9
+how10 value1
+zow1 value8 value8
+zow3 value2
+zow2 value9
+zow10 value1
+zaw1 value8 value8
+zaw3 value2
+zaw2 value9
+zaw10 value1
+haw1 value8 value8
+haw3 value2
+haw2 value9
+haw10 value1
+low1 value8 value8
+low3 value2
+low2 value9
+low10 value1
+law1 value8 value8
+law3 value2
+law2 value9
+law10 value1
+EOF
+
+echo "uploading example.tsv to HDFS"
+"${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -mkdir example
+"${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -copyFromLocal "${working_dir}/example.tsv" "example/"
+
+echo "Importing TSV via shaded client artifact for HBase - MapReduce integration."
+# hbase_thirdparty_jars=("${component_install}"/lib/htrace-core4*.jar \
+# "${component_install}"/lib/slf4j-api-*.jar \
+# "${component_install}"/lib/commons-logging-*.jar \
+# "${component_install}"/lib/slf4j-log4j12-*.jar \
+# "${component_install}"/lib/log4j-1.2.*.jar \
+# "${working_dir}/hbase-conf/log4j.properties")
+# hbase_dep_classpath=$(IFS=:; echo "${hbase_thirdparty_jars[*]}")
+hbase_dep_classpath="$("${hbase_client}/bin/hbase" --config "${working_dir}/hbase-conf/" mapredcp)"
+HADOOP_CLASSPATH="${hbase_dep_classpath}" redirect_and_run "${working_dir}/mr-importtsv" \
+ "${hadoop_exec}" --config "${working_dir}/hbase-conf/" jar "${hbase_client}/lib/shaded-clients/hbase-shaded-mapreduce-${hbase_version}.jar" importtsv -Dimporttsv.columns=HBASE_ROW_KEY,family1:column1,family1:column4,family1:column3 test:example example/ -libjars "${hbase_dep_classpath}"
+"${hbase_client}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive >"${working_dir}/scan_import.out" 2>"${working_dir}/scan_import.err" <<EOF
+ scan 'test:example'
+EOF
+
+echo "Verifying row count from import."
+import_rowcount=$(echo 'count "test:example"' | "${hbase_client}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive 2>/dev/null | tail -n 1)
+if [ ! "${import_rowcount}" -eq 48 ]; then
+ echo "ERROR: Instead of finding 48 rows, we found ${import_rowcount}."
+ exit 2
+fi
+
+if [ -z "${hadoop_jars}" ]; then
+ echo "Hadoop client jars not given; getting them from 'hadoop classpath' for the example."
+ hadoop_jars=$("${hadoop_exec}" --config "${working_dir}/hbase-conf/" classpath)
+fi
+
+echo "Building shaded client example."
+cat >"${working_dir}/HBaseClientReadWriteExample.java" <<EOF
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellBuilder;
+import org.apache.hadoop.hbase.CellBuilderFactory;
+import org.apache.hadoop.hbase.CellBuilderType;
+import org.apache.hadoop.hbase.ClusterMetrics;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.RegionMetrics;
+import org.apache.hadoop.hbase.ServerMetrics;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import java.util.LinkedList;
+import java.util.List;
+
+
+public class HBaseClientReadWriteExample {
+ private static final byte[] FAMILY_BYTES = Bytes.toBytes("family2");
+
+ public static void main(String[] args) throws Exception {
+ Configuration hbase = HBaseConfiguration.create();
+ Configuration hadoop = new Configuration();
+ try (Connection connection = ConnectionFactory.createConnection(hbase)) {
+ System.out.println("Generating list of regions");
+ final List<String> regions = new LinkedList<>();
+ try (Admin admin = connection.getAdmin()) {
+ final ClusterMetrics cluster = admin.getClusterMetrics();
+ System.out.println(String.format("\tCluster reports version %s, ave load %f, region count %d", cluster.getHBaseVersion(), cluster.getAverageLoad(), cluster.getRegionCount()));
+ for (ServerMetrics server : cluster.getLiveServerMetrics().values()) {
+ for (RegionMetrics region : server.getRegionMetrics().values()) {
+ regions.add(region.getNameAsString());
+ }
+ }
+ }
+ final Path listing = new Path("example-region-listing.data");
+ System.out.println("Writing list to HDFS");
+ try (FileSystem fs = FileSystem.newInstance(hadoop)) {
+ final Path path = fs.makeQualified(listing);
+ try (FSDataOutputStream out = fs.create(path)) {
+ out.writeInt(regions.size());
+ for (String region : regions) {
+ out.writeUTF(region);
+ }
+ out.hsync();
+ }
+ }
+ final List<Put> puts = new LinkedList<>();
+ final Put marker = new Put(new byte[] { (byte)0 });
+ System.out.println("Reading list from HDFS");
+ try (FileSystem fs = FileSystem.newInstance(hadoop)) {
+ final Path path = fs.makeQualified(listing);
+ final CellBuilder builder = CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY);
+ try (FSDataInputStream in = fs.open(path)) {
+ final int count = in.readInt();
+ marker.addColumn(FAMILY_BYTES, Bytes.toBytes("count"), Bytes.toBytes(count));
+ for(int i = 0; i < count; i++) {
+ builder.clear();
+ final byte[] row = Bytes.toBytes(in.readUTF());
+ final Put put = new Put(row);
+ builder.setRow(row);
+ builder.setFamily(FAMILY_BYTES);
+ builder.setType(Cell.Type.Put);
+ put.add(builder.build());
+ puts.add(put);
+ }
+ }
+ }
+ System.out.println("Writing list into HBase table");
+ try (Table table = connection.getTable(TableName.valueOf("test:example"))) {
+ table.put(marker);
+ table.put(puts);
+ }
+ }
+ }
+}
+EOF
+redirect_and_run "${working_dir}/hbase-shaded-client-compile" \
+ javac -cp "${hbase_client}/lib/shaded-clients/hbase-shaded-client-byo-hadoop-${hbase_version}.jar:${hadoop_jars}" "${working_dir}/HBaseClientReadWriteExample.java"
+echo "Running shaded client example. It'll fetch the set of regions, round-trip them to a file in HDFS, then write them one-per-row into the test table."
+# The order of classpath entries here is important. if we're using non-shaded Hadoop 3 / 2.9.0 jars, we have to work around YARN-2190.
+redirect_and_run "${working_dir}/hbase-shaded-client-example" \
+ java -cp "${working_dir}/hbase-conf/:${hbase_client}/lib/shaded-clients/hbase-shaded-client-byo-hadoop-${hbase_version}.jar:${hbase_dep_classpath}:${working_dir}:${hadoop_jars}" HBaseClientReadWriteExample
+
+echo "Checking on results of example program."
+"${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -copyToLocal "example-region-listing.data" "${working_dir}/example-region-listing.data"
+
+"${hbase_client}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive >"${working_dir}/scan_example.out" 2>"${working_dir}/scan_example.err" <<EOF
+ scan 'test:example'
+EOF
+
+echo "Verifying row count from example."
+example_rowcount=$(echo 'count "test:example"' | "${hbase_client}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive 2>/dev/null | tail -n 1)
+if [ "${example_rowcount}" -gt "1050" ]; then
+ echo "Found ${example_rowcount} rows, which is enough to cover 48 for import, 1000 example's use of user table regions, 2 for example's use of meta/root regions, and 1 for example's count record"
+else
+ echo "ERROR: Only found ${example_rowcount} rows."
+fi
+
+)
http://git-wip-us.apache.org/repos/asf/hbase/blob/e6f89414/dev-support/hbase_nightly_source-artifact.sh
----------------------------------------------------------------------
diff --git a/dev-support/hbase_nightly_source-artifact.sh b/dev-support/hbase_nightly_source-artifact.sh
index 9458324..c435c48 100755
--- a/dev-support/hbase_nightly_source-artifact.sh
+++ b/dev-support/hbase_nightly_source-artifact.sh
@@ -174,9 +174,13 @@ cd "${unpack_dir}"
echo "Follow the ref guide section on making a RC: Step 8 Build the binary tarball."
if mvn -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${m2_tarbuild}" clean install \
assembly:single >"${working_dir}/srctarball_install.log" 2>&1; then
- echo "Building a binary tarball from the source tarball succeeded."
-else
- echo "Building a binary tarball from the source tarball failed. see srtarball_install.log for details."
- exit 1
+ for artifact in "${unpack_dir}"/hbase-assembly/target/hbase-*-bin.tar.gz; do
+ if [ -f "${artifact}" ]; then
+ # TODO check the layout of the binary artifact we just made.
+ echo "Building a binary tarball from the source tarball succeeded."
+ exit 0
+ fi
+ done
fi
-# TODO check the layout of the binary artifact we just made.
+echo "Building a binary tarball from the source tarball failed. see srtarball_install.log for details."
+exit 1
http://git-wip-us.apache.org/repos/asf/hbase/blob/e6f89414/dev-support/jenkins-scripts/cache-apache-project-artifact.sh
----------------------------------------------------------------------
diff --git a/dev-support/jenkins-scripts/cache-apache-project-artifact.sh b/dev-support/jenkins-scripts/cache-apache-project-artifact.sh
new file mode 100755
index 0000000..57853c3
--- /dev/null
+++ b/dev-support/jenkins-scripts/cache-apache-project-artifact.sh
@@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+function usage {
+ echo "Usage: ${0} [options] /path/to/download/file.tar.gz download/fragment/eg/project/subdir/some-artifact-version.tar.gz"
+ echo ""
+ echo " --force for a redownload even if /path/to/download/file.tar.gz exists."
+ echo " --working-dir /path/to/use Path for writing tempfiles. must exist."
+ echo " defaults to making a directory via mktemp that we clean."
+ echo " --keys url://to/project/KEYS where to get KEYS. needed to check signature on download."
+ echo ""
+ exit 1
+}
+# if no args specified, show usage
+if [ $# -lt 2 ]; then
+ usage
+fi
+
+
+# Get arguments
+declare done_if_cached="true"
+declare working_dir
+declare cleanup="true"
+declare keys
+while [ $# -gt 0 ]
+do
+ case "$1" in
+ --force) shift; done_if_cached="false";;
+ --working-dir) shift; working_dir=$1; cleanup="false"; shift;;
+ --keys) shift; keys=$1; shift;;
+ --) shift; break;;
+ -*) usage ;;
+ *) break;; # terminate while loop
+ esac
+done
+
+# should still have required args
+if [ $# -lt 2 ]; then
+ usage
+fi
+
+target="$1"
+artifact="$2"
+
+if [ -f "${target}" ] && [ "true" = "${done_if_cached}" ]; then
+ echo "Reusing existing download of '${artifact}'."
+ exit 0
+fi
+
+if [ -z "${working_dir}" ]; then
+ if ! working_dir="$(mktemp -d -t hbase-download-apache-artifact)" ; then
+ echo "Failed to create temporary working directory. Please specify via --working-dir" >&2
+ exit 1
+ fi
+else
+ # absolutes please
+ working_dir="$(cd "$(dirname "${working_dir}")"; pwd)/$(basename "${working_dir}")"
+ if [ ! -d "${working_dir}" ]; then
+ echo "passed working directory '${working_dir}' must already exist." >&2
+ exit 1
+ fi
+fi
+
+function cleanup {
+ if [ "true" = "${cleanup}" ]; then
+ echo "cleaning up temp space."
+ rm -rf "${working_dir}"
+ fi
+}
+trap cleanup EXIT SIGQUIT
+
+echo "New download of '${artifact}'"
+
+# N.B. this comes first so that if gpg falls over we skip the expensive download.
+if [ -n "${keys}" ]; then
+ if [ ! -d "${working_dir}/.gpg" ]; then
+ rm -rf "${working_dir}/.gpg"
+ mkdir -p "${working_dir}/.gpg"
+ chmod -R 700 "${working_dir}/.gpg"
+ fi
+
+ echo "installing project KEYS"
+ curl -L --fail -o "${working_dir}/KEYS" "${keys}"
+ if ! gpg --homedir "${working_dir}/.gpg" --import "${working_dir}/KEYS" ; then
+ echo "ERROR importing the keys via gpg failed. If the output above mentions this error:" >&2
+ echo " gpg: can't connect to the agent: File name too long" >&2
+ # we mean to give them the command to run, not to run it.
+ #shellcheck disable=SC2016
+ echo 'then you prolly need to create /var/run/user/$(id -u)' >&2
+ echo "see this thread on gnupg-users: https://s.apache.org/uI7x" >&2
+ exit 2
+ fi
+
+ echo "downloading signature"
+ curl -L --fail -o "${working_dir}/artifact.asc" "https://archive.apache.org/dist/${artifact}.asc"
+fi
+
+echo "downloading artifact"
+if ! curl --dump-header "${working_dir}/artifact_download_headers.txt" -L --fail -o "${working_dir}/artifact" "https://www.apache.org/dyn/closer.lua?filename=${artifact}&action=download" ; then
+ echo "Artifact wasn't in mirror system. falling back to archive.a.o."
+ curl --dump-header "${working_dir}/artifact_fallback_headers.txt" -L --fail -o "${working_dir}/artifact" "http://archive.apache.org/dist/${artifact}"
+fi
+
+if [ -n "${keys}" ]; then
+ echo "verifying artifact signature"
+ gpg --homedir "${working_dir}/.gpg" --verify "${working_dir}/artifact.asc"
+ echo "signature good."
+fi
+
+echo "moving artifact into place at '${target}'"
+# ensure we're on the same filesystem
+mv "${working_dir}/artifact" "${target}.copying"
+# attempt atomic move
+mv "${target}.copying" "${target}"
+echo "all done!"