You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2017/04/04 16:01:25 UTC
[1/3] beam git commit: [BEAM-1550] Enable parallel test execution for
HBaseIO
Repository: beam
Updated Branches:
refs/heads/master 45f63eb6c -> 498ce9f40
[BEAM-1550] Enable parallel test execution for HBaseIO
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/2b939638
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/2b939638
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/2b939638
Branch: refs/heads/master
Commit: 2b939638a894bfafd2a10d35dee7de65fa6d1021
Parents: 6d502fa
Author: Isma�l Mej�a <ie...@apache.org>
Authored: Fri Mar 31 12:08:22 2017 +0200
Committer: Dan Halperin <dh...@google.com>
Committed: Tue Apr 4 09:01:14 2017 -0700
----------------------------------------------------------------------
sdks/java/io/hbase/pom.xml | 10 ++++++++
.../src/test/resources/log4j-test.properties | 27 ++++++++++++++++++++
2 files changed, 37 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/2b939638/sdks/java/io/hbase/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/hbase/pom.xml b/sdks/java/io/hbase/pom.xml
index 7b5fc6b..612742b 100644
--- a/sdks/java/io/hbase/pom.xml
+++ b/sdks/java/io/hbase/pom.xml
@@ -40,6 +40,16 @@
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <parallel>all</parallel>
+ <threadCount>4</threadCount>
+ <!-- Need to redefine log configuration because it gets lost on parallel tests -->
+ <argLine>-Dlog4j.configuration=log4j-test.properties -XX:-UseGCOverheadLimit ${beamSurefireArgline}</argLine>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
http://git-wip-us.apache.org/repos/asf/beam/blob/2b939638/sdks/java/io/hbase/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/sdks/java/io/hbase/src/test/resources/log4j-test.properties b/sdks/java/io/hbase/src/test/resources/log4j-test.properties
new file mode 100644
index 0000000..4c74d85
--- /dev/null
+++ b/sdks/java/io/hbase/src/test/resources/log4j-test.properties
@@ -0,0 +1,27 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# Set root logger level to OFF to not flood build logs
+# set manually to INFO for debugging purposes
+log4j.rootLogger=OFF, testlogger
+
+# A1 is set to be a ConsoleAppender.
+log4j.appender.testlogger=org.apache.log4j.ConsoleAppender
+log4j.appender.testlogger.target = System.err
+log4j.appender.testlogger.layout=org.apache.log4j.PatternLayout
+log4j.appender.testlogger.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
[3/3] beam git commit: This closes #2370
Posted by dh...@apache.org.
This closes #2370
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/498ce9f4
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/498ce9f4
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/498ce9f4
Branch: refs/heads/master
Commit: 498ce9f40e984d145177675d22e7537cab1d8049
Parents: 45f63eb 2b93963
Author: Dan Halperin <dh...@google.com>
Authored: Tue Apr 4 09:01:17 2017 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Tue Apr 4 09:01:17 2017 -0700
----------------------------------------------------------------------
sdks/java/io/hbase/pom.xml | 83 +++++++-------------
.../apache/beam/sdk/io/hbase/HBaseIOTest.java | 2 -
.../src/test/resources/log4j-test.properties | 27 +++++++
3 files changed, 55 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
[2/3] beam git commit: [BEAM-1550] Fix HBaseIO tests and change
HBaseIO to use shaded dependencies
Posted by dh...@apache.org.
[BEAM-1550] Fix HBaseIO tests and change HBaseIO to use shaded dependencies
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/6d502faa
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/6d502faa
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/6d502faa
Branch: refs/heads/master
Commit: 6d502faa318f126552151693dacb6a569b535fba
Parents: 45f63eb
Author: Isma�l Mej�a <ie...@apache.org>
Authored: Thu Mar 30 11:04:51 2017 +0200
Committer: Dan Halperin <dh...@google.com>
Committed: Tue Apr 4 09:01:14 2017 -0700
----------------------------------------------------------------------
sdks/java/io/hbase/pom.xml | 73 +++++---------------
.../apache/beam/sdk/io/hbase/HBaseIOTest.java | 2 -
2 files changed, 18 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/6d502faa/sdks/java/io/hbase/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/hbase/pom.xml b/sdks/java/io/hbase/pom.xml
index 29671a2..7b5fc6b 100644
--- a/sdks/java/io/hbase/pom.xml
+++ b/sdks/java/io/hbase/pom.xml
@@ -32,9 +32,7 @@
<properties>
<hbase.version>1.3.0</hbase.version>
- <hbase.guava.version>12.0.1</hbase.guava.version>
- <hbase.protobuf.version>2.5.0</hbase.protobuf.version>
- <hadoop.version>2.5.1</hadoop.version>
+ <hbase.hadoop.version>2.5.1</hbase.hadoop.version>
</properties>
<build>
@@ -51,30 +49,9 @@
<goal>shade</goal>
</goals>
<configuration>
+ <!-- Disable the creation of the reduced pom to avoid an issue with the shade
+ plugin. Ref. https://issues.apache.org/jira/browse/MSHADE-148 -->
<createDependencyReducedPom>false</createDependencyReducedPom>
- <artifactSet>
- <includes>
- <include>com.google.guava:guava</include>
- <include>com.google.protobuf:protobuf-java</include>
- </includes>
- </artifactSet>
- <relocations>
- <relocation>
- <pattern>com.google.common</pattern>
- <shadedPattern>org.apache.beam.sdk.io.hbase.repackaged.com.google.common</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.google.thirdparty</pattern>
- <shadedPattern>org.apache.beam.sdk.io.hbase.repackaged.com.google.thirdparty</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.google.protobuf</pattern>
- <shadedPattern>org.apache.beam.sdk.io.hbase.repackaged.com.google.protobuf</shadedPattern>
- </relocation>
- </relocations>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
- </transformers>
</configuration>
</execution>
</executions>
@@ -113,39 +90,19 @@
</dependency>
<dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-shaded-client</artifactId>
+ <version>${hbase.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>${hbase.guava.version}</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- <scope>provided</scope>
- <version>${hbase.protobuf.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <version>${hbase.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-protocol</artifactId>
- <version>${hbase.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-common</artifactId>
- <version>${hbase.version}</version>
+ <artifactId>protobuf-lite</artifactId>
</dependency>
<dependency>
@@ -154,15 +111,14 @@
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
</dependency>
<!-- test dependencies -->
<dependency>
<groupId>org.apache.hbase</groupId>
- <artifactId>hbase-server</artifactId>
+ <artifactId>hbase-shaded-server</artifactId>
<version>${hbase.version}</version>
<scope>test</scope>
</dependency>
@@ -178,7 +134,14 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
- <version>${hadoop.version}</version>
+ <version>${hbase.hadoop.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hbase.hadoop.version}</version>
<scope>test</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/beam/blob/6d502faa/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java b/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java
index 774e17e..7bfdd25 100644
--- a/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java
+++ b/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseIOTest.java
@@ -66,7 +66,6 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.hamcrest.Matchers;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -77,7 +76,6 @@ import org.junit.runners.JUnit4;
/**
* Test HBaseIO.
*/
-@Ignore
@RunWith(JUnit4.class)
public class HBaseIOTest {
@Rule public final transient TestPipeline p = TestPipeline.create();