You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by em...@apache.org on 2020/05/22 03:49:37 UTC
[arrow] branch master updated: ARROW-8696: [Java] Convert tests to
maven failsafe
This is an automated email from the ASF dual-hosted git repository.
emkornfield pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 93ba086 ARROW-8696: [Java] Convert tests to maven failsafe
93ba086 is described below
commit 93ba086820f5336d67177acd0c35cb11e7c66ece
Author: Ryan Murray <ry...@dremio.com>
AuthorDate: Thu May 21 20:49:08 2020 -0700
ARROW-8696: [Java] Convert tests to maven failsafe
Some tests are run via main() and can be run as integration tests
instead. This makes running as part of an automated job easier.
Closes #7100 from rymurr/ARROW-8696
Authored-by: Ryan Murray <ry...@dremio.com>
Signed-off-by: Micah Kornfield <em...@gmail.com>
---
java/README.md | 13 ++++
java/memory/pom.xml | 32 ++++++++
...LargeArrowBuf.java => ITTestLargeArrowBuf.java} | 38 ++++++----
java/vector/pom.xml | 36 ++++++++-
...TestLargeVector.java => ITTestLargeVector.java} | 85 +++++++++++-----------
5 files changed, 145 insertions(+), 59 deletions(-)
diff --git a/java/README.md b/java/README.md
index 9851308..773d2ba 100644
--- a/java/README.md
+++ b/java/README.md
@@ -112,6 +112,19 @@ mvn -Dlogback.configurationFile=file:<path-of-logback-file>
See [Logback Configuration][1] for more details.
+## Integration Tests
+
+Integration tests which require more time or more memory can be run by activating
+the `integration-tests` profile. This activates the [maven failsafe][4] plugin
+and any class prefixed with `IT` will be run during the testing phase. The integration
+tests currently require a larger amount of memory (>4GB) and time to complete. To activate
+the profile:
+
+```bash
+mvn -Pintegration-tests <rest of mvn arguments>
+```
+
[1]: https://logback.qos.ch/manual/configuration.html
[2]: https://github.com/apache/arrow/blob/master/cpp/README.md
[3]: http://google.github.io/styleguide/javaguide.html
+[4]: https://maven.apache.org/surefire/maven-failsafe-plugin/
\ No newline at end of file
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index 45d1c98..b3418e6 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -46,4 +46,36 @@
<build>
</build>
+ <profiles>
+ <profile>
+ <!-- This profile turns on integration testing. It activates the failsafe plugin and will run any tests
+ with the 'IT' prefix. This should be run in a separate CI build or on developers machines as it potentially
+ uses quite a bit of memory. Activate the tests by adding -Pintegration-tests to your maven command line -->
+ <id>integration-tests</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <configuration>
+ <systemPropertyVariables>
+ <java.io.tmpdir>${project.build.directory}</java.io.tmpdir>
+ <io.netty.tryReflectionSetAccessible>true</io.netty.tryReflectionSetAccessible>
+ <user.timezone>UTC</user.timezone>
+ </systemPropertyVariables>
+ <argLine></argLine>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
</project>
diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestLargeArrowBuf.java b/java/memory/src/test/java/org/apache/arrow/memory/ITTestLargeArrowBuf.java
similarity index 65%
rename from java/memory/src/test/java/org/apache/arrow/memory/TestLargeArrowBuf.java
rename to java/memory/src/test/java/org/apache/arrow/memory/ITTestLargeArrowBuf.java
index 7d85187..fa8d510 100644
--- a/java/memory/src/test/java/org/apache/arrow/memory/TestLargeArrowBuf.java
+++ b/java/memory/src/test/java/org/apache/arrow/memory/ITTestLargeArrowBuf.java
@@ -19,46 +19,54 @@ package org.apache.arrow.memory;
import static org.junit.Assert.assertEquals;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
/**
* Integration test for large (more than 2GB) {@link org.apache.arrow.memory.ArrowBuf}.
* To run this test, please make sure there is at least 4GB memory in the system.
- * <p>
- * Please note that this is not a standard test case, so please run it by manually invoking the
- * main method.
- * </p>
*/
-public class TestLargeArrowBuf {
+public class ITTestLargeArrowBuf {
+ private static final Logger logger = LoggerFactory.getLogger(ITTestLargeArrowBuf.class);
- private static void testLargeArrowBuf(long bufSize) {
+ private void run(long bufSize) {
try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
ArrowBuf largeBuf = allocator.buffer(bufSize)) {
assertEquals(bufSize, largeBuf.capacity());
- System.out.println("Successfully allocated a buffer with capacity " + largeBuf.capacity());
+ logger.trace("Successfully allocated a buffer with capacity {}", largeBuf.capacity());
for (long i = 0; i < bufSize / 8; i++) {
largeBuf.setLong(i * 8, i);
if ((i + 1) % 10000 == 0) {
- System.out.println("Successfully written " + (i + 1) + " long words");
+ logger.trace("Successfully written {} long words", i + 1);
}
}
- System.out.println("Successfully written " + (bufSize / 8) + " long words");
+ logger.trace("Successfully written {} long words", bufSize / 8);
for (long i = 0; i < bufSize / 8; i++) {
long val = largeBuf.getLong(i * 8);
assertEquals(i, val);
if ((i + 1) % 10000 == 0) {
- System.out.println("Successfully read " + (i + 1) + " long words");
+ logger.trace("Successfully read {} long words", i + 1);
}
}
- System.out.println("Successfully read " + (bufSize / 8) + " long words");
+ logger.trace("Successfully read {} long words", bufSize / 8);
}
- System.out.println("Successfully released the large buffer.");
+ logger.trace("Successfully released the large buffer.");
}
- public static void main(String[] args) {
- testLargeArrowBuf(4 * 1024 * 1024 * 1024L);
- testLargeArrowBuf(Integer.MAX_VALUE);
+ @Test
+ public void testLargeArrowBuf() {
+ run(4 * 1024 * 1024 * 1024L);
}
+
+ @Test
+ public void testMaxIntArrowBuf() {
+ run(Integer.MAX_VALUE);
+ }
+
}
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index 7d2fb85..bfd5d54 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -9,7 +9,8 @@
License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
OF ANY KIND, either express or implied. See the License for the specific
language governing permissions and limitations under the License. -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.arrow</groupId>
@@ -205,5 +206,38 @@
</build>
+ <profiles>
+ <profile>
+ <!-- This profile turns on integration testing. It activates the failsafe plugin and will run any tests
+ with the 'IT' prefix. This should be run in a separate CI build or on developers machines as it potentially
+ uses quite a bit of memory. Activate the tests by adding -Pintegration-tests to your maven command line -->
+ <id>integration-tests</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <configuration>
+ <forkedProcessTimeoutInSeconds>3600</forkedProcessTimeoutInSeconds>
+ <systemPropertyVariables>
+ <java.io.tmpdir>${project.build.directory}</java.io.tmpdir>
+ <io.netty.tryReflectionSetAccessible>true</io.netty.tryReflectionSetAccessible>
+ <user.timezone>UTC</user.timezone>
+ </systemPropertyVariables>
+ <argLine></argLine>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
</project>
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVector.java b/java/vector/src/test/java/org/apache/arrow/vector/ITTestLargeVector.java
similarity index 61%
rename from java/vector/src/test/java/org/apache/arrow/vector/TestLargeVector.java
rename to java/vector/src/test/java/org/apache/arrow/vector/ITTestLargeVector.java
index c33acb6..2736d25 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ITTestLargeVector.java
@@ -23,20 +23,22 @@ import static org.junit.Assert.assertEquals;
import org.apache.arrow.memory.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* Integration test for a vector with a large (more than 2GB) {@link org.apache.arrow.memory.ArrowBuf} as
* the data buffer.
* To run this test, please make sure there is at least 4GB free memory in the system.
- * <p>
- * Please note that this is not a standard test case, so please run it by manually invoking the
- * main method.
- * </p>
*/
-public class TestLargeVector {
+public class ITTestLargeVector {
+ private static final Logger logger = LoggerFactory.getLogger(ITTestLargeVector.class);
- private static void testLargeLongVector() {
- System.out.println("Testing large big int vector.");
+ @Test
+ public void testLargeLongVector() {
+ logger.trace("Testing large big int vector.");
final long bufSize = 4 * 1024 * 1024 * 1024L;
final int vecLength = (int) (bufSize / BigIntVector.TYPE_WIDTH);
@@ -45,32 +47,33 @@ public class TestLargeVector {
BigIntVector largeVec = new BigIntVector("vec", allocator)) {
largeVec.allocateNew(vecLength);
- System.out.println("Successfully allocated a vector with capacity " + vecLength);
+ logger.trace("Successfully allocated a vector with capacity {}", vecLength);
for (int i = 0; i < vecLength; i++) {
largeVec.set(i, i * 10L);
if ((i + 1) % 10000 == 0) {
- System.out.println("Successfully written " + (i + 1) + " values");
+ logger.trace("Successfully written {} values", i + 1);
}
}
- System.out.println("Successfully written " + vecLength + " values");
+ logger.trace("Successfully written {} values", vecLength);
for (int i = 0; i < vecLength; i++) {
long val = largeVec.get(i);
assertEquals(i * 10L, val);
if ((i + 1) % 10000 == 0) {
- System.out.println("Successfully read " + (i + 1) + " values");
+ logger.trace("Successfully read {} values", i + 1);
}
}
- System.out.println("Successfully read " + vecLength + " values");
+ logger.trace("Successfully read {} values", vecLength);
}
- System.out.println("Successfully released the large vector.");
+ logger.trace("Successfully released the large vector.");
}
- private static void testLargeIntVector() {
- System.out.println("Testing large int vector.");
+ @Test
+ public void testLargeIntVector() {
+ logger.trace("Testing large int vector.");
final long bufSize = 4 * 1024 * 1024 * 1024L;
final int vecLength = (int) (bufSize / IntVector.TYPE_WIDTH);
@@ -79,32 +82,33 @@ public class TestLargeVector {
IntVector largeVec = new IntVector("vec", allocator)) {
largeVec.allocateNew(vecLength);
- System.out.println("Successfully allocated a vector with capacity " + vecLength);
+ logger.trace("Successfully allocated a vector with capacity {}", vecLength);
for (int i = 0; i < vecLength; i++) {
largeVec.set(i, i);
if ((i + 1) % 10000 == 0) {
- System.out.println("Successfully written " + (i + 1) + " values");
+ logger.trace("Successfully written {} values", i + 1);
}
}
- System.out.println("Successfully written " + vecLength + " values");
+ logger.trace("Successfully written {} values", vecLength);
for (int i = 0; i < vecLength; i++) {
long val = largeVec.get(i);
assertEquals(i, val);
if ((i + 1) % 10000 == 0) {
- System.out.println("Successfully read " + (i + 1) + " values");
+ logger.trace("Successfully read {} values", i + 1);
}
}
- System.out.println("Successfully read " + vecLength + " values");
+ logger.trace("Successfully read {} values", vecLength);
}
- System.out.println("Successfully released the large vector.");
+ logger.trace("Successfully released the large vector.");
}
- private static void testLargeDecimalVector() {
- System.out.println("Testing large decimal vector.");
+ @Test
+ public void testLargeDecimalVector() {
+ logger.trace("Testing large decimal vector.");
final long bufSize = 4 * 1024 * 1024 * 1024L;
final int vecLength = (int) (bufSize / DecimalVector.TYPE_WIDTH);
@@ -113,16 +117,16 @@ public class TestLargeVector {
DecimalVector largeVec = new DecimalVector("vec", allocator, 38, 16)) {
largeVec.allocateNew(vecLength);
- System.out.println("Successfully allocated a vector with capacity " + vecLength);
+ logger.trace("Successfully allocated a vector with capacity {}", vecLength);
for (int i = 0; i < vecLength; i++) {
largeVec.set(i, 0);
if ((i + 1) % 10000 == 0) {
- System.out.println("Successfully written " + (i + 1) + " values");
+ logger.trace("Successfully written {} values", i + 1);
}
}
- System.out.println("Successfully written " + vecLength + " values");
+ logger.trace("Successfully written {} values", vecLength);
for (int i = 0; i < vecLength; i++) {
ArrowBuf buf = largeVec.get(i);
@@ -131,16 +135,17 @@ public class TestLargeVector {
assertEquals(0, buf.getLong(8));
if ((i + 1) % 10000 == 0) {
- System.out.println("Successfully read " + (i + 1) + " values");
+ logger.trace("Successfully read {} values", i + 1);
}
}
- System.out.println("Successfully read " + vecLength + " values");
+ logger.trace("Successfully read {} values", vecLength);
}
- System.out.println("Successfully released the large vector.");
+ logger.trace("Successfully released the large vector.");
}
- private static void testLargeFixedSizeBinaryVector() {
- System.out.println("Testing large fixed size binary vector.");
+ @Test
+ public void testLargeFixedSizeBinaryVector() {
+ logger.trace("Testing large fixed size binary vector.");
final long bufSize = 4 * 1024 * 1024 * 1024L;
final int typeWidth = 8;
@@ -150,17 +155,17 @@ public class TestLargeVector {
FixedSizeBinaryVector largeVec = new FixedSizeBinaryVector("vec", allocator, typeWidth)) {
largeVec.allocateNew(vecLength);
- System.out.println("Successfully allocated a vector with capacity " + vecLength);
+ logger.trace("Successfully allocated a vector with capacity {}", vecLength);
byte[] value = new byte[] {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'};
for (int i = 0; i < vecLength; i++) {
largeVec.set(i, value);
if ((i + 1) % 10000 == 0) {
- System.out.println("Successfully written " + (i + 1) + " values");
+ logger.trace("Successfully written {} values", i + 1);
}
}
- System.out.println("Successfully written " + vecLength + " values");
+ logger.trace("Successfully written {} values", vecLength);
for (int i = 0; i < vecLength; i++) {
byte[] buf = largeVec.get(i);
@@ -168,18 +173,12 @@ public class TestLargeVector {
assertArrayEquals(buf, value);
if ((i + 1) % 10000 == 0) {
- System.out.println("Successfully read " + (i + 1) + " values");
+ logger.trace("Successfully read {} values", i + 1);
}
}
- System.out.println("Successfully read " + vecLength + " values");
+ logger.trace("Successfully read {} values", vecLength);
}
- System.out.println("Successfully released the large vector.");
+ logger.trace("Successfully released the large vector.");
}
- public static void main(String[] args) {
- testLargeLongVector();
- testLargeIntVector();
- testLargeDecimalVector();
- testLargeFixedSizeBinaryVector();
- }
}