You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kg...@apache.org on 2020/05/05 15:37:17 UTC
[hive] branch master updated: HIVE-23323: Add qsplits profile
(Zoltan Haindrich reviewed by Miklos Gergely)
This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 772bfda HIVE-23323: Add qsplits profile (Zoltan Haindrich reviewed by Miklos Gergely)
772bfda is described below
commit 772bfdadab6f3d8c3a54431fe7b6e8b115e99b29
Author: Zoltan Haindrich <ki...@rxd.hu>
AuthorDate: Tue May 5 15:35:31 2020 +0000
HIVE-23323: Add qsplits profile (Zoltan Haindrich reviewed by Miklos Gergely)
Signed-off-by: Zoltan Haindrich <zh...@cloudera.com>
---
itests/bin/generate-cli-splits.sh | 26 ++++++++
itests/qtest-spark/pom.xml | 50 +++++++++++++++-
.../hive/cli/TestMiniSparkOnYarnCliDriver.java | 7 ++-
.../apache/hadoop/hive/cli/TestSparkCliDriver.java | 7 ++-
itests/qtest/pom.xml | 51 +++++++++++++++-
.../org/apache/hadoop/hive/cli/TestCliDriver.java | 5 +-
.../hive/cli/TestEncryptedHDFSCliDriver.java | 5 +-
.../hadoop/hive/cli/TestMiniLlapCliDriver.java | 5 +-
.../hive/cli/TestMiniLlapLocalCliDriver.java | 5 +-
.../hadoop/hive/cli/control/SplitSupport.java | 69 ++++++++++++++++++++++
.../hadoop/hive/cli/control/TestSplitSupport.java | 48 +++++++++++++++
.../control/splitsupport/SplitSupportDummy.java | 5 ++
.../splitsupport/split0/SplitSupportDummy.java | 5 ++
.../splitsupport/split125/SplitSupportDummy.java | 5 ++
14 files changed, 281 insertions(+), 12 deletions(-)
diff --git a/itests/bin/generate-cli-splits.sh b/itests/bin/generate-cli-splits.sh
new file mode 100755
index 0000000..c16d369
--- /dev/null
+++ b/itests/bin/generate-cli-splits.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+usage() {
+ echo "$0 <from> <to>"
+ exit 1
+}
+
+[ "$1" == "" ] && usage
+[ "$2" == "" ] && usage
+
+
+inDir="$1"
+outDir="$2"
+
+git grep SplitSupport.process | grep "$1" | cut -d ':' -f1 | while read f;do
+
+ echo "processing: $f"
+ n="`grep N_SPLITS "$f" | cut -d= -f2 | tr -c -d '0-9'`"
+ echo " * nSplits: $n"
+
+ for((i=0;i<n;i++)) {
+ oDir="`dirname $f | sed "s|$inDir|$outDir|"`/split$i"
+ mkdir -p $oDir
+ cat $f | sed -r "s|^(package.*);$|\1.split$i;|g" > $oDir/`basename $f`
+ }
+done
diff --git a/itests/qtest-spark/pom.xml b/itests/qtest-spark/pom.xml
index 60d032d..4f97e29 100644
--- a/itests/qtest-spark/pom.xml
+++ b/itests/qtest-spark/pom.xml
@@ -417,5 +417,53 @@
</plugin>
</plugins>
</build>
-
+ <profiles>
+ <profile>
+ <id>qsplits</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>generate-split-tests</id>
+ <phase>generate-sources</phase>
+ <configuration>
+ <target>
+ <exec executable="bash" failonerror="true">
+ <arg value="${basedir}/../bin/generate-cli-splits.sh"/>
+ <arg value="src/test/java"/>
+ <arg value="target/generated-test-sources"/>
+ </exec>
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-test-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-test-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>target/generated-test-sources</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
</project>
diff --git a/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestMiniSparkOnYarnCliDriver.java b/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestMiniSparkOnYarnCliDriver.java
index 889d0f2..c19d4db 100644
--- a/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestMiniSparkOnYarnCliDriver.java
+++ b/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestMiniSparkOnYarnCliDriver.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
+
package org.apache.hadoop.hive.cli;
import java.io.File;
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.hadoop.hive.cli.control.CliAdapter;
import org.apache.hadoop.hive.cli.control.CliConfigs;
+import org.apache.hadoop.hive.cli.control.SplitSupport;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
@@ -34,11 +35,13 @@ import org.junit.runners.Parameterized.Parameters;
@RunWith(Parameterized.class)
public class TestMiniSparkOnYarnCliDriver {
+ private static final int N_SPLITS = 5;
+
static CliAdapter adapter = new CliConfigs.SparkOnYarnCliConfig().getCliAdapter();
@Parameters(name = "{0}")
public static List<Object[]> getParameters() throws Exception {
- return adapter.getParameters();
+ return SplitSupport.process(adapter.getParameters(), TestMiniSparkOnYarnCliDriver.class, N_SPLITS);
}
@ClassRule
diff --git a/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestSparkCliDriver.java b/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestSparkCliDriver.java
index def73d4..bf200b6 100644
--- a/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestSparkCliDriver.java
+++ b/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestSparkCliDriver.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
+
package org.apache.hadoop.hive.cli;
import java.io.File;
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.hadoop.hive.cli.control.CliAdapter;
import org.apache.hadoop.hive.cli.control.CliConfigs;
+import org.apache.hadoop.hive.cli.control.SplitSupport;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
@@ -34,11 +35,13 @@ import org.junit.runners.Parameterized.Parameters;
@RunWith(Parameterized.class)
public class TestSparkCliDriver {
+ private static final int N_SPLITS = 11;
+
static CliAdapter adapter = new CliConfigs.SparkCliConfig().getCliAdapter();
@Parameters(name = "{0}")
public static List<Object[]> getParameters() throws Exception {
- return adapter.getParameters();
+ return SplitSupport.process(adapter.getParameters(), TestSparkCliDriver.class, N_SPLITS);
}
@ClassRule
diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml
index 1cc562b..d9492fa 100644
--- a/itests/qtest/pom.xml
+++ b/itests/qtest/pom.xml
@@ -434,9 +434,55 @@
<test.warehouse.scheme></test.warehouse.scheme>
</properties>
</profile>
+ <profile>
+ <id>qsplits</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>generate-split-tests</id>
+ <phase>generate-sources</phase>
+ <configuration>
+ <target>
+ <exec executable="bash" failonerror="true">
+ <arg value="${basedir}/../bin/generate-cli-splits.sh"/>
+ <arg value="src/test/java"/>
+ <arg value="target/generated-test-sources"/>
+ </exec>
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-test-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-test-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>target/generated-test-sources</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
-
- <build>
+ <build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
@@ -539,5 +585,4 @@
</plugin>
</plugins>
</build>
-
</project>
diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestCliDriver.java b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestCliDriver.java
index 4617d31..ccc8a66 100644
--- a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestCliDriver.java
+++ b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestCliDriver.java
@@ -22,6 +22,7 @@ import java.util.List;
import org.apache.hadoop.hive.cli.control.CliAdapter;
import org.apache.hadoop.hive.cli.control.CliConfigs;
+import org.apache.hadoop.hive.cli.control.SplitSupport;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
@@ -33,11 +34,13 @@ import org.junit.runners.Parameterized.Parameters;
@RunWith(Parameterized.class)
public class TestCliDriver {
+ private static final int N_SPLITS = 17;
+
static CliAdapter adapter = new CliConfigs.CliConfig().getCliAdapter();
@Parameters(name = "{0}")
public static List<Object[]> getParameters() throws Exception {
- return adapter.getParameters();
+ return SplitSupport.process(adapter.getParameters(), TestCliDriver.class, N_SPLITS);
}
@ClassRule
diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestEncryptedHDFSCliDriver.java b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestEncryptedHDFSCliDriver.java
index 76fce63..8b6efdb 100644
--- a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestEncryptedHDFSCliDriver.java
+++ b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestEncryptedHDFSCliDriver.java
@@ -22,6 +22,7 @@ import java.util.List;
import org.apache.hadoop.hive.cli.control.CliAdapter;
import org.apache.hadoop.hive.cli.control.CliConfigs;
+import org.apache.hadoop.hive.cli.control.SplitSupport;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
@@ -33,11 +34,13 @@ import org.junit.runners.Parameterized.Parameters;
@RunWith(Parameterized.class)
public class TestEncryptedHDFSCliDriver {
+ private static final int N_SPLITS = 3;
+
static CliAdapter adapter = new CliConfigs.EncryptedHDFSCliConfig().getCliAdapter();
@Parameters(name = "{0}")
public static List<Object[]> getParameters() throws Exception {
- return adapter.getParameters();
+ return SplitSupport.process(adapter.getParameters(), TestEncryptedHDFSCliDriver.class, N_SPLITS);
}
@ClassRule
diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapCliDriver.java b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapCliDriver.java
index cc05c11..c0dd2bf 100644
--- a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapCliDriver.java
+++ b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapCliDriver.java
@@ -22,6 +22,7 @@ import java.util.List;
import org.apache.hadoop.hive.cli.control.CliAdapter;
import org.apache.hadoop.hive.cli.control.CliConfigs;
+import org.apache.hadoop.hive.cli.control.SplitSupport;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
@@ -33,11 +34,13 @@ import org.junit.runners.Parameterized.Parameters;
@RunWith(Parameterized.class)
public class TestMiniLlapCliDriver {
+ private static final int N_SPLITS = 3;
+
static CliAdapter adapter = new CliConfigs.MiniLlapCliConfig().getCliAdapter();
@Parameters(name = "{0}")
public static List<Object[]> getParameters() throws Exception {
- return adapter.getParameters();
+ return SplitSupport.process(adapter.getParameters(), TestMiniLlapCliDriver.class, N_SPLITS);
}
@ClassRule
diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java
index 66d7bda..06b802b 100644
--- a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java
+++ b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java
@@ -22,6 +22,7 @@ import java.util.List;
import org.apache.hadoop.hive.cli.control.CliAdapter;
import org.apache.hadoop.hive.cli.control.CliConfigs;
+import org.apache.hadoop.hive.cli.control.SplitSupport;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
@@ -35,9 +36,11 @@ public class TestMiniLlapLocalCliDriver {
static CliAdapter adapter = new CliConfigs.MiniLlapLocalCliConfig().getCliAdapter();
+ private static int N_SPLITS = 32;
+
@Parameters(name = "{0}")
public static List<Object[]> getParameters() throws Exception {
- return adapter.getParameters();
+ return SplitSupport.process(adapter.getParameters(), TestMiniLlapLocalCliDriver.class, N_SPLITS);
}
@ClassRule
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/SplitSupport.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/SplitSupport.java
new file mode 100644
index 0000000..a51c5e4
--- /dev/null
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/SplitSupport.java
@@ -0,0 +1,69 @@
+package org.apache.hadoop.hive.cli.control;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.google.common.annotations.VisibleForTesting;
+
+public class SplitSupport {
+
+ public static List<Object[]> process(List<Object[]> parameters, Class<?> currentClass, int nSplits) {
+ if (!isSplitExecution(currentClass)) {
+ return parameters;
+ }
+ // auto-disable primary test in case splits are present
+ if (isSplit0ClassExistsFor(currentClass)) {
+ return new ArrayList<>();
+ }
+ int i = getSplitIndex(currentClass);
+ return getSplitParams(parameters, i, nSplits);
+ }
+
+ private static boolean isSplitExecution(Class<?> currentClass) {
+ return isSplitClass(currentClass) || isSplit0ClassExistsFor(currentClass);
+ }
+
+ @VisibleForTesting
+ static List<Object[]> getSplitParams(List<Object[]> parameters, int i, int nSplits) {
+ if(i<0 || i>=nSplits) {
+ throw new IllegalArgumentException("unexpected");
+ }
+ int n = parameters.size();
+ int st = i * n / nSplits;
+ int ed = (i + 1) * n / nSplits;
+
+ return parameters.subList(st, ed);
+ }
+
+ @VisibleForTesting
+ static boolean isSplitClass(Class<?> currentClass) {
+ Package p = currentClass.getPackage();
+ return p.getName().matches(".*split[0-9]+$");
+ }
+
+ @VisibleForTesting
+ static int getSplitIndex(Class<?> currentClass) {
+ Package p = currentClass.getPackage();
+ Pattern pat = Pattern.compile("(.*split)([0-9]+)$");
+ Matcher matcher = pat.matcher(p.getName());
+ if (matcher.find()) {
+ return Integer.parseInt(matcher.group(2));
+ }
+ throw new IllegalArgumentException("cant get splitindex for: " + p);
+ }
+
+ @VisibleForTesting
+ static boolean isSplit0ClassExistsFor(Class<?> clazz) {
+ Package p = clazz.getPackage();
+ String split1 = p.getName() + ".split0." + clazz.getSimpleName();
+ try {
+ Class<?> c = Class.forName(split1);
+ return c != null;
+ } catch (ClassNotFoundException e) {
+ return false;
+ }
+ }
+
+}
diff --git a/itests/util/src/test/java/org/apache/hadoop/hive/cli/control/TestSplitSupport.java b/itests/util/src/test/java/org/apache/hadoop/hive/cli/control/TestSplitSupport.java
new file mode 100644
index 0000000..2e5eede
--- /dev/null
+++ b/itests/util/src/test/java/org/apache/hadoop/hive/cli/control/TestSplitSupport.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.cli.control;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+public class TestSplitSupport {
+
+ @Test
+ public void testIsSplitClass1() {
+ Class<?> mainClass = org.apache.hadoop.hive.cli.control.splitsupport.SplitSupportDummy.class;
+ Class<?> split0Class =
+ org.apache.hadoop.hive.cli.control.splitsupport.split0.SplitSupportDummy.class;
+ assertFalse(SplitSupport.isSplitClass(mainClass));
+ assertTrue(SplitSupport.isSplitClass(split0Class));
+ }
+
+ @Test
+ public void testGetSplitIndex0() {
+ Class<?> split0Class = org.apache.hadoop.hive.cli.control.splitsupport.split0.SplitSupportDummy.class;
+ assertEquals(0, SplitSupport.getSplitIndex(split0Class));
+ }
+
+ @Test
+ public void testGetSplitIndex125() {
+ Class<?> split0Class = org.apache.hadoop.hive.cli.control.splitsupport.split125.SplitSupportDummy.class;
+ assertEquals(125, SplitSupport.getSplitIndex(split0Class));
+ }
+}
diff --git a/itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/SplitSupportDummy.java b/itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/SplitSupportDummy.java
new file mode 100644
index 0000000..335a50e
--- /dev/null
+++ b/itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/SplitSupportDummy.java
@@ -0,0 +1,5 @@
+package org.apache.hadoop.hive.cli.control.splitsupport;
+
+public class SplitSupportDummy {
+
+}
diff --git a/itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/split0/SplitSupportDummy.java b/itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/split0/SplitSupportDummy.java
new file mode 100644
index 0000000..4ea134d
--- /dev/null
+++ b/itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/split0/SplitSupportDummy.java
@@ -0,0 +1,5 @@
+package org.apache.hadoop.hive.cli.control.splitsupport.split0;
+
+public class SplitSupportDummy {
+
+}
diff --git a/itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/split125/SplitSupportDummy.java b/itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/split125/SplitSupportDummy.java
new file mode 100644
index 0000000..254f8aa
--- /dev/null
+++ b/itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/split125/SplitSupportDummy.java
@@ -0,0 +1,5 @@
+package org.apache.hadoop.hive.cli.control.splitsupport.split125;
+
+public class SplitSupportDummy {
+
+}