You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2017/03/23 17:27:47 UTC
[16/50] [abbrv] bigtop git commit: ODPI-185. Test [HIVE_HCATALOG]
ODPI-185. Test [HIVE_HCATALOG]
(cherry picked from commit b493a57dd679c482daa1465fd57a9854550be549)
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/47c31dca
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/47c31dca
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/47c31dca
Branch: refs/heads/master
Commit: 47c31dcaa0d409f63e6869a54331c29e26f1b840
Parents: a41bb2d
Author: Roman Shaposhnik <rv...@apache.org>
Authored: Tue Nov 8 19:24:44 2016 -0800
Committer: Roman Shaposhnik <rv...@apache.org>
Committed: Thu Mar 23 10:27:13 2017 -0700
----------------------------------------------------------------------
bigtop-tests/spec-tests/runtime/build.gradle | 13 +++++++++++++
.../java/org/odpi/specs/runtime/hive/HCatalogMR.java | 11 -----------
.../org/odpi/specs/runtime/hive/TestHCatalog.java | 15 +++++----------
3 files changed, 18 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/47c31dca/bigtop-tests/spec-tests/runtime/build.gradle
----------------------------------------------------------------------
diff --git a/bigtop-tests/spec-tests/runtime/build.gradle b/bigtop-tests/spec-tests/runtime/build.gradle
index f0166c9..a88a3b6 100644
--- a/bigtop-tests/spec-tests/runtime/build.gradle
+++ b/bigtop-tests/spec-tests/runtime/build.gradle
@@ -43,7 +43,20 @@ dependencies {
if (System.env.HADOOP_CONF_DIR) testRuntime files(System.env.HADOOP_CONF_DIR)
}
+jar {
+ from {
+ (configurations.runtime).grep{it.toString() =~ /(hive|libfb303)-.*[jw]ar$/}.collect {
+ zipTree(it)
+ }
+ }
+
+ exclude 'META-INF/*.RSA', 'META-INF/*.SF','META-INF/*.DSA'
+}
+
test {
// Change the default location where test data is picked up
systemProperty 'test.resources.dir', "${buildDir}/resources/test/"
+ systemProperty 'odpi.test.hive.hcat.job.jar', jar.archivePath
+ systemProperty 'odpi.test.hive.hcat.core.jar', (configurations.runtime).find { it.toString() =~ /hive-hcatalog-core-.*jar$/ }
}
+test.dependsOn jar
http://git-wip-us.apache.org/repos/asf/bigtop/blob/47c31dca/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java
----------------------------------------------------------------------
diff --git a/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java b/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java
index a6ff375..7cb9bbe 100644
--- a/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java
+++ b/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java
@@ -72,17 +72,6 @@ public class HCatalogMR extends Configured implements Tool {
HCatOutputFormat.setSchema(job, HCatSchemaUtils.getHCatSchema(outputSchemaStr));
job.setOutputFormatClass(HCatOutputFormat.class);
- // TODO All four of these jars need to be in the distributed cache of the job for the job to
- // succeed. I loaded them into a known location in HDFS to get them in the cache. There may
- // be a way to load them from a file on the gateway machine. We could also put in a hdfs dfs
- // -put operation into a gradle step as part of the build so that the jars are picked up from
- // the distribution and put in a known location in HDFS from when they can be picked up in
- // the distributed cache.
- job.addCacheArchive(new URI("hdfs:/user/gates/hive-hcatalog-core-1.2.1.jar"));
- job.addCacheArchive(new URI("hdfs:/user/gates/hive-metastore-1.2.1.jar"));
- job.addCacheArchive(new URI("hdfs:/user/gates/hive-exec-1.2.1.jar"));
- job.addCacheArchive(new URI("hdfs:/user/gates/libfb303-0.9.2.jar"));
-
return job.waitForCompletion(true) ? 0 : 1;
http://git-wip-us.apache.org/repos/asf/bigtop/blob/47c31dca/bigtop-tests/spec-tests/runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java
----------------------------------------------------------------------
diff --git a/bigtop-tests/spec-tests/runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java b/bigtop-tests/spec-tests/runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java
index 87e3eb0..b51db02 100644
--- a/bigtop-tests/spec-tests/runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java
+++ b/bigtop-tests/spec-tests/runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java
@@ -51,6 +51,8 @@ import java.util.Random;
public class TestHCatalog {
+ private static final String JOBJAR = "odpi.test.hive.hcat.job.jar";
+ private static final String HCATCORE = "odpi.test.hive.hcat.core.jar";
private static final Log LOG = LogFactory.getLog(TestHCatalog.class.getName());
@@ -132,18 +134,11 @@ public class TestHCatalog {
out.close();
Map<String, String> env = new HashMap<>();
- // TODO These need to be set from the environment rather than hard wired
- env.put("HADOOP_HOME","/Users/gates/grid/odpi-testing/hadoop-2.7.3");
- env.put("HADOOP_CLASSPATH", "/Users/gates/grid/odpi-testing/apache-hive-1.2.1-bin/hcatalog/share/hcatalog/hive-hcatalog-core-1.2.1.jar");
- env.put("HIVE_HOME", "/Users/gates/grid/odpi-testing/apache-hive-1.2.1-bin");
- Map<String, String> results = HiveHelper.execCommand(new CommandLine("/Users/gates/grid/odpi-testing/apache-hive-1.2.1-bin/bin/hive")
+ env.put("HADOOP_CLASSPATH", System.getProperty(HCATCORE, ""));
+ Map<String, String> results = HiveHelper.execCommand(new CommandLine("hive")
.addArgument("--service")
.addArgument("jar")
- // TODO This is the jar built by gradle, but I didn't know how to take the jar built in
- // the build phase and reference it in the test phase. Perhaps a move operation could be
- // put in the middle so the jar is moved to a known location that can be referenced here,
- // or maybe gradle can pass in its working directory so that we can reference it from there.
- .addArgument("/Users/gates/git/bigtop/runtime-1.2.0-SNAPSHOT.jar")
+ .addArgument(System.getProperty(JOBJAR))
.addArgument(HCatalogMR.class.getName())
.addArgument(inputTable)
.addArgument(outputTable)