You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2017/03/23 17:27:46 UTC
[15/50] [abbrv] bigtop git commit: Added TODOs to outline remaining
work.
Added TODOs to outline remaining work.
(cherry picked from commit 67c51056610260ad99dd2f194a33ec7af4b89c9e)
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/a41bb2dd
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/a41bb2dd
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/a41bb2dd
Branch: refs/heads/master
Commit: a41bb2dd74e526512ca5e8eb26e0a8d379cec56f
Parents: ab4f414
Author: Alan Gates <ga...@hortonworks.com>
Authored: Tue Nov 8 07:42:28 2016 -0800
Committer: Roman Shaposhnik <rv...@apache.org>
Committed: Thu Mar 23 10:27:13 2017 -0700
----------------------------------------------------------------------
.../src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java | 6 ++++++
.../test/java/org/odpi/specs/runtime/hive/TestHCatalog.java | 7 ++++++-
2 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a41bb2dd/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java
----------------------------------------------------------------------
diff --git a/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java b/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java
index 6fcfe37..a6ff375 100644
--- a/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java
+++ b/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java
@@ -72,6 +72,12 @@ public class HCatalogMR extends Configured implements Tool {
HCatOutputFormat.setSchema(job, HCatSchemaUtils.getHCatSchema(outputSchemaStr));
job.setOutputFormatClass(HCatOutputFormat.class);
+ // TODO All four of these jars need to be in the distributed cache of the job for the job to
+ // succeed. I loaded them into a known location in HDFS to get them in the cache. There may
+ // be a way to load them from a file on the gateway machine. We could also put in a hdfs dfs
+ // -put operation into a gradle step as part of the build so that the jars are picked up from
+ // the distribution and put in a known location in HDFS from when they can be picked up in
+ // the distributed cache.
job.addCacheArchive(new URI("hdfs:/user/gates/hive-hcatalog-core-1.2.1.jar"));
job.addCacheArchive(new URI("hdfs:/user/gates/hive-metastore-1.2.1.jar"));
job.addCacheArchive(new URI("hdfs:/user/gates/hive-exec-1.2.1.jar"));
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a41bb2dd/bigtop-tests/spec-tests/runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java
----------------------------------------------------------------------
diff --git a/bigtop-tests/spec-tests/runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java b/bigtop-tests/spec-tests/runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java
index bb237d8..87e3eb0 100644
--- a/bigtop-tests/spec-tests/runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java
+++ b/bigtop-tests/spec-tests/runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java
@@ -118,7 +118,7 @@ public class TestHCatalog {
new HCatFieldSchema("word", HCatFieldSchema.Type.STRING, ""),
new HCatFieldSchema("count", HCatFieldSchema.Type.INT, "")));
- // TODO Could I use HCatWriter hear and the reader to read it?
+ // LATER Could I use HCatWriter here and the reader to read it?
// Write some stuff into a file in the location of the table
table = client.getTable("default", inputTable);
String inputFile = table.getSd().getLocation() + "/input";
@@ -132,12 +132,17 @@ public class TestHCatalog {
out.close();
Map<String, String> env = new HashMap<>();
+ // TODO These need to be set from the environment rather than hard wired
env.put("HADOOP_HOME","/Users/gates/grid/odpi-testing/hadoop-2.7.3");
env.put("HADOOP_CLASSPATH", "/Users/gates/grid/odpi-testing/apache-hive-1.2.1-bin/hcatalog/share/hcatalog/hive-hcatalog-core-1.2.1.jar");
env.put("HIVE_HOME", "/Users/gates/grid/odpi-testing/apache-hive-1.2.1-bin");
Map<String, String> results = HiveHelper.execCommand(new CommandLine("/Users/gates/grid/odpi-testing/apache-hive-1.2.1-bin/bin/hive")
.addArgument("--service")
.addArgument("jar")
+ // TODO This is the jar built by gradle, but I didn't know how to take the jar built in
+ // the build phase and reference it in the test phase. Perhaps a move operation could be
+ // put in the middle so the jar is moved to a known location that can be referenced here,
+ // or maybe gradle can pass in its working directory so that we can reference it from there.
.addArgument("/Users/gates/git/bigtop/runtime-1.2.0-SNAPSHOT.jar")
.addArgument(HCatalogMR.class.getName())
.addArgument(inputTable)