You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2016/10/11 06:52:42 UTC
oozie git commit: OOZIE-2613 Upgrade hive version from 0.13.1 to
1.2.0 (abhishekbafna via rkanter)
Repository: oozie
Updated Branches:
refs/heads/master e0016c93a -> bd5445280
OOZIE-2613 Upgrade hive version from 0.13.1 to 1.2.0 (abhishekbafna via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/bd544528
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/bd544528
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/bd544528
Branch: refs/heads/master
Commit: bd54452808335cbadc48d4aacd0bd23374bed928
Parents: e0016c9
Author: Robert Kanter <rk...@cloudera.com>
Authored: Mon Oct 10 23:52:34 2016 -0700
Committer: Robert Kanter <rk...@cloudera.com>
Committed: Mon Oct 10 23:52:34 2016 -0700
----------------------------------------------------------------------
.../action/hadoop/TestFsActionExecutor.java | 3 ++
.../command/coord/TestCoordRerunXCommand.java | 3 ++
.../oozie/dependency/TestHCatURIHandler.java | 3 ++
.../org/apache/oozie/test/MiniHCatServer.java | 10 ----
docs/src/site/twiki/DG_QuickStart.twiki | 2 +-
docs/src/site/twiki/ENG_Building.twiki | 2 +-
pom.xml | 14 +++--
release-log.txt | 1 +
.../oozie/action/hadoop/TestHiveMain.java | 4 +-
sharelib/hive2/pom.xml | 56 ++++++++++++++++++++
.../action/hadoop/TestHive2ActionExecutor.java | 1 +
11 files changed, 76 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/bd544528/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java
index 5345ae6..b27ede7 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java
@@ -35,6 +35,7 @@ import org.apache.oozie.service.HadoopAccessorService;
import org.apache.oozie.service.Services;
import org.apache.oozie.service.URIHandlerService;
import org.apache.oozie.service.WorkflowAppService;
+import org.apache.oozie.test.XTestCase;
import org.apache.oozie.util.XConfiguration;
import org.apache.oozie.util.XmlUtils;
import org.jdom.Element;
@@ -303,6 +304,8 @@ public class TestFsActionExecutor extends ActionExecutorTestCase {
}
public void testDeleteHcat() throws Exception {
+ //setting current user as test user because directory structure created by HCat have current user permissions (755).
+ setSystemProperty(XTestCase.TEST_USER1_PROP, System.getProperty("user.name"));
createTestTable();
addPartition(db, table, "year=2012;month=12;dt=02;country=us");
URI hcatURI = getHCatURI(db, table, "country=us;year=2012;month=12;dt=02");
http://git-wip-us.apache.org/repos/asf/oozie/blob/bd544528/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java
index 891e6ec..9a05a85 100644
--- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java
@@ -60,6 +60,7 @@ import org.apache.oozie.service.StatusTransitService;
import org.apache.oozie.service.URIHandlerService;
import org.apache.oozie.store.StoreException;
import org.apache.oozie.test.XDataTestCase;
+import org.apache.oozie.test.XTestCase;
import org.apache.oozie.util.DateUtils;
import org.apache.oozie.util.IOUtils;
import org.apache.oozie.util.XLog;
@@ -569,6 +570,8 @@ public class TestCoordRerunXCommand extends XDataTestCase {
* @throws Exception
*/
public void testCoordRerunCleanupForHCat() throws Exception {
+ //setting current user as test user because directory structure created by HCat have current user permissions (755).
+ setSystemProperty(XTestCase.TEST_USER1_PROP, System.getProperty("user.name"));
super.setupHCatalogServer();
services = super.setupServicesForHCatalog();
services.init();
http://git-wip-us.apache.org/repos/asf/oozie/blob/bd544528/core/src/test/java/org/apache/oozie/dependency/TestHCatURIHandler.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/dependency/TestHCatURIHandler.java b/core/src/test/java/org/apache/oozie/dependency/TestHCatURIHandler.java
index a49eba5..615f5e1 100644
--- a/core/src/test/java/org/apache/oozie/dependency/TestHCatURIHandler.java
+++ b/core/src/test/java/org/apache/oozie/dependency/TestHCatURIHandler.java
@@ -25,6 +25,7 @@ import org.apache.oozie.service.HCatAccessorService;
import org.apache.oozie.service.Services;
import org.apache.oozie.service.URIHandlerService;
import org.apache.oozie.test.XHCatTestCase;
+import org.apache.oozie.test.XTestCase;
import org.junit.Test;
public class TestHCatURIHandler extends XHCatTestCase {
@@ -67,6 +68,8 @@ public class TestHCatURIHandler extends XHCatTestCase {
@Test
public void testExists() throws Exception {
+ //setting current user as test user because directory structure created by HCat have current user permissions (755).
+ setSystemProperty(XTestCase.TEST_USER1_PROP, System.getProperty("user.name"));
createTestTable();
addPartition(db, table, "year=2012;month=12;dt=02;country=us");
http://git-wip-us.apache.org/repos/asf/oozie/blob/bd544528/core/src/test/java/org/apache/oozie/test/MiniHCatServer.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/test/MiniHCatServer.java b/core/src/test/java/org/apache/oozie/test/MiniHCatServer.java
index 8699ff8..9b7a004 100644
--- a/core/src/test/java/org/apache/oozie/test/MiniHCatServer.java
+++ b/core/src/test/java/org/apache/oozie/test/MiniHCatServer.java
@@ -38,7 +38,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStore;
-import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hive.hcatalog.api.HCatAddPartitionDesc;
import org.apache.hive.hcatalog.api.HCatClient;
@@ -85,7 +84,6 @@ public class MiniHCatServer {
initMetastoreServerConf();
}
hcatClient = HCatClient.create(hiveConf);
- resetDefaultDBCreation();
}
public void shutdown() throws Exception {
@@ -147,14 +145,6 @@ public class MiniHCatServer {
Thread.sleep(10000L);
}
- public static void resetDefaultDBCreation() throws Exception {
- // Need to do this, else default db will not be created for local metastores.
- // TestHiveMain will fail with InvalidObjectException(message:There is no database named default)
- Field declaredField = HMSHandler.class.getDeclaredField("createDefaultDB");
- declaredField.setAccessible(true);
- declaredField.set(null, false);
- }
-
public static void resetHiveConfStaticVariables() throws Exception {
// HiveConf initializes location of hive-site.xml in static block.
// So this is needed so that tests like TestHiveMain that create hive-site.xml don't fail.
http://git-wip-us.apache.org/repos/asf/oozie/blob/bd544528/docs/src/site/twiki/DG_QuickStart.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/DG_QuickStart.twiki b/docs/src/site/twiki/DG_QuickStart.twiki
index 3938ff4..1eb02a9 100644
--- a/docs/src/site/twiki/DG_QuickStart.twiki
+++ b/docs/src/site/twiki/DG_QuickStart.twiki
@@ -54,7 +54,7 @@ profile depending on the hadoop version used.
-Dopenjpa.version=<version> - default 2.2.2
-Dxerces.version=<version> - default 2.10.0
-Dcurator.version=<version> - default 2.5.0
--Dhive.version=<version> - default 0.13.1
+-Dhive.version=<version> - default 0.13.1 for hadoop-1, 1.2.0 for hadoop-2 and hadoop-3 profile
-Dhbase.version=<version> - default 0.94.2
</verbatim>
http://git-wip-us.apache.org/repos/asf/oozie/blob/bd544528/docs/src/site/twiki/ENG_Building.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/ENG_Building.twiki b/docs/src/site/twiki/ENG_Building.twiki
index f6c88d6..3560a9b 100644
--- a/docs/src/site/twiki/ENG_Building.twiki
+++ b/docs/src/site/twiki/ENG_Building.twiki
@@ -225,7 +225,7 @@ profile depending on the hadoop version used.
-Dopenjpa.version=<version> - default 2.2.2
-Dxerces.version=<version> - default 2.10.0
-Dcurator.version=<version> - default 2.5.0
--Dhive.version=<version> - default 0.13.1
+-Dhive.version=<version> - default 0.13.1 for hadoop-1, 1.2.0 for hadoop-2 and hadoop-3 profile
-Dhbase.version=<version> - default 0.94.2
</verbatim>
http://git-wip-us.apache.org/repos/asf/oozie/blob/bd544528/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a33d2c5..83a1d42 100644
--- a/pom.xml
+++ b/pom.xml
@@ -89,8 +89,8 @@
<clover.license>/home/jenkins/tools/clover/latest/lib/clover.license</clover.license>
<!-- Sharelib component versions -->
- <hive.version>0.13.1</hive.version>
- <hive.jline.version>0.9.94</hive.jline.version>
+ <hive.version>1.2.0</hive.version>
+ <hive.jline.version>2.12</hive.jline.version>
<pig.version>0.16.0</pig.version>
<pig.classifier></pig.classifier>
<sqoop.version>1.4.3</sqoop.version>
@@ -106,7 +106,7 @@
<!-- Tomcat version -->
<tomcat.version>6.0.44</tomcat.version>
-
+ <jline.version>0.9.94</jline.version>
<openjpa.version>2.4.1</openjpa.version>
<xerces.version>2.10.0</xerces.version>
<curator.version>2.5.0</curator.version>
@@ -878,7 +878,7 @@
<dependency>
<groupId>jline</groupId>
<artifactId>jline</artifactId>
- <version>0.9.94</version>
+ <version>${jline.version}</version>
<exclusions>
<exclusion>
<groupId>junit</groupId>
@@ -987,10 +987,6 @@
<groupId>org.apache.hive</groupId>
</exclusion>
<exclusion>
- <artifactId>hive-common</artifactId>
- <groupId>org.apache.hive</groupId>
- </exclusion>
- <exclusion>
<artifactId>hive-serde</artifactId>
<groupId>org.apache.hive</groupId>
</exclusion>
@@ -1860,6 +1856,8 @@
<hadoop.majorversion>1</hadoop.majorversion>
<pig.classifier></pig.classifier>
<sqoop.classifier>hadoop100</sqoop.classifier>
+ <hive.version>0.13.1</hive.version>
+ <hive.jline.version>0.9.94</hive.jline.version>
</properties>
</profile>
<profile>
http://git-wip-us.apache.org/repos/asf/oozie/blob/bd544528/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index a34525e..f73fbb8 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -5,6 +5,7 @@ OOZIE-2634 Queue dump command message is confusing when the queue is empty (andr
-- Oozie 4.3.0 release
+OOZIE-2613 Upgrade hive version from 0.13.1 to 1.2.0 (abhishekbafna via rkanter)
OOZIE-2658 --driver-class-path can overwrite the classpath in SparkMain (gezapeti via rkanter)
OOZIE-1814 Oozie should mask any passwords in logs and REST interfaces (andras.piros via rkanter)
OOZIE-2622 ExtJS 2.2 is no longer available (rkanter)
http://git-wip-us.apache.org/repos/asf/oozie/blob/bd544528/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
----------------------------------------------------------------------
diff --git a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
index 879ae2d..3026311 100644
--- a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
+++ b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
@@ -50,6 +50,7 @@ public class TestHiveMain extends MainTestCase {
private String getHiveScript(String inputPath, String outputPath) {
StringBuilder buffer = new StringBuilder(NEW_LINE);
buffer.append("set -v;").append(NEW_LINE);
+ buffer.append("CREATE DATABASE IF NOT EXISTS default;").append(NEW_LINE);
buffer.append("DROP TABLE IF EXISTS test;").append(NEW_LINE);
buffer.append("CREATE EXTERNAL TABLE test (a INT) STORED AS");
buffer.append(NEW_LINE).append("TEXTFILE LOCATION '");
@@ -57,7 +58,6 @@ public class TestHiveMain extends MainTestCase {
buffer.append("INSERT OVERWRITE DIRECTORY '");
buffer.append(outputPath).append("'").append(NEW_LINE);
buffer.append("SELECT (a-1) FROM test;").append(NEW_LINE);
-
return buffer.toString();
}
@@ -133,7 +133,6 @@ public class TestHiveMain extends MainTestCase {
os = new FileOutputStream(hiveSite);
jobConf.writeXml(os);
os.close();
- MiniHCatServer.resetDefaultDBCreation();
MiniHCatServer.resetHiveConfStaticVariables();
HiveMain.main(null);
}
@@ -166,5 +165,4 @@ public class TestHiveMain extends MainTestCase {
}
return null;
}
-
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/bd544528/sharelib/hive2/pom.xml
----------------------------------------------------------------------
diff --git a/sharelib/hive2/pom.xml b/sharelib/hive2/pom.xml
index bc94fd5..3dd9a4d 100644
--- a/sharelib/hive2/pom.xml
+++ b/sharelib/hive2/pom.xml
@@ -40,6 +40,62 @@
<dependencies>
<dependency>
<groupId>org.apache.hive</groupId>
+ <artifactId>hive-cli</artifactId>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-hwi</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-jdbc</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-anttasks</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hive</groupId>
<artifactId>hive-beeline</artifactId>
<scope>compile</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/oozie/blob/bd544528/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java
----------------------------------------------------------------------
diff --git a/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java b/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java
index b023b79..4818bb6 100644
--- a/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java
+++ b/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java
@@ -156,6 +156,7 @@ public class TestHive2ActionExecutor extends ActionExecutorTestCase {
private String getHive2Script(String inputPath, String outputPath) {
StringBuilder buffer = new StringBuilder(NEW_LINE);
buffer.append("set -v;").append(NEW_LINE);
+ buffer.append("CREATE DATABASE IF NOT EXISTS default;").append(NEW_LINE);
buffer.append("DROP TABLE IF EXISTS test;").append(NEW_LINE);
buffer.append("CREATE EXTERNAL TABLE test (a INT) STORED AS");
buffer.append(NEW_LINE).append("TEXTFILE LOCATION '");