You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sp...@apache.org on 2016/12/06 21:00:11 UTC
hive git commit: HIVE-15363: Execute hive-blobstore tests using
ProxyLocalFileSystem (Sergio Pena, reviewed by Mohit Sabharwal)
Repository: hive
Updated Branches:
refs/heads/master 2f1015365 -> f317d738f
HIVE-15363: Execute hive-blobstore tests using ProxyLocalFileSystem (Sergio Pena, reviewed by Mohit Sabharwal)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f317d738
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f317d738
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f317d738
Branch: refs/heads/master
Commit: f317d738f526cf3a43955358c48612fab2cfa681
Parents: 2f10153
Author: Sergio Pena <se...@cloudera.com>
Authored: Tue Dec 6 14:59:42 2016 -0600
Committer: Sergio Pena <se...@cloudera.com>
Committed: Tue Dec 6 14:59:42 2016 -0600
----------------------------------------------------------------------
itests/hive-blobstore/pom.xml | 25 --------------------
.../src/test/resources/hive-site.xml | 19 +++++++++++++++
.../control/AbstractCoreBlobstoreCliDriver.java | 13 +++++++++-
.../org/apache/hadoop/hive/ql/QTestUtil.java | 2 +-
4 files changed, 32 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f317d738/itests/hive-blobstore/pom.xml
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/pom.xml b/itests/hive-blobstore/pom.xml
index a62885f..edc0872 100644
--- a/itests/hive-blobstore/pom.xml
+++ b/itests/hive-blobstore/pom.xml
@@ -39,31 +39,6 @@
<test.dfs.mkdir>-mkdir -p</test.dfs.mkdir>
</properties>
- <profiles>
- <profile>
- <id>tests-off</id>
- <activation>
- <file>
- <missing>src/test/resources/blobstore-conf.xml</missing>
- </file>
- </activation>
- <properties>
- <maven.test.skip>true</maven.test.skip>
- </properties>
- </profile>
- <profile>
- <id>tests-on</id>
- <activation>
- <file>
- <exists>src/test/resources/blobstore-conf.xml</exists>
- </file>
- </activation>
- <properties>
- <maven.test.skip>false</maven.test.skip>
- </properties>
- </profile>
- </profiles>
-
<dependencies>
<!-- dependencies are always listed in sorted order by groupId, artifactId -->
<dependency>
http://git-wip-us.apache.org/repos/asf/hive/blob/f317d738/itests/hive-blobstore/src/test/resources/hive-site.xml
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/resources/hive-site.xml b/itests/hive-blobstore/src/test/resources/hive-site.xml
index ffac40a..406b3b8 100644
--- a/itests/hive-blobstore/src/test/resources/hive-site.xml
+++ b/itests/hive-blobstore/src/test/resources/hive-site.xml
@@ -261,6 +261,25 @@
</property>
<!--
+ The following 3 properties are used for testing blobstore changes on the local filesystem.
+ If tests are required to run on an real Blobstore system, then create the blobstore-conf.xml.
+ -->
+ <property>
+ <name>fs.pblob.impl</name>
+ <value>org.apache.hadoop.fs.ProxyLocalFileSystem</value>
+ </property>
+
+ <property>
+ <name>hive.blobstore.supported.schemes</name>
+ <value>pblob,s3a,s3c,s3</value>
+ </property>
+
+ <property>
+ <name>test.blobstore.path</name>
+ <value>pblob:${system:test.tmp.dir}/bucket</value>
+ </property>
+
+ <!--
To run these tests:
# Create a file blobstore-conf.xml - DO NOT ADD TO REVISION CONTROL
# Add the credentials for the service you are testing against
http://git-wip-us.apache.org/repos/asf/hive/blob/f317d738/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
index 2f57683..9c97c31 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
@@ -23,7 +23,12 @@ import static org.junit.Assert.fail;
import com.google.common.base.Strings;
import java.text.SimpleDateFormat;
import java.util.Calendar;
+import java.util.Map;
+
import org.apache.hadoop.hive.cli.control.AbstractCliConfig.MetastoreType;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveVariableSource;
+import org.apache.hadoop.hive.conf.VariableSubstitution;
import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.junit.After;
@@ -157,7 +162,13 @@ public abstract class AbstractCoreBlobstoreCliDriver extends CliAdapter {
* the same ${test.blobstore.path} (possible if test runs are controlled by an automated system)
*/
private void setupUniqueTestPath() {
- String testBlobstorePath = qt.getConf().get(HCONF_TEST_BLOBSTORE_PATH);
+ String testBlobstorePath = new VariableSubstitution(new HiveVariableSource() {
+ @Override
+ public Map<String, String> getHiveVariable() {
+ return null;
+ }
+ }).substitute(new HiveConf(), qt.getConf().get(HCONF_TEST_BLOBSTORE_PATH));
+
testBlobstorePath = QTestUtil.ensurePathEndsInSlash(testBlobstorePath);
testBlobstorePath += QTestUtil.ensurePathEndsInSlash(this.getClass().getSimpleName()); // name of child class
String uid = new SimpleDateFormat("yyyyMMdd.HHmmss.SSS").format(Calendar.getInstance().getTime())
http://git-wip-us.apache.org/repos/asf/hive/blob/f317d738/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 5bf23e7..8b383c7 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -1731,7 +1731,7 @@ public class QTestUtil {
/* This list may be modified by specific cli drivers to mask strings that change on every test */
private List<Pair<Pattern, String>> patternsWithMaskComments = new ArrayList<Pair<Pattern, String>>() {{
- add(toPatternPair("(s3.?|swift|wasb.?).*hive-staging.*","### BLOBSTORE_STAGING_PATH ###"));
+ add(toPatternPair("(pblob|s3.?|swift|wasb.?).*hive-staging.*","### BLOBSTORE_STAGING_PATH ###"));
}};
private Pair<Pattern, String> toPatternPair(String patternStr, String maskComment) {