You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2013/10/23 05:59:30 UTC
svn commit: r1534903 [5/5] - in /hive/branches/tez: ./
common/src/java/org/apache/hadoop/hive/conf/ conf/ data/files/
hbase-handler/src/java/org/apache/hadoop/hive/hbase/ hcatalog/bin/
hcatalog/src/test/e2e/templeton/inpdir/ hcatalog/src/test/e2e/templ...
Modified: hive/branches/tez/ql/src/test/results/compiler/plan/udf_when.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/compiler/plan/udf_when.q.xml?rev=1534903&r1=1534902&r2=1534903&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/compiler/plan/udf_when.q.xml (original)
+++ hive/branches/tez/ql/src/test/results/compiler/plan/udf_when.q.xml Wed Oct 23 03:59:27 2013
@@ -377,11 +377,11 @@
<void method="put">
<string>_col1</string>
<object id="ExprNodeGenericFuncDesc0" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc">
- <void property="childExprs">
+ <void property="children">
<object class="java.util.ArrayList">
<void method="add">
<object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc">
- <void property="childExprs">
+ <void property="children">
<object class="java.util.ArrayList">
<void method="add">
<object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc">
@@ -429,7 +429,7 @@
</void>
<void method="add">
<object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc">
- <void property="childExprs">
+ <void property="children">
<object class="java.util.ArrayList">
<void method="add">
<object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc">
@@ -484,11 +484,11 @@
<void method="put">
<string>_col0</string>
<object id="ExprNodeGenericFuncDesc1" class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc">
- <void property="childExprs">
+ <void property="children">
<object class="java.util.ArrayList">
<void method="add">
<object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc">
- <void property="childExprs">
+ <void property="children">
<object class="java.util.ArrayList">
<void method="add">
<object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc">
@@ -532,7 +532,7 @@
</void>
<void method="add">
<object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc">
- <void property="childExprs">
+ <void property="children">
<object class="java.util.ArrayList">
<void method="add">
<object class="org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc">
Modified: hive/branches/tez/ql/src/test/results/compiler/plan/union.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/compiler/plan/union.q.xml?rev=1534903&r1=1534902&r2=1534903&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/compiler/plan/union.q.xml (original)
+++ hive/branches/tez/ql/src/test/results/compiler/plan/union.q.xml Wed Oct 23 03:59:27 2013
@@ -1046,7 +1046,7 @@
<object class="org.apache.hadoop.hive.ql.plan.FilterDesc">
<void property="predicate">
<object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc">
- <void property="childExprs">
+ <void property="children">
<object class="java.util.ArrayList">
<void method="add">
<object class="org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc">
@@ -1492,7 +1492,7 @@
<object class="org.apache.hadoop.hive.ql.plan.FilterDesc">
<void property="predicate">
<object class="org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc">
- <void property="childExprs">
+ <void property="children">
<object class="java.util.ArrayList">
<void method="add">
<object class="org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc">
Modified: hive/branches/tez/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java?rev=1534903&r1=1534902&r2=1534903&view=diff
==============================================================================
--- hive/branches/tez/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java (original)
+++ hive/branches/tez/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java Wed Oct 23 03:59:27 2013
@@ -107,8 +107,10 @@ public class HiveCommandOperation extend
response = commandProcessor.run(commandArgs);
int returnCode = response.getResponseCode();
- String sqlState = response.getSQLState();
- String errorMessage = response.getErrorMessage();
+ if (returnCode != 0) {
+ throw new HiveSQLException("Error while processing statement: "
+ + response.getErrorMessage(), response.getSQLState(), response.getResponseCode());
+ }
Schema schema = response.getSchema();
if (schema != null) {
setHasResultSet(true);
@@ -117,6 +119,9 @@ public class HiveCommandOperation extend
setHasResultSet(false);
resultSchema = new TableSchema();
}
+ } catch (HiveSQLException e) {
+ setState(OperationState.ERROR);
+ throw e;
} catch (Exception e) {
setState(OperationState.ERROR);
throw new HiveSQLException("Error running query: " + e.toString(), e);
Modified: hive/branches/tez/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1534903&r1=1534902&r2=1534903&view=diff
==============================================================================
--- hive/branches/tez/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (original)
+++ hive/branches/tez/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java Wed Oct 23 03:59:27 2013
@@ -43,6 +43,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.ProxyFileSystem;
import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil;
@@ -783,4 +784,8 @@ public class Hadoop20Shims implements Ha
public WebHCatJTShim getWebHCatShim(Configuration conf, UserGroupInformation ugi) throws IOException {
throw new UnsupportedOperationException("WebHCat does not support Hadoop 0.20.x");
}
+ @Override
+ public FileSystem createProxyFileSystem(FileSystem fs, URI uri) {
+ return new ProxyFileSystem(fs, uri);
+ }
}
Modified: hive/branches/tez/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1534903&r1=1534902&r2=1534903&view=diff
==============================================================================
--- hive/branches/tez/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (original)
+++ hive/branches/tez/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java Wed Oct 23 03:59:27 2013
@@ -22,6 +22,7 @@ import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Iterator;
+import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
@@ -30,6 +31,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.ProxyFileSystem;
import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.mapred.JobTracker;
@@ -372,4 +374,8 @@ public class Hadoop20SShims extends Hado
return fs.getFileBlockLocations(status, 0, status.getLen());
}
+ @Override
+ public FileSystem createProxyFileSystem(FileSystem fs, URI uri) {
+ return new ProxyFileSystem(fs, uri);
+ }
}
Modified: hive/branches/tez/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1534903&r1=1534902&r2=1534903&view=diff
==============================================================================
--- hive/branches/tez/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original)
+++ hive/branches/tez/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Wed Oct 23 03:59:27 2013
@@ -24,6 +24,8 @@ import java.net.MalformedURLException;
import java.net.URL;
import java.util.Iterator;
import java.util.Map;
+import java.net.URI;
+import java.io.FileNotFoundException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
@@ -33,6 +35,9 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.ProxyFileSystem;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -407,4 +412,40 @@ public class Hadoop23Shims extends Hadoo
}
}
+ class ProxyFileSystem23 extends ProxyFileSystem {
+ public ProxyFileSystem23(FileSystem fs) {
+ super(fs);
+ }
+ public ProxyFileSystem23(FileSystem fs, URI uri) {
+ super(fs, uri);
+ }
+
+ @Override
+ public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f)
+ throws FileNotFoundException, IOException {
+ return new RemoteIterator<LocatedFileStatus>() {
+ private RemoteIterator<LocatedFileStatus> stats =
+ ProxyFileSystem23.super.listLocatedStatus(
+ ProxyFileSystem23.super.swizzleParamPath(f));
+
+ @Override
+ public boolean hasNext() throws IOException {
+ return stats.hasNext();
+ }
+
+ @Override
+ public LocatedFileStatus next() throws IOException {
+ LocatedFileStatus result = stats.next();
+ return new LocatedFileStatus(
+ ProxyFileSystem23.super.swizzleFileStatus(result, false),
+ result.getBlockLocations());
+ }
+ };
+ }
+ }
+
+ @Override
+ public FileSystem createProxyFileSystem(FileSystem fs, URI uri) {
+ return new ProxyFileSystem23(fs, uri);
+ }
}
Modified: hive/branches/tez/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java?rev=1534903&r1=1534902&r2=1534903&view=diff
==============================================================================
--- hive/branches/tez/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java (original)
+++ hive/branches/tez/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java Wed Oct 23 03:59:27 2013
@@ -622,4 +622,7 @@ public abstract class HadoopShimsSecure
@Override
abstract public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration conf)
throws IOException;
+
+ @Override
+ abstract public FileSystem createProxyFileSystem(FileSystem fs, URI uri);
}
Modified: hive/branches/tez/shims/src/common/java/org/apache/hadoop/fs/ProxyFileSystem.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/shims/src/common/java/org/apache/hadoop/fs/ProxyFileSystem.java?rev=1534903&r1=1534902&r2=1534903&view=diff
==============================================================================
--- hive/branches/tez/shims/src/common/java/org/apache/hadoop/fs/ProxyFileSystem.java (original)
+++ hive/branches/tez/shims/src/common/java/org/apache/hadoop/fs/ProxyFileSystem.java Wed Oct 23 03:59:27 2013
@@ -45,7 +45,7 @@ public class ProxyFileSystem extends Fil
- private Path swizzleParamPath(Path p) {
+ protected Path swizzleParamPath(Path p) {
String pathUriString = p.toUri().toString();
URI newPathUri = URI.create(pathUriString);
return new Path (realScheme, realAuthority, newPathUri.getPath());
@@ -57,7 +57,7 @@ public class ProxyFileSystem extends Fil
return new Path (myScheme, myAuthority, newPathUri.getPath());
}
- private FileStatus swizzleFileStatus(FileStatus orig, boolean isParam) {
+ protected FileStatus swizzleFileStatus(FileStatus orig, boolean isParam) {
FileStatus ret =
new FileStatus(orig.getLen(), orig.isDir(), orig.getReplication(),
orig.getBlockSize(), orig.getModificationTime(),
Modified: hive/branches/tez/shims/src/common/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/shims/src/common/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java?rev=1534903&r1=1534902&r2=1534903&view=diff
==============================================================================
--- hive/branches/tez/shims/src/common/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java (original)
+++ hive/branches/tez/shims/src/common/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java Wed Oct 23 03:59:27 2013
@@ -23,6 +23,8 @@ import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.hive.shims.HadoopShims;
/****************************************************************
* A Proxy for LocalFileSystem
@@ -61,7 +63,9 @@ public class ProxyLocalFileSystem extend
String authority = name.getAuthority() != null ? name.getAuthority() : "";
String proxyUriString = nameUriString + "://" + authority + "/";
- fs = new ProxyFileSystem(localFs, URI.create(proxyUriString));
+
+ fs = ShimLoader.getHadoopShims().createProxyFileSystem(
+ localFs, URI.create(proxyUriString));
fs.initialize(name, conf);
}
Modified: hive/branches/tez/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1534903&r1=1534902&r2=1534903&view=diff
==============================================================================
--- hive/branches/tez/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java (original)
+++ hive/branches/tez/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java Wed Oct 23 03:59:27 2013
@@ -562,4 +562,10 @@ public interface HadoopShims {
*/
public void close();
}
+
+ /**
+ * Create a proxy file system that can serve a given scheme/authority using some
+ * other file system.
+ */
+ public FileSystem createProxyFileSystem(FileSystem fs, URI uri);
}
Modified: hive/branches/tez/shims/src/common/java/org/apache/hadoop/hive/thrift/TUGIContainingTransport.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/shims/src/common/java/org/apache/hadoop/hive/thrift/TUGIContainingTransport.java?rev=1534903&r1=1534902&r2=1534903&view=diff
==============================================================================
--- hive/branches/tez/shims/src/common/java/org/apache/hadoop/hive/thrift/TUGIContainingTransport.java (original)
+++ hive/branches/tez/shims/src/common/java/org/apache/hadoop/hive/thrift/TUGIContainingTransport.java Wed Oct 23 03:59:27 2013
@@ -82,8 +82,15 @@ public class TUGIContainingTransport ext
// UGI information is not available at connection setup time, it will be set later
// via set_ugi() rpc.
- transMap.putIfAbsent(trans, new TUGIContainingTransport(trans));
- return transMap.get(trans);
+ TUGIContainingTransport tugiTrans = transMap.get(trans);
+ if (tugiTrans == null) {
+ tugiTrans = new TUGIContainingTransport(trans);
+ TUGIContainingTransport prev = transMap.putIfAbsent(trans, tugiTrans);
+ if (prev != null) {
+ return prev;
+ }
+ }
+ return tugiTrans;
}
}
}
Modified: hive/branches/tez/testutils/ptest2/src/main/resources/batch-exec.vm
URL: http://svn.apache.org/viewvc/hive/branches/tez/testutils/ptest2/src/main/resources/batch-exec.vm?rev=1534903&r1=1534902&r2=1534903&view=diff
==============================================================================
--- hive/branches/tez/testutils/ptest2/src/main/resources/batch-exec.vm (original)
+++ hive/branches/tez/testutils/ptest2/src/main/resources/batch-exec.vm Wed Oct 23 03:59:27 2013
@@ -62,7 +62,7 @@ then
testModule=./
fi
pushd $testModule
- timeout 2h mvn -o test -Dmaven.repo.local=$localDir/$instanceName/maven \
+ timeout 2h mvn -B -o test -Dmaven.repo.local=$localDir/$instanceName/maven \
$mavenArgs $mavenTestArgs $testArguments 1>$logDir/maven-test.txt 2>&1 </dev/null &
#[[
pid=$!
Modified: hive/branches/tez/testutils/ptest2/src/main/resources/smart-apply-patch.sh
URL: http://svn.apache.org/viewvc/hive/branches/tez/testutils/ptest2/src/main/resources/smart-apply-patch.sh?rev=1534903&r1=1534902&r2=1534903&view=diff
==============================================================================
--- hive/branches/tez/testutils/ptest2/src/main/resources/smart-apply-patch.sh (original)
+++ hive/branches/tez/testutils/ptest2/src/main/resources/smart-apply-patch.sh Wed Oct 23 03:59:27 2013
@@ -84,7 +84,7 @@ elif $PATCH -p1 -E --dry-run < $PATCH_FI
elif $PATCH -p2 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
PLEVEL=2
else
- echo "The patch does not appear to apply with p0 to p2";
+ echo "The patch does not appear to apply with p0, p1, or p2";
cleanup 1;
fi
Modified: hive/branches/tez/testutils/ptest2/src/main/resources/source-prep.vm
URL: http://svn.apache.org/viewvc/hive/branches/tez/testutils/ptest2/src/main/resources/source-prep.vm?rev=1534903&r1=1534902&r2=1534903&view=diff
==============================================================================
--- hive/branches/tez/testutils/ptest2/src/main/resources/source-prep.vm (original)
+++ hive/branches/tez/testutils/ptest2/src/main/resources/source-prep.vm Wed Oct 23 03:59:27 2013
@@ -85,11 +85,11 @@ cd $workingDir/
if [[ "${buildTool}" == "maven" ]]
then
rm -rf $workingDir/maven/org/apache/hive
- mvn clean install -DskipTests -Dmaven.repo.local=$workingDir/maven
- mvn test -Dmaven.repo.local=$workingDir/maven -Dtest=nothing
+ mvn -B clean install -DskipTests -Dmaven.repo.local=$workingDir/maven
+ mvn -B test -Dmaven.repo.local=$workingDir/maven -Dtest=TestDummy
cd itests
- mvn clean install -DskipTests -Dmaven.repo.local=$workingDir/maven
- mvn test -Dmaven.repo.local=$workingDir/maven -Dtest=nothing
+ mvn -B clean install -DskipTests -Dmaven.repo.local=$workingDir/maven
+ mvn -B test -Dmaven.repo.local=$workingDir/maven -Dtest=TestDummy
elif [[ "${buildTool}" == "ant" ]]
then
ant $antArgs -Divy.default.ivy.user.dir=$workingDir/ivy \