You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jb...@apache.org on 2016/09/13 15:11:27 UTC
[1/3] incubator-impala git commit: IMPALA-4096: Allow clean.sh to
work from snapshots
Repository: incubator-impala
Updated Branches:
refs/heads/master 78e129c92 -> 94fc6a3d0
IMPALA-4096: Allow clean.sh to work from snapshots
buildall.sh calls bin/clean.sh, which fails when not in a git
directory. This skips the "git clean" steps when not in a git
checkout.
Todd Lipcon found this when testing a snapshot created using git
archive.
Change-Id: I12dd9035298151557491009680d66d25c8f58c1d
Reviewed-on: http://gerrit.cloudera.org:8080/4336
Tested-by: Internal Jenkins
Reviewed-by: Jim Apple <jb...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/f3f72fa0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/f3f72fa0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/f3f72fa0
Branch: refs/heads/master
Commit: f3f72fa0e130aa8ac5f6cc57ae3385811c81b6f8
Parents: 78e129c
Author: Jim Apple <jb...@cloudera.com>
Authored: Thu Sep 8 08:59:59 2016 -0700
Committer: Jim Apple <jb...@cloudera.com>
Committed: Tue Sep 13 01:44:35 2016 +0000
----------------------------------------------------------------------
bin/clean.sh | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f3f72fa0/bin/clean.sh
----------------------------------------------------------------------
diff --git a/bin/clean.sh b/bin/clean.sh
index 9ac73cf..ec5a658 100755
--- a/bin/clean.sh
+++ b/bin/clean.sh
@@ -52,13 +52,13 @@ popd
# clean be
pushd "$IMPALA_HOME/be"
# remove everything listed in .gitignore
-git clean -Xdfq
+git rev-parse 2>/dev/null && git clean -Xdfq
popd
# clean shell build artifacts
pushd "$IMPALA_HOME/shell"
# remove everything listed in .gitignore
-git clean -Xdfq
+git rev-parse 2>/dev/null && git clean -Xdfq
popd
# Clean stale .pyc, .pyo files and __pycache__ directories.
@@ -73,7 +73,9 @@ rm -f "$IMPALA_HOME/be/generated-sources/impala-ir/"*
# Cleanup Impala-lzo
if [ -e "$IMPALA_LZO" ]; then
- pushd "$IMPALA_LZO"; git clean -fdx .; popd
+ pushd "$IMPALA_LZO"
+ git rev-parse 2>/dev/null && git clean -fdx
+ popd
fi
# When switching to and from toolchain, make sure to remove all CMake generated files
[2/3] incubator-impala git commit: IMPALA-3918: Fix straggler
Cloudera -> ASF license headers
Posted by jb...@apache.org.
IMPALA-3918: Fix straggler Cloudera -> ASF license headers
beeswax.thrift was originally copied from Hive, which is also an ASF
project.
Change-Id: Ief3e09ac93137f870b7fb31a0bc851f24a17573b
Reviewed-on: http://gerrit.cloudera.org:8080/4386
Reviewed-by: Jim Apple <jb...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/f9bcb91a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/f9bcb91a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/f9bcb91a
Branch: refs/heads/master
Commit: f9bcb91a0217867db70a00fbf53a1f5d7aa390bf
Parents: f3f72fa
Author: Jim Apple <jb...@cloudera.com>
Authored: Mon Sep 12 14:22:46 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Tue Sep 13 02:10:07 2016 +0000
----------------------------------------------------------------------
common/thrift/beeswax.thrift | 37 ++++++++++----------
.../impala/common/FrontendTestBase.java | 25 +++++++------
2 files changed, 32 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f9bcb91a/common/thrift/beeswax.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/beeswax.thrift b/common/thrift/beeswax.thrift
index a0ca5a7..6e0bb12 100644
--- a/common/thrift/beeswax.thrift
+++ b/common/thrift/beeswax.thrift
@@ -1,22 +1,21 @@
-/*
- * Licensed to Cloudera, Inc. under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. Cloudera, Inc. licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * Interface for interacting with Beeswax Server
- */
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Interface for interacting with Beeswax Server
namespace java com.cloudera.beeswax.api
namespace py beeswaxd
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f9bcb91a/fe/src/test/java/com/cloudera/impala/common/FrontendTestBase.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/com/cloudera/impala/common/FrontendTestBase.java b/fe/src/test/java/com/cloudera/impala/common/FrontendTestBase.java
index 2d54182..228dd49 100644
--- a/fe/src/test/java/com/cloudera/impala/common/FrontendTestBase.java
+++ b/fe/src/test/java/com/cloudera/impala/common/FrontendTestBase.java
@@ -1,16 +1,19 @@
-// Copyright (c) 2016 Cloudera, Inc. All rights reserved.
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
package com.cloudera.impala.common;
[3/3] incubator-impala git commit: IMPALA-4099: Fix the error message
while loading UDFs with no JARs
Posted by jb...@apache.org.
IMPALA-4099: Fix the error message while loading UDFs with no JARs
Currently we throw a generic exception due to a missing check in
the code for cases where the UDF has no resources associated with
it. This is not super useful from supportability point of view.
This commit fixes that. This also cleans up the code a little
by moving these checks out of the extractFunctions() method.
Change-Id: I167426ea96b0a41374227ef238b6f60e4184a9d7
Reviewed-on: http://gerrit.cloudera.org:8080/4365
Reviewed-by: Alex Behm <al...@cloudera.com>
Reviewed-by: Marcel Kornacker <ma...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/94fc6a3d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/94fc6a3d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/94fc6a3d
Branch: refs/heads/master
Commit: 94fc6a3d07aaf19591b832d4fc88eee831db9f99
Parents: f9bcb91
Author: Bharath Vissapragada <bh...@cloudera.com>
Authored: Fri Sep 9 17:22:28 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Tue Sep 13 03:40:02 2016 +0000
----------------------------------------------------------------------
.../impala/catalog/CatalogServiceCatalog.java | 70 +++++++++++++-------
1 file changed, 45 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/94fc6a3d/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java b/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java
index e8f5577..f43938b 100644
--- a/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java
+++ b/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java
@@ -391,6 +391,49 @@ public class CatalogServiceCatalog extends Catalog {
}
/**
+ * Checks if the Hive function 'fn' is Impala compatible. A function is Impala
+ * compatible iff
+ *
+ * 1. The function is JAVA based,
+ * 2. Has exactly one binary resource associated (We don't support loading
+ * dependencies yet) and
+ * 3. The binary is of type JAR.
+ *
+ * Returns true if compatible and false otherwise. In case of incompatible
+ * functions 'incompatMsg' has the reason for the incompatibility.
+ * */
+ public static boolean isFunctionCompatible(
+ org.apache.hadoop.hive.metastore.api.Function fn, StringBuilder incompatMsg) {
+ boolean isCompatible = true;
+ if (fn.getFunctionType() != FunctionType.JAVA) {
+ isCompatible = false;
+ incompatMsg.append("Function type: " + fn.getFunctionType().name()
+ + " is not supported. Only " + FunctionType.JAVA.name() + " functions "
+ + "are supported.");
+ } else if (fn.getResourceUrisSize() == 0) {
+ isCompatible = false;
+ incompatMsg.append("No executable binary resource (like a JAR file) is " +
+ "associated with this function. To fix this, recreate the function by " +
+ "specifying a 'location' in the function create statement.");
+ } else if (fn.getResourceUrisSize() != 1) {
+ isCompatible = false;
+ List<String> resourceUris = Lists.newArrayList();
+ for (ResourceUri resource: fn.getResourceUris()) {
+ resourceUris.add(resource.getUri());
+ }
+ incompatMsg.append("Impala does not support multiple Jars for dependencies."
+ + "(" + Joiner.on(",").join(resourceUris) + ") ");
+ } else if (fn.getResourceUris().get(0).getResourceType() != ResourceType.JAR) {
+ isCompatible = false;
+ incompatMsg.append("Function binary type: " +
+ fn.getResourceUris().get(0).getResourceType().name()
+ + " is not supported. Only " + ResourceType.JAR.name()
+ + " type is supported.");
+ }
+ return isCompatible;
+ }
+
+ /**
* Returns a list of Impala Functions, one per compatible "evaluate" method in the UDF
* class referred to by the given Java function. This method copies the UDF Jar
* referenced by "function" to a temporary file in "LOCAL_LIBRARY_PATH" and loads it
@@ -404,32 +447,9 @@ public class CatalogServiceCatalog extends Catalog {
throws ImpalaRuntimeException{
List<Function> result = Lists.newArrayList();
List<String> addedSignatures = Lists.newArrayList();
- boolean compatible = true;
StringBuilder warnMessage = new StringBuilder();
- if (function.getFunctionType() != FunctionType.JAVA) {
- compatible = false;
- warnMessage.append("Function type: " + function.getFunctionType().name()
- + " is not supported. Only " + FunctionType.JAVA.name() + " functions "
- + "are supported.");
- }
- if (function.getResourceUrisSize() != 1) {
- compatible = false;
- List<String> resourceUris = Lists.newArrayList();
- for (ResourceUri resource: function.getResourceUris()) {
- resourceUris.add(resource.getUri());
- }
- warnMessage.append("Impala does not support multiple Jars for dependencies."
- + "(" + Joiner.on(",").join(resourceUris) + ") ");
- }
- if (function.getResourceUris().get(0).getResourceType() != ResourceType.JAR) {
- compatible = false;
- warnMessage.append("Function binary type: " +
- function.getResourceUris().get(0).getResourceType().name()
- + " is not supported. Only " + ResourceType.JAR.name()
- + " type is supported.");
- }
- if (!compatible) {
- LOG.warn("Skipping load of incompatible Java function: " +
+ if (!isFunctionCompatible(function, warnMessage)) {
+ LOG.warn("Skipping load of incompatible function: " +
function.getFunctionName() + ". " + warnMessage.toString());
return result;
}