You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by he...@apache.org on 2016/09/22 04:53:15 UTC
[1/2] incubator-impala git commit: IMPALA-4171: Remove JAR from repo.
Repository: incubator-impala
Updated Branches:
refs/heads/master a8a5f6583 -> 1b9d9ea7c
IMPALA-4171: Remove JAR from repo.
By ASF rules, we can't have JARs in releases. The releases are just
tarballs of the repo.
This patch removes from the repo the single JAR there, which was a
version of a JAR that is built during data load, with one string
changed. The JAR is used only for testing.
Instead of building that jar with the different string and saving the
result in git, daa loading will now build the jar twice, with one Java
source file slightly changed.
Change-Id: Icee7b8c32b08e064dea4a14624acff6021ef5ce1
Reviewed-on: http://gerrit.cloudera.org:8080/4499
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/57fcbf7a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/57fcbf7a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/57fcbf7a
Branch: refs/heads/master
Commit: 57fcbf7a28339e55a344dece10d87d16631939fa
Parents: a8a5f65
Author: Jim Apple <jb...@cloudera.com>
Authored: Tue Sep 20 13:54:55 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Thu Sep 22 02:00:50 2016 +0000
----------------------------------------------------------------------
.gitignore | 3 +
bin/check-rat-report.py | 3 +-
testdata/bin/copy-udfs-udas.sh | 62 +++++++++++--------
testdata/udfs/README | 8 +--
testdata/udfs/impala-hive-udfs.jar | Bin 3801 -> 0 bytes
.../java/com/cloudera/impala/TestUpdateUdf.java | 12 +---
6 files changed, 46 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/57fcbf7a/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index ec52448..68a43cb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -52,3 +52,6 @@ toolchain/
core
core.*
hs_err_pid*.log
+
+# Binaries disallowed by ASF rules
+*.jar
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/57fcbf7a/bin/check-rat-report.py
----------------------------------------------------------------------
diff --git a/bin/check-rat-report.py b/bin/check-rat-report.py
index 06c822d..a2f3b52 100755
--- a/bin/check-rat-report.py
+++ b/bin/check-rat-report.py
@@ -67,7 +67,8 @@ for r in resources:
break
if not excluded:
typename = r.findall('type')[0].attrib['name']
- if not (clean_name[0:9] == 'testdata/' and typename in ['archive', 'binary']):
+ if not (clean_name[0:9] == 'testdata/' and typename in ['archive', 'binary']
+ and clean_name[-4:] <> '.jar'):
sys.stderr.write(
"%s: %s\n" %
('UNAPPROVED' if approvals else "NO APPROVALS; " + typename, clean_name))
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/57fcbf7a/testdata/bin/copy-udfs-udas.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/copy-udfs-udas.sh b/testdata/bin/copy-udfs-udas.sh
index 4440db1..57c0bc7 100755
--- a/testdata/bin/copy-udfs-udas.sh
+++ b/testdata/bin/copy-udfs-udas.sh
@@ -26,7 +26,7 @@ if [ x${JAVA_HOME} == x ]; then
echo JAVA_HOME not set
exit 1
fi
-. ${IMPALA_HOME}/bin/impala-config.sh > /dev/null 2>&1
+. "${IMPALA_HOME}/bin/impala-config.sh" > /dev/null 2>&1
BUILD=0
@@ -47,11 +47,19 @@ done
if [ $BUILD -eq 1 ]
then
- pushd $IMPALA_HOME
+ pushd "${IMPALA_HOME}"
"${MAKE_CMD:-make}" "-j${IMPALA_BUILD_THREADS:-4}" \
TestUdas TestUdfs test-udfs-ir udfsample udasample udf-sample-ir uda-sample-ir
- cd $IMPALA_HOME/tests/test-hive-udfs
- ${IMPALA_HOME}/bin/mvn-quiet.sh package
+ cd "${IMPALA_HOME}/tests/test-hive-udfs"
+ "${IMPALA_HOME}/bin/mvn-quiet.sh" package
+ cp target/test-hive-udfs-1.0.jar "${IMPALA_HOME}/testdata/udfs/impala-hive-udfs.jar"
+ # Change one of the Java files to make a new jar for testing
+ # purposes, then change it back
+ find . -type f -name 'TestUpdateUdf.java' -execdir \
+ bash -c "sed -i s/'Old UDF'/'New UDF'/g '{}'" \;
+ "${IMPALA_HOME}/bin/mvn-quiet.sh" package
+ find . -type f -name 'TestUpdateUdf.java' -execdir \
+ bash -c "sed -i s/'New UDF'/'Old UDF'/g '{}'" \;
popd
fi
@@ -63,27 +71,27 @@ fi
# impala-hive-udfs.jar
# test-udfs.ll
# udf/uda samples (.so/.ll)
-hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/testutil/libTestUdas.so\
- ${FILESYSTEM_PREFIX}/test-warehouse
-hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so\
- ${FILESYSTEM_PREFIX}/test-warehouse
-hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so\
- ${FILESYSTEM_PREFIX}/test-warehouse/libTestUdfs.SO
-hadoop fs -mkdir -p ${FILESYSTEM_PREFIX}/test-warehouse/udf_test
-hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so\
- ${FILESYSTEM_PREFIX}/test-warehouse/udf_test/libTestUdfs.so
-hadoop fs -put -f ${HIVE_HOME}/lib/hive-exec-${IMPALA_HIVE_VERSION}.jar\
- ${FILESYSTEM_PREFIX}/test-warehouse/hive-exec.jar
-hadoop fs -put -f ${IMPALA_HOME}/tests/test-hive-udfs/target/test-hive-udfs-1.0.jar\
- ${FILESYSTEM_PREFIX}/test-warehouse/impala-hive-udfs.jar
-hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/testutil/test-udfs.ll\
- ${FILESYSTEM_PREFIX}/test-warehouse
-hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/udf_samples/libudfsample.so\
- ${FILESYSTEM_PREFIX}/test-warehouse
-hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/udf_samples/udf-sample.ll\
- ${FILESYSTEM_PREFIX}/test-warehouse
-hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/udf_samples/libudasample.so\
- ${FILESYSTEM_PREFIX}/test-warehouse
-hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/udf_samples/uda-sample.ll\
- ${FILESYSTEM_PREFIX}/test-warehouse
+hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/testutil/libTestUdas.so"\
+ "${FILESYSTEM_PREFIX}/test-warehouse"
+hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so"\
+ "${FILESYSTEM_PREFIX}/test-warehouse"
+hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so"\
+ "${FILESYSTEM_PREFIX}/test-warehouse/libTestUdfs.SO"
+hadoop fs -mkdir -p "${FILESYSTEM_PREFIX}/test-warehouse/udf_test"
+hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so"\
+ "${FILESYSTEM_PREFIX}/test-warehouse/udf_test/libTestUdfs.so"
+hadoop fs -put -f "${HIVE_HOME}/lib/hive-exec-${IMPALA_HIVE_VERSION}.jar"\
+ "${FILESYSTEM_PREFIX}/test-warehouse/hive-exec.jar"
+hadoop fs -put -f "${IMPALA_HOME}/tests/test-hive-udfs/target/test-hive-udfs-1.0.jar"\
+ "${FILESYSTEM_PREFIX}/test-warehouse/impala-hive-udfs.jar"
+hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/testutil/test-udfs.ll"\
+ "${FILESYSTEM_PREFIX}/test-warehouse"
+hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/udf_samples/libudfsample.so"\
+ "${FILESYSTEM_PREFIX}/test-warehouse"
+hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/udf_samples/udf-sample.ll"\
+ "${FILESYSTEM_PREFIX}/test-warehouse"
+hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/udf_samples/libudasample.so"\
+ "${FILESYSTEM_PREFIX}/test-warehouse"
+hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/udf_samples/uda-sample.ll"\
+ "${FILESYSTEM_PREFIX}/test-warehouse"
echo "Done copying udf/uda libraries."
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/57fcbf7a/testdata/udfs/README
----------------------------------------------------------------------
diff --git a/testdata/udfs/README b/testdata/udfs/README
index d8817bb..31f03b2 100644
--- a/testdata/udfs/README
+++ b/testdata/udfs/README
@@ -1,7 +1,5 @@
-Most of the test UDFs are generated as part of the build. This folder
-contains prebuilt UDFs to facilitate tests that verify the UDF binary
-can be upgraded.
-
-The jar is generated from $IMPALA_HOME/tests/test-hive-udfs.
+Most of the test UDFs are generated as part of the build. After
+copy-udfs-udas.sh is run, This folder contains prebuilt UDFs to
+facilitate tests that verify the UDF binary can be upgraded.
This is used by test_udfs.py.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/57fcbf7a/testdata/udfs/impala-hive-udfs.jar
----------------------------------------------------------------------
diff --git a/testdata/udfs/impala-hive-udfs.jar b/testdata/udfs/impala-hive-udfs.jar
deleted file mode 100644
index d916937..0000000
Binary files a/testdata/udfs/impala-hive-udfs.jar and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/57fcbf7a/tests/test-hive-udfs/src/main/java/com/cloudera/impala/TestUpdateUdf.java
----------------------------------------------------------------------
diff --git a/tests/test-hive-udfs/src/main/java/com/cloudera/impala/TestUpdateUdf.java b/tests/test-hive-udfs/src/main/java/com/cloudera/impala/TestUpdateUdf.java
index f29e8af..2d76d73 100644
--- a/tests/test-hive-udfs/src/main/java/com/cloudera/impala/TestUpdateUdf.java
+++ b/tests/test-hive-udfs/src/main/java/com/cloudera/impala/TestUpdateUdf.java
@@ -21,8 +21,9 @@ import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.Text;
/**
- * Simple UDF for testing updating a UDF in HDFS. This file is built and
- * the old version of the jar is saved to $IMPALA_HOME/testdata/udfs/
+ * Simple UDF for testing updating a UDF in HDFS. This file is built
+ * twice, with one version of the jar having TestUpdateUdf.evaluate()
+ * return "Old UDF" and the other "New UDF".
*
* The build produces a new version of the UDF. The tests make sure the
* jar can be updated without having to restart Impalad.
@@ -32,13 +33,6 @@ public class TestUpdateUdf extends UDF {
}
public Text evaluate() {
- return new Text("New UDF");
- }
-
- // Use this function instead of the other evaluate to regenerate the checked in jar.
- /*
- public Text evaluate() {
return new Text("Old UDF");
}
- */
}
[2/2] incubator-impala git commit: IMPALA-4160: Remove some leftover
Llama references
Posted by he...@apache.org.
IMPALA-4160: Remove some leftover Llama references
Change-Id: I62e12363ab3ecca42bf7a82be3c2df01bc47cdca
Reviewed-on: http://gerrit.cloudera.org:8080/4493
Reviewed-by: Matthew Jacobs <mj...@cloudera.com>
Reviewed-by: Henry Robinson <he...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/1b9d9ea7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/1b9d9ea7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/1b9d9ea7
Branch: refs/heads/master
Commit: 1b9d9ea7c151c35b034b5e2097017794849a1b7b
Parents: 57fcbf7
Author: Henry Robinson <he...@cloudera.com>
Authored: Tue Sep 20 23:13:21 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Thu Sep 22 02:10:32 2016 +0000
----------------------------------------------------------------------
testdata/cluster/admin | 27 ++-------------------------
1 file changed, 2 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/1b9d9ea7/testdata/cluster/admin
----------------------------------------------------------------------
diff --git a/testdata/cluster/admin b/testdata/cluster/admin
index d263999..98969ca 100755
--- a/testdata/cluster/admin
+++ b/testdata/cluster/admin
@@ -18,7 +18,7 @@
# specific language governing permissions and limitations
# under the License.
-# This will create/control/destroy a local hdfs+yarn+llama cluster.
+# This will create/control/destroy a local hdfs+yarn cluster.
#
# The original idea was to run each node on a different loopback address but
# https://jira.cloudera.com/browse/CDH-16602 makes that impossible for now. So all roles
@@ -79,7 +79,6 @@ KUDU_TS_RPC_FREE_PORT_START=31200
# existing cluster.
export HDFS_WEBUI_PORT=5070 # changed from 50070 so it is not ephemeral
export YARN_WEBUI_PORT=8088 # same as default
-export LLAMA_WEBUI_PORT=1501 # same as default
export KMS_WEBUI_PORT=16000 # same as default
export KUDU_WEBUI_PORT=8051 # same as default
@@ -205,24 +204,6 @@ function create_cluster {
${MINIKDC_INIT} stop
fi
- # Llama needs a mapping of DataNodes to NodeManagers and for that we'll need to know
- # the hostname hadoop has chosen. It should be the first entry in /etc/hosts for
- # 127.0.0.1.
- #
- # It is possible this hostname must also mactch the impala hostname, which will
- # never be "localhost".
- if $IS_OSX; then
- HADOOP_HOSTNAME=$(dscacheutil -q host -a ip_address 127.0.0.1 | head -n 1 \
- | awk '{print $2}')
- else
- if getent hosts 127.0.0.1 1>/dev/null; then
- HADOOP_HOSTNAME=$(getent hosts 127.0.0.1 | awk '{print $2}')
- else
- # This may be an error case...
- HADOOP_HOSTNAME=127.0.0.1
- fi
- fi
-
# For consistency, the first node will host all the master roles.
for ((NODE_IDX=$NODE_COUNT; NODE_IDX >= 1; NODE_IDX--)); do
NODE=${NODE_PREFIX}$NODE_IDX
@@ -259,11 +240,7 @@ function create_cluster {
echo "$NODE will use ports DATANODE_PORT=$DATANODE_PORT," \
"NODEMANAGER_PORT=$NODEMANAGER_PORT, and KUDU_TS_RPC_PORT=$KUDU_TS_RPC_PORT"
- # Escape the first : to workaround https://jira.cloudera.com/browse/CDH-16840
- LLAMA_PORT_MAPPINGS+="$HADOOP_HOSTNAME\\:$DATANODE_PORT="
- LLAMA_PORT_MAPPINGS+="$HADOOP_HOSTNAME:$NODEMANAGER_PORT
-"
- export NODE NODE_DIR DATANODE_PORT NODEMANAGER_PORT LLAMA_PORT_MAPPINGS
+ export NODE NODE_DIR DATANODE_PORT NODEMANAGER_PORT
export KUDU_TS_RPC_PORT
for TEMPLATE_PATH in $(find "$NODE_DIR" -name "*$TEMPLATE_SUFFIX"); do
ACTUAL_PATH="${TEMPLATE_PATH%$TEMPLATE_SUFFIX}"