You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by gr...@apache.org on 2020/03/03 23:09:08 UTC
[kudu] branch master updated (e2301c0 -> c35871d)
This is an automated email from the ASF dual-hosted git repository.
granthenke pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git.
from e2301c0 util: deadline for BlockingQueue::Blocking{Put,Get}
new 8fb170b [hms] Fix testAlterTableHandler in later Hive versions
new c35871d [build] Move the echo subprocess to the kudu-subprocess jar.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
build-support/dist_test.py | 2 +-
build-support/run_dist_test.py | 4 +-
build-support/verify_jars.pl | 5 ++-
.../hive/metastore/TestKuduMetastorePlugin.java | 11 +++--
java/kudu-subprocess-echo/build.gradle | 51 ----------------------
.../kudu/subprocess/echo/EchoProtocolHandler.java | 0
.../kudu/subprocess/echo/EchoSubprocessMain.java | 0
.../kudu/subprocess/echo/TestEchoSubprocess.java | 0
java/settings.gradle | 1 -
src/kudu/subprocess/CMakeLists.txt | 14 +++---
src/kudu/subprocess/subprocess_server-test.cc | 7 +--
11 files changed, 25 insertions(+), 70 deletions(-)
delete mode 100644 java/kudu-subprocess-echo/build.gradle
rename java/{kudu-subprocess-echo => kudu-subprocess}/src/main/java/org/apache/kudu/subprocess/echo/EchoProtocolHandler.java (100%)
rename java/{kudu-subprocess-echo => kudu-subprocess}/src/main/java/org/apache/kudu/subprocess/echo/EchoSubprocessMain.java (100%)
rename java/{kudu-subprocess-echo => kudu-subprocess}/src/test/java/org/apache/kudu/subprocess/echo/TestEchoSubprocess.java (100%)
[kudu] 02/02: [build] Move the echo subprocess to the
kudu-subprocess jar.
Posted by gr...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
granthenke pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit c35871df7983a4a1273310483c3cde67553bc8bd
Author: Grant Henke <gr...@apache.org>
AuthorDate: Tue Mar 3 15:35:18 2020 -0600
[build] Move the echo subprocess to the kudu-subprocess jar.
This patch moves the echo subprocess to the kudu-subprocess
jar. This simplifies the build so that a single module is used
for all subprocess implementations.
Change-Id: Id42e55185007905bf831fb9214d5292817e0f8f4
Reviewed-on: http://gerrit.cloudera.org:8080/15353
Reviewed-by: Andrew Wong <aw...@cloudera.com>
Tested-by: Kudu Jenkins
---
build-support/dist_test.py | 2 +-
build-support/run_dist_test.py | 4 +-
build-support/verify_jars.pl | 5 ++-
java/kudu-subprocess-echo/build.gradle | 51 ----------------------
.../kudu/subprocess/echo/EchoProtocolHandler.java | 0
.../kudu/subprocess/echo/EchoSubprocessMain.java | 0
.../kudu/subprocess/echo/TestEchoSubprocess.java | 0
java/settings.gradle | 1 -
src/kudu/subprocess/CMakeLists.txt | 14 +++---
src/kudu/subprocess/subprocess_server-test.cc | 7 +--
10 files changed, 18 insertions(+), 66 deletions(-)
diff --git a/build-support/dist_test.py b/build-support/dist_test.py
index a7424c4..588a396 100755
--- a/build-support/dist_test.py
+++ b/build-support/dist_test.py
@@ -111,7 +111,7 @@ DEPS_FOR_ALL = \
"build/latest/bin/sentry-home",
# Add the Kudu echo subprocess.
- "build/latest/bin/kudu-subprocess-echo.jar",
+ "build/latest/bin/kudu-subprocess.jar",
# Add the Kudu HMS plugin.
"build/latest/bin/hms-plugin.jar",
diff --git a/build-support/run_dist_test.py b/build-support/run_dist_test.py
index 9ee8e8c..d613b44 100755
--- a/build-support/run_dist_test.py
+++ b/build-support/run_dist_test.py
@@ -162,8 +162,8 @@ def main():
os.path.join(bin_path, "chronyd"))
os.symlink(
glob.glob(os.path.join(ROOT,
- "java/kudu-subprocess-echo/build/libs/kudu-subprocess-echo*"))[0],
- os.path.join(bin_path, "kudu-subprocess-echo.jar"))
+ "java/kudu-subprocess/build/libs/kudu-subprocess*"))[0],
+ os.path.join(bin_path, "kudu-subprocess.jar"))
env['LD_LIBRARY_PATH'] = ":".join(
[os.path.join(ROOT, "build/dist-test-system-libs/")] +
diff --git a/build-support/verify_jars.pl b/build-support/verify_jars.pl
index 4209c44..fdb66ae 100755
--- a/build-support/verify_jars.pl
+++ b/build-support/verify_jars.pl
@@ -78,7 +78,10 @@ chomp(my @jars = `find . -type f -name \*.jar |
# Ignored because it's test only and unpublished.
grep -v kudu-jepsen.*\.jar |
# Ignored because it's a tool jar that shades everything.
- grep -v kudu-backup-tools.*\.jar`);
+ grep -v kudu-backup-tools.*\.jar |
+ # Ignored because it's an internal jar that shades everything.
+ grep -v kudu-subprocess.*\.jar`
+ );
my $num_errors = 0;
diff --git a/java/kudu-subprocess-echo/build.gradle b/java/kudu-subprocess-echo/build.gradle
deleted file mode 100644
index 8d32e68..0000000
--- a/java/kudu-subprocess-echo/build.gradle
+++ /dev/null
@@ -1,51 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-apply from: "$rootDir/gradle/protobuf.gradle"
-apply from: "$rootDir/gradle/shadow.gradle"
-
-dependencies {
- // Note: We don't use the shaded version to avoid protobuf message type
- // got shaded and causes incompatible bounds error for type casting.
- compile (project(path: ":kudu-subprocess"))
- compile libs.protobufJava
- compile libs.protobufJavaUtil
-
- optional libs.yetusAnnotations
-
- testCompile project(path: ":kudu-test-utils", configuration: "shadow")
- testCompile project(path: ":kudu-subprocess", configuration: "test")
- testCompile libs.junit
- testCompile libs.log4j
- testCompile libs.log4jSlf4jImpl
-}
-jar {
- manifest {
- attributes(
- 'Main-Class': 'org.apache.kudu.subprocess.echo.EchoSubprocessMain'
- )
- }
-}
-
-// kudu-subprocess-echo has no public Javadoc.
-javadoc {
- enabled = false
-}
-
-// Skip publishing kudu-subprocess-echo artifact because it's not intended for external use.
-uploadArchives.enabled = false
-install.enabled = false
diff --git a/java/kudu-subprocess-echo/src/main/java/org/apache/kudu/subprocess/echo/EchoProtocolHandler.java b/java/kudu-subprocess/src/main/java/org/apache/kudu/subprocess/echo/EchoProtocolHandler.java
similarity index 100%
rename from java/kudu-subprocess-echo/src/main/java/org/apache/kudu/subprocess/echo/EchoProtocolHandler.java
rename to java/kudu-subprocess/src/main/java/org/apache/kudu/subprocess/echo/EchoProtocolHandler.java
diff --git a/java/kudu-subprocess-echo/src/main/java/org/apache/kudu/subprocess/echo/EchoSubprocessMain.java b/java/kudu-subprocess/src/main/java/org/apache/kudu/subprocess/echo/EchoSubprocessMain.java
similarity index 100%
rename from java/kudu-subprocess-echo/src/main/java/org/apache/kudu/subprocess/echo/EchoSubprocessMain.java
rename to java/kudu-subprocess/src/main/java/org/apache/kudu/subprocess/echo/EchoSubprocessMain.java
diff --git a/java/kudu-subprocess-echo/src/test/java/org/apache/kudu/subprocess/echo/TestEchoSubprocess.java b/java/kudu-subprocess/src/test/java/org/apache/kudu/subprocess/echo/TestEchoSubprocess.java
similarity index 100%
rename from java/kudu-subprocess-echo/src/test/java/org/apache/kudu/subprocess/echo/TestEchoSubprocess.java
rename to java/kudu-subprocess/src/test/java/org/apache/kudu/subprocess/echo/TestEchoSubprocess.java
diff --git a/java/settings.gradle b/java/settings.gradle
index e48c62e..a6c3355 100644
--- a/java/settings.gradle
+++ b/java/settings.gradle
@@ -30,5 +30,4 @@ include "kudu-mapreduce"
include "kudu-spark"
include "kudu-spark-tools"
include "kudu-subprocess"
-include "kudu-subprocess-echo"
include "kudu-test-utils"
diff --git a/src/kudu/subprocess/CMakeLists.txt b/src/kudu/subprocess/CMakeLists.txt
index e0b7af0..62f277c 100644
--- a/src/kudu/subprocess/CMakeLists.txt
+++ b/src/kudu/subprocess/CMakeLists.txt
@@ -35,19 +35,19 @@ target_link_libraries(subprocess_proto
#######################################
# kudu_subprocess
#######################################
-set(ECHO_JAR ${EXECUTABLE_OUTPUT_PATH}/kudu-subprocess-echo.jar)
+set(SUBPROCESS_JAR ${EXECUTABLE_OUTPUT_PATH}/kudu-subprocess.jar)
# Add the EXTRA_GRADLE_FLAGS environment variable to a cmake list and calls `separate_arguments`.
# This solves the problem of escaped spaces in the EXTRA_GRADLE_FLAGS environment variable.
list(APPEND EXTRA_GRADLE_FLAGS $ENV{EXTRA_GRADLE_FLAGS})
separate_arguments(EXTRA_GRADLE_FLAGS)
-add_custom_command(OUTPUT ${ECHO_JAR}
- COMMAND ./gradlew :kudu-subprocess-echo:jar ${EXTRA_GRADLE_FLAGS}
+add_custom_command(OUTPUT ${SUBPROCESS_JAR}
+ COMMAND ./gradlew :kudu-subprocess:jar ${EXTRA_GRADLE_FLAGS}
COMMAND ln -nsf
- "${JAVA_DIR}/kudu-subprocess-echo/build/libs/kudu-subprocess-echo-${KUDU_VERSION_NUMBER}.jar"
- "${ECHO_JAR}"
+ "${JAVA_DIR}/kudu-subprocess/build/libs/kudu-subprocess-${KUDU_VERSION_NUMBER}.jar"
+ "${SUBPROCESS_JAR}"
WORKING_DIRECTORY "${JAVA_DIR}")
-add_custom_target(echo_subprocess DEPENDS ${ECHO_JAR})
+add_custom_target(subprocess_jar DEPENDS ${SUBPROCESS_JAR})
add_library(kudu_subprocess
server.cc
@@ -60,7 +60,7 @@ target_link_libraries(kudu_subprocess
tool_proto
${KUDU_BASE_LIBS}
)
-add_dependencies(kudu_subprocess echo_subprocess)
+add_dependencies(kudu_subprocess subprocess_jar)
#######################################
# Unit tests
diff --git a/src/kudu/subprocess/subprocess_server-test.cc b/src/kudu/subprocess/subprocess_server-test.cc
index d81f1b4..2de5ec7 100644
--- a/src/kudu/subprocess/subprocess_server-test.cc
+++ b/src/kudu/subprocess/subprocess_server-test.cc
@@ -83,8 +83,8 @@ class SubprocessServerTest : public KuduTest {
// Resets the subprocess server to account for any new configuration.
Status ResetSubprocessServer(int java_queue_size = 0,
int java_parser_threads = 0) {
- // Set up a subprocess server pointing at the kudu-subprocess-echo.jar that
- // contains an echo server.
+ // Set up a subprocess server pointing at the kudu-subprocess.jar that
+ // contains an echo handler and call EchoSubprocessMain.
string exe;
RETURN_NOT_OK(env_->GetExecutablePath(&exe));
const string bin_dir = DirName(exe);
@@ -92,7 +92,8 @@ class SubprocessServerTest : public KuduTest {
RETURN_NOT_OK(FindHomeDir("java", bin_dir, &java_home));
vector<string> argv = {
Substitute("$0/bin/java", java_home),
- "-jar", Substitute("$0/kudu-subprocess-echo.jar", bin_dir)
+ "-cp", Substitute("$0/kudu-subprocess.jar", bin_dir),
+ "org.apache.kudu.subprocess.echo.EchoSubprocessMain"
};
if (java_queue_size > 0) {
argv.emplace_back("q");
[kudu] 01/02: [hms] Fix testAlterTableHandler in later Hive versions
Posted by gr...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
granthenke pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 8fb170b004b8298429ada0ec5622d6f566477285
Author: Grant Henke <gr...@apache.org>
AuthorDate: Tue Mar 3 08:49:41 2020 -0600
[hms] Fix testAlterTableHandler in later Hive versions
This patch adjusts testAlterTableHandler to stop reusing
already created table objects when creating new tables.
This prevents issues if read-only properties are set.
An example of a read-only property is the ID field added in
HIVE-20556.
The error seen was:
`testAlterTableHandler(org.apache.kudu.hive.metastore.TestKuduMetastorePlugin)InvalidObjectException(message:Id shouldn't be set but table default.table has the Id set to 1. It's a read-only option)`
Change-Id: I8f8ddbe30a3e51957cafd8f1b596745b3262ca36
Reviewed-on: http://gerrit.cloudera.org:8080/15349
Reviewed-by: Hao Hao <ha...@cloudera.com>
Tested-by: Grant Henke <gr...@apache.org>
---
.../apache/kudu/hive/metastore/TestKuduMetastorePlugin.java | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java b/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java
index 2dc358a..0558746 100644
--- a/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java
+++ b/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java
@@ -233,10 +233,10 @@ public class TestKuduMetastorePlugin {
@Test
public void testAlterTableHandler() throws Exception {
// Test altering a Kudu (or a legacy) table.
- Table table = newTable("table");
- client.createTable(table, masterContext());
+ Table initTable = newTable("table");
+ client.createTable(initTable, masterContext());
// Get the table from the HMS in case any translation occurred.
- table = client.getTable(table.getDbName(), table.getTableName());
+ Table table = client.getTable(initTable.getDbName(), initTable.getTableName());
Table legacyTable = newLegacyTable("legacy_table");
client.createTable(legacyTable, masterContext());
// Get the table from the HMS in case any translation occurred.
@@ -380,8 +380,10 @@ public class TestKuduMetastorePlugin {
// Test altering a non-Kudu table.
{
+ table = initTable.deepCopy();
table.getParameters().clear();
client.createTable(table);
+ table = client.getTable(table.getDbName(), table.getTableName());
try {
// Try to alter the table and add a Kudu table ID.
@@ -431,11 +433,12 @@ public class TestKuduMetastorePlugin {
// Test altering an unsynchronized table is accepted.
{
- table.getParameters().clear();
+ table = initTable.deepCopy();
table.setTableType(TableType.EXTERNAL_TABLE.name());
table.putToParameters(KuduMetastorePlugin.EXTERNAL_TABLE_KEY, "TRUE");
table.putToParameters(KuduMetastorePlugin.EXTERNAL_PURGE_KEY, "FALSE");
client.createTable(table);
+ table = client.getTable(table.getDbName(), table.getTableName());
try {
client.alter_table(table.getDbName(), table.getTableName(), table);
} finally {