You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by st...@apache.org on 2022/10/07 13:02:09 UTC

[impala] branch master updated (e142e48f3 -> 276759271)

This is an automated email from the ASF dual-hosted git repository.

stigahuang pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git


    from e142e48f3 IMPALA-11631 Fix impala crashes in impala::TopNNode::Heap::Close()
     new ded49055d IMPALA-9448: (Addendum) Avoid generic recreate for Ozone bucket
     new 276759271 IMPALA-9823: Make use_local_catalog and related flags visible

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 be/src/runtime/exec-env.cc                   | 13 +++++++------
 testdata/bin/load-test-warehouse-snapshot.sh | 11 +++++++----
 tests/common/environ.py                      |  1 -
 tests/custom_cluster/test_exchange_delays.py |  5 +++--
 tests/util/filesystem_utils.py               |  1 +
 5 files changed, 18 insertions(+), 13 deletions(-)


[impala] 01/02: IMPALA-9448: (Addendum) Avoid generic recreate for Ozone bucket

Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit ded49055d3c21e4a0f92a6b827b6997dc73a09c8
Author: Michael Smith <mi...@cloudera.com>
AuthorDate: Thu Sep 29 16:46:41 2022 -0700

    IMPALA-9448: (Addendum) Avoid generic recreate for Ozone bucket
    
    Avoids using `hadoop fs` to recreate an Ozone bucket during snapshot
    load, as this removes the encryption key.
    
    Change-Id: I31a3813fac6f2d68859ba7fed53b5c5a82502780
    Reviewed-on: http://gerrit.cloudera.org:8080/19082
    Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 testdata/bin/load-test-warehouse-snapshot.sh | 11 +++++++----
 tests/custom_cluster/test_exchange_delays.py |  5 +++--
 tests/util/filesystem_utils.py               |  1 +
 3 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/testdata/bin/load-test-warehouse-snapshot.sh b/testdata/bin/load-test-warehouse-snapshot.sh
index cef8dc6ca..11f60a237 100755
--- a/testdata/bin/load-test-warehouse-snapshot.sh
+++ b/testdata/bin/load-test-warehouse-snapshot.sh
@@ -70,17 +70,20 @@ if [[ "$REPLY" =~ ^[Yy]$ ]]; then
   else
     # Either isilon or hdfs, no change in procedure.
     if hadoop fs -test -d ${FILESYSTEM_PREFIX}${TEST_WAREHOUSE_DIR}; then
-      echo "Removing existing ${TEST_WAREHOUSE_DIR} directory"
+      echo "Removing existing ${TEST_WAREHOUSE_DIR} directory contents"
       # For filesystems that don't allow 'rm' without 'x', chmod to 777 for the
       # subsequent 'rm -r'.
       if [ "${TARGET_FILESYSTEM}" = "isilon" ] || \
           [ "${TARGET_FILESYSTEM}" = "local" ]; then
         hadoop fs -chmod -R 777 ${FILESYSTEM_PREFIX}${TEST_WAREHOUSE_DIR}
       fi
-      hadoop fs -rm -r -skipTrash ${FILESYSTEM_PREFIX}${TEST_WAREHOUSE_DIR}
+      hadoop fs -rm -f -r -skipTrash "${FILESYSTEM_PREFIX}${TEST_WAREHOUSE_DIR}/*"
+    else
+      # Only create the directory if it doesn't exist. Some filesystems - such as Ozone -
+      # are created with extra properties.
+      echo "Creating ${TEST_WAREHOUSE_DIR} directory"
+      hadoop fs -mkdir -p ${FILESYSTEM_PREFIX}${TEST_WAREHOUSE_DIR}
     fi
-    echo "Creating ${TEST_WAREHOUSE_DIR} directory"
-    hadoop fs -mkdir -p ${FILESYSTEM_PREFIX}${TEST_WAREHOUSE_DIR}
     if [[ -n "${HDFS_ERASURECODE_POLICY:-}" ]]; then
       hdfs ec -enablePolicy -policy "${HDFS_ERASURECODE_POLICY}"
       hdfs ec -setPolicy -policy "${HDFS_ERASURECODE_POLICY}" \
diff --git a/tests/custom_cluster/test_exchange_delays.py b/tests/custom_cluster/test_exchange_delays.py
index 1df77b31e..b6aa044c5 100644
--- a/tests/custom_cluster/test_exchange_delays.py
+++ b/tests/custom_cluster/test_exchange_delays.py
@@ -19,7 +19,7 @@ import pytest
 from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
 from tests.common.environ import build_flavor_timeout
 from tests.common.skip import SkipIfBuildType
-from tests.util.filesystem_utils import IS_S3, IS_ADLS, IS_ISILON
+from tests.util.filesystem_utils import IS_S3, IS_ADLS, IS_ISILON, IS_OZONE, IS_ENCRYPTED
 
 # IMPALA-6100: add additional margin for error for slow build types.
 SLOW_BUILD_TIMEOUT=20000
@@ -67,8 +67,9 @@ class TestExchangeDelays(CustomClusterTestSuite):
   # can't time out). Use a longer delay for platforms that have slow scans:
   # IMPALA-6811: S3/ADLS have slow scans.
   # IMPALA-6866: Isilon has slow scans (and is counted as a slow build above).
+  # Scans with Ozone encryption are also slower.
   SLOW_SCAN_EXTRA_DELAY_MS = 10000
-  if IS_S3 or IS_ADLS or IS_ISILON:
+  if IS_S3 or IS_ADLS or IS_ISILON or (IS_OZONE and IS_ENCRYPTED):
     DELAY_MS += SLOW_SCAN_EXTRA_DELAY_MS
 
   @pytest.mark.execute_serially
diff --git a/tests/util/filesystem_utils.py b/tests/util/filesystem_utils.py
index a9f0fff74..88f5bfd0a 100644
--- a/tests/util/filesystem_utils.py
+++ b/tests/util/filesystem_utils.py
@@ -35,6 +35,7 @@ IS_GCS = FILESYSTEM == "gs"
 IS_COS = FILESYSTEM == "cosn"
 IS_OZONE = FILESYSTEM == "ozone"
 IS_EC = os.getenv("ERASURE_CODING") == "true"
+IS_ENCRYPTED = os.getenv("USE_OZONE_ENCRYPTION") == "true"
 # This condition satisfies both the states where one can assume a default fs
 #   - The environment variable is set to an empty string.
 #   - Tne environment variables is unset ( None )


[impala] 02/02: IMPALA-9823: Make use_local_catalog and related flags visible

Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit 276759271d6a75b6de1b565e7bb7843c382fbb31
Author: stiga-huang <hu...@gmail.com>
AuthorDate: Thu Jun 23 21:06:53 2022 +0800

    IMPALA-9823: Make use_local_catalog and related flags visible
    
    use_local_catalog and related flags shouldn't be hidden and should be
    show up on the Web UI. This makes them visible. Also updates the
    description of use_local_catalog.
    
    Change-Id: Ic5a39321b1fee4bc34266f235ee2dd1374778083
    Reviewed-on: http://gerrit.cloudera.org:8080/18660
    Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 be/src/runtime/exec-env.cc | 13 +++++++------
 tests/common/environ.py    |  1 -
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/be/src/runtime/exec-env.cc b/be/src/runtime/exec-env.cc
index dd1bc076c..34820d450 100644
--- a/be/src/runtime/exec-env.cc
+++ b/be/src/runtime/exec-env.cc
@@ -100,21 +100,22 @@ DEFINE_int32(admission_control_slots, 0,
     "that it needs. Defaults to number of cores / -num_cores for executors, and 8x that "
     "value for dedicated coordinators).");
 
-DEFINE_bool_hidden(use_local_catalog, false,
-    "Use experimental implementation of a local catalog. If this is set, "
-    "the catalog service is not used and does not need to be started.");
-DEFINE_int32_hidden(local_catalog_cache_mb, -1,
+DEFINE_bool(use_local_catalog, false,
+    "Use the on-demand metadata feature in coordinators. If this is set, coordinators "
+    "pull metadata as needed from catalogd and cache it locally. The cached metadata "
+    "gets evicted automatically under memory pressure or after an expiration time.");
+DEFINE_int32(local_catalog_cache_mb, -1,
     "If --use_local_catalog is enabled, configures the size of the catalog "
     "cache within each impalad. If this is set to -1, the cache is auto-"
     "configured to 60% of the configured Java heap size. Note that the Java "
     "heap size is distinct from and typically smaller than the overall "
     "Impala memory limit.");
-DEFINE_int32_hidden(local_catalog_cache_expiration_s, 60 * 60,
+DEFINE_int32(local_catalog_cache_expiration_s, 60 * 60,
     "If --use_local_catalog is enabled, configures the expiration time "
     "of the catalog cache within each impalad. Even if the configured "
     "cache capacity has not been reached, items are removed from the cache "
     "if they have not been accessed in this amount of time.");
-DEFINE_int32_hidden(local_catalog_max_fetch_retries, 40,
+DEFINE_int32(local_catalog_max_fetch_retries, 40,
     "If --use_local_catalog is enabled, configures the maximum number of times "
     "the frontend retries when fetching a metadata object from the impalad "
     "coordinator's local catalog cache.");
diff --git a/tests/common/environ.py b/tests/common/environ.py
index 831809247..405572b8d 100644
--- a/tests/common/environ.py
+++ b/tests/common/environ.py
@@ -369,7 +369,6 @@ class ImpalaTestClusterProperties(object):
     """Checks whether we use local catalog."""
     try:
       key = "use_local_catalog"
-      # --use_local_catalog is hidden so does not appear in JSON if disabled.
       return key in self.runtime_flags and self.runtime_flags[key]["current"] == "true"
     except Exception:
       if self.is_remote_cluster():