You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2017/10/31 17:14:50 UTC

[1/2] incubator-impala git commit: IMPALA-6124: Fix alter table ddl updates and test

Repository: incubator-impala
Updated Branches:
  refs/heads/master f11181cbe -> d1594298e


IMPALA-6124: Fix alter table ddl updates and test

Impala would previously update the ddl time of a table when dropping a
partition but not when adding one. This change removes updates to the
ddl time when partitions are added or removed to be consistent with
Hive.

Additionally the check in the ddl update test would fail if some
operations took longer than 20 seconds. Instead, this change makes sure
that the ddl time increases as intended.

To test this change I ran test_last_ddl_time_update in exhaustive mode
and also ran a private S3 build.

Change-Id: I3126252e7709304d3e1fa4bb06a0b847180bd6cf
Reviewed-on: http://gerrit.cloudera.org:8080/8411
Reviewed-by: Bharath Vissapragada <bh...@cloudera.com>
Tested-by: Impala Public Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/471285be
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/471285be
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/471285be

Branch: refs/heads/master
Commit: 471285bee3b8d4e18f97153fa0bd95641bba0d78
Parents: f11181c
Author: Lars Volker <lv...@cloudera.com>
Authored: Sat Oct 28 09:47:39 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Oct 31 04:00:37 2017 +0000

----------------------------------------------------------------------
 .../org/apache/impala/service/CatalogOpExecutor.java    |  3 ---
 tests/metadata/test_last_ddl_time_update.py             | 12 ++++++------
 2 files changed, 6 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/471285be/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
index bb466dd..1683cc0 100644
--- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
+++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
@@ -2148,7 +2148,6 @@ public class CatalogOpExecutor {
               " ifExists is true.", e, tableName));
         }
       }
-      updateLastDdlTime(msTbl, msClient);
     } catch (TException e) {
       throw new ImpalaRuntimeException(
           String.format(HMS_RPC_ERROR_FORMAT_STR, "dropPartition"), e);
@@ -2628,7 +2627,6 @@ public class CatalogOpExecutor {
           MetastoreShim.alterPartitions(msClient.getHiveClient(), tableName.getDb(),
               tableName.getTbl(), hmsAddedPartitions);
         }
-        updateLastDdlTime(msTbl, msClient);
       }
     } catch (TException e) {
       throw new ImpalaRuntimeException(
@@ -3264,7 +3262,6 @@ public class CatalogOpExecutor {
                   }
                 }
               }
-              updateLastDdlTime(msTbl, msClient);
             }
           } catch (AlreadyExistsException e) {
             throw new InternalException(

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/471285be/tests/metadata/test_last_ddl_time_update.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_last_ddl_time_update.py b/tests/metadata/test_last_ddl_time_update.py
index 0ec57ee..b1bcfbb 100644
--- a/tests/metadata/test_last_ddl_time_update.py
+++ b/tests/metadata/test_last_ddl_time_update.py
@@ -49,12 +49,12 @@ class TestLastDdlTimeUpdate(ImpalaTestSuite):
 
     # add/drop partitions
     self.run_test("alter table %s add partition (j=1, s='2012')" % FQ_TBL_NAME,
-                  unique_database, TBL_NAME, True)
+                  unique_database, TBL_NAME, False)
     self.run_test("alter table %s add if not exists "
                   "partition (j=1, s='2012')" % FQ_TBL_NAME,
                   unique_database, TBL_NAME, False)
     self.run_test("alter table %s drop partition (j=1, s='2012')" % FQ_TBL_NAME,
-                  unique_database, TBL_NAME, True)
+                  unique_database, TBL_NAME, False)
     self.run_test("alter table %s drop if exists "
                   "partition (j=2, s='2012')" % FQ_TBL_NAME,
                   unique_database, TBL_NAME, False)
@@ -78,10 +78,10 @@ class TestLastDdlTimeUpdate(ImpalaTestSuite):
                        "partitioned by (j int, s string)" % FQ_TBL_NAME)
     # static partition insert
     self.run_test("insert into %s partition(j=1, s='2012') "
-                  "select 10" % FQ_TBL_NAME, unique_database, TBL_NAME, True)
+                  "select 10" % FQ_TBL_NAME, unique_database, TBL_NAME, False)
     # dynamic partition insert
     self.run_test("insert into %s partition(j, s) "
-                  "select 10, 2, '2013'" % FQ_TBL_NAME, unique_database, TBL_NAME, True)
+                  "select 10, 2, '2013'" % FQ_TBL_NAME, unique_database, TBL_NAME, False)
     # dynamic partition insert changing no partitions (empty input)
     self.run_test("insert into %s partition(j, s) "
                   "select * from (select 10 as i, 2 as j, '2013' as s) as t "
@@ -116,7 +116,7 @@ class TestLastDdlTimeUpdate(ImpalaTestSuite):
     afterDdlTime = table.parameters[HIVE_LAST_DDL_TIME_PARAM_KEY]
 
     if expect_changed:
-      # check that the time difference is within 20s
-      assert long(afterDdlTime) - long(beforeDdlTime) <= 20
+      # check that the new ddlTime is strictly greater than the old one.
+      assert long(afterDdlTime) > long(beforeDdlTime)
     else:
       assert long(afterDdlTime) == long(beforeDdlTime)


[2/2] incubator-impala git commit: IMPALA-6118: Fix assertion failure in coordinator bloom filter updating

Posted by ta...@apache.org.
IMPALA-6118: Fix assertion failure in coordinator bloom filter updating

Coordinator::UpdateFilter wrongly assumed std::string::capacity() to be
0 after std::string::clear(), resulting in an DCHECK failure in memory
tracking. This patch tracks size() instead of capacity() to fix the bug.

Change-Id: Ia28f9eb8d0b3e795d798949a1cfcfeaaec7040ed
Reviewed-on: http://gerrit.cloudera.org:8080/8410
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Impala Public Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/d1594298
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/d1594298
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/d1594298

Branch: refs/heads/master
Commit: d1594298eac2f3b5b4e9a1ce72de499df2b3ecc1
Parents: 471285b
Author: Tianyi Wang <tw...@cloudera.com>
Authored: Fri Oct 27 11:02:40 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Oct 31 05:23:28 2017 +0000

----------------------------------------------------------------------
 be/src/runtime/coordinator.cc | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/d1594298/be/src/runtime/coordinator.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/coordinator.cc b/be/src/runtime/coordinator.cc
index 5f6bd08..4a131b9 100644
--- a/be/src/runtime/coordinator.cc
+++ b/be/src/runtime/coordinator.cc
@@ -1124,12 +1124,12 @@ void Coordinator::UpdateFilter(const TUpdateFilterParams& params) {
 
     // Assign outgoing bloom filter.
     TBloomFilter& aggregated_filter = state->bloom_filter();
-    filter_mem_tracker_->Release(aggregated_filter.directory.capacity());
+    filter_mem_tracker_->Release(aggregated_filter.directory.size());
     swap(rpc_params.bloom_filter, aggregated_filter);
     DCHECK(rpc_params.bloom_filter.always_false || rpc_params.bloom_filter.always_true ||
-        rpc_params.bloom_filter.directory.size() != 0);
+        !rpc_params.bloom_filter.directory.empty());
+    DCHECK(aggregated_filter.directory.empty());
     rpc_params.__isset.bloom_filter = true;
-    DCHECK_EQ(aggregated_filter.directory.capacity(), 0);
     // Filter is complete, and can be released.
     state->Disable(filter_mem_tracker_);
   }
@@ -1158,7 +1158,7 @@ void Coordinator::FilterState::ApplyUpdate(const TUpdateFilterParams& params,
   if (params.bloom_filter.always_true) {
     Disable(coord->filter_mem_tracker_);
   } else if (bloom_filter_.always_false) {
-    int64_t heap_space = params.bloom_filter.directory.capacity();
+    int64_t heap_space = params.bloom_filter.directory.size();
     if (!coord->filter_mem_tracker_->TryConsume(heap_space)) {
       VLOG_QUERY << "Not enough memory to allocate filter: "
                  << PrettyPrinter::Print(heap_space, TUnit::BYTES)
@@ -1186,9 +1186,9 @@ void Coordinator::FilterState::ApplyUpdate(const TUpdateFilterParams& params,
 void Coordinator::FilterState::Disable(MemTracker* tracker) {
   bloom_filter_.always_true = true;
   bloom_filter_.always_false = false;
-  int64_t capacity = bloom_filter_.directory.capacity();
+  tracker->Release(bloom_filter_.directory.size());
   bloom_filter_.directory.clear();
-  tracker->Release(capacity);
+  bloom_filter_.directory.shrink_to_fit();
 }
 
 const TUniqueId& Coordinator::query_id() const {