You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2016/11/26 00:28:13 UTC

[1/3] incubator-impala git commit: IMPALA-4523: Correct max VARCHAR size to 65535 (2^16 - 1).

Repository: incubator-impala
Updated Branches:
  refs/heads/master 16552f6ed -> fe8d994f0


IMPALA-4523: Correct max VARCHAR size to 65535 (2^16 - 1).

Change-Id: If76eb45b01692ed360fad8fa1722d56fa06c6c00
Reviewed-on: http://gerrit.cloudera.org:8080/5204
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Internal Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/fc61f2a3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/fc61f2a3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/fc61f2a3

Branch: refs/heads/master
Commit: fc61f2a3f733ecd8fb75d8c5a7ac7e8c25eca5c8
Parents: 16552f6
Author: Alex Behm <al...@cloudera.com>
Authored: Tue Nov 22 22:56:42 2016 -0800
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Thu Nov 24 22:28:01 2016 +0000

----------------------------------------------------------------------
 be/src/exprs/expr-test.cc                                    | 2 +-
 be/src/runtime/types.h                                       | 6 +++---
 fe/src/main/java/org/apache/impala/catalog/ScalarType.java   | 8 ++++----
 .../test/java/org/apache/impala/analysis/AnalyzeDDLTest.java | 7 ++++---
 .../java/org/apache/impala/analysis/AnalyzeExprsTest.java    | 4 ++--
 5 files changed, 14 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fc61f2a3/be/src/exprs/expr-test.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/expr-test.cc b/be/src/exprs/expr-test.cc
index 2a4e4ff..1340f55 100644
--- a/be/src/exprs/expr-test.cc
+++ b/be/src/exprs/expr-test.cc
@@ -2241,7 +2241,7 @@ TEST_F(ExprTest, StringFunctions) {
   TestStringValue("lower(cast('HELLO' as CHAR(3)))", "hel");
   TestStringValue("lower(cast(123456 as CHAR(3)))", "123");
   TestStringValue("cast(cast(123456 as CHAR(3)) as VARCHAR(3))", "123");
-  TestStringValue("cast(cast(123456 as CHAR(3)) as VARCHAR(65355))", "123");
+  TestStringValue("cast(cast(123456 as CHAR(3)) as VARCHAR(65535))", "123");
   TestIsNull("cast(NULL as CHAR(3))", ColumnType::CreateCharType(3));
 
   TestCharValue("cast('HELLO' as CHAR(255))",

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fc61f2a3/be/src/runtime/types.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/types.h b/be/src/runtime/types.h
index f2edbee..f265705 100644
--- a/be/src/runtime/types.h
+++ b/be/src/runtime/types.h
@@ -74,9 +74,9 @@ struct ColumnType {
   PrimitiveType type;
   /// Only set if type == TYPE_CHAR or type == TYPE_VARCHAR
   int len;
-  static const int MAX_VARCHAR_LENGTH = 65355;
-  static const int MAX_CHAR_LENGTH = 255;
-  static const int MAX_CHAR_INLINE_LENGTH = 128;
+  static const int MAX_VARCHAR_LENGTH = (1 << 16) - 1; // 65535
+  static const int MAX_CHAR_LENGTH = (1 << 8) - 1; // 255
+  static const int MAX_CHAR_INLINE_LENGTH = (1 << 7); // 128
 
   /// Only set if type == TYPE_DECIMAL
   int precision, scale;

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fc61f2a3/fe/src/main/java/org/apache/impala/catalog/ScalarType.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/ScalarType.java b/fe/src/main/java/org/apache/impala/catalog/ScalarType.java
index c75ef16..052fd24 100644
--- a/fe/src/main/java/org/apache/impala/catalog/ScalarType.java
+++ b/fe/src/main/java/org/apache/impala/catalog/ScalarType.java
@@ -18,12 +18,12 @@
 package org.apache.impala.catalog;
 
 import org.apache.commons.lang3.StringUtils;
-
 import org.apache.impala.analysis.TypesUtil;
 import org.apache.impala.thrift.TColumnType;
 import org.apache.impala.thrift.TScalarType;
 import org.apache.impala.thrift.TTypeNode;
 import org.apache.impala.thrift.TTypeNodeType;
+
 import com.google.common.base.Preconditions;
 
 /**
@@ -56,12 +56,12 @@ public class ScalarType extends Type {
   public static final int DEFAULT_SCALE = 0; // SQL standard
 
   // Longest supported VARCHAR and CHAR, chosen to match Hive.
-  public static final int MAX_VARCHAR_LENGTH = 65355;
-  public static final int MAX_CHAR_LENGTH = 255;
+  public static final int MAX_VARCHAR_LENGTH = (1 << 16) - 1; // 65535
+  public static final int MAX_CHAR_LENGTH = (1 << 8) - 1; // 255
 
   // Longest CHAR that we in line in the tuple.
   // Keep consistent with backend ColumnType::CHAR_INLINE_LENGTH
-  public static final int CHAR_INLINE_LENGTH = 128;
+  public static final int CHAR_INLINE_LENGTH = (1 << 7); // 128
 
   // Hive, mysql, sql server standard.
   public static final int MAX_PRECISION = 38;

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fc61f2a3/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
index dbfdd60..94d1c1d 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
@@ -1553,11 +1553,12 @@ public class AnalyzeDDLTest extends FrontendTestBase {
     AnalysisError("create table new_table (i int) PARTITIONED BY (d decimal(40,1))",
         "Decimal precision must be <= 38: 40");
 
-    AnalyzesOk("create table new_table(s1 varchar(1), s2 varchar(32672))");
+    AnalyzesOk("create table new_table(s1 varchar(1), s2 varchar(32672), " +
+        "s3 varchar(65535))");
     AnalysisError("create table new_table(s1 varchar(0))",
         "Varchar size must be > 0: 0");
-    AnalysisError("create table new_table(s1 varchar(65356))",
-        "Varchar size must be <= 65355: 65356");
+    AnalysisError("create table new_table(s1 varchar(65536))",
+        "Varchar size must be <= 65535: 65536");
     AnalysisError("create table new_table(s1 char(0))",
         "Char size must be > 0: 0");
     AnalysisError("create table new_table(s1 Char(256))",

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fc61f2a3/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java
index 6bc493b..e2d0058 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java
@@ -404,8 +404,8 @@ public class AnalyzeExprsTest extends AnalyzerTest {
 
     AnalysisError("select cast('foo' as varchar(0))",
         "Varchar size must be > 0: 0");
-    AnalysisError("select cast('foo' as varchar(65356))",
-        "Varchar size must be <= 65355: 65356");
+    AnalysisError("select cast('foo' as varchar(65536))",
+        "Varchar size must be <= 65535: 65536");
     AnalysisError("select cast('foo' as char(0))",
         "Char size must be > 0: 0");
     AnalysisError("select cast('foo' as char(256))",


[3/3] incubator-impala git commit: IMPALA-4541: fix test dimensions for test_codegen_mem_limit

Posted by ta...@apache.org.
IMPALA-4541: fix test dimensions for test_codegen_mem_limit

The test should only be run with codegen enabled.

Change-Id: Iac460d2a1b69de638c557d7c8aa318a73ad0507b
Reviewed-on: http://gerrit.cloudera.org:8080/5221
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Internal Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/fe8d994f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/fe8d994f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/fe8d994f

Branch: refs/heads/master
Commit: fe8d994f0f320c541ff22fd15660a3d926c7d90b
Parents: 3c65fa1
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Fri Nov 25 00:13:58 2016 -0800
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Fri Nov 25 23:32:24 2016 +0000

----------------------------------------------------------------------
 tests/query_test/test_query_mem_limit.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fe8d994f/tests/query_test/test_query_mem_limit.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_query_mem_limit.py b/tests/query_test/test_query_mem_limit.py
index ce12e41..161f714 100644
--- a/tests/query_test/test_query_mem_limit.py
+++ b/tests/query_test/test_query_mem_limit.py
@@ -26,9 +26,9 @@ from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
 from tests.common.impala_test_suite import ImpalaTestSuite
 from tests.common.test_dimensions import (
     TestDimension,
+    create_single_exec_option_dimension,
     create_uncompressed_text_dimension)
 
-
 class TestQueryMemLimit(ImpalaTestSuite):
   """Test class to do functional validation of per query memory limits.
 
@@ -115,6 +115,7 @@ class TestCodegenMemLimit(ImpalaTestSuite):
   @classmethod
   def add_test_dimensions(cls):
     super(TestCodegenMemLimit, cls).add_test_dimensions()
+    cls.TestMatrix.add_dimension(create_single_exec_option_dimension())
     # Only run the query for parquet
     cls.TestMatrix.add_constraint(
       lambda v: v.get_value('table_format').file_format == 'parquet')


[2/3] incubator-impala git commit: IMPALA-4540: Function call in DCHECK crashes scheduler

Posted by ta...@apache.org.
IMPALA-4540: Function call in DCHECK crashes scheduler

The fix for IMPALA-4494 wrapped the call to
backend_config_.LookUpBackendIp in simple-scheduler.cc:922 in a DCHECK,
effectively disabling it in release builds. This leads to a crash when
backend_ip is empty.

This change makes sure that the function is always called.

Change-Id: Iefbf719e86a59c09c45a5418f6ec0aa5cc4d8c44
Reviewed-on: http://gerrit.cloudera.org:8080/5218
Reviewed-by: Michael Ho <kw...@cloudera.com>
Tested-by: Internal Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/3c65fa1d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/3c65fa1d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/3c65fa1d

Branch: refs/heads/master
Commit: 3c65fa1d7e91177770d745940e84645cf9c49ba8
Parents: fc61f2a
Author: Lars Volker <lv...@cloudera.com>
Authored: Thu Nov 24 22:02:31 2016 +0100
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Fri Nov 25 15:26:48 2016 +0000

----------------------------------------------------------------------
 be/src/scheduling/simple-scheduler.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3c65fa1d/be/src/scheduling/simple-scheduler.cc
----------------------------------------------------------------------
diff --git a/be/src/scheduling/simple-scheduler.cc b/be/src/scheduling/simple-scheduler.cc
index 1c8af72..a0b4a35 100644
--- a/be/src/scheduling/simple-scheduler.cc
+++ b/be/src/scheduling/simple-scheduler.cc
@@ -919,7 +919,8 @@ void SimpleScheduler::AssignmentCtx::RecordScanRangeAssignment(
   }
 
   IpAddr backend_ip;
-  DCHECK(backend_config_.LookUpBackendIp(backend.address.hostname, &backend_ip));
+  bool ret = backend_config_.LookUpBackendIp(backend.address.hostname, &backend_ip);
+  DCHECK(ret);
   DCHECK(!backend_ip.empty());
   assignment_heap_.InsertOrUpdate(backend_ip, scan_range_length,
       GetBackendRank(backend_ip));