You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/01/11 01:25:21 UTC
[1/2] impala git commit: IMPALA-6371: Additional check for delimiters
Repository: impala
Updated Branches:
refs/heads/master 3394eb1be -> c0c1202db
IMPALA-6371: Additional check for delimiters
The check validates the codepoint of the Java char.
Testing:
- Added tests for valid/invalid unicode in HdfsStorageDescriptorTest.
Change-Id: If8dc335d39dd02f602cf93682bccf84b2c099dde
Reviewed-on: http://gerrit.cloudera.org:8080/8959
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/a34df684
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/a34df684
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/a34df684
Branch: refs/heads/master
Commit: a34df684f78ede6925ebeff1ed174b00828a980f
Parents: 3394eb1
Author: Adam Holley <ah...@cloudera.com>
Authored: Sun Jan 7 19:13:31 2018 -0600
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed Jan 10 22:31:32 2018 +0000
----------------------------------------------------------------------
.../impala/catalog/HdfsStorageDescriptor.java | 7 ++++++-
.../impala/catalog/HdfsStorageDescriptorTest.java | 18 ++++++++++++++++++
2 files changed, 24 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/a34df684/fe/src/main/java/org/apache/impala/catalog/HdfsStorageDescriptor.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsStorageDescriptor.java b/fe/src/main/java/org/apache/impala/catalog/HdfsStorageDescriptor.java
index 4554a31..b4e2564 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsStorageDescriptor.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsStorageDescriptor.java
@@ -140,7 +140,12 @@ public class HdfsStorageDescriptor {
// need support from the Hive side.
return Byte.parseByte(delimVal);
} catch (NumberFormatException e) {
- if (delimVal.length() == 1) return (byte) delimVal.charAt(0);
+ if (delimVal.length() == 1) {
+ // Adding additional check as Java chars are two bytes.
+ // e.g. \u1111 as delimVal will return a valid byte '11'
+ int cp = Character.codePointAt(delimVal, 0);
+ if (cp >= 0 && cp <= 255) return (byte) cp;
+ }
}
return null;
}
http://git-wip-us.apache.org/repos/asf/impala/blob/a34df684/fe/src/test/java/org/apache/impala/catalog/HdfsStorageDescriptorTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/catalog/HdfsStorageDescriptorTest.java b/fe/src/test/java/org/apache/impala/catalog/HdfsStorageDescriptorTest.java
index 0934fed..bb133a1 100644
--- a/fe/src/test/java/org/apache/impala/catalog/HdfsStorageDescriptorTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/HdfsStorageDescriptorTest.java
@@ -157,5 +157,23 @@ public class HdfsStorageDescriptorTest {
"single character or as a decimal value in the range [-128:127]",
e.getMessage());
}
+
+ // Test that a unicode character out of the valid range will not be accepted.
+ sd.getSerdeInfo().setParameters(new HashMap<String,String>());
+ sd.getSerdeInfo().putToParameters(serdeConstants.LINE_DELIM, "\u1111");
+ try {
+ HdfsStorageDescriptor.fromStorageDescriptor("fake", sd);
+ fail();
+ } catch (HdfsStorageDescriptor.InvalidStorageDescriptorException e) {
+ assertEquals("Invalid delimiter: '\u1111'. Delimiter must be specified as a " +
+ "single character or as a decimal value in the range [-128:127]",
+ e.getMessage());
+ }
+
+ // Validate that unicode character in the valid range will be accepted.
+ sd.getSerdeInfo().setParameters(new HashMap<String,String>());
+ sd.getSerdeInfo().putToParameters(serdeConstants.FIELD_DELIM, "\u0001");
+ assertNotNull(HdfsStorageDescriptor.fromStorageDescriptor("fakeTbl", sd));
+
}
}
[2/2] impala git commit: IMPALA-6381: increase test_exchange_delays
timeout for isilon
Posted by ta...@apache.org.
IMPALA-6381: increase test_exchange_delays timeout for isilon
Change-Id: Ie82030403fa238b673b0a3ccdc7731b0d78b63af
Reviewed-on: http://gerrit.cloudera.org:8080/8993
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/c0c1202d
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/c0c1202d
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/c0c1202d
Branch: refs/heads/master
Commit: c0c1202dbfd1ffbd36a1053aa59e92d53b23e67a
Parents: a34df68
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Wed Jan 10 11:39:41 2018 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Thu Jan 11 00:42:02 2018 +0000
----------------------------------------------------------------------
tests/custom_cluster/test_exchange_delays.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/c0c1202d/tests/custom_cluster/test_exchange_delays.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_exchange_delays.py b/tests/custom_cluster/test_exchange_delays.py
index 01acaef..dfab1fe 100644
--- a/tests/custom_cluster/test_exchange_delays.py
+++ b/tests/custom_cluster/test_exchange_delays.py
@@ -19,9 +19,14 @@ import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.environ import specific_build_type_timeout
from tests.common.skip import SkipIfBuildType
+from tests.util.filesystem_utils import IS_ISILON
# IMPALA-6100: add additional margin for error for slow build types.
-DELAY_MS = specific_build_type_timeout(10000, slow_build_timeout=20000)
+SLOW_BUILD_TIMEOUT=20000
+DELAY_MS = specific_build_type_timeout(10000, slow_build_timeout=SLOW_BUILD_TIMEOUT)
+# IMPALA-6381: Isilon can behave as a slow build.
+if IS_ISILON:
+ DELAY_MS = SLOW_BUILD_TIMEOUT
@SkipIfBuildType.not_dev_build
class TestExchangeDelays(CustomClusterTestSuite):