You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mj...@apache.org on 2017/08/04 20:12:48 UTC
incubator-impala git commit: IMPALA-5757: Make tbl property toSql
deterministic
Repository: incubator-impala
Updated Branches:
refs/heads/master b55ec3f64 -> dced73167
IMPALA-5757: Make tbl property toSql deterministic
On Ubuntu 16.04, it seems that table properties may be
returned in a different order from that expected by
TestShowCreateTable in test_kudu.py.
The fix is to change ToSqlUtils to print properties in
a deterministic way.
Change-Id: I2391e04eb40e398098704b77588ad352d514df7d
Reviewed-on: http://gerrit.cloudera.org:8080/7580
Reviewed-by: Matthew Jacobs <mj...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/dced7316
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/dced7316
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/dced7316
Branch: refs/heads/master
Commit: dced731671d28249f35d707854605e7ada82a6e0
Parents: b55ec3f
Author: Matthew Jacobs <mj...@cloudera.com>
Authored: Thu Aug 3 15:31:36 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Fri Aug 4 19:51:46 2017 +0000
----------------------------------------------------------------------
.../java/org/apache/impala/analysis/ToSqlUtils.java | 14 ++++++++++++--
.../java/org/apache/impala/analysis/ToSqlTest.java | 8 ++++----
2 files changed, 16 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/dced7316/fe/src/main/java/org/apache/impala/analysis/ToSqlUtils.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/ToSqlUtils.java b/fe/src/main/java/org/apache/impala/analysis/ToSqlUtils.java
index 01517e1..471947c 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ToSqlUtils.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ToSqlUtils.java
@@ -18,10 +18,12 @@
package org.apache.impala.analysis;
import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.Collections;
+import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
@@ -32,6 +34,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.Token;
+import org.apache.commons.lang.ObjectUtils;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.metastore.TableType;
@@ -425,8 +428,15 @@ public class ToSqlUtils {
}
private static String propertyMapToSql(Map<String, String> propertyMap) {
+ // Sort entries on the key to ensure output is deterministic for tests (IMPALA-5757).
+ List<Entry<String, String>> mapEntries = Lists.newArrayList(propertyMap.entrySet());
+ Collections.sort(mapEntries, new Comparator<Entry<String, String>>() {
+ public int compare(Entry<String, String> o1, Entry<String, String> o2) {
+ return ObjectUtils.compare(o1.getKey(), o2.getKey());
+ } });
+
List<String> properties = Lists.newArrayList();
- for (Map.Entry<String, String> entry: propertyMap.entrySet()) {
+ for (Map.Entry<String, String> entry: mapEntries) {
properties.add(String.format("'%s'='%s'", entry.getKey(),
// Properties may contain characters that need to be escaped.
// e.g. If the row format escape delimiter is '\', the map of serde properties
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/dced7316/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java b/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java
index 6ef4b04..4cd8934 100644
--- a/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java
@@ -314,8 +314,8 @@ public class ToSqlTest extends FrontendTestBase {
"CREATE TABLE default.p ( a BIGINT PRIMARY KEY, b TIMESTAMP " +
"DEFAULT '1987-05-19' ) PARTITION BY HASH (a) PARTITIONS 3 " +
"STORED AS KUDU TBLPROPERTIES ('kudu.master_addresses'='foo', " +
- "'storage_handler'='com.cloudera.kudu.hive.KuduStorageHandler', " +
- "'kudu.table_name'='impala::default.p')", true);
+ "'kudu.table_name'='impala::default.p', " +
+ "'storage_handler'='com.cloudera.kudu.hive.KuduStorageHandler')", true);
}
@Test
@@ -347,8 +347,8 @@ public class ToSqlTest extends FrontendTestBase {
"CREATE TABLE default.p PRIMARY KEY (a, b) PARTITION BY HASH (a) PARTITIONS 3, " +
"RANGE (b) (PARTITION VALUE = 1) STORED AS KUDU TBLPROPERTIES " +
"('kudu.master_addresses'='foo', " +
- "'storage_handler'='com.cloudera.kudu.hive.KuduStorageHandler', " +
- "'kudu.table_name'='impala::default.p') AS " +
+ "'kudu.table_name'='impala::default.p', " +
+ "'storage_handler'='com.cloudera.kudu.hive.KuduStorageHandler') AS " +
"SELECT int_col a, bigint_col b FROM functional.alltypes", true);
}