You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2016/10/11 23:10:17 UTC
[22/22] hive git commit: HIVE-14639 : handle SKEWED BY for MM tables
(Sergey Shelukhin)
HIVE-14639 : handle SKEWED BY for MM tables (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b9e81572
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b9e81572
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b9e81572
Branch: refs/heads/hive-14535
Commit: b9e815722686f5bcdae40caadae34f84509d86c2
Parents: 7d50a15
Author: Sergey Shelukhin <se...@apache.org>
Authored: Tue Oct 11 16:09:28 2016 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Tue Oct 11 16:09:28 2016 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/exec/FileSinkOperator.java | 31 +-
.../apache/hadoop/hive/ql/exec/Utilities.java | 2 +-
.../hadoop/hive/ql/io/HiveInputFormat.java | 2 +
.../apache/hadoop/hive/ql/metadata/Hive.java | 6 +-
ql/src/test/queries/clientpositive/mm_all.q | 76 ++-
ql/src/test/queries/clientpositive/mm_current.q | 45 +-
.../results/clientpositive/llap/mm_all.q.out | 473 +++++++++++--------
.../clientpositive/llap/mm_current.q.out | 215 +++++----
8 files changed, 473 insertions(+), 377 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b9e81572/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index 00115fe..5902036 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@ -1189,6 +1189,7 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
Path specPath = conf.getDirName();
String unionSuffix = null;
DynamicPartitionCtx dpCtx = conf.getDynPartCtx();
+ ListBucketingCtx lbCtx = conf.getLbCtx();
if (conf.isLinkedFileSink() && (dpCtx != null)) {
specPath = conf.getParentDir();
Utilities.LOG14535.info("Setting specPath to " + specPath + " for dynparts");
@@ -1197,7 +1198,7 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
if (!conf.isMmTable()) {
Utilities.mvFileToFinalPath(specPath, hconf, success, LOG, dpCtx, conf, reporter); // TODO# other callers
} else {
- handleMmTable(specPath, unionSuffix, hconf, success, dpCtx, conf, reporter);
+ handleMmTable(specPath, unionSuffix, hconf, success, dpCtx, lbCtx, conf, reporter);
}
}
} catch (IOException e) {
@@ -1207,22 +1208,26 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
}
private static FileStatus[] getMmDirectoryCandidates(FileSystem fs, Path path,
- int dpLevels, String unionSuffix, PathFilter filter) throws IOException {
+ DynamicPartitionCtx dpCtx, ListBucketingCtx lbCtx, String unionSuffix, PathFilter filter)
+ throws IOException {
StringBuilder sb = new StringBuilder(path.toUri().getPath());
- for (int i = 0; i < dpLevels; i++) {
+ int dpLevels = dpCtx == null ? 0 : dpCtx.getNumDPCols(),
+ lbLevels = lbCtx == null ? 0 : lbCtx.getSkewedColNames().size();
+ for (int i = 0; i < dpLevels + lbLevels; i++) {
sb.append(Path.SEPARATOR).append("*");
}
if (unionSuffix != null) {
sb.append(Path.SEPARATOR).append(unionSuffix);
}
sb.append(Path.SEPARATOR).append("*"); // TODO: we could add exact mm prefix here
+ Utilities.LOG14535.info("Looking for files via: " + sb.toString());
Path pathPattern = new Path(path, sb.toString());
return fs.globStatus(pathPattern, filter);
}
private void handleMmTable(Path specPath, String unionSuffix, Configuration hconf,
- boolean success, DynamicPartitionCtx dpCtx, FileSinkDesc conf, Reporter reporter)
- throws IOException, HiveException {
+ boolean success, DynamicPartitionCtx dpCtx, ListBucketingCtx lbCtx, FileSinkDesc conf,
+ Reporter reporter) throws IOException, HiveException {
FileSystem fs = specPath.getFileSystem(hconf);
// Manifests would be at the root level, but the results at target level.
// TODO# special case - doesn't take bucketing into account
@@ -1230,7 +1235,7 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
ValidWriteIds.IdPathFilter filter = new ValidWriteIds.IdPathFilter(conf.getMmWriteId(), true);
if (!success) {
- tryDeleteAllMmFiles(fs, specPath, manifestDir, dpCtx, unionSuffix, filter);
+ tryDeleteAllMmFiles(fs, specPath, manifestDir, dpCtx, lbCtx, unionSuffix, filter);
return;
}
FileStatus[] files = HiveStatsUtils.getFileStatusRecurse(manifestDir, 1, fs, filter);
@@ -1240,14 +1245,14 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
for (FileStatus status : files) {
Path path = status.getPath();
if (path.getName().endsWith(MANIFEST_EXTENSION)) {
+ Utilities.LOG14535.info("Reading manifest " + path);
manifests.add(path);
}
}
}
Utilities.LOG14535.info("Looking for files in: " + specPath);
- files = getMmDirectoryCandidates(fs, specPath,
- dpCtx == null ? 0 : dpCtx.getNumDPCols(), unionSuffix, filter);
+ files = getMmDirectoryCandidates(fs, specPath, dpCtx, lbCtx, unionSuffix, filter);
ArrayList<FileStatus> results = new ArrayList<>();
if (files != null) {
for (FileStatus status : files) {
@@ -1307,8 +1312,11 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
}
if (results.isEmpty()) return;
- FileStatus[] finalResults = results.toArray(new FileStatus[results.size()]);
+ // TODO: see HIVE-14886 - removeTempOrDuplicateFiles is broken for list bucketing,
+ // so maintain parity here by not calling it at all.
+ if (lbCtx != null) return;
+ FileStatus[] finalResults = results.toArray(new FileStatus[results.size()]);
List<Path> emptyBuckets = Utilities.removeTempOrDuplicateFiles(
fs, finalResults, dpCtx, conf, hconf);
// create empty buckets if necessary
@@ -1318,10 +1326,9 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
}
private void tryDeleteAllMmFiles(FileSystem fs, Path specPath, Path manifestDir,
- DynamicPartitionCtx dpCtx, String unionSuffix,
+ DynamicPartitionCtx dpCtx, ListBucketingCtx lbCtx, String unionSuffix,
ValidWriteIds.IdPathFilter filter) throws IOException {
- FileStatus[] files = getMmDirectoryCandidates(fs, specPath,
- dpCtx == null ? 0 : dpCtx.getNumDPCols(), unionSuffix, filter);
+ FileStatus[] files = getMmDirectoryCandidates(fs, specPath, dpCtx, lbCtx, unionSuffix, filter);
if (files != null) {
for (FileStatus status : files) {
Utilities.LOG14535.info("Deleting " + status.getPath() + " on failure");
http://git-wip-us.apache.org/repos/asf/hive/blob/b9e81572/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 03abdc1..a2eff8b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -1599,7 +1599,7 @@ public final class Utilities {
}
} else {
String taskId = getPrefixedTaskIdFromFilename(one.getPath().getName());
- Utilities.LOG14535.info("removeTempOrDuplicateFiles pondering " + one.getPath() + ", taskId " + taskId/*, new Exception()*/);
+ Utilities.LOG14535.info("removeTempOrDuplicateFiles pondering " + one.getPath() + ", taskId " + taskId);
FileStatus otherFile = taskIdToFile.get(taskId);
if (otherFile == null) {
http://git-wip-us.apache.org/repos/asf/hive/blob/b9e81572/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
index c3e2681..a539799 100755
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
@@ -399,6 +399,7 @@ public class HiveInputFormat<K extends WritableComparable, V extends Writable>
private void processForWriteIds(Path dir, JobConf conf,
ValidWriteIds writeIds, List<Path> finalPaths) throws IOException {
FileSystem fs = dir.getFileSystem(conf);
+ Utilities.LOG14535.warn("Checking " + dir + " (root) for inputs");
FileStatus[] files = fs.listStatus(dir); // TODO: batch?
LinkedList<Path> subdirs = new LinkedList<>();
for (FileStatus file : files) {
@@ -415,6 +416,7 @@ public class HiveInputFormat<K extends WritableComparable, V extends Writable>
private void handleNonMmDirChild(FileStatus file, ValidWriteIds writeIds,
LinkedList<Path> subdirs, List<Path> finalPaths) {
Path path = file.getPath();
+ Utilities.LOG14535.warn("Checking " + path + " for inputs");
if (!file.isDirectory()) {
Utilities.LOG14535.warn("Ignoring a file not in MM directory " + path);
return;
http://git-wip-us.apache.org/repos/asf/hive/blob/b9e81572/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 73a3b19..84c1e7b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -1587,12 +1587,12 @@ public class Hive {
List<Path> newFiles = null;
if (mmWriteId != null) {
Utilities.LOG14535.info("not moving " + loadPath + " to " + newPartPath);
- assert !isAcid && !replace;
+ assert !isAcid;
if (areEventsForDmlNeeded(tbl, oldPart)) {
newFiles = listFilesCreatedByQuery(loadPath, mmWriteId);
}
- if (replace) {
- Path tableDest = tbl.getPath();
+ Utilities.LOG14535.info("maybe deleting stuff from " + oldPartPath + " (new " + newPartPath + ") for replace");
+ if (replace && oldPartPath != null) { // TODO# is this correct? ignore until iow jira
deleteOldPathForReplace(newPartPath, oldPartPath,
getConf(), new ValidWriteIds.IdPathFilter(mmWriteId, false));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b9e81572/ql/src/test/queries/clientpositive/mm_all.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/mm_all.q b/ql/src/test/queries/clientpositive/mm_all.q
index cc44c19..1f85c48 100644
--- a/ql/src/test/queries/clientpositive/mm_all.q
+++ b/ql/src/test/queries/clientpositive/mm_all.q
@@ -9,8 +9,8 @@ set hive.exec.dynamic.partition.mode=nonstrict;
-- Force multiple writers when reading
drop table intermediate;
create table intermediate(key int) partitioned by (p int) stored as orc;
-insert into table intermediate partition(p='455') select key from src limit 2;
-insert into table intermediate partition(p='456') select key from src limit 2;
+insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2;
+insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2;
drop table part_mm;
create table part_mm(key int) partitioned by (key_mm int) stored as orc tblproperties ('hivecommit'='true');
@@ -18,21 +18,20 @@ explain insert into table part_mm partition(key_mm='455') select key from interm
insert into table part_mm partition(key_mm='455') select key from intermediate;
insert into table part_mm partition(key_mm='456') select key from intermediate;
insert into table part_mm partition(key_mm='455') select key from intermediate;
-select * from part_mm;
+select * from part_mm order by key;
drop table part_mm;
drop table simple_mm;
create table simple_mm(key int) stored as orc tblproperties ('hivecommit'='true');
insert into table simple_mm select key from intermediate;
insert overwrite table simple_mm select key from intermediate;
-select * from simple_mm;
+select * from simple_mm order by key;
insert into table simple_mm select key from intermediate;
-select * from simple_mm;
+select * from simple_mm order by key;
drop table simple_mm;
--- simple DP (no bucketing, no sorting?)
-drop table dp_no_mm;
+-- simple DP (no bucketing)
drop table dp_mm;
set hive.exec.dynamic.partition.mode=nonstrict;
@@ -41,18 +40,13 @@ set hive.merge.mapredfiles=false;
set hive.merge.sparkfiles=false;
set hive.merge.tezfiles=false;
-create table dp_no_mm (key int) partitioned by (key1 string, key2 int) stored as orc;
create table dp_mm (key int) partitioned by (key1 string, key2 int) stored as orc
tblproperties ('hivecommit'='true');
-insert into table dp_no_mm partition (key1='123', key2) select key, key from intermediate;
-
insert into table dp_mm partition (key1='123', key2) select key, key from intermediate;
-select * from dp_no_mm;
-select * from dp_mm;
+select * from dp_mm order by key;
-drop table dp_no_mm;
drop table dp_mm;
@@ -108,10 +102,34 @@ select key as p, key from intermediate
union all
select key + 1 as p, key + 1 from intermediate ) temps;
-select * from partunion_mm;
+select * from partunion_mm order by id;
drop table partunion_mm;
--- TODO# from here, fix it
+
+
+create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3))
+ stored as directories tblproperties ('hivecommit'='true');
+
+insert into table skew_mm
+select key, key, key from intermediate;
+
+select * from skew_mm order by k2;
+drop table skew_mm;
+
+
+create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int)
+skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ('hivecommit'='true');
+
+insert into table skew_dp_union_mm partition (k3)
+select key as i, key as j, key as k, key as l from intermediate
+union all
+select key +1 as i, key +2 as j, key +3 as k, key +4 as l from intermediate;
+
+
+select * from skew_dp_union_mm order by k2;
+drop table skew_dp_union_mm;
+
+
@@ -122,24 +140,12 @@ drop table partunion_mm;
---drop table partunion_mm;
--drop table merge_mm;
--drop table ctas_mm;
---drop table T1;
---drop table T2;
---drop table skew_mm;
--
--
--create table ctas_mm tblproperties ('hivecommit'='true') as select * from src limit 3;
--
---create table partunion_mm(id_mm int) partitioned by (key_mm int) tblproperties ('hivecommit'='true');
---
---
---insert into table partunion_mm partition(key_mm)
---select temps.* from (
---select key as key_mm, key from ctas_mm
---union all
---select key as key_mm, key from simple_mm ) temps;
--
--set hive.merge.mapredfiles=true;
--set hive.merge.sparkfiles=true;
@@ -158,22 +164,6 @@ drop table partunion_mm;
-- FROM src;
--
--
---set hive.optimize.skewjoin.compiletime = true;
----- the test case is wrong?
---
---CREATE TABLE T1(key STRING, val STRING)
---SKEWED BY (key) ON ((2)) STORED AS TEXTFILE;
---LOAD DATA LOCAL INPATH '../../data/files/T1.txt' INTO TABLE T1;
---CREATE TABLE T2(key STRING, val STRING)
---SKEWED BY (key) ON ((3)) STORED AS TEXTFILE;
---LOAD DATA LOCAL INPATH '../../data/files/T2.txt' INTO TABLE T2;
---
---EXPLAIN
---SELECT a.*, b.* FROM T1 a JOIN T2 b ON a.key = b.key;
---
---create table skew_mm(k1 string, k2 string, k3 string, k4 string) SKEWED BY (key) ON ((2)) tblproperties ('hivecommit'='true');
---INSERT OVERWRITE TABLE skew_mm
---SELECT a.key as k1, a.val as k2, b.key as k3, b.val as k4 FROM T1 a JOIN T2 b ON a.key = b.key;
--
---- TODO load, multi-insert etc
--
http://git-wip-us.apache.org/repos/asf/hive/blob/b9e81572/ql/src/test/queries/clientpositive/mm_current.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/mm_current.q b/ql/src/test/queries/clientpositive/mm_current.q
index e1fb3d9..ceb7a1a 100644
--- a/ql/src/test/queries/clientpositive/mm_current.q
+++ b/ql/src/test/queries/clientpositive/mm_current.q
@@ -12,26 +12,41 @@ insert into table intermediate partition(p='455') select key from src limit 2;
insert into table intermediate partition(p='456') select key from src limit 2;
-create table partunion_no_mm(id int) partitioned by (key int);
-insert into table partunion_no_mm partition(key)
-select temps.* from (
-select key as p, key from intermediate
-union all
-select key + 1 as p, key + 1 from intermediate ) temps;
+set hive.optimize.skewjoin.compiletime = true;
+
+create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3))
+ stored as directories tblproperties ('hivecommit'='false');
+
+insert into table skew_mm
+select key, key, key from intermediate;
+
+drop table skew_mm;
+
+
+create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3))
+ stored as directories tblproperties ('hivecommit'='true');
-select * from partunion_no_mm;
-drop table partunion_no_mm;
+insert into table skew_mm
+select key, key, key from intermediate;
+select * from skew_mm;
+drop table skew_mm;
-create table partunion_mm(id int) partitioned by (key int) tblproperties ('hivecommit'='true');
-insert into table partunion_mm partition(key)
-select temps.* from (
-select key as p, key from intermediate
+
+
+
+
+create table skew_mm(k1 int, k2 int, k4 int) partitioned by (k3 int)
+skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ('hivecommit'='true');
+
+insert into table skew_mm partition (k3)
+select key as i, key as j, key as k, key as l from intermediate
union all
-select key + 1 as p, key + 1 from intermediate ) temps;
+select key +1 as i, key +2 as j, key +3 as k, key +4 as l from intermediate;
+
-select * from partunion_mm;
-drop table partunion_mm;
+select * from skew_mm;
+drop table skew_mm;
http://git-wip-us.apache.org/repos/asf/hive/blob/b9e81572/ql/src/test/results/clientpositive/llap/mm_all.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/mm_all.q.out b/ql/src/test/results/clientpositive/llap/mm_all.q.out
index 0a8bb40..b70ae3c 100644
--- a/ql/src/test/results/clientpositive/llap/mm_all.q.out
+++ b/ql/src/test/results/clientpositive/llap/mm_all.q.out
@@ -12,20 +12,20 @@ POSTHOOK: query: create table intermediate(key int) partitioned by (p int) store
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@intermediate
-PREHOOK: query: insert into table intermediate partition(p='455') select key from src limit 2
+PREHOOK: query: insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@intermediate@p=455
-POSTHOOK: query: insert into table intermediate partition(p='455') select key from src limit 2
+POSTHOOK: query: insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@intermediate@p=455
POSTHOOK: Lineage: intermediate PARTITION(p=455).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
-PREHOOK: query: insert into table intermediate partition(p='456') select key from src limit 2
+PREHOOK: query: insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@intermediate@p=456
-POSTHOOK: query: insert into table intermediate partition(p='456') select key from src limit 2
+POSTHOOK: query: insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@intermediate@p=456
@@ -135,30 +135,30 @@ POSTHOOK: Input: default@intermediate@p=455
POSTHOOK: Input: default@intermediate@p=456
POSTHOOK: Output: default@part_mm@key_mm=455
POSTHOOK: Lineage: part_mm PARTITION(key_mm=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from part_mm
+PREHOOK: query: select * from part_mm order by key
PREHOOK: type: QUERY
PREHOOK: Input: default@part_mm
PREHOOK: Input: default@part_mm@key_mm=455
PREHOOK: Input: default@part_mm@key_mm=456
#### A masked pattern was here ####
-POSTHOOK: query: select * from part_mm
+POSTHOOK: query: select * from part_mm order by key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@part_mm
POSTHOOK: Input: default@part_mm@key_mm=455
POSTHOOK: Input: default@part_mm@key_mm=456
#### A masked pattern was here ####
0 455
-455 455
0 455
-455 455
-0 455
-455 455
-0 455
-455 455
0 456
-455 456
-0 456
-455 456
+10 456
+10 455
+10 455
+97 456
+97 455
+97 455
+98 455
+98 455
+98 456
PREHOOK: query: drop table part_mm
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@part_mm
@@ -205,18 +205,18 @@ POSTHOOK: Input: default@intermediate@p=455
POSTHOOK: Input: default@intermediate@p=456
POSTHOOK: Output: default@simple_mm
POSTHOOK: Lineage: simple_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from simple_mm
+PREHOOK: query: select * from simple_mm order by key
PREHOOK: type: QUERY
PREHOOK: Input: default@simple_mm
#### A masked pattern was here ####
-POSTHOOK: query: select * from simple_mm
+POSTHOOK: query: select * from simple_mm order by key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@simple_mm
#### A masked pattern was here ####
0
-455
-0
-455
+10
+97
+98
PREHOOK: query: insert into table simple_mm select key from intermediate
PREHOOK: type: QUERY
PREHOOK: Input: default@intermediate
@@ -230,22 +230,22 @@ POSTHOOK: Input: default@intermediate@p=455
POSTHOOK: Input: default@intermediate@p=456
POSTHOOK: Output: default@simple_mm
POSTHOOK: Lineage: simple_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from simple_mm
+PREHOOK: query: select * from simple_mm order by key
PREHOOK: type: QUERY
PREHOOK: Input: default@simple_mm
#### A masked pattern was here ####
-POSTHOOK: query: select * from simple_mm
+POSTHOOK: query: select * from simple_mm order by key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@simple_mm
#### A masked pattern was here ####
0
-455
0
-455
-0
-455
-0
-455
+10
+10
+97
+97
+98
+98
PREHOOK: query: drop table simple_mm
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@simple_mm
@@ -254,24 +254,12 @@ POSTHOOK: query: drop table simple_mm
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@simple_mm
POSTHOOK: Output: default@simple_mm
-PREHOOK: query: -- simple DP (no bucketing, no sorting?)
-drop table dp_no_mm
+PREHOOK: query: -- simple DP (no bucketing)
+drop table dp_mm
PREHOOK: type: DROPTABLE
-POSTHOOK: query: -- simple DP (no bucketing, no sorting?)
-drop table dp_no_mm
+POSTHOOK: query: -- simple DP (no bucketing)
+drop table dp_mm
POSTHOOK: type: DROPTABLE
-PREHOOK: query: drop table dp_mm
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table dp_mm
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table dp_no_mm (key int) partitioned by (key1 string, key2 int) stored as orc
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@dp_no_mm
-POSTHOOK: query: create table dp_no_mm (key int) partitioned by (key1 string, key2 int) stored as orc
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@dp_no_mm
PREHOOK: query: create table dp_mm (key int) partitioned by (key1 string, key2 int) stored as orc
tblproperties ('hivecommit'='true')
PREHOOK: type: CREATETABLE
@@ -282,21 +270,6 @@ POSTHOOK: query: create table dp_mm (key int) partitioned by (key1 string, key2
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@dp_mm
-PREHOOK: query: insert into table dp_no_mm partition (key1='123', key2) select key, key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Output: default@dp_no_mm@key1=123
-POSTHOOK: query: insert into table dp_no_mm partition (key1='123', key2) select key, key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Output: default@dp_no_mm@key1=123/key2=0
-POSTHOOK: Output: default@dp_no_mm@key1=123/key2=455
-POSTHOOK: Lineage: dp_no_mm PARTITION(key1=123,key2=0).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: dp_no_mm PARTITION(key1=123,key2=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
PREHOOK: query: insert into table dp_mm partition (key1='123', key2) select key, key from intermediate
PREHOOK: type: QUERY
PREHOOK: Input: default@intermediate
@@ -309,49 +282,33 @@ POSTHOOK: Input: default@intermediate
POSTHOOK: Input: default@intermediate@p=455
POSTHOOK: Input: default@intermediate@p=456
POSTHOOK: Output: default@dp_mm@key1=123/key2=0
-POSTHOOK: Output: default@dp_mm@key1=123/key2=455
+POSTHOOK: Output: default@dp_mm@key1=123/key2=10
+POSTHOOK: Output: default@dp_mm@key1=123/key2=97
+POSTHOOK: Output: default@dp_mm@key1=123/key2=98
POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=0).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from dp_no_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@dp_no_mm
-PREHOOK: Input: default@dp_no_mm@key1=123/key2=0
-PREHOOK: Input: default@dp_no_mm@key1=123/key2=455
-#### A masked pattern was here ####
-POSTHOOK: query: select * from dp_no_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@dp_no_mm
-POSTHOOK: Input: default@dp_no_mm@key1=123/key2=0
-POSTHOOK: Input: default@dp_no_mm@key1=123/key2=455
-#### A masked pattern was here ####
-455 123 455
-455 123 455
-0 123 0
-0 123 0
-PREHOOK: query: select * from dp_mm
+POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=10).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=97).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=98).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from dp_mm order by key
PREHOOK: type: QUERY
PREHOOK: Input: default@dp_mm
PREHOOK: Input: default@dp_mm@key1=123/key2=0
-PREHOOK: Input: default@dp_mm@key1=123/key2=455
+PREHOOK: Input: default@dp_mm@key1=123/key2=10
+PREHOOK: Input: default@dp_mm@key1=123/key2=97
+PREHOOK: Input: default@dp_mm@key1=123/key2=98
#### A masked pattern was here ####
-POSTHOOK: query: select * from dp_mm
+POSTHOOK: query: select * from dp_mm order by key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@dp_mm
POSTHOOK: Input: default@dp_mm@key1=123/key2=0
-POSTHOOK: Input: default@dp_mm@key1=123/key2=455
+POSTHOOK: Input: default@dp_mm@key1=123/key2=10
+POSTHOOK: Input: default@dp_mm@key1=123/key2=97
+POSTHOOK: Input: default@dp_mm@key1=123/key2=98
#### A masked pattern was here ####
-455 123 455
-455 123 455
-0 123 0
0 123 0
-PREHOOK: query: drop table dp_no_mm
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@dp_no_mm
-PREHOOK: Output: default@dp_no_mm
-POSTHOOK: query: drop table dp_no_mm
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@dp_no_mm
-POSTHOOK: Output: default@dp_no_mm
+10 123 10
+97 123 97
+98 123 98
PREHOOK: query: drop table dp_mm
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@dp_mm
@@ -402,13 +359,13 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@union_mm
#### A masked pattern was here ####
0
-0
1
-1
-455
-455
-456
-456
+10
+11
+97
+98
+98
+99
PREHOOK: query: insert into table union_mm
select p from
(
@@ -448,20 +405,23 @@ POSTHOOK: Input: default@union_mm
#### A masked pattern was here ####
0
0
-0
-1
1
1
2
-2
-455
-455
-455
-456
-456
-456
-457
-457
+10
+10
+11
+11
+12
+97
+97
+98
+98
+98
+99
+99
+99
+100
PREHOOK: query: insert into table union_mm
SELECT p FROM
(
@@ -516,25 +476,32 @@ POSTHOOK: Input: default@union_mm
0
0
0
-0
-1
1
1
1
2
2
-2
-455
-455
-455
-455
-456
-456
-456
-456
-457
-457
-457
+10
+10
+10
+11
+11
+11
+12
+12
+97
+97
+97
+98
+98
+98
+98
+99
+99
+99
+99
+100
+100
PREHOOK: query: drop table union_mm
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@union_mm
@@ -572,36 +539,48 @@ POSTHOOK: Input: default@intermediate@p=455
POSTHOOK: Input: default@intermediate@p=456
POSTHOOK: Output: default@partunion_mm@key=0
POSTHOOK: Output: default@partunion_mm@key=1
-POSTHOOK: Output: default@partunion_mm@key=455
-POSTHOOK: Output: default@partunion_mm@key=456
+POSTHOOK: Output: default@partunion_mm@key=10
+POSTHOOK: Output: default@partunion_mm@key=11
+POSTHOOK: Output: default@partunion_mm@key=97
+POSTHOOK: Output: default@partunion_mm@key=98
+POSTHOOK: Output: default@partunion_mm@key=99
POSTHOOK: Lineage: partunion_mm PARTITION(key=0).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: partunion_mm PARTITION(key=10).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: partunion_mm PARTITION(key=11).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
POSTHOOK: Lineage: partunion_mm PARTITION(key=1).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: partunion_mm PARTITION(key=455).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: partunion_mm PARTITION(key=456).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from partunion_mm
+POSTHOOK: Lineage: partunion_mm PARTITION(key=97).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: partunion_mm PARTITION(key=98).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: partunion_mm PARTITION(key=99).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from partunion_mm order by id
PREHOOK: type: QUERY
PREHOOK: Input: default@partunion_mm
PREHOOK: Input: default@partunion_mm@key=0
PREHOOK: Input: default@partunion_mm@key=1
-PREHOOK: Input: default@partunion_mm@key=455
-PREHOOK: Input: default@partunion_mm@key=456
+PREHOOK: Input: default@partunion_mm@key=10
+PREHOOK: Input: default@partunion_mm@key=11
+PREHOOK: Input: default@partunion_mm@key=97
+PREHOOK: Input: default@partunion_mm@key=98
+PREHOOK: Input: default@partunion_mm@key=99
#### A masked pattern was here ####
-POSTHOOK: query: select * from partunion_mm
+POSTHOOK: query: select * from partunion_mm order by id
POSTHOOK: type: QUERY
POSTHOOK: Input: default@partunion_mm
POSTHOOK: Input: default@partunion_mm@key=0
POSTHOOK: Input: default@partunion_mm@key=1
-POSTHOOK: Input: default@partunion_mm@key=455
-POSTHOOK: Input: default@partunion_mm@key=456
+POSTHOOK: Input: default@partunion_mm@key=10
+POSTHOOK: Input: default@partunion_mm@key=11
+POSTHOOK: Input: default@partunion_mm@key=97
+POSTHOOK: Input: default@partunion_mm@key=98
+POSTHOOK: Input: default@partunion_mm@key=99
#### A masked pattern was here ####
0 0
-0 0
-1 1
1 1
-455 455
-455 455
-456 456
-456 456
+10 10
+11 11
+97 97
+98 98
+98 98
+99 99
PREHOOK: query: drop table partunion_mm
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@partunion_mm
@@ -610,35 +589,164 @@ POSTHOOK: query: drop table partunion_mm
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@partunion_mm
POSTHOOK: Output: default@partunion_mm
-PREHOOK: query: -- TODO# from here, fix it
-
-
-
-
--- future
+PREHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3))
+ stored as directories tblproperties ('hivecommit'='true')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@skew_mm
+POSTHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3))
+ stored as directories tblproperties ('hivecommit'='true')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@skew_mm
+PREHOOK: query: insert into table skew_mm
+select key, key, key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Output: default@skew_mm
+POSTHOOK: query: insert into table skew_mm
+select key, key, key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Output: default@skew_mm
+POSTHOOK: Lineage: skew_mm.k1 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_mm.k2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_mm.k4 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from skew_mm order by k2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@skew_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from skew_mm order by k2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@skew_mm
+#### A masked pattern was here ####
+0 0 0
+10 10 10
+97 97 97
+98 98 98
+PREHOOK: query: drop table skew_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@skew_mm
+PREHOOK: Output: default@skew_mm
+POSTHOOK: query: drop table skew_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@skew_mm
+POSTHOOK: Output: default@skew_mm
+PREHOOK: query: create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int)
+skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ('hivecommit'='true')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@skew_dp_union_mm
+POSTHOOK: query: create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int)
+skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ('hivecommit'='true')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@skew_dp_union_mm
+PREHOOK: query: insert into table skew_dp_union_mm partition (k3)
+select key as i, key as j, key as k, key as l from intermediate
+union all
+select key +1 as i, key +2 as j, key +3 as k, key +4 as l from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Output: default@skew_dp_union_mm
+POSTHOOK: query: insert into table skew_dp_union_mm partition (k3)
+select key as i, key as j, key as k, key as l from intermediate
+union all
+select key +1 as i, key +2 as j, key +3 as k, key +4 as l from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Output: default@skew_dp_union_mm@k3=0
+POSTHOOK: Output: default@skew_dp_union_mm@k3=10
+POSTHOOK: Output: default@skew_dp_union_mm@k3=101
+POSTHOOK: Output: default@skew_dp_union_mm@k3=102
+POSTHOOK: Output: default@skew_dp_union_mm@k3=14
+POSTHOOK: Output: default@skew_dp_union_mm@k3=4
+POSTHOOK: Output: default@skew_dp_union_mm@k3=97
+POSTHOOK: Output: default@skew_dp_union_mm@k3=98
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=0).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=0).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=0).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=101).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=101).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=101).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=102).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=102).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=102).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=10).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=10).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=10).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=14).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=14).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=14).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=4).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=4).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=4).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=97).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=97).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=97).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=98).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=98).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=98).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from skew_dp_union_mm order by k2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@skew_dp_union_mm
+PREHOOK: Input: default@skew_dp_union_mm@k3=0
+PREHOOK: Input: default@skew_dp_union_mm@k3=10
+PREHOOK: Input: default@skew_dp_union_mm@k3=101
+PREHOOK: Input: default@skew_dp_union_mm@k3=102
+PREHOOK: Input: default@skew_dp_union_mm@k3=14
+PREHOOK: Input: default@skew_dp_union_mm@k3=4
+PREHOOK: Input: default@skew_dp_union_mm@k3=97
+PREHOOK: Input: default@skew_dp_union_mm@k3=98
+#### A masked pattern was here ####
+POSTHOOK: query: select * from skew_dp_union_mm order by k2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@skew_dp_union_mm
+POSTHOOK: Input: default@skew_dp_union_mm@k3=0
+POSTHOOK: Input: default@skew_dp_union_mm@k3=10
+POSTHOOK: Input: default@skew_dp_union_mm@k3=101
+POSTHOOK: Input: default@skew_dp_union_mm@k3=102
+POSTHOOK: Input: default@skew_dp_union_mm@k3=14
+POSTHOOK: Input: default@skew_dp_union_mm@k3=4
+POSTHOOK: Input: default@skew_dp_union_mm@k3=97
+POSTHOOK: Input: default@skew_dp_union_mm@k3=98
+#### A masked pattern was here ####
+0 0 0 0
+1 2 3 4
+10 10 10 10
+11 12 13 14
+97 97 97 97
+98 98 98 98
+98 99 100 101
+99 100 101 102
+PREHOOK: query: drop table skew_dp_union_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@skew_dp_union_mm
+PREHOOK: Output: default@skew_dp_union_mm
+POSTHOOK: query: drop table skew_dp_union_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@skew_dp_union_mm
+POSTHOOK: Output: default@skew_dp_union_mm
+PREHOOK: query: -- future
---drop table partunion_mm;
--drop table merge_mm;
--drop table ctas_mm;
---drop table T1;
---drop table T2;
---drop table skew_mm;
--
--
--create table ctas_mm tblproperties ('hivecommit'='true') as select * from src limit 3;
--
---create table partunion_mm(id_mm int) partitioned by (key_mm int) tblproperties ('hivecommit'='true');
---
---
---insert into table partunion_mm partition(key_mm)
---select temps.* from (
---select key as key_mm, key from ctas_mm
---union all
---select key as key_mm, key from simple_mm ) temps;
--
--set hive.merge.mapredfiles=true;
--set hive.merge.sparkfiles=true;
@@ -657,22 +765,6 @@ PREHOOK: query: -- TODO# from here, fix it
-- FROM src;
--
--
---set hive.optimize.skewjoin.compiletime = true;
----- the test case is wrong?
---
---CREATE TABLE T1(key STRING, val STRING)
---SKEWED BY (key) ON ((2)) STORED AS TEXTFILE;
---LOAD DATA LOCAL INPATH '../../data/files/T1.txt' INTO TABLE T1;
---CREATE TABLE T2(key STRING, val STRING)
---SKEWED BY (key) ON ((3)) STORED AS TEXTFILE;
---LOAD DATA LOCAL INPATH '../../data/files/T2.txt' INTO TABLE T2;
---
---EXPLAIN
---SELECT a.*, b.* FROM T1 a JOIN T2 b ON a.key = b.key;
---
---create table skew_mm(k1 string, k2 string, k3 string, k4 string) SKEWED BY (key) ON ((2)) tblproperties ('hivecommit'='true');
---INSERT OVERWRITE TABLE skew_mm
---SELECT a.key as k1, a.val as k2, b.key as k3, b.val as k4 FROM T1 a JOIN T2 b ON a.key = b.key;
--
---- TODO load, multi-insert etc
--
@@ -682,35 +774,18 @@ drop table intermediate
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@intermediate
PREHOOK: Output: default@intermediate
-POSTHOOK: query: -- TODO# from here, fix it
-
-
-
-
--- future
+POSTHOOK: query: -- future
---drop table partunion_mm;
--drop table merge_mm;
--drop table ctas_mm;
---drop table T1;
---drop table T2;
---drop table skew_mm;
--
--
--create table ctas_mm tblproperties ('hivecommit'='true') as select * from src limit 3;
--
---create table partunion_mm(id_mm int) partitioned by (key_mm int) tblproperties ('hivecommit'='true');
---
---
---insert into table partunion_mm partition(key_mm)
---select temps.* from (
---select key as key_mm, key from ctas_mm
---union all
---select key as key_mm, key from simple_mm ) temps;
--
--set hive.merge.mapredfiles=true;
--set hive.merge.sparkfiles=true;
@@ -729,22 +804,6 @@ POSTHOOK: query: -- TODO# from here, fix it
-- FROM src;
--
--
---set hive.optimize.skewjoin.compiletime = true;
----- the test case is wrong?
---
---CREATE TABLE T1(key STRING, val STRING)
---SKEWED BY (key) ON ((2)) STORED AS TEXTFILE;
---LOAD DATA LOCAL INPATH '../../data/files/T1.txt' INTO TABLE T1;
---CREATE TABLE T2(key STRING, val STRING)
---SKEWED BY (key) ON ((3)) STORED AS TEXTFILE;
---LOAD DATA LOCAL INPATH '../../data/files/T2.txt' INTO TABLE T2;
---
---EXPLAIN
---SELECT a.*, b.* FROM T1 a JOIN T2 b ON a.key = b.key;
---
---create table skew_mm(k1 string, k2 string, k3 string, k4 string) SKEWED BY (key) ON ((2)) tblproperties ('hivecommit'='true');
---INSERT OVERWRITE TABLE skew_mm
---SELECT a.key as k1, a.val as k2, b.key as k3, b.val as k4 FROM T1 a JOIN T2 b ON a.key = b.key;
--
---- TODO load, multi-insert etc
--
http://git-wip-us.apache.org/repos/asf/hive/blob/b9e81572/ql/src/test/results/clientpositive/llap/mm_current.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/mm_current.q.out b/ql/src/test/results/clientpositive/llap/mm_current.q.out
index 87214ba..d6d31ea 100644
--- a/ql/src/test/results/clientpositive/llap/mm_current.q.out
+++ b/ql/src/test/results/clientpositive/llap/mm_current.q.out
@@ -28,140 +28,163 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@intermediate@p=456
POSTHOOK: Lineage: intermediate PARTITION(p=456).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
-PREHOOK: query: create table partunion_no_mm(id int) partitioned by (key int)
+PREHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3))
+ stored as directories tblproperties ('hivecommit'='false')
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
-PREHOOK: Output: default@partunion_no_mm
-POSTHOOK: query: create table partunion_no_mm(id int) partitioned by (key int)
+PREHOOK: Output: default@skew_mm
+POSTHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3))
+ stored as directories tblproperties ('hivecommit'='false')
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
-POSTHOOK: Output: default@partunion_no_mm
-PREHOOK: query: insert into table partunion_no_mm partition(key)
-select temps.* from (
-select key as p, key from intermediate
-union all
-select key + 1 as p, key + 1 from intermediate ) temps
+POSTHOOK: Output: default@skew_mm
+PREHOOK: query: insert into table skew_mm
+select key, key, key from intermediate
PREHOOK: type: QUERY
PREHOOK: Input: default@intermediate
PREHOOK: Input: default@intermediate@p=455
PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Output: default@partunion_no_mm
-POSTHOOK: query: insert into table partunion_no_mm partition(key)
-select temps.* from (
-select key as p, key from intermediate
-union all
-select key + 1 as p, key + 1 from intermediate ) temps
+PREHOOK: Output: default@skew_mm
+POSTHOOK: query: insert into table skew_mm
+select key, key, key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Output: default@skew_mm
+POSTHOOK: Lineage: skew_mm.k1 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_mm.k2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_mm.k4 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: drop table skew_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@skew_mm
+PREHOOK: Output: default@skew_mm
+POSTHOOK: query: drop table skew_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@skew_mm
+POSTHOOK: Output: default@skew_mm
+PREHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3))
+ stored as directories tblproperties ('hivecommit'='true')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@skew_mm
+POSTHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3))
+ stored as directories tblproperties ('hivecommit'='true')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@skew_mm
+PREHOOK: query: insert into table skew_mm
+select key, key, key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Output: default@skew_mm
+POSTHOOK: query: insert into table skew_mm
+select key, key, key from intermediate
POSTHOOK: type: QUERY
POSTHOOK: Input: default@intermediate
POSTHOOK: Input: default@intermediate@p=455
POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Output: default@partunion_no_mm@key=0
-POSTHOOK: Output: default@partunion_no_mm@key=1
-POSTHOOK: Output: default@partunion_no_mm@key=455
-POSTHOOK: Output: default@partunion_no_mm@key=456
-POSTHOOK: Lineage: partunion_no_mm PARTITION(key=0).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: partunion_no_mm PARTITION(key=1).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: partunion_no_mm PARTITION(key=455).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: partunion_no_mm PARTITION(key=456).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from partunion_no_mm
+POSTHOOK: Output: default@skew_mm
+POSTHOOK: Lineage: skew_mm.k1 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_mm.k2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_mm.k4 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from skew_mm
PREHOOK: type: QUERY
-PREHOOK: Input: default@partunion_no_mm
-PREHOOK: Input: default@partunion_no_mm@key=0
-PREHOOK: Input: default@partunion_no_mm@key=1
-PREHOOK: Input: default@partunion_no_mm@key=455
-PREHOOK: Input: default@partunion_no_mm@key=456
+PREHOOK: Input: default@skew_mm
#### A masked pattern was here ####
-POSTHOOK: query: select * from partunion_no_mm
+POSTHOOK: query: select * from skew_mm
POSTHOOK: type: QUERY
-POSTHOOK: Input: default@partunion_no_mm
-POSTHOOK: Input: default@partunion_no_mm@key=0
-POSTHOOK: Input: default@partunion_no_mm@key=1
-POSTHOOK: Input: default@partunion_no_mm@key=455
-POSTHOOK: Input: default@partunion_no_mm@key=456
+POSTHOOK: Input: default@skew_mm
#### A masked pattern was here ####
-0 0
-0 0
-1 1
-1 1
-455 455
-455 455
-456 456
-456 456
-PREHOOK: query: drop table partunion_no_mm
+455 455 455
+455 455 455
+0 0 0
+0 0 0
+PREHOOK: query: drop table skew_mm
PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@partunion_no_mm
-PREHOOK: Output: default@partunion_no_mm
-POSTHOOK: query: drop table partunion_no_mm
+PREHOOK: Input: default@skew_mm
+PREHOOK: Output: default@skew_mm
+POSTHOOK: query: drop table skew_mm
POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@partunion_no_mm
-POSTHOOK: Output: default@partunion_no_mm
-PREHOOK: query: create table partunion_mm(id int) partitioned by (key int) tblproperties ('hivecommit'='true')
+POSTHOOK: Input: default@skew_mm
+POSTHOOK: Output: default@skew_mm
+PREHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) partitioned by (k3 int)
+skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ('hivecommit'='true')
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
-PREHOOK: Output: default@partunion_mm
-POSTHOOK: query: create table partunion_mm(id int) partitioned by (key int) tblproperties ('hivecommit'='true')
+PREHOOK: Output: default@skew_mm
+POSTHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) partitioned by (k3 int)
+skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ('hivecommit'='true')
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
-POSTHOOK: Output: default@partunion_mm
-PREHOOK: query: insert into table partunion_mm partition(key)
-select temps.* from (
-select key as p, key from intermediate
+POSTHOOK: Output: default@skew_mm
+PREHOOK: query: insert into table skew_mm partition (k3)
+select key as i, key as j, key as k, key as l from intermediate
union all
-select key + 1 as p, key + 1 from intermediate ) temps
+select key +1 as i, key +2 as j, key +3 as k, key +4 as l from intermediate
PREHOOK: type: QUERY
PREHOOK: Input: default@intermediate
PREHOOK: Input: default@intermediate@p=455
PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Output: default@partunion_mm
-POSTHOOK: query: insert into table partunion_mm partition(key)
-select temps.* from (
-select key as p, key from intermediate
+PREHOOK: Output: default@skew_mm
+POSTHOOK: query: insert into table skew_mm partition (k3)
+select key as i, key as j, key as k, key as l from intermediate
union all
-select key + 1 as p, key + 1 from intermediate ) temps
+select key +1 as i, key +2 as j, key +3 as k, key +4 as l from intermediate
POSTHOOK: type: QUERY
POSTHOOK: Input: default@intermediate
POSTHOOK: Input: default@intermediate@p=455
POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Output: default@partunion_mm@key=0
-POSTHOOK: Output: default@partunion_mm@key=1
-POSTHOOK: Output: default@partunion_mm@key=455
-POSTHOOK: Output: default@partunion_mm@key=456
-POSTHOOK: Lineage: partunion_mm PARTITION(key=0).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: partunion_mm PARTITION(key=1).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: partunion_mm PARTITION(key=455).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: partunion_mm PARTITION(key=456).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from partunion_mm
+POSTHOOK: Output: default@skew_mm@k3=0
+POSTHOOK: Output: default@skew_mm@k3=4
+POSTHOOK: Output: default@skew_mm@k3=455
+POSTHOOK: Output: default@skew_mm@k3=459
+POSTHOOK: Lineage: skew_mm PARTITION(k3=0).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_mm PARTITION(k3=0).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_mm PARTITION(k3=0).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_mm PARTITION(k3=455).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_mm PARTITION(k3=455).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_mm PARTITION(k3=455).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_mm PARTITION(k3=459).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_mm PARTITION(k3=459).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_mm PARTITION(k3=459).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_mm PARTITION(k3=4).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_mm PARTITION(k3=4).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_mm PARTITION(k3=4).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from skew_mm
PREHOOK: type: QUERY
-PREHOOK: Input: default@partunion_mm
-PREHOOK: Input: default@partunion_mm@key=0
-PREHOOK: Input: default@partunion_mm@key=1
-PREHOOK: Input: default@partunion_mm@key=455
-PREHOOK: Input: default@partunion_mm@key=456
+PREHOOK: Input: default@skew_mm
+PREHOOK: Input: default@skew_mm@k3=0
+PREHOOK: Input: default@skew_mm@k3=4
+PREHOOK: Input: default@skew_mm@k3=455
+PREHOOK: Input: default@skew_mm@k3=459
#### A masked pattern was here ####
-POSTHOOK: query: select * from partunion_mm
+POSTHOOK: query: select * from skew_mm
POSTHOOK: type: QUERY
-POSTHOOK: Input: default@partunion_mm
-POSTHOOK: Input: default@partunion_mm@key=0
-POSTHOOK: Input: default@partunion_mm@key=1
-POSTHOOK: Input: default@partunion_mm@key=455
-POSTHOOK: Input: default@partunion_mm@key=456
+POSTHOOK: Input: default@skew_mm
+POSTHOOK: Input: default@skew_mm@k3=0
+POSTHOOK: Input: default@skew_mm@k3=4
+POSTHOOK: Input: default@skew_mm@k3=455
+POSTHOOK: Input: default@skew_mm@k3=459
#### A masked pattern was here ####
-0 0
-0 0
-1 1
-1 1
-455 455
-455 455
-456 456
-456 456
-PREHOOK: query: drop table partunion_mm
+0 0 0 0
+0 0 0 0
+1 2 3 4
+1 2 3 4
+455 455 455 455
+455 455 455 455
+456 457 458 459
+456 457 458 459
+PREHOOK: query: drop table skew_mm
PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@partunion_mm
-PREHOOK: Output: default@partunion_mm
-POSTHOOK: query: drop table partunion_mm
+PREHOOK: Input: default@skew_mm
+PREHOOK: Output: default@skew_mm
+POSTHOOK: query: drop table skew_mm
POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@partunion_mm
-POSTHOOK: Output: default@partunion_mm
+POSTHOOK: Input: default@skew_mm
+POSTHOOK: Output: default@skew_mm
PREHOOK: query: drop table intermediate
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@intermediate