You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/07/11 02:04:33 UTC
[01/32] hive git commit: HIVE-18118: Explain Extended should indicate
if a file being read is an EC file (Andrew Sherman, reviewed by Sahil Takiar)
Repository: hive
Updated Branches:
refs/heads/master-txnstats f384d4cac -> 04ea1455f
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreUtils.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreUtils.java
index 55ff150..d5ae5d1 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreUtils.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreUtils.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.metastore.utils;
import com.google.common.collect.ImmutableMap;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
@@ -43,6 +44,7 @@ import java.util.Map;
import static org.apache.hadoop.hive.common.StatsSetupConst.COLUMN_STATS_ACCURATE;
import static org.apache.hadoop.hive.common.StatsSetupConst.NUM_FILES;
+import static org.apache.hadoop.hive.common.StatsSetupConst.NUM_ERASURE_CODED_FILES;
import static org.apache.hadoop.hive.common.StatsSetupConst.STATS_GENERATED;
import static org.apache.hadoop.hive.common.StatsSetupConst.TOTAL_SIZE;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.updateTableStatsSlow;
@@ -60,7 +62,11 @@ public class TestMetaStoreUtils {
private static final String DB_NAME = "db1";
private static final String TABLE_NAME = "tbl1";
- private final Map<String, String> paramsWithStats = ImmutableMap.of(NUM_FILES, "1", TOTAL_SIZE, "2");
+ private final Map<String, String> paramsWithStats = ImmutableMap.of(
+ NUM_FILES, "1",
+ TOTAL_SIZE, "2",
+ NUM_ERASURE_CODED_FILES, "0"
+ );
private Database db;
@@ -120,7 +126,7 @@ public class TestMetaStoreUtils {
* <li>Create database</li>
* <li>Create unpartitioned table</li>
* <li>Create unpartitioned table which has params</li>
- * <li>Call updateTableStatsSlow with arguments which should caue stats calculation</li>
+ * <li>Call updateTableStatsSlow with arguments which should cause stats calculation</li>
* <li>Verify table statistics using mocked warehouse</li>
* <li>Create table which already have stats</li>
* <li>Call updateTableStatsSlow forcing stats recompute</li>
@@ -141,18 +147,17 @@ public class TestMetaStoreUtils {
// Set up mock warehouse
- FileStatus fs1 = new FileStatus(1, true, 2, 3,
- 4, new Path("/tmp/0"));
- FileStatus fs2 = new FileStatus(fileLength, false, 3, 4,
- 5, new Path("/tmp/1"));
- FileStatus fs3 = new FileStatus(fileLength, false, 3, 4,
- 5, new Path("/tmp/1"));
+ FileStatus fs1 = getFileStatus(1, true, 2, 3, 4, "/tmp/0", false);
+ FileStatus fs2 = getFileStatus(fileLength, false, 3, 4, 5, "/tmp/1", true);
+ FileStatus fs3 = getFileStatus(fileLength, false, 3, 4, 5, "/tmp/1", false);
List<FileStatus> fileStatus = Arrays.asList(fs1, fs2, fs3);
Warehouse wh = mock(Warehouse.class);
when(wh.getFileStatusesForUnpartitionedTable(db, tbl)).thenReturn(fileStatus);
Map<String, String> expected = ImmutableMap.of(NUM_FILES, "2",
- TOTAL_SIZE, String.valueOf(2 * fileLength));
+ TOTAL_SIZE, String.valueOf(2 * fileLength),
+ NUM_ERASURE_CODED_FILES, "1"
+ );
updateTableStatsSlow(db, tbl, wh, false, false, null);
assertThat(tbl.getParameters(), is(expected));
@@ -195,6 +200,7 @@ public class TestMetaStoreUtils {
Map<String, String> expected1 = ImmutableMap.of(NUM_FILES, "2",
TOTAL_SIZE, String.valueOf(2 * fileLength),
+ NUM_ERASURE_CODED_FILES, "1",
COLUMN_STATS_ACCURATE, "{\"BASIC_STATS\":\"true\"}");
assertThat(tbl3.getParameters(), is(expected1));
}
@@ -227,7 +233,7 @@ public class TestMetaStoreUtils {
}
/**
- * Verify that updateTableStatsSlow() does not calculate tabe statistics when
+ * Verify that updateTableStatsSlow() does not calculate table statistics when
* <ol>
* <li>newDir is true</li>
* <li>Table is partitioned</li>
@@ -270,5 +276,16 @@ public class TestMetaStoreUtils {
updateTableStatsSlow(db, tbl2, wh, false, false, null);
verify(wh, never()).getFileStatusesForUnpartitionedTable(db, tbl2);
}
+
+ /**
+ * Build a FileStatus object.
+ */
+ private static FileStatus getFileStatus(long fileLength, boolean isdir, int blockReplication,
+ int blockSize, int modificationTime, String pathString, boolean isErasureCoded) {
+ return new FileStatus(fileLength, isdir, blockReplication, blockSize, modificationTime,
+ 0L, (FsPermission)null, (String)null, (String)null, null,
+ new Path(pathString), false, false, isErasureCoded);
+ }
+
}
[19/32] hive git commit: HIVE-20099: Fix logger for LlapServlet
(Rajkumar Singh via Gopal V)
Posted by se...@apache.org.
HIVE-20099: Fix logger for LlapServlet (Rajkumar Singh via Gopal V)
Signed-off-by: Gopal V <go...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/eb91fa49
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/eb91fa49
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/eb91fa49
Branch: refs/heads/master-txnstats
Commit: eb91fa49db99396c3cf0e79d02962dbbaef55e06
Parents: d3bba71
Author: Raj Kumar Singh <ra...@hortonworks.com>
Authored: Mon Jul 9 09:19:07 2018 -0700
Committer: Gopal V <go...@apache.org>
Committed: Mon Jul 9 09:19:07 2018 -0700
----------------------------------------------------------------------
service/src/java/org/apache/hive/http/LlapServlet.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/eb91fa49/service/src/java/org/apache/hive/http/LlapServlet.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/http/LlapServlet.java b/service/src/java/org/apache/hive/http/LlapServlet.java
index a0eb409..2bc59b5 100644
--- a/service/src/java/org/apache/hive/http/LlapServlet.java
+++ b/service/src/java/org/apache/hive/http/LlapServlet.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.hive.llap.cli.LlapStatusServiceDriver;
@SuppressWarnings("serial")
public class LlapServlet extends HttpServlet {
- private static final Log LOG = LogFactory.getLog(JMXJsonServlet.class);
+ private static final Log LOG = LogFactory.getLog(LlapServlet.class);
/**
* Initialize this servlet.
[25/32] hive git commit: HIVE-20126: OrcInputFormat does not pass
conf to orc reader options (Prasanth Jayachandran reviewed by Sergey
Shelukhin)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/18fb1b3f/data/files/studenttab10k
----------------------------------------------------------------------
diff --git a/data/files/studenttab10k b/data/files/studenttab10k
new file mode 100644
index 0000000..30226f0
--- /dev/null
+++ b/data/files/studenttab10k
@@ -0,0 +1,10000 @@
+ulysses thompson 64 1.90
+katie carson 25 3.65
+luke king 65 0.73
+holly davidson 57 2.43
+fred miller 55 3.77
+holly white 43 0.24
+luke steinbeck 51 1.14
+nick underhill 31 2.46
+holly davidson 59 1.26
+calvin brown 56 0.72
+rachel robinson 62 2.25
+tom carson 35 0.56
+tom johnson 72 0.99
+irene garcia 54 1.06
+oscar nixon 39 3.60
+holly allen 32 2.58
+oscar hernandez 19 0.05
+alice ichabod 65 2.25
+wendy thompson 30 2.39
+priscilla hernandez 73 0.23
+gabriella van buren 68 1.32
+yuri thompson 42 3.65
+yuri laertes 60 1.16
+sarah young 23 2.76
+zach white 32 0.20
+nick van buren 68 1.75
+xavier underhill 41 1.51
+bob ichabod 56 2.81
+zach steinbeck 61 2.22
+alice garcia 42 2.03
+jessica king 29 3.61
+calvin nixon 37 0.30
+fred polk 66 3.69
+bob zipper 40 0.28
+alice young 75 0.31
+nick underhill 37 1.65
+mike white 57 0.69
+calvin ovid 41 3.02
+fred steinbeck 47 3.57
+sarah ovid 65 0.00
+wendy nixon 63 0.62
+gabriella zipper 77 1.51
+david king 40 1.99
+jessica white 30 3.82
+alice robinson 37 3.69
+zach nixon 74 2.75
+irene davidson 27 1.22
+priscilla xylophone 43 1.60
+oscar zipper 25 2.43
+fred falkner 38 2.23
+ulysses polk 58 0.01
+katie hernandez 47 3.80
+zach steinbeck 55 0.68
+fred laertes 69 3.62
+quinn laertes 70 3.66
+nick garcia 50 0.12
+oscar young 55 2.22
+bob underhill 47 0.24
+calvin young 77 1.60
+mike allen 65 2.95
+david young 77 0.26
+oscar garcia 69 1.59
+ulysses ichabod 26 0.95
+wendy laertes 76 1.13
+sarah laertes 20 0.24
+zach ichabod 60 1.60
+tom robinson 62 0.78
+zach steinbeck 69 1.01
+quinn garcia 57 0.98
+yuri van buren 32 1.97
+luke carson 39 0.76
+calvin ovid 73 0.82
+luke ellison 27 0.56
+oscar zipper 50 1.31
+fred steinbeck 52 3.14
+katie xylophone 76 1.38
+luke king 54 2.30
+ethan white 72 1.43
+yuri ovid 37 3.64
+jessica garcia 54 1.08
+luke young 29 0.80
+mike miller 39 3.35
+fred hernandez 63 0.17
+priscilla hernandez 52 0.35
+ethan garcia 43 1.70
+quinn hernandez 25 2.58
+calvin nixon 33 1.01
+yuri xylophone 47 1.36
+ulysses steinbeck 63 1.05
+jessica nixon 25 2.13
+bob johnson 53 3.31
+jessica ichabod 56 2.21
+zach miller 63 3.87
+priscilla white 66 2.82
+ulysses allen 21 1.68
+katie falkner 47 1.49
+tom king 51 1.91
+bob laertes 60 3.33
+luke nixon 27 3.54
+quinn johnson 42 2.24
+wendy quirinius 71 0.10
+victor polk 55 3.63
+rachel robinson 32 1.11
+sarah king 57 1.37
+victor young 38 1.72
+priscilla steinbeck 38 2.11
+fred brown 19 2.72
+xavier underhill 55 3.56
+irene ovid 67 3.80
+calvin brown 37 2.22
+katie thompson 20 3.27
+katie carson 66 3.55
+tom miller 57 2.83
+rachel brown 56 0.74
+holly johnson 38 2.51
+irene steinbeck 29 1.97
+wendy falkner 37 0.14
+ethan white 29 3.62
+bob underhill 26 1.10
+jessica king 64 0.69
+luke steinbeck 19 1.16
+luke laertes 70 3.58
+rachel polk 74 0.92
+calvin xylophone 52 0.58
+luke white 57 3.86
+calvin van buren 52 3.13
+holly quirinius 59 1.70
+mike brown 44 1.93
+yuri ichabod 61 0.70
+ulysses miller 56 3.53
+victor hernandez 64 2.52
+oscar young 34 0.34
+luke ovid 36 3.17
+quinn ellison 50 1.13
+quinn xylophone 72 2.07
+nick underhill 48 0.15
+rachel miller 23 3.38
+mike van buren 68 1.74
+zach van buren 38 0.34
+irene zipper 32 0.54
+sarah garcia 31 3.87
+rachel van buren 56 0.35
+fred davidson 69 1.57
+nick hernandez 19 2.11
+irene polk 40 3.89
+katie young 26 2.88
+priscilla ovid 49 3.28
+jessica hernandez 39 3.13
+yuri allen 29 3.51
+victor garcia 66 3.45
+zach johnson 77 0.95
+yuri zipper 48 3.44
+alice falkner 28 3.72
+gabriella allen 58 3.61
+bob nixon 34 3.34
+bob white 67 2.93
+holly steinbeck 57 1.81
+wendy van buren 40 1.09
+calvin brown 61 2.08
+irene young 25 2.66
+holly van buren 40 2.37
+katie underhill 30 0.63
+quinn hernandez 73 0.31
+fred nixon 53 1.76
+luke ellison 59 1.10
+quinn nixon 24 0.30
+ethan underhill 68 2.25
+fred underhill 28 3.88
+jessica brown 59 3.66
+katie falkner 49 3.96
+calvin ellison 27 2.23
+zach carson 59 0.46
+priscilla polk 47 2.99
+rachel zipper 49 3.26
+holly king 73 1.23
+zach carson 64 2.60
+fred xylophone 61 3.15
+gabriella miller 43 1.73
+david laertes 56 3.43
+tom garcia 63 2.78
+ethan king 66 3.13
+david hernandez 26 2.52
+wendy ichabod 57 2.81
+alice young 69 0.25
+tom xylophone 50 2.78
+ulysses carson 62 2.26
+nick garcia 43 2.23
+gabriella ellison 33 1.18
+ethan miller 28 2.15
+tom zipper 19 2.56
+wendy white 19 1.12
+luke ovid 31 1.68
+wendy xylophone 75 2.58
+quinn garcia 22 3.65
+holly ellison 68 0.26
+yuri hernandez 75 2.50
+tom underhill 71 2.68
+ulysses king 31 1.76
+fred thompson 46 1.55
+gabriella ichabod 26 1.59
+xavier ichabod 22 3.57
+zach ichabod 54 0.62
+priscilla robinson 36 1.48
+holly hernandez 30 3.53
+tom hernandez 68 2.60
+david white 52 0.75
+rachel ellison 23 2.21
+quinn ellison 69 1.73
+oscar falkner 62 3.78
+nick quirinius 47 2.87
+ethan johnson 55 0.75
+calvin ovid 38 2.50
+zach hernandez 22 2.32
+rachel white 43 3.17
+holly king 44 2.24
+quinn zipper 52 1.08
+katie underhill 31 1.37
+victor ichabod 41 2.73
+david quirinius 18 0.09
+mike allen 73 0.41
+luke garcia 34 0.33
+fred xylophone 50 2.79
+zach zipper 59 2.53
+priscilla young 48 0.93
+xavier steinbeck 58 2.99
+tom nixon 58 2.27
+bob polk 67 2.62
+luke xylophone 27 2.25
+victor xylophone 71 2.75
+gabriella white 25 3.08
+rachel brown 37 1.26
+nick carson 41 2.77
+victor brown 18 3.18
+priscilla miller 55 1.78
+calvin ichabod 76 2.08
+katie miller 25 0.21
+mike robinson 29 1.17
+alice quirinius 75 3.08
+ulysses van buren 20 1.97
+tom brown 72 1.16
+oscar nixon 40 0.98
+holly hernandez 35 3.62
+bob davidson 74 1.97
+xavier king 23 0.62
+luke ovid 25 0.46
+gabriella davidson 54 3.73
+ethan xylophone 39 0.32
+victor ichabod 62 1.45
+holly xylophone 28 3.61
+bob hernandez 52 1.46
+alice johnson 52 2.42
+bob van buren 65 0.94
+xavier miller 64 2.89
+quinn garcia 70 0.59
+jessica miller 39 3.50
+quinn king 44 0.34
+mike nixon 61 3.40
+holly hernandez 61 0.97
+tom garcia 41 0.66
+alice thompson 62 2.50
+ulysses underhill 77 2.30
+sarah johnson 25 0.90
+ethan allen 28 0.24
+oscar falkner 77 3.46
+david zipper 30 1.72
+yuri davidson 24 1.83
+jessica davidson 69 3.06
+nick quirinius 42 1.25
+tom ichabod 39 2.37
+gabriella ichabod 18 3.10
+xavier johnson 51 3.48
+zach garcia 26 1.69
+fred robinson 49 2.18
+alice miller 49 3.15
+gabriella van buren 74 2.92
+ethan ellison 50 2.32
+fred white 46 1.16
+mike zipper 20 0.67
+tom ellison 29 3.70
+bob ellison 41 0.49
+bob brown 43 2.88
+quinn white 55 2.31
+bob nixon 62 1.86
+wendy white 37 0.52
+calvin underhill 47 3.67
+ulysses steinbeck 47 0.40
+oscar king 29 2.86
+priscilla carson 70 0.07
+priscilla garcia 36 1.88
+gabriella nixon 77 1.16
+quinn ichabod 45 0.79
+calvin garcia 59 0.67
+holly garcia 76 0.59
+victor davidson 34 1.05
+holly nixon 26 2.99
+calvin hernandez 68 0.42
+ethan miller 48 1.72
+gabriella carson 37 1.61
+oscar young 37 2.65
+fred zipper 45 3.90
+xavier white 26 3.19
+luke laertes 18 2.41
+nick allen 50 3.45
+bob underhill 74 1.31
+nick garcia 36 3.12
+xavier underhill 28 0.63
+gabriella quirinius 71 0.65
+quinn carson 51 1.80
+holly garcia 30 0.39
+fred allen 23 2.71
+mike underhill 63 0.77
+quinn white 27 3.10
+rachel thompson 28 1.90
+quinn king 67 2.65
+victor laertes 43 2.25
+bob hernandez 30 0.42
+tom thompson 34 3.83
+david miller 23 2.46
+oscar laertes 26 1.19
+gabriella brown 58 2.52
+calvin king 72 2.90
+quinn young 71 3.28
+priscilla polk 45 2.71
+quinn allen 26 0.24
+irene robinson 35 3.29
+priscilla johnson 76 0.66
+ethan van buren 74 3.81
+katie underhill 34 1.22
+nick nixon 21 1.12
+xavier xylophone 67 2.53
+xavier garcia 49 2.39
+zach miller 58 1.99
+ulysses thompson 49 2.05
+fred allen 61 1.42
+victor falkner 36 0.38
+ethan ellison 52 1.39
+calvin davidson 25 0.01
+gabriella nixon 31 1.33
+irene nixon 27 1.53
+gabriella davidson 74 0.35
+david ellison 26 2.55
+sarah davidson 69 0.94
+yuri allen 75 1.76
+ethan hernandez 75 2.07
+jessica underhill 18 0.88
+gabriella davidson 27 3.48
+nick miller 69 2.82
+holly ovid 77 3.55
+david robinson 34 0.06
+katie ovid 29 0.92
+rachel xylophone 25 0.96
+katie thompson 41 2.52
+luke van buren 43 1.42
+tom zipper 25 0.97
+quinn zipper 47 3.50
+luke johnson 75 0.87
+david miller 26 2.60
+victor polk 55 2.32
+nick laertes 18 2.90
+tom ellison 58 2.39
+yuri ichabod 43 1.43
+rachel brown 24 0.74
+victor brown 58 3.22
+mike white 24 0.05
+katie davidson 68 2.92
+jessica young 26 1.53
+yuri garcia 53 2.74
+fred zipper 64 0.76
+gabriella nixon 68 1.62
+alice young 36 2.77
+quinn brown 42 1.38
+priscilla polk 21 0.52
+katie carson 76 0.66
+katie polk 63 2.96
+sarah thompson 77 2.89
+xavier young 56 0.18
+luke garcia 19 1.62
+jessica king 70 3.58
+victor white 72 2.28
+ethan ovid 59 0.60
+oscar king 19 0.67
+wendy van buren 70 1.96
+luke hernandez 21 2.65
+mike carson 72 1.47
+sarah nixon 34 3.05
+rachel miller 42 3.15
+fred laertes 51 1.77
+quinn johnson 44 0.54
+gabriella carson 61 1.38
+nick zipper 38 0.16
+oscar van buren 53 2.51
+sarah king 32 3.51
+holly davidson 63 3.80
+david hernandez 40 1.18
+oscar king 19 1.65
+calvin carson 30 3.47
+yuri miller 70 3.46
+sarah hernandez 20 3.23
+quinn brown 68 1.03
+jessica xylophone 48 1.88
+tom quirinius 19 1.43
+bob van buren 39 0.81
+zach garcia 61 1.63
+fred garcia 49 0.25
+wendy ichabod 39 0.46
+zach miller 32 2.99
+katie falkner 48 2.45
+calvin laertes 54 0.36
+ethan carson 76 3.68
+alice young 32 1.62
+xavier johnson 24 2.97
+sarah carson 48 0.18
+priscilla carson 56 1.70
+tom polk 40 1.91
+quinn falkner 26 0.86
+sarah xylophone 24 1.39
+mike ichabod 54 0.17
+quinn miller 22 0.17
+tom steinbeck 53 1.78
+calvin garcia 75 1.64
+gabriella steinbeck 35 1.27
+ethan ovid 52 2.03
+wendy falkner 19 3.04
+quinn brown 49 1.39
+irene carson 75 1.21
+holly white 23 3.77
+irene nixon 29 1.87
+irene polk 39 0.58
+ulysses miller 23 3.15
+david ovid 64 0.24
+jessica carson 34 3.99
+jessica ellison 48 0.91
+zach king 73 1.74
+nick ichabod 35 3.81
+ulysses ichabod 61 1.40
+calvin quirinius 72 2.93
+luke white 31 0.51
+xavier underhill 42 0.01
+jessica white 63 1.00
+nick steinbeck 49 3.80
+xavier young 46 0.46
+priscilla carson 23 2.29
+fred carson 71 2.97
+priscilla carson 46 2.89
+victor young 75 3.09
+sarah robinson 61 1.40
+bob brown 75 0.45
+ulysses carson 49 3.13
+rachel thompson 44 0.04
+katie zipper 62 1.93
+victor carson 62 1.54
+victor robinson 55 1.13
+victor allen 40 2.80
+katie king 62 3.56
+katie king 63 3.20
+alice steinbeck 30 0.13
+ulysses young 68 2.52
+priscilla white 58 1.51
+calvin falkner 30 3.87
+alice miller 54 0.81
+ethan white 62 3.40
+bob zipper 65 1.62
+zach zipper 24 2.64
+alice thompson 18 1.88
+nick king 68 1.80
+luke quirinius 52 2.75
+zach ichabod 67 1.64
+calvin hernandez 34 3.60
+zach brown 65 0.35
+xavier white 59 3.87
+tom zipper 61 1.41
+wendy falkner 73 0.07
+rachel carson 22 3.35
+luke garcia 52 2.42
+katie king 58 0.19
+victor robinson 21 3.51
+tom allen 58 0.76
+fred underhill 47 3.45
+luke zipper 56 0.66
+yuri allen 19 0.43
+quinn van buren 66 0.94
+tom xylophone 75 4.00
+zach garcia 44 0.98
+holly zipper 74 0.64
+mike allen 75 1.54
+david davidson 74 2.36
+fred miller 54 0.07
+bob davidson 54 1.29
+david thompson 36 0.66
+ulysses laertes 25 2.59
+mike xylophone 59 3.56
+bob xylophone 36 0.80
+jessica allen 19 3.69
+alice white 40 1.89
+wendy laertes 58 2.47
+oscar robinson 40 2.81
+holly nixon 24 1.95
+yuri brown 18 1.33
+wendy ellison 36 1.27
+gabriella ellison 75 3.04
+fred nixon 63 0.61
+irene quirinius 46 1.42
+irene robinson 38 1.04
+alice johnson 72 1.25
+irene allen 48 1.83
+bob young 63 1.11
+katie nixon 24 0.59
+xavier van buren 73 1.09
+sarah brown 50 2.75
+victor thompson 39 3.63
+luke johnson 76 3.23
+tom thompson 40 2.09
+wendy thompson 72 0.29
+wendy miller 72 1.76
+priscilla young 55 2.06
+zach nixon 44 0.64
+quinn underhill 50 0.22
+david allen 50 0.07
+nick quirinius 71 1.32
+ulysses ovid 48 2.01
+oscar white 43 0.74
+xavier ellison 62 0.55
+tom king 21 0.91
+luke johnson 64 2.54
+irene xylophone 34 1.65
+yuri polk 35 3.19
+sarah xylophone 51 3.52
+bob hernandez 63 1.78
+jessica thompson 38 3.72
+ulysses ichabod 35 2.12
+ethan hernandez 48 1.30
+fred van buren 25 1.90
+quinn thompson 48 3.14
+katie van buren 66 2.66
+luke miller 52 3.85
+priscilla robinson 65 0.06
+jessica hernandez 64 3.65
+ulysses hernandez 35 1.86
+alice laertes 25 2.18
+irene johnson 45 1.07
+xavier davidson 48 1.37
+wendy van buren 66 0.16
+oscar zipper 57 0.56
+david garcia 19 2.06
+yuri van buren 36 2.75
+rachel nixon 75 0.51
+wendy xylophone 65 0.85
+rachel white 40 2.48
+katie ichabod 41 0.84
+katie king 71 0.54
+nick allen 50 1.24
+sarah davidson 58 1.13
+ulysses polk 59 0.13
+alice garcia 62 0.36
+quinn nixon 55 0.07
+alice white 74 3.46
+wendy thompson 22 1.28
+mike johnson 42 1.36
+luke laertes 67 0.53
+victor polk 65 3.15
+xavier brown 51 1.97
+sarah xylophone 60 0.69
+tom ichabod 74 0.94
+ulysses robinson 48 2.81
+mike carson 32 3.14
+xavier ichabod 19 2.80
+xavier ellison 55 0.52
+jessica brown 61 3.98
+victor ichabod 39 2.88
+yuri davidson 24 2.66
+david ovid 30 3.27
+alice hernandez 49 1.41
+david robinson 68 1.28
+oscar allen 64 0.04
+wendy king 41 3.77
+david underhill 57 1.60
+quinn brown 20 0.41
+gabriella steinbeck 46 1.58
+priscilla nixon 35 3.72
+wendy carson 39 2.62
+nick ellison 57 0.81
+luke van buren 77 3.92
+jessica hernandez 37 0.92
+ethan young 51 1.28
+david zipper 22 0.65
+jessica thompson 70 0.59
+jessica brown 63 1.49
+quinn hernandez 18 2.60
+gabriella zipper 44 0.96
+priscilla brown 41 0.09
+calvin nixon 23 3.06
+victor laertes 27 3.61
+bob allen 40 1.00
+victor garcia 64 3.68
+rachel davidson 49 1.73
+david johnson 18 0.54
+priscilla johnson 23 1.24
+katie miller 40 0.47
+xavier steinbeck 23 0.59
+nick falkner 42 1.37
+ethan hernandez 37 0.09
+jessica miller 18 2.17
+katie robinson 44 2.49
+rachel falkner 23 2.75
+sarah young 38 3.66
+quinn garcia 37 1.19
+david allen 18 2.87
+zach ellison 74 3.36
+luke carson 65 0.85
+katie white 37 0.51
+fred quirinius 40 3.43
+irene ellison 76 2.66
+irene laertes 49 1.06
+xavier johnson 20 0.81
+alice thompson 74 0.38
+priscilla nixon 77 3.09
+holly ovid 35 0.66
+alice davidson 28 3.52
+irene king 23 2.05
+rachel robinson 23 2.71
+luke thompson 77 1.66
+victor thompson 39 1.05
+wendy quirinius 62 3.62
+bob xylophone 38 2.32
+xavier underhill 24 2.71
+bob davidson 73 3.02
+yuri hernandez 57 3.68
+holly young 40 0.09
+fred underhill 37 3.72
+quinn garcia 63 3.78
+xavier underhill 24 0.57
+ethan van buren 58 3.60
+irene ovid 49 2.46
+victor polk 39 1.82
+jessica polk 63 3.00
+fred laertes 69 2.48
+oscar brown 44 2.29
+priscilla steinbeck 34 2.55
+priscilla van buren 62 2.90
+victor brown 22 1.56
+victor young 21 3.29
+ulysses hernandez 56 1.10
+rachel laertes 42 3.27
+fred ellison 50 3.54
+quinn johnson 41 2.18
+wendy allen 32 2.38
+fred thompson 30 2.66
+david allen 65 3.21
+gabriella falkner 70 3.18
+alice davidson 49 1.24
+gabriella laertes 54 3.19
+mike miller 73 0.36
+rachel brown 19 3.03
+tom white 46 1.99
+tom ichabod 21 1.77
+bob laertes 31 3.07
+ethan robinson 74 2.67
+oscar xylophone 25 1.96
+priscilla davidson 43 1.38
+priscilla garcia 67 2.84
+bob quirinius 48 2.52
+zach quirinius 35 0.47
+oscar steinbeck 45 0.07
+ulysses underhill 32 3.18
+jessica xylophone 60 3.90
+jessica laertes 62 3.91
+calvin xylophone 18 0.53
+ethan johnson 75 0.91
+xavier robinson 42 3.71
+fred garcia 67 2.23
+wendy zipper 30 4.00
+wendy brown 49 0.07
+alice thompson 36 2.70
+priscilla young 41 1.60
+holly brown 76 1.15
+victor laertes 57 3.59
+oscar davidson 77 0.55
+david white 37 2.47
+ethan underhill 32 1.95
+xavier nixon 43 0.53
+holly xylophone 38 0.03
+bob miller 40 1.81
+zach ichabod 69 3.00
+quinn van buren 44 0.10
+quinn davidson 66 3.54
+luke ovid 60 2.91
+tom thompson 40 0.29
+calvin ichabod 64 0.35
+mike carson 26 2.32
+bob thompson 51 1.26
+nick king 63 0.13
+zach carson 65 0.65
+fred allen 53 3.46
+zach white 19 0.03
+priscilla hernandez 68 1.91
+nick quirinius 34 3.88
+quinn zipper 60 1.16
+victor young 56 1.77
+jessica young 30 3.60
+calvin thompson 53 3.34
+holly steinbeck 47 3.33
+wendy king 38 3.62
+yuri xylophone 63 1.57
+xavier johnson 57 2.55
+victor polk 24 1.98
+tom thompson 25 0.81
+sarah miller 25 1.79
+tom davidson 73 0.75
+mike van buren 52 0.45
+wendy king 37 3.75
+wendy carson 50 3.76
+luke falkner 58 2.24
+sarah laertes 40 3.95
+priscilla underhill 39 3.37
+irene davidson 22 0.40
+mike young 49 0.56
+gabriella quirinius 54 0.58
+sarah zipper 42 0.12
+oscar quirinius 34 1.25
+victor underhill 59 1.52
+xavier garcia 27 1.01
+irene young 33 1.27
+irene allen 52 0.05
+oscar steinbeck 46 2.02
+xavier thompson 26 2.32
+katie thompson 53 3.84
+ulysses king 71 0.29
+quinn steinbeck 22 2.84
+oscar falkner 55 2.00
+alice carson 19 1.88
+irene ichabod 54 1.55
+zach young 59 2.26
+katie ichabod 21 1.58
+zach young 68 1.51
+alice underhill 46 2.43
+ulysses quirinius 43 2.59
+rachel hernandez 36 0.00
+mike zipper 50 0.19
+ulysses hernandez 43 0.77
+holly garcia 53 2.60
+alice zipper 21 2.17
+wendy thompson 25 3.83
+priscilla miller 32 0.99
+david xylophone 37 1.97
+alice johnson 77 0.74
+nick davidson 76 0.25
+calvin underhill 38 3.24
+mike davidson 51 0.60
+nick allen 72 2.60
+sarah miller 76 0.43
+oscar xylophone 59 2.19
+priscilla laertes 19 0.91
+ethan xylophone 62 0.18
+oscar falkner 65 0.11
+holly young 71 0.25
+victor thompson 50 2.17
+nick ellison 48 0.96
+calvin xylophone 67 3.26
+tom young 46 0.85
+alice laertes 65 0.74
+mike quirinius 72 1.28
+katie nixon 21 1.99
+quinn zipper 72 0.74
+rachel ellison 50 3.43
+david king 62 3.97
+luke xylophone 65 0.24
+katie miller 43 0.85
+quinn robinson 75 1.79
+mike allen 27 3.80
+nick johnson 32 2.00
+luke miller 28 0.72
+calvin brown 29 0.54
+ulysses nixon 30 2.18
+wendy allen 29 1.38
+alice carson 28 2.08
+quinn carson 62 1.94
+mike nixon 58 1.34
+luke davidson 32 1.14
+xavier van buren 64 0.40
+sarah laertes 28 1.39
+oscar robinson 51 1.25
+ethan allen 51 3.23
+sarah nixon 22 0.20
+ulysses king 73 1.97
+holly hernandez 32 3.49
+ulysses allen 72 3.72
+irene van buren 21 2.89
+gabriella white 67 1.20
+mike thompson 70 3.46
+alice zipper 45 1.80
+xavier ovid 62 0.87
+jessica steinbeck 76 3.86
+sarah miller 53 1.08
+jessica nixon 63 1.22
+ethan robinson 43 0.64
+ethan zipper 48 3.98
+holly robinson 56 1.30
+bob davidson 57 3.79
+luke ichabod 63 2.76
+mike hernandez 32 3.97
+katie king 50 0.66
+tom polk 34 3.44
+sarah davidson 51 3.40
+gabriella johnson 29 3.91
+mike nixon 62 1.21
+fred quirinius 28 2.25
+david garcia 22 3.14
+ethan hernandez 55 3.07
+jessica davidson 46 2.01
+bob johnson 37 3.15
+ethan brown 56 1.96
+bob polk 68 0.59
+mike miller 74 3.48
+zach garcia 58 2.63
+oscar king 51 0.07
+gabriella robinson 29 3.37
+rachel thompson 52 0.89
+gabriella steinbeck 28 0.07
+sarah young 71 1.37
+mike thompson 70 2.22
+david zipper 49 0.25
+wendy davidson 64 1.21
+mike king 35 1.31
+victor carson 37 2.27
+katie falkner 38 2.85
+priscilla miller 57 2.22
+alice allen 27 1.95
+fred falkner 53 2.60
+sarah ellison 45 1.72
+victor ichabod 58 1.94
+mike white 72 1.86
+katie young 68 1.18
+bob allen 21 2.27
+mike johnson 71 2.74
+luke johnson 64 1.85
+xavier quirinius 60 2.09
+quinn van buren 48 0.42
+bob laertes 55 1.63
+zach davidson 63 0.05
+bob king 52 3.09
+oscar hernandez 36 0.43
+rachel ovid 32 2.00
+xavier nixon 47 3.29
+wendy nixon 51 2.43
+ethan steinbeck 23 3.68
+katie allen 69 1.59
+wendy xylophone 28 2.40
+victor king 77 1.12
+zach nixon 50 0.81
+xavier young 56 1.20
+fred xylophone 65 0.39
+bob robinson 60 1.44
+sarah ellison 75 0.23
+holly carson 30 3.62
+fred young 41 3.02
+gabriella carson 35 3.91
+holly carson 65 3.52
+nick allen 40 1.30
+katie davidson 20 2.75
+oscar quirinius 31 1.62
+xavier garcia 54 2.88
+luke miller 39 0.36
+priscilla quirinius 60 3.70
+gabriella white 45 3.93
+calvin hernandez 30 0.82
+ulysses van buren 26 1.31
+mike allen 31 0.50
+rachel polk 22 2.67
+priscilla carson 59 1.47
+victor ellison 54 2.13
+victor steinbeck 31 0.63
+bob white 65 2.68
+ulysses xylophone 76 3.80
+nick young 45 2.10
+sarah carson 44 1.19
+ethan laertes 49 3.86
+jessica steinbeck 71 3.81
+rachel nixon 37 2.06
+wendy young 70 3.76
+priscilla ellison 48 2.72
+zach robinson 72 2.25
+rachel white 50 1.43
+ulysses king 74 0.15
+katie miller 46 2.31
+david ellison 19 2.87
+priscilla ovid 25 1.65
+luke underhill 24 2.80
+rachel laertes 64 0.72
+ulysses davidson 47 3.63
+sarah nixon 43 2.51
+oscar king 23 2.59
+katie white 63 1.13
+fred hernandez 46 0.07
+quinn brown 35 2.77
+tom zipper 52 1.86
+tom hernandez 57 3.26
+katie garcia 50 1.14
+calvin thompson 51 1.87
+holly hernandez 57 3.31
+wendy davidson 53 1.51
+jessica johnson 70 2.70
+oscar ovid 38 3.92
+ulysses johnson 67 0.71
+victor brown 63 3.59
+ethan laertes 39 1.02
+david miller 54 2.69
+sarah laertes 26 2.02
+gabriella johnson 63 0.67
+mike thompson 31 1.50
+fred polk 59 3.04
+xavier miller 67 0.76
+jessica garcia 54 1.29
+nick steinbeck 51 2.91
+yuri ovid 70 3.39
+tom brown 44 0.40
+quinn allen 77 1.67
+bob falkner 24 2.39
+wendy nixon 36 0.58
+yuri hernandez 41 1.56
+quinn laertes 60 2.81
+xavier thompson 30 3.44
+katie davidson 61 3.80
+quinn king 47 2.67
+wendy young 34 1.14
+quinn king 44 2.30
+sarah ichabod 61 1.05
+nick thompson 59 2.03
+david white 47 0.86
+sarah ichabod 67 0.83
+mike young 64 2.56
+zach xylophone 38 1.87
+bob young 24 0.94
+sarah allen 33 3.20
+nick van buren 73 2.89
+yuri thompson 47 0.30
+fred ovid 68 2.77
+katie thompson 57 2.76
+wendy ellison 24 3.97
+irene quirinius 25 1.82
+victor ellison 62 1.61
+mike underhill 22 2.19
+ethan carson 47 2.80
+mike nixon 45 2.22
+calvin young 43 1.65
+irene king 58 3.32
+calvin brown 23 0.24
+irene davidson 46 0.62
+sarah nixon 51 3.84
+rachel robinson 36 3.82
+luke johnson 63 0.41
+tom ovid 38 0.43
+tom ellison 65 0.51
+zach polk 20 3.08
+quinn falkner 37 3.16
+priscilla miller 66 1.89
+holly nixon 26 3.68
+quinn ichabod 29 0.49
+ulysses thompson 32 1.34
+zach garcia 22 2.17
+katie underhill 28 3.24
+ulysses davidson 30 2.38
+alice johnson 33 3.05
+irene king 27 3.57
+nick nixon 60 3.81
+sarah ichabod 68 3.00
+jessica thompson 73 0.70
+ethan nixon 62 3.69
+victor garcia 76 2.51
+nick thompson 41 1.64
+fred garcia 67 1.21
+irene allen 66 2.41
+rachel allen 28 2.52
+wendy ichabod 64 3.27
+ulysses young 77 0.92
+bob steinbeck 61 1.26
+fred underhill 27 2.49
+bob robinson 71 2.94
+oscar xylophone 65 1.74
+wendy johnson 44 0.12
+xavier quirinius 46 2.76
+priscilla underhill 47 3.18
+luke johnson 36 2.02
+holly davidson 36 0.89
+yuri van buren 26 2.55
+sarah hernandez 52 2.67
+zach ovid 22 0.70
+ethan robinson 25 1.05
+rachel carson 60 2.94
+ethan ellison 60 3.51
+jessica young 41 2.22
+tom underhill 66 0.64
+victor hernandez 59 3.80
+ethan hernandez 37 1.28
+holly ovid 70 2.71
+luke king 67 3.67
+katie ovid 40 3.67
+alice miller 47 3.13
+sarah carson 21 2.99
+zach hernandez 19 3.59
+holly young 36 0.51
+xavier garcia 61 0.29
+holly robinson 61 2.91
+oscar brown 62 3.57
+alice polk 47 0.45
+nick zipper 22 2.19
+wendy steinbeck 61 0.48
+david ichabod 25 3.47
+calvin white 50 2.47
+david ellison 42 0.34
+rachel ellison 36 0.38
+priscilla underhill 28 2.15
+gabriella zipper 66 0.12
+xavier nixon 54 3.57
+bob underhill 32 1.31
+zach robinson 20 0.90
+xavier zipper 70 2.18
+bob johnson 77 2.41
+rachel thompson 32 0.80
+tom miller 33 2.91
+ethan garcia 76 3.35
+oscar ellison 52 2.41
+nick johnson 39 0.78
+wendy robinson 29 0.26
+gabriella robinson 50 0.99
+bob brown 50 2.18
+ulysses xylophone 54 2.69
+wendy allen 20 1.47
+luke hernandez 56 2.89
+priscilla falkner 29 1.79
+jessica thompson 69 1.44
+priscilla carson 60 1.23
+tom hernandez 72 0.09
+bob van buren 45 3.55
+david ellison 51 0.88
+david allen 61 3.11
+sarah hernandez 53 3.79
+rachel nixon 39 0.56
+zach steinbeck 30 2.14
+yuri xylophone 18 3.00
+nick carson 59 2.37
+priscilla ichabod 32 1.28
+rachel allen 44 1.50
+quinn johnson 20 2.06
+xavier ellison 75 3.49
+wendy ichabod 58 3.30
+yuri johnson 77 3.30
+jessica zipper 69 3.02
+nick white 68 3.28
+calvin underhill 37 2.76
+david ichabod 29 2.24
+ethan ovid 64 0.37
+jessica davidson 28 1.79
+irene laertes 18 2.43
+nick garcia 57 2.77
+oscar white 70 0.51
+quinn thompson 37 0.44
+jessica hernandez 77 2.97
+holly underhill 30 0.07
+luke underhill 55 0.40
+calvin quirinius 30 2.75
+priscilla ovid 25 2.45
+zach falkner 35 3.94
+priscilla nixon 38 2.52
+gabriella johnson 37 0.21
+quinn laertes 38 2.51
+gabriella robinson 45 0.39
+alice robinson 75 1.02
+yuri robinson 51 2.39
+rachel laertes 65 0.34
+david garcia 53 2.27
+mike allen 54 2.67
+yuri nixon 55 3.19
+bob van buren 33 2.82
+jessica steinbeck 20 1.46
+rachel young 22 3.43
+priscilla garcia 41 0.40
+alice young 23 0.05
+sarah ichabod 62 0.26
+bob nixon 67 2.53
+jessica ovid 34 2.49
+bob zipper 50 0.53
+mike davidson 19 0.31
+victor robinson 37 0.56
+ulysses nixon 51 1.64
+fred white 26 3.24
+ethan davidson 62 0.75
+irene underhill 77 1.63
+holly davidson 40 1.05
+mike davidson 69 2.96
+ethan young 67 1.99
+ethan miller 59 0.17
+ethan quirinius 59 1.60
+irene van buren 69 2.00
+holly steinbeck 47 0.67
+ethan ellison 64 2.15
+calvin steinbeck 36 1.72
+holly johnson 24 2.72
+ulysses nixon 57 2.29
+ethan hernandez 33 3.61
+katie quirinius 27 1.70
+bob white 44 3.89
+alice carson 35 3.10
+katie davidson 36 2.14
+yuri polk 75 0.26
+tom robinson 45 1.50
+mike brown 60 1.70
+fred white 31 2.14
+zach allen 31 0.53
+alice brown 59 2.94
+irene young 28 1.82
+jessica johnson 67 3.55
+xavier laertes 40 3.32
+yuri miller 59 1.00
+sarah ovid 33 0.95
+mike garcia 56 0.97
+gabriella ellison 73 2.92
+bob thompson 19 2.96
+ethan zipper 57 0.15
+sarah ellison 40 0.74
+nick king 49 1.15
+fred thompson 45 2.93
+alice white 57 3.76
+bob white 62 0.90
+bob ovid 27 0.95
+david ellison 49 2.85
+mike miller 36 2.27
+sarah thompson 34 3.72
+irene underhill 47 1.35
+nick underhill 60 0.24
+gabriella van buren 25 1.13
+holly hernandez 26 3.59
+wendy robinson 56 0.87
+katie johnson 45 2.16
+luke nixon 58 2.85
+yuri king 58 2.03
+tom king 53 1.44
+calvin underhill 19 1.45
+zach steinbeck 43 2.92
+fred young 43 2.54
+calvin ellison 74 1.79
+jessica garcia 71 0.70
+katie polk 21 1.15
+nick white 69 3.20
+alice johnson 20 2.31
+victor thompson 26 2.08
+gabriella zipper 46 2.86
+calvin johnson 66 1.60
+gabriella garcia 38 0.44
+priscilla underhill 75 2.36
+nick polk 63 1.46
+gabriella thompson 60 3.94
+ulysses miller 58 0.60
+ulysses hernandez 41 1.35
+wendy white 57 2.97
+gabriella king 74 3.21
+holly ichabod 72 3.66
+katie nixon 75 1.06
+tom ellison 22 1.93
+victor ovid 65 3.88
+calvin ellison 76 1.95
+ulysses white 42 1.84
+rachel polk 30 3.30
+wendy allen 76 0.26
+luke hernandez 38 2.89
+fred nixon 32 2.76
+sarah van buren 59 1.79
+tom hernandez 61 3.99
+luke allen 38 1.14
+zach van buren 49 2.45
+oscar ichabod 33 3.16
+alice brown 66 1.57
+rachel quirinius 19 0.47
+sarah steinbeck 75 1.75
+tom ellison 71 1.02
+jessica ovid 22 3.69
+nick allen 49 3.12
+luke xylophone 55 2.40
+katie thompson 74 3.65
+ulysses johnson 37 1.04
+katie johnson 66 1.91
+nick ovid 47 0.96
+zach steinbeck 34 2.32
+nick allen 53 2.50
+fred young 43 2.94
+priscilla laertes 63 2.42
+rachel ovid 25 1.65
+sarah carson 28 0.37
+sarah underhill 19 3.98
+irene garcia 67 2.47
+quinn zipper 29 1.54
+irene miller 18 0.42
+bob white 19 2.56
+david falkner 77 1.06
+david falkner 35 1.14
+alice quirinius 32 3.56
+calvin davidson 36 0.72
+victor young 53 2.14
+david laertes 51 0.45
+victor quirinius 18 2.89
+quinn thompson 51 2.54
+katie allen 22 0.16
+bob xylophone 75 3.96
+luke carson 65 0.88
+tom xylophone 54 1.35
+holly zipper 20 2.39
+rachel johnson 55 3.95
+holly white 76 0.43
+holly zipper 20 2.69
+ethan ovid 47 1.44
+priscilla thompson 38 1.93
+wendy thompson 40 0.63
+wendy zipper 69 2.08
+irene ovid 58 1.14
+luke nixon 20 3.65
+sarah carson 67 0.91
+bob davidson 69 1.49
+gabriella ichabod 32 0.22
+sarah young 44 0.38
+ethan ichabod 31 2.44
+fred white 22 1.74
+ulysses zipper 66 2.83
+irene carson 64 1.80
+calvin robinson 48 0.44
+ulysses ovid 62 0.51
+irene xylophone 71 2.35
+calvin polk 45 3.07
+irene robinson 51 2.25
+bob ichabod 18 3.40
+gabriella allen 68 1.65
+irene garcia 41 1.69
+nick brown 65 2.91
+nick laertes 71 3.65
+mike xylophone 48 2.16
+ulysses davidson 56 0.47
+nick nixon 36 2.95
+tom falkner 24 3.83
+ethan thompson 41 2.86
+mike underhill 34 2.36
+irene xylophone 52 2.23
+xavier johnson 45 1.33
+bob quirinius 67 3.67
+zach miller 38 3.45
+nick zipper 68 0.60
+ulysses thompson 35 0.81
+calvin young 29 0.29
+jessica thompson 25 3.78
+yuri polk 45 1.08
+zach robinson 19 1.50
+katie king 36 2.89
+zach white 56 0.42
+ethan nixon 74 0.36
+victor zipper 65 0.43
+nick laertes 64 3.93
+gabriella van buren 40 0.36
+wendy king 19 0.44
+gabriella young 57 1.62
+quinn carson 39 1.23
+victor miller 28 3.73
+victor brown 72 0.03
+david van buren 73 2.19
+gabriella miller 20 3.74
+rachel xylophone 69 0.84
+gabriella ellison 33 1.68
+quinn robinson 67 3.47
+irene carson 45 0.11
+alice ovid 44 3.57
+katie miller 76 3.21
+holly zipper 75 0.04
+calvin steinbeck 49 3.50
+tom nixon 51 0.65
+tom falkner 50 3.32
+zach van buren 73 0.85
+quinn carson 52 2.68
+priscilla miller 36 3.25
+xavier carson 54 0.13
+nick robinson 63 1.28
+rachel davidson 47 2.90
+ethan allen 47 0.59
+ulysses carson 18 2.38
+quinn nixon 64 2.24
+quinn xylophone 39 0.66
+priscilla falkner 25 1.81
+ulysses brown 44 2.10
+katie garcia 59 1.38
+rachel garcia 72 1.68
+jessica xylophone 52 1.91
+gabriella steinbeck 47 2.46
+bob zipper 45 2.26
+tom young 54 0.38
+yuri steinbeck 71 2.06
+alice underhill 67 3.64
+ethan nixon 37 0.69
+calvin nixon 23 0.00
+rachel davidson 70 3.83
+quinn young 51 2.40
+alice ellison 40 0.09
+calvin davidson 55 1.89
+jessica zipper 29 2.02
+irene ellison 24 2.19
+fred xylophone 29 1.00
+fred falkner 37 0.86
+xavier ellison 54 1.35
+gabriella polk 40 3.56
+quinn white 52 2.72
+tom quirinius 63 1.27
+alice ovid 76 3.75
+wendy thompson 54 1.53
+fred miller 44 0.04
+holly van buren 76 0.74
+priscilla davidson 55 1.59
+nick hernandez 55 1.84
+wendy laertes 34 3.17
+mike brown 70 2.05
+gabriella falkner 34 1.53
+rachel van buren 26 0.76
+ulysses white 62 1.82
+yuri miller 69 2.32
+oscar xylophone 60 0.37
+sarah johnson 31 2.20
+alice davidson 56 3.51
+tom hernandez 76 1.77
+nick quirinius 62 3.48
+luke steinbeck 46 2.98
+nick steinbeck 64 0.95
+priscilla falkner 74 3.38
+luke white 30 1.31
+ethan underhill 21 1.94
+rachel allen 65 0.84
+priscilla falkner 66 3.68
+calvin polk 18 0.54
+priscilla brown 22 1.63
+fred johnson 24 2.72
+gabriella falkner 32 3.85
+mike falkner 52 3.26
+sarah xylophone 49 0.55
+rachel garcia 26 2.15
+rachel robinson 52 3.61
+rachel brown 20 0.87
+calvin quirinius 19 2.37
+fred van buren 64 1.18
+rachel quirinius 26 0.93
+ethan king 32 2.08
+sarah laertes 29 2.26
+fred davidson 60 2.01
+ethan laertes 52 1.96
+nick quirinius 27 3.59
+xavier johnson 21 2.74
+jessica falkner 19 1.06
+luke thompson 73 2.44
+david ichabod 43 0.92
+irene miller 25 1.16
+mike steinbeck 22 1.24
+zach laertes 24 2.79
+wendy davidson 28 1.98
+alice hernandez 67 3.30
+gabriella ichabod 71 1.62
+ulysses quirinius 59 0.23
+alice underhill 27 2.78
+irene ellison 47 3.01
+david young 28 0.41
+mike white 30 2.13
+ulysses johnson 21 1.98
+holly ichabod 36 0.59
+gabriella quirinius 76 0.32
+ethan van buren 46 1.83
+oscar ichabod 36 3.53
+david thompson 55 1.70
+holly davidson 53 2.25
+ulysses young 54 3.37
+fred davidson 70 2.20
+victor steinbeck 44 3.57
+irene quirinius 75 2.43
+victor white 63 2.11
+oscar van buren 73 3.30
+irene laertes 48 0.28
+katie quirinius 77 0.94
+sarah davidson 52 1.81
+jessica garcia 28 1.21
+victor hernandez 69 3.80
+katie hernandez 39 2.58
+priscilla van buren 30 1.20
+david carson 44 2.24
+sarah king 65 2.72
+sarah davidson 65 0.32
+yuri allen 19 0.39
+priscilla falkner 55 2.26
+oscar laertes 68 1.23
+wendy davidson 74 0.06
+bob king 64 2.48
+wendy ichabod 31 2.71
+irene miller 19 2.27
+priscilla nixon 24 2.96
+alice allen 42 2.46
+jessica king 24 1.42
+irene thompson 48 3.59
+tom ichabod 55 2.17
+holly king 21 0.76
+holly quirinius 18 3.84
+nick king 49 0.42
+gabriella ovid 71 0.12
+quinn king 74 3.78
+victor allen 25 1.09
+tom ovid 77 1.53
+katie king 64 1.32
+yuri miller 76 2.52
+calvin king 61 2.48
+rachel king 45 2.58
+quinn nixon 37 0.64
+quinn garcia 52 1.66
+mike zipper 57 2.87
+david hernandez 22 1.27
+mike zipper 18 0.17
+yuri king 61 0.40
+ulysses polk 24 2.52
+zach laertes 59 0.35
+luke ellison 50 0.65
+ethan thompson 43 3.18
+fred allen 27 1.85
+xavier garcia 57 0.98
+jessica brown 53 2.47
+bob ovid 25 3.74
+ethan davidson 62 2.65
+ethan van buren 50 2.43
+priscilla robinson 39 0.25
+bob miller 60 2.59
+wendy zipper 32 0.31
+holly ovid 66 0.76
+holly johnson 27 0.01
+yuri zipper 21 3.31
+calvin zipper 72 0.48
+victor miller 51 0.22
+mike laertes 49 1.02
+victor white 37 1.85
+zach king 45 2.78
+calvin laertes 51 0.29
+quinn johnson 25 1.26
+nick ovid 49 0.16
+zach laertes 74 3.04
+calvin ovid 29 1.95
+zach falkner 75 2.48
+yuri thompson 75 0.32
+sarah davidson 53 2.49
+quinn underhill 31 2.08
+xavier hernandez 37 0.04
+holly xylophone 56 0.61
+yuri ellison 28 1.68
+sarah robinson 34 1.62
+bob johnson 47 3.34
+fred davidson 58 3.56
+nick garcia 26 1.23
+victor van buren 57 2.42
+yuri nixon 43 2.01
+luke johnson 35 3.42
+katie garcia 50 1.40
+katie robinson 59 0.41
+ethan steinbeck 49 1.77
+ulysses xylophone 26 1.27
+quinn carson 49 0.57
+katie ellison 73 3.90
+ulysses ovid 51 3.07
+sarah ichabod 61 1.00
+holly falkner 71 2.49
+xavier robinson 57 2.82
+irene brown 31 0.46
+tom allen 39 0.43
+luke quirinius 25 2.53
+irene allen 68 3.60
+zach xylophone 62 1.68
+wendy king 68 0.90
+yuri white 55 0.66
+calvin white 28 1.46
+xavier ichabod 63 2.61
+david king 76 3.51
+oscar white 31 1.56
+david white 76 2.96
+david ovid 39 2.98
+holly king 39 1.33
+katie hernandez 37 3.43
+quinn thompson 31 0.45
+ethan brown 70 0.83
+luke garcia 23 1.85
+zach laertes 72 1.97
+ulysses brown 58 1.26
+ethan steinbeck 60 0.92
+ulysses young 57 2.88
+mike nixon 56 1.07
+calvin van buren 26 3.43
+luke steinbeck 58 0.87
+xavier laertes 60 2.20
+ulysses nixon 53 2.75
+luke xylophone 35 2.67
+bob thompson 42 0.22
+zach polk 59 2.31
+holly miller 26 1.80
+gabriella steinbeck 48 0.31
+yuri hernandez 57 2.32
+victor xylophone 53 0.35
+wendy young 57 3.73
+ethan brown 53 3.10
+nick thompson 66 0.17
+luke carson 34 2.62
+jessica king 39 1.44
+bob brown 46 2.54
+nick underhill 71 2.11
+zach hernandez 72 3.94
+jessica garcia 66 3.20
+mike van buren 39 2.53
+calvin xylophone 54 2.44
+quinn carson 59 1.07
+jessica van buren 73 1.43
+victor polk 50 1.12
+zach young 56 3.00
+david steinbeck 21 2.44
+calvin ellison 40 0.84
+alice hernandez 27 1.08
+priscilla steinbeck 18 0.68
+tom ichabod 51 1.53
+mike steinbeck 69 1.28
+ulysses zipper 54 1.17
+zach white 18 0.88
+gabriella robinson 48 2.16
+quinn ichabod 39 0.12
+luke xylophone 33 0.69
+alice davidson 44 2.03
+calvin ellison 18 0.83
+ethan miller 68 3.34
+calvin young 60 3.24
+xavier xylophone 76 1.00
+bob zipper 64 0.54
+gabriella xylophone 49 0.07
+priscilla garcia 62 2.42
+alice ichabod 60 2.60
+katie johnson 29 3.30
+xavier allen 70 0.03
+jessica ellison 69 1.13
+ethan garcia 47 1.08
+mike van buren 29 1.48
+zach steinbeck 24 2.36
+irene garcia 67 1.42
+rachel laertes 49 2.70
+ulysses johnson 20 1.50
+victor king 56 1.42
+katie garcia 60 2.47
+rachel ellison 25 3.31
+ethan garcia 27 1.85
+xavier laertes 61 3.93
+holly garcia 36 1.57
+zach brown 70 0.07
+rachel zipper 43 0.99
+sarah young 44 1.22
+holly underhill 27 1.88
+ethan robinson 50 1.23
+wendy hernandez 72 3.12
+alice white 60 0.82
+yuri thompson 30 3.65
+jessica miller 39 2.99
+fred johnson 43 3.33
+bob van buren 66 3.15
+fred johnson 72 0.77
+irene thompson 53 3.56
+ethan steinbeck 70 2.19
+gabriella miller 65 1.59
+katie allen 51 1.28
+ethan young 50 1.24
+victor king 25 1.19
+wendy allen 57 1.70
+calvin miller 73 3.81
+rachel johnson 39 0.81
+rachel laertes 57 3.26
+xavier ichabod 31 3.92
+david nixon 46 2.43
+david miller 40 0.94
+alice nixon 51 3.55
+bob polk 23 3.29
+mike quirinius 67 2.01
+tom robinson 59 0.90
+alice nixon 38 0.92
+gabriella young 20 0.80
+xavier underhill 32 2.56
+ulysses steinbeck 42 1.37
+quinn steinbeck 76 1.15
+mike van buren 66 1.80
+tom ichabod 33 0.48
+rachel ellison 59 3.57
+zach white 57 0.39
+victor ichabod 21 1.99
+oscar johnson 61 3.46
+tom miller 35 3.56
+david hernandez 48 1.97
+jessica falkner 26 2.54
+ulysses brown 77 0.77
+priscilla falkner 62 3.77
+luke miller 40 0.82
+priscilla nixon 64 0.87
+priscilla johnson 66 3.68
+bob davidson 70 0.88
+holly polk 18 2.90
+mike king 58 0.54
+bob falkner 35 3.90
+victor allen 48 1.86
+ulysses ichabod 52 3.78
+xavier xylophone 23 0.78
+calvin underhill 29 2.60
+luke white 58 1.44
+ethan johnson 31 3.60
+xavier ichabod 49 3.27
+calvin king 69 0.03
+quinn nixon 41 2.75
+sarah zipper 26 0.92
+jessica ellison 20 2.59
+oscar quirinius 28 0.20
+quinn polk 21 0.42
+jessica king 38 0.38
+fred white 26 3.79
+katie falkner 75 2.74
+sarah laertes 18 1.92
+priscilla robinson 24 2.73
+holly white 23 3.85
+david underhill 64 2.36
+irene allen 56 0.31
+gabriella ovid 64 2.33
+luke steinbeck 71 2.20
+luke garcia 32 1.97
+katie hernandez 30 0.33
+priscilla falkner 38 2.44
+nick xylophone 58 3.15
+victor king 71 1.95
+priscilla zipper 41 0.95
+calvin hernandez 31 3.55
+mike falkner 18 1.52
+rachel king 64 1.62
+sarah robinson 72 2.22
+wendy hernandez 69 1.07
+katie nixon 35 0.03
+wendy polk 64 1.41
+tom hernandez 19 1.61
+sarah hernandez 24 1.52
+fred allen 45 0.05
+ethan polk 59 3.72
+wendy brown 55 1.78
+quinn polk 59 3.63
+calvin young 53 0.14
+gabriella ellison 47 3.06
+oscar thompson 27 0.79
+alice van buren 32 3.57
+yuri quirinius 22 1.93
+mike zipper 23 1.35
+rachel polk 67 0.77
+victor nixon 38 0.46
+nick zipper 39 3.87
+oscar white 67 1.06
+calvin king 54 0.06
+mike davidson 68 3.62
+ulysses ellison 60 0.30
+bob brown 44 2.08
+nick carson 74 2.93
+sarah johnson 54 3.68
+ethan xylophone 34 0.41
+quinn miller 20 0.93
+yuri ichabod 48 1.69
+wendy polk 69 2.88
+katie thompson 48 2.67
+ulysses nixon 45 0.18
+holly miller 26 3.42
+ulysses miller 35 0.81
+rachel ovid 50 1.07
+rachel van buren 27 0.81
+tom polk 58 2.21
+luke underhill 46 0.84
+holly young 37 0.84
+gabriella laertes 37 3.32
+rachel hernandez 65 3.00
+priscilla quirinius 22 3.54
+mike ovid 49 2.11
+fred hernandez 20 0.27
+priscilla quirinius 55 3.89
+jessica johnson 26 2.33
+ulysses garcia 38 3.05
+holly ichabod 38 0.07
+irene ichabod 69 1.21
+katie ichabod 34 1.62
+tom quirinius 18 2.91
+quinn carson 19 2.83
+irene steinbeck 71 2.02
+ethan carson 57 2.74
+priscilla laertes 47 2.66
+sarah ellison 25 1.91
+yuri white 60 2.40
+ethan nixon 73 3.88
+zach laertes 51 1.54
+gabriella steinbeck 44 2.07
+tom brown 23 2.20
+ulysses falkner 64 1.36
+zach van buren 42 1.17
+sarah laertes 76 1.93
+priscilla white 77 0.65
+holly falkner 49 0.42
+priscilla garcia 49 1.71
+ethan johnson 47 1.44
+david young 30 1.42
+rachel king 73 0.89
+gabriella laertes 29 3.57
+rachel hernandez 20 0.25
+ethan garcia 76 1.58
+nick nixon 33 3.91
+yuri brown 28 1.14
+wendy allen 25 1.64
+bob king 55 3.07
+jessica zipper 39 3.06
+gabriella ichabod 41 0.18
+xavier miller 49 1.88
+katie xylophone 72 3.06
+xavier falkner 73 0.08
+victor steinbeck 39 3.27
+yuri miller 38 1.67
+yuri brown 38 3.88
+xavier ichabod 49 2.45
+calvin nixon 22 0.85
+oscar quirinius 53 1.47
+katie nixon 64 0.81
+nick garcia 26 1.27
+calvin garcia 40 2.43
+irene van buren 40 3.56
+victor king 19 2.64
+ethan davidson 68 0.36
+quinn carson 23 3.46
+ethan carson 50 0.49
+gabriella nixon 28 0.89
+quinn thompson 42 1.22
+rachel steinbeck 42 0.31
+gabriella xylophone 39 2.70
+xavier carson 29 0.96
+xavier laertes 53 0.88
+nick king 36 2.97
+david ellison 50 3.17
+calvin zipper 50 1.04
+nick king 46 0.92
+alice quirinius 22 3.93
+nick miller 66 1.76
+yuri ellison 32 1.80
+gabriella robinson 37 3.11
+tom quirinius 24 2.75
+fred garcia 54 1.05
+ulysses van buren 47 3.01
+jessica zipper 46 3.63
+tom falkner 54 2.90
+zach ellison 62 2.41
+xavier white 58 3.93
+priscilla nixon 43 3.47
+wendy allen 68 2.30
+sarah thompson 32 0.15
+tom underhill 51 1.66
+katie xylophone 36 1.03
+victor king 25 0.13
+gabriella young 44 0.46
+jessica falkner 75 2.29
+rachel miller 20 2.23
+ulysses king 54 3.42
+yuri hernandez 31 0.82
+xavier van buren 75 1.05
+ulysses zipper 18 0.56
+tom ichabod 51 1.91
+bob thompson 71 2.86
+xavier young 66 0.26
+ulysses thompson 33 0.06
+bob garcia 68 0.73
+xavier van buren 25 0.20
+ulysses davidson 68 3.35
+tom hernandez 29 3.81
+irene robinson 55 1.39
+nick ovid 18 0.19
+mike king 23 3.74
+katie polk 56 2.86
+wendy brown 43 2.61
+tom robinson 76 3.30
+ethan robinson 50 0.70
+mike ellison 43 2.17
+ulysses ovid 45 1.46
+victor hernandez 37 0.83
+sarah brown 77 3.24
+fred quirinius 18 0.61
+mike ichabod 43 0.45
+holly thompson 51 0.81
+katie miller 33 3.91
+katie king 72 3.46
+zach king 40 2.95
+nick king 55 0.81
+irene ichabod 71 1.74
+yuri garcia 56 2.46
+katie van buren 43 0.44
+alice miller 76 3.43
+nick carson 67 2.06
+alice ichabod 70 3.89
+holly davidson 30 0.34
+wendy quirinius 74 3.26
+calvin laertes 51 2.53
+calvin ovid 50 1.85
+alice king 51 3.58
+quinn ellison 66 2.96
+alice brown 69 3.04
+gabriella king 41 2.98
+luke ichabod 62 2.77
+bob steinbeck 53 1.50
+irene allen 77 2.90
+jessica ovid 34 1.92
+yuri carson 19 0.22
+sarah van buren 47 1.92
+fred van buren 77 0.91
+holly ovid 69 3.78
+calvin van buren 67 2.48
+calvin thompson 47 3.81
+ulysses ovid 52 3.72
+tom allen 71 2.90
+rachel brown 38 0.89
+david polk 40 1.89
+rachel van buren 49 0.19
+bob xylophone 73 0.10
+holly miller 50 0.59
+ethan ovid 48 1.77
+alice quirinius 71 0.94
+sarah carson 39 1.76
+fred hernandez 22 1.59
+luke quirinius 72 0.44
+xavier van buren 24 3.04
+david nixon 23 0.96
+sarah white 22 0.36
+yuri johnson 33 0.34
+david davidson 71 0.92
+priscilla davidson 20 1.92
+xavier ovid 41 1.46
+david king 49 2.77
+bob garcia 68 1.34
+zach johnson 56 2.13
+yuri ovid 47 1.77
+calvin underhill 56 3.46
+ethan underhill 56 1.73
+nick brown 60 3.18
+holly brown 73 2.77
+sarah robinson 64 1.91
+ethan ichabod 49 3.61
+ethan ellison 46 2.90
+bob davidson 41 3.48
+yuri brown 23 1.42
+ulysses ichabod 46 2.29
+fred young 59 1.72
+wendy allen 54 3.42
+victor van buren 19 1.57
+holly underhill 27 2.87
+sarah underhill 23 1.84
+wendy young 50 2.28
+irene king 46 1.75
+holly quirinius 61 1.39
+oscar falkner 47 2.25
+fred ovid 64 3.71
+fred van buren 39 3.72
+bob davidson 59 3.39
+bob ichabod 47 1.89
+alice nixon 19 3.79
+mike thompson 53 1.84
+priscilla hernandez 70 0.65
+yuri robinson 46 0.37
+ethan xylophone 25 1.61
+luke ovid 32 2.71
+irene king 18 0.31
+priscilla steinbeck 36 2.37
+zach ovid 64 2.65
+ethan underhill 52 1.15
+alice quirinius 33 1.31
+david underhill 24 0.98
+sarah hernandez 30 2.30
+tom steinbeck 31 0.05
+david nixon 25 2.15
+tom young 64 0.37
+tom underhill 42 2.10
+victor brown 54 3.09
+bob underhill 27 0.99
+katie nixon 34 2.89
+oscar white 47 0.25
+nick falkner 42 1.88
+quinn garcia 59 2.22
+jessica garcia 75 3.06
+mike laertes 42 2.88
+fred ovid 61 0.52
+ethan laertes 45 0.25
+zach xylophone 24 3.52
+ethan robinson 23 3.55
+bob garcia 25 1.04
+oscar van buren 55 0.92
+calvin ichabod 69 1.32
+yuri king 66 3.59
+nick hernandez 73 2.89
+mike ichabod 67 1.11
+oscar davidson 71 1.81
+katie nixon 25 2.71
+quinn king 27 1.78
+rachel white 26 0.95
+nick steinbeck 61 2.37
+rachel allen 18 2.41
+luke nixon 32 3.05
+quinn van buren 33 3.40
+holly thompson 47 1.21
+xavier nixon 54 1.81
+ethan xylophone 33 1.11
+oscar brown 66 1.35
+tom polk 35 3.64
+holly steinbeck 31 2.20
+rachel polk 34 1.68
+calvin davidson 30 1.10
+wendy ichabod 71 2.85
+sarah brown 22 2.09
+bob steinbeck 35 0.83
+holly ichabod 23 0.50
+sarah davidson 24 1.73
+ethan falkner 20 1.03
+david white 52 3.32
+zach garcia 31 0.44
+david polk 29 2.79
+victor ellison 56 3.90
+quinn underhill 51 0.16
+alice van buren 67 1.99
+luke robinson 75 2.53
+mike miller 51 2.07
+xavier carson 50 2.14
+quinn young 76 2.00
+quinn van buren 63 3.52
+bob laertes 72 1.01
+wendy falkner 67 3.01
+ulysses xylophone 36 1.33
+quinn hernandez 34 2.45
+mike garcia 58 0.86
+nick quirinius 27 3.23
+xavier ichabod 68 0.61
+tom robinson 67 1.46
+xavier garcia 33 1.38
+mike johnson 45 1.89
+nick johnson 35 3.63
+alice garcia 55 2.86
+ethan nixon 77 0.01
+jessica ovid 32 1.66
+wendy miller 74 1.27
+victor king 33 0.75
+fred johnson 75 0.26
+nick carson 36 0.52
+mike steinbeck 41 2.13
+xavier van buren 36 3.58
+mike johnson 63 1.25
+holly johnson 48 1.05
+calvin robinson 65 0.78
+ulysses laertes 63 3.04
+luke laertes 58 3.69
+david zipper 61 0.99
+luke ichabod 57 3.62
+katie allen 66 1.56
+irene steinbeck 53 0.91
+alice johnson 32 1.88
+alice ovid 26 1.44
+xavier ichabod 25 3.54
+oscar ellison 66 0.46
+calvin miller 36 1.61
+quinn xylophone 46 2.72
+irene robinson 54 0.76
+priscilla underhill 24 2.76
+luke ellison 54 1.41
+jessica brown 62 2.51
+wendy carson 32 3.11
+gabriella thompson 32 0.35
+katie davidson 60 3.41
+irene white 51 3.15
+holly johnson 62 3.00
+victor carson 37 2.23
+bob laertes 62 3.19
+irene garcia 23 3.64
+calvin xylophone 69 2.58
+ethan carson 73 0.83
+luke thompson 32 3.27
+holly white 42 3.78
+nick van buren 56 2.16
+wendy falkner 20 0.08
+quinn white 44 2.31
+zach thompson 31 2.12
+jessica garcia 22 1.12
+fred white 66 0.16
+irene polk 55 3.66
+rachel falkner 58 3.62
+fred carson 65 3.35
+gabriella ichabod 19 3.39
+wendy polk 47 0.20
+holly king 41 2.14
+rachel ichabod 70 0.84
+wendy van buren 58 2.81
+bob steinbeck 39 1.05
+victor thompson 26 3.72
+katie falkner 65 1.33
+luke hernandez 23 3.97
+katie miller 69 3.12
+katie davidson 67 1.76
+mike zipper 29 3.50
+victor robinson 37 3.38
+oscar allen 38 2.99
+tom falkner 54 3.92
+david nixon 77 3.99
+zach nixon 75 0.21
+xavier falkner 33 2.27
+sarah johnson 45 3.97
+xavier white 74 1.29
+mike garcia 50 2.78
+priscilla nixon 24 3.76
+gabriella ichabod 35 0.49
+mike thompson 57 3.05
+holly miller 24 0.22
+zach xylophone 62 1.58
+yuri nixon 54 1.47
+priscilla carson 64 3.85
+nick hernandez 55 2.93
+oscar quirinius 32 0.72
+ulysses hernandez 30 2.74
+wendy robinson 67 1.84
+mike allen 39 0.42
+yuri zipper 52 3.04
+fred young 57 1.45
+rachel falkner 62 3.41
+tom laertes 49 1.76
+gabriella ichabod 52 1.25
+david zipper 21 2.38
+yuri carson 43 2.25
+alice carson 46 1.55
+zach zipper 32 3.81
+zach garcia 64 0.23
+ulysses ovid 49 0.01
+ulysses brown 51 2.60
+priscilla johnson 39 2.41
+mike quirinius 21 1.92
+alice allen 38 0.81
+zach robinson 41 0.00
+wendy brown 58 2.87
+zach quirinius 50 1.77
+xavier brown 65 0.93
+bob zipper 71 0.06
+wendy ovid 38 1.15
+priscilla brown 35 3.84
+bob zipper 53 3.89
+irene robinson 56 0.60
+sarah miller 47 3.33
+gabriella carson 55 2.08
+rachel hernandez 21 0.22
+priscilla carson 38 0.19
+katie zipper 67 1.21
+mike ovid 24 2.73
+calvin ellison 45 0.11
+calvin hernandez 33 0.73
+bob johnson 62 0.40
+victor davidson 43 1.56
+wendy zipper 51 0.38
+luke ichabod 26 1.05
+tom garcia 48 2.64
+sarah garcia 70 2.30
+ethan ichabod 69 3.80
+fred laertes 71 3.87
+quinn van buren 40 1.01
+quinn van buren 60 3.14
+katie brown 63 2.22
+jessica underhill 19 3.74
+wendy thompson 48 0.01
+oscar ellison 67 3.23
+xavier young 49 3.67
+fred ichabod 52 0.53
+ulysses ellison 40 1.03
+gabriella allen 30 2.42
+zach robinson 68 0.77
+irene thompson 27 3.76
+gabriella white 75 3.19
+wendy quirinius 38 0.00
+mike king 43 2.59
+victor hernandez 41 2.59
+ulysses robinson 56 2.15
+tom brown 33 3.66
+bob young 64 1.42
+sarah steinbeck 46 0.74
+wendy ellison 74 2.79
+nick king 34 0.54
+fred quirinius 25 2.51
+mike underhill 22 0.98
+holly brown 39 2.42
+calvin thompson 41 3.57
+quinn brown 42 2.18
+ulysses ellison 64 0.64
+david xylophone 72 2.31
+sarah laertes 61 2.80
+mike garcia 36 0.53
+katie robinson 43 0.77
+zach white 34 2.41
+fred king 71 1.14
+bob polk 28 0.01
+zach underhill 39 0.27
+quinn xylophone 46 2.82
+victor allen 63 2.19
+mike robinson 28 2.11
+rachel miller 47 3.71
+irene robinson 48 1.00
+ethan van buren 41 3.30
+irene falkner 67 3.41
+rachel quirinius 39 0.22
+rachel hernandez 59 2.34
+david ovid 61 3.96
+fred johnson 25 1.40
+holly carson 71 0.05
+quinn steinbeck 23 2.86
+bob thompson 45 0.38
+gabriella allen 55 1.07
+priscilla quirinius 49 0.21
+bob garcia 58 3.91
+calvin brown 44 2.40
+calvin hernandez 31 2.78
+tom laertes 56 3.86
+zach nixon 22 3.49
+holly laertes 24 1.64
+irene zipper 60 0.63
+sarah polk 68 2.80
+luke steinbeck 64 2.16
+nick zipper 76 1.01
+fred ovid 18 1.42
+rachel polk 28 1.68
+luke ichabod 50 1.99
+ethan xylophone 37 0.52
+quinn thompson 75 3.71
+wendy hernandez 49 3.47
+quinn king 19 2.69
+ethan robinson 39 3.46
+ulysses underhill 19 1.08
+ethan robinson 72 0.55
+quinn davidson 20 3.38
+katie allen 58 3.88
+david falkner 74 2.94
+ulysses thompson 72 1.16
+yuri garcia 25 3.95
+luke polk 63 1.54
+nick garcia 64 3.06
+bob miller 20 1.48
+zach nixon 71 0.66
+holly laertes 61 3.12
+yuri xylophone 38 3.22
+zach zipper 51 3.12
+fred nixon 64 3.85
+gabriella underhill 61 1.96
+david polk 37 0.17
+bob xylophone 66 0.26
+luke garcia 77 2.43
+fred ichabod 54 1.14
+luke davidson 46 1.08
+sarah davidson 45 2.22
+zach garcia 54 0.97
+gabriella falkner 66 2.01
+sarah xylophone 41 1.99
+jessica robinson 77 1.73
+jessica polk 47 0.18
+yuri van buren 46 1.24
+alice quirinius 41 0.19
+holly garcia 42 0.62
+holly laertes 29 0.42
+ethan laertes 38 0.19
+katie young 77 1.16
+victor davidson 75 1.33
+nick ovid 76 1.21
+victor polk 18 3.91
+oscar zipper 19 1.01
+calvin garcia 38 1.79
+quinn nixon 72 2.31
+holly young 45 3.60
+quinn miller 58 2.90
+alice robinson 65 0.16
+victor polk 44 1.23
+bob steinbeck 69 1.49
+tom ellison 65 2.61
+wendy miller 73 1.00
+oscar hernandez 69 2.95
+gabriella davidson 39 2.39
+xavier steinbeck 34 3.56
+bob steinbeck 27 3.35
+alice king 66 2.56
+ulysses brown 27 0.70
+luke underhill 40 2.56
+holly steinbeck 32 0.18
+holly robinson 47 2.60
+calvin polk 74 0.97
+alice thompson 31 0.43
+luke laertes 27 3.86
+tom young 39 3.95
+jessica quirinius 69 3.11
+calvin king 77 1.03
+ethan laertes 47 1.08
+luke robinson 42 1.87
+fred miller 38 0.13
+calvin king 42 2.60
+mike hernandez 68 1.66
+calvin brown 47 0.12
+gabriella steinbeck 37 1.50
+tom nixon 57 3.02
+calvin white 75 2.96
+alice white 71 0.82
+zach quirinius 24 0.13
+oscar carson 53 1.01
+ulysses allen 69 0.26
+luke white 50 0.87
+holly underhill 51 2.03
+gabriella nixon 28 0.68
+calvin carson 73 2.59
+fred robinson 45 2.65
+david falkner 38 2.56
+jessica ovid 42 3.62
+jessica quirinius 58 2.72
+bob thompson 75 3.84
+luke falkner 65 2.42
+oscar ovid 77 1.83
+ethan xylophone 68 2.62
+david robinson 35 0.03
+ulysses brown 30 1.22
+bob falkner 65 1.75
+wendy steinbeck 75 1.00
+david nixon 39 2.95
+bob brown 64 3.92
+nick falkner 56 2.60
+gabriella xylophone 49 2.44
+fred white 59 2.11
+ethan falkner 52 3.78
+victor davidson 26 1.48
+calvin van buren 46 0.64
+zach robinson 22 2.46
+nick carson 30 3.23
+rachel ellison 40 2.18
+yuri brown 46 1.28
+alice davidson 33 0.11
+alice thompson 20 2.37
+yuri polk 70 2.31
+gabriella hernandez 35 2.96
+bob falkner 59 2.61
+gabriella ovid 22 2.29
+tom nixon 27 1.77
+mike zipper 42 3.86
+fred miller 36 1.64
+yuri carson 19 3.11
+oscar laertes 57 0.89
+xavier white 39 2.53
+oscar ichabod 66 3.02
+luke laertes 22 1.94
+xavier king 68 1.61
+wendy ellison 58 2.33
+gabriella ichabod 66 3.55
+alice carson 35 0.46
+ethan robinson 18 1.83
+xavier underhill 39 2.13
+ulysses van buren 74 1.65
+zach davidson 70 1.52
+sarah xylophone 68 2.22
+rachel brown 24 0.63
+david young 60 1.50
+quinn quirinius 48 2.46
+tom garcia 62 1.83
+david carson 25 3.00
+zach ellison 61 3.39
+ethan ichabod 53 2.33
+yuri johnson 66 3.69
+quinn johnson 58 3.21
+yuri underhill 60 2.16
+priscilla quirinius 30 0.96
+alice van buren 52 0.73
+victor allen 29 0.59
+zach ovid 22 0.35
+gabriella ellison 73 0.97
+fred hernandez 20 1.20
+priscilla zipper 44 2.22
+ethan thompson 41 0.02
+ethan young 69 0.60
+priscilla xylophone 61 1.91
+holly laertes 72 3.59
+irene johnson 63 1.75
+holly zipper 58 2.58
+fred ichabod 59 2.61
+zach davidson 52 1.01
+irene quirinius 74 0.86
+tom hernandez 38 0.79
+luke van buren 56 3.65
+wendy white 61 2.44
+holly underhill 73 3.42
+quinn van buren 72 4.00
+tom van buren 61 2.26
+irene davidson 47 0.05
+wendy king 22 1.88
+luke laertes 45 0.50
+zach brown 54 3.76
+yuri johnson 22 1.20
+bob xylophone 22 3.07
+priscilla zipper 36 0.76
+xavier polk 67 0.84
+oscar ovid 21 3.24
+oscar quirinius 52 2.41
+victor king 68 0.05
+oscar polk 56 3.29
+bob polk 36 2.75
+jessica miller 70 3.01
+xavier falkner 45 0.06
+holly ichabod 70 1.43
+tom thompson 19 2.62
+mike thompson 27 3.36
+fred king 76 1.64
+quinn king 34 3.39
+holly laertes 39 1.53
+ulysses xylophone 44 2.67
+fred steinbeck 22 1.48
+tom ellison 51 3.71
+victor white 41 3.45
+xavier ovid 65 1.19
+sarah miller 55 2.91
+irene davidson 40 1.32
+bob hernandez 43 1.48
+nick johnson 55 2.70
+oscar carson 34 3.76
+irene polk 38 2.29
+fred king 22 3.71
+bob laertes 40 2.62
+yuri nixon 36 1.31
+oscar thompson 27 3.59
+irene allen 34 1.54
+holly hernandez 51 2.69
+sarah brown 71 3.36
+jessica allen 58 3.12
+victor carson 50 1.97
+yuri xylophone 22 1.93
+katie xylophone 27 3.40
+quinn polk 24 2.24
+wendy van buren 63 2.43
+david zipper 38 2.57
+priscilla underhill 53 2.93
+david johnson 60 0.15
+tom ellison 59 3.63
+zach thompson 58 2.90
+victor van buren 71 0.98
+priscilla hernandez 18 0.29
+fred garcia 64 0.88
+victor polk 71 1.61
+priscilla miller 29 3.69
+jessica johnson 77 2.07
+nick falkner 60 3.60
+holly steinbeck 69 3.45
+victor allen 30 0.21
+oscar white 21 0.27
+quinn underhill 45 1.21
+tom johnson 77 1.11
+tom falkner 66 3.69
+mike young 61 3.17
+alice laertes 27 3.33
+luke ovid 43 0.82
+mike allen 26 1.55
+irene davidson 75 1.95
+tom carson 21 2.34
+quinn zipper 51 0.71
+calvin ellison 24 3.95
+oscar hernandez 30 0.38
+ulysses hernandez 19 0.26
+katie underhill 36 3.47
+ulysses hernandez 77 0.55
+tom carson 26 3.20
+alice young 57 2.14
+alice king 75 2.29
+ulysses ellison 44 0.50
+gabriella allen 57 2.45
+calvin polk 28 3.64
+xavier robinson 64 3.61
+jessica polk 28 3.69
+fred ichabod 72 1.95
+gabriella xylophone 68 1.26
+ulysses quirinius 56 3.63
+ethan nixon 22 0.35
+holly thompson 26 1.25
+oscar brown 47 2.74
+oscar allen 32 0.32
+katie brown 51 0.92
+gabriella underhill 66 1.33
+priscilla steinbeck 31 0.21
+holly falkner 68 3.56
+calvin carson 36 3.26
+rachel laertes 25 3.21
+fred king 51 1.46
+rachel hernandez 39 3.37
+wendy van buren 69 1.20
+xavier nixon 74 3.94
+tom ellison 59 0.00
+calvin johnson 51 0.86
+rachel white 57 3.47
+david laertes 76 2.93
+holly quirinius 27 1.34
+irene falkner 52 3.68
+quinn davidson 42 0.94
+nick underhill 38 1.27
+oscar brown 64 0.03
+fred allen 54 0.80
+oscar thompson 75 1.33
+oscar robinson 22 0.89
+priscilla carson 21 2.94
+zach xylophone 28 0.34
+jessica johnson 19 3.26
+yuri steinbeck 37 0.24
+bob ichabod 55 3.41
+luke laertes 73 3.22
+tom nixon 50 2.75
+yuri king 76 3.34
+irene ovid 59 3.60
+ethan steinbeck 61 2.39
+yuri young 36 2.19
+priscilla white 28 0.88
+priscilla laertes 19 3.74
+priscilla van buren 73 2.39
+rachel miller 35 0.64
+sarah falkner 22 2.29
+victor ichabod 38 1.66
+david ellison 65 3.12
+rachel robinson 73 1.77
+oscar king 45 3.21
+nick brown 24 3.19
+holly nixon 38 0.39
+victor young 69 0.12
+yuri miller 40 1.68
+ethan brown 74 0.96
+ethan underhill 44 1.14
+katie underhill 43 3.99
+rachel laertes 63 1.14
+ethan johnson 28 0.01
+oscar young 63 0.18
+wendy miller 53 1.74
+calvin garcia 63 3.68
+katie steinbeck 44 1.84
+bob falkner 60 1.96
+victor hernandez 18 3.68
+wendy steinbeck 30 1.65
+xavier polk 24 3.41
+quinn ichabod 66 0.38
+irene ellison 65 0.51
+calvin ichabod 42 1.09
+irene miller 76 0.47
+gabriella xylophone 72 2.29
+calvin ichabod 63 0.25
+wendy hernandez 36 3.00
+yuri quirinius 67 1.90
+alice garcia 29 1.51
+gabriella johnson 54 2.85
+nick van buren 56 1.95
+nick king 37 2.55
+yuri hernandez 20 2.99
+sarah ovid 73 0.32
+calvin garcia 49 2.14
+fred robinson 50 2.91
+rachel white 22 2.08
+bob quirinius 34 3.45
+zach ichabod 68 0.24
+tom falkner 37 1.47
+irene miller 73 0.57
+mike ovid 57 1.27
+calvin xylophone 28 1.27
+oscar miller 71 1.89
+gabriella young 51 2.18
+ethan davidson 77 3.97
+david xylophone 21 2.74
+xavier xylophone 56 2.83
+gabriella king 43 3.47
+quinn laertes 60 1.66
+katie nixon 28 3.46
+irene garcia 72 2.76
+mike nixon 42 0.74
+quinn davidson 25 3.65
+nick falkner 45 2.02
+oscar miller 38 1.31
+ethan polk 47 2.84
+jessica ovid 60 0.54
+jessica white 24 3.52
+katie king 73 3.75
+victor ovid 66 0.49
+rachel ellison 21 3.88
+bob brown 47 1.56
+gabriella hernandez 71 3.98
+wendy ichabod 29 0.07
+katie xylophone 36 3.67
+rachel johnson 73 3.93
+holly underhill 29 2.67
+gabriella allen 50 2.47
+gabriella allen 38 0.53
+jessica miller 18 3.08
+yuri thompson 77 1.99
+zach ichabod 54 0.88
+victor carson 71 2.60
+katie quirinius 26 1.93
+yuri robinson 53 3.51
+victor carson 21 0.24
+priscilla polk 53 2.28
+zach zipper 19 1.91
+rachel polk 62 0.66
+rachel brown 64 2.11
+david brown 36 0.65
+oscar young 62 1.81
+jessica quirinius 76 2.48
+david underhill 74 3.73
+priscilla quirinius 53 0.79
+calvin johnson 40 0.56
+ethan hernandez 63 1.92
+sarah ellison 36 1.11
+mike young 74 3.82
+fred garcia 40 2.21
+fred steinbeck 49 3.41
+david ovid 44 2.71
+holly zipper 25 2.79
+jessica robinson 25 2.43
+nick ichabod 51 1.16
+david carson 37 1.80
+ethan allen 75 2.50
+zach quirinius 42 0.70
+alice van buren 69 0.57
+mike ovid 32 3.89
+david ellison 23 2.02
+ethan white 69 3.63
+bob robinson 71 0.52
+david laertes 71 3.35
+ethan steinbeck 71 2.21
+fred steinbeck 32 1.42
+katie davidson 30 2.83
+calvin quirinius 35 3.70
+luke carson 36 1.08
+ulysses polk 55 1.00
+gabriella polk 38 0.51
+katie laertes 60 1.51
+gabriella steinbeck 38 1.06
+priscilla johnson 41 3.88
+david zipper 26 1.30
+sarah ellison 76 2.67
+oscar miller 54 1.20
+priscilla ichabod 70 2.18
+priscilla carson 69 0.02
+tom laertes 49 0.59
+rachel zipper 19 0.28
+jessica ichabod 75 2.90
+victor quirinius 43 2.24
+oscar quirinius 38 3.48
+gabriella brown 54 1.87
+katie davidson 77 3.90
+victor xylophone 58 2.75
+katie laertes 54 2.04
+wendy carson 73 1.45
+victor garcia 67 1.70
+ulysses miller 55 3.46
+holly thompson 51 2.66
+priscilla miller 64 0.34
+holly steinbeck 67 0.40
+wendy miller 19 0.05
+jessica nixon 19 3.73
+rachel miller 72 0.81
+tom king 29 0.11
+gabriella johnson 55 0.10
+yuri thompson 47 1.87
+quinn ovid 29 1.06
+sarah nixon 67 3.94
+fred xylophone 29 0.08
+luke garcia 74 3.83
+bob robinson 34 2.32
+yuri hernandez 70 3.97
+zach young 70 1.73
+ethan young 49 0.87
+xavier miller 56 0.97
+oscar thompson 30 3.48
+david xylophone 53 1.41
+jessica johnson 67 3.33
+ethan thompson 69 3.69
+katie garcia 76 1.56
+david ovid 63 2.89
+oscar miller 48 2.16
+alice thompson 30 0.74
+victor miller 56 3.54
+calvin robinson 20 1.03
+sarah carson 49 1.24
+gabriella van buren 21 1.26
+holly nixon 34 1.33
+quinn xylophone 48 1.06
+zach nixon 75 1.17
+irene allen 46 3.78
+nick allen 19 1.14
+gabriella ovid 73 2.71
+xavier van buren 38 0.37
+alice steinbeck 43 3.76
+bob ellison 76 0.63
+oscar hernandez 35 3.95
+jessica johnson 32 0.18
+calvin underhill 23 1.30
+priscilla young 67 2.92
+xavier white 67 3.72
+oscar ichabod 18 1.81
+irene falkner 31 1.02
+ethan garcia 29 3.65
+calvin steinbeck 71 0.20
+gabriella garcia 33 3.83
+irene underhill 69 1.12
+irene falkner 41 0.36
+gabriella polk 69 2.08
+priscilla robinson 47 2.02
+quinn allen 41 1.55
+priscilla davidson 69 2.49
+yuri ovid 47 3.49
+wendy xylophone 71 2.28
+wendy ellison 37 2.47
+david allen 72 1.19
+bob underhill 52 1.99
+jessica hernandez 77 2.52
+fred laertes 54 0.91
+mike steinbeck 33 3.36
+oscar xylophone 21 0.06
+alice young 22 0.02
+wendy underhill 70 2.62
+fred thompson 41 2.93
+katie laertes 48 3.86
+jessica van buren 44 2.35
+calvin carson 65 3.17
+nick garcia 20 3.95
+luke robinson 72 1.52
+gabriella van buren 55 0.32
+calvin white 35 1.41
+wendy garcia 25 2.51
+oscar robinson 30 0.60
+gabriella ovid 45 3.00
+oscar van buren 52 1.06
+yuri steinbeck 71 3.88
+yuri king 66 1.20
+katie white 52 3.31
+gabriella young 64 1.72
+xavier van buren 35 0.12
+bob ichabod 28 2.31
+katie underhill 58 0.14
+mike thompson 35 1.12
+rachel underhill 54 2.37
+gabriella robinson 30 3.76
+ulysses polk 45 3.69
+yuri ellison 51 3.75
+jessica falkner 21 3.16
+quinn johnson 18 3.45
+rachel polk 46 3.84
+luke zipper 18 3.75
+mike underhill 70 0.97
+zach johnson 25 2.26
+yuri hernandez 72 2.15
+jessica underhill 64 3.26
+katie hernandez 66 0.31
+katie brown 53 2.27
+tom ellison 33 1.58
+holly allen 69 0.06
+fred miller 46 1.52
+mike johnson 24 0.61
+luke xylophone 38 0.32
+tom ichabod 28 3.66
+sarah xylophone 63 3.75
+xavier hernandez 46 2.75
+oscar laertes 19 1.80
+tom quirinius 54 3.74
+ulysses nixon 26 1.14
+mike ellison 67 3.40
+quinn ellison 71 1.44
+fred laertes 76 0.03
+victor falkner 68 1.59
+sarah xylophone 37 0.86
+priscilla quirinius 43 0.08
+jessica ellison 50 3.67
+ulysses robinson 71 2.46
+xavier davidson 62 3.10
+victor young 45 2.66
+david underhill 45 0.95
+calvin ichabod 73 2.29
+mike nixon 46 1.22
+ethan underhill 31 2.08
+nick van buren 39 3.57
+holly carson 73 0.96
+alice laertes 43 2.72
+mike polk 18 0.96
+sarah steinbeck 68 2.35
+quinn king 34 1.42
+oscar allen 52 2.18
+jessica falkner 34 0.98
+wendy zipper 57 2.44
+irene falkner 23 3.42
+gabriella nixon 21 3.79
+katie hernandez 72 2.71
+katie nixon 70 3.09
+quinn brown 43 0.34
+nick ellison 47 3.27
+alice ellison 62 3.61
+quinn laertes 33 1.90
+david brown 74 3.53
+sarah robinson 39 1.75
+luke thompson 57 2.91
+luke underhill 56 0.61
+bob falkner 27 3.24
+tom white 20 1.01
+rachel johnson 73 3.29
+oscar falkner 25 2.50
+luke king 32 0.67
+sarah robinson 63 2.75
+ulysses steinbeck 47 0.01
+calvin nixon 29 0.31
+jessica ichabod 21 2.13
+calvin young 31 3.20
+gabriella ovid 42 1.22
+quinn steinbeck 33 2.80
+rachel king 24 3.79
+ethan underhill 32 3.83
+fred robinson 52 0.42
+david van buren 34 0.89
+ulysses white 22 0.12
+ulysses zipper 28 3.48
+sarah allen 58 2.53
+katie polk 21 3.44
+oscar zipper 56 2.43
+oscar brown 26 2.49
+luke polk 68 2.64
+yuri van buren 74 1.64
+priscilla johnson 54 2.71
+calvin davidson 48 2.40
+rachel davidson 27 0.98
+bob garcia 56 2.07
+calvin underhill 31 2.93
+bob garcia 58 1.63
+sarah davidson 53 3.03
+zach young 37 1.47
+xavier quirinius 62 3.35
+nick young 55 0.77
+ulysses carson 57 2.36
+irene robinson 59 1.13
+gabriella thompson 28 3.72
+david thompson 49 3.91
+priscilla allen 61 1.23
+wendy miller 24 0.00
+nick quirinius 75 1.03
+priscilla van buren 34 3.52
+david hernandez 30 0.41
+tom davidson 38 2.57
+ethan ichabod 27 3.77
+katie allen 20 1.50
+oscar quirinius 54 3.22
+mike nixon 40 2.41
+calvin miller 73 0.11
+bob underhill 76 3.81
+irene zipper 47 3.46
+fred ellison 42 3.49
+quinn garcia 75 2.23
+zach quirinius 55 3.20
+irene allen 61 3.98
+irene zipper 18 2.26
+wendy xylophone 65 2.15
+david zipper 73 1.67
+ulysses xylophone 44 1.72
+nick johnson 66 2.54
+mike carson 20 3.12
+sarah garcia 56 0.98
+sarah van buren 67 3.89
+jessica nixon 50 0.15
+bob allen 52 0.07
+tom garcia 56 2.94
+sarah thompson 46 1.18
+irene nixon 30 3.38
+calvin laertes 39 3.23
+katie zipper 45 1.85
+david xylophone 18 0.64
+david van buren 48 0.30
+david white 26 0.28
+victor laertes 77 0.63
+yuri van buren 21 2.12
+ethan ovid 74 0.89
+victor laertes 77 1.16
+tom underhill 27 3.22
+sarah xylophone 28 2.89
+tom king 34 3.03
+calvin polk 26 3.05
+ulysses white 26 0.67
+victor underhill 63 0.35
+sarah thompson 31 0.37
+katie garcia 24 1.44
+calvin garcia 35 1.53
+irene ellison 45 3.50
+yuri king 43 3.08
+wendy underhill 52 2.97
+bob nixon 68 2.02
+xavier van buren 51 1.00
+luke van buren 39 3.54
+rachel carson 62 3.86
+david garcia 33 0.41
+victor underhill 21 2.48
+victor zipper 45 3.89
+fred polk 36 3.60
+rachel polk 49 1.27
+nick van buren 71 3.02
+david ichabod 55 3.55
+wendy steinbeck 66 3.89
+ethan garcia 71 0.07
+holly garcia 19 1.46
+ulysses allen 31 1.71
+alice quirinius 36 0.22
+yuri ichabod 69 3.49
+jessica johnson 64 0.26
+ulysses white 69 2.73
+gabriella van buren 71 2.30
+sarah falkner 47 2.23
+priscilla white 41 3.90
+jessica thompson 54 1.62
+holly brown 64 0.70
+fred king 23 1.57
+quinn young 57 1.24
+xavier quirinius 63 3.38
+ulysses underhill 50 1.82
+fred ichabod 24 0.37
+ethan white 38 2.94
+alice ichabod 73 3.49
+calvin carson 68 2.60
+tom zipper 62 1.93
+rachel brown 43 1.51
+rachel ovid 25 1.34
+mike ichabod 23 1.08
+luke miller 42 1.32
+quinn miller 60 2.81
+victor ellison 50 3.73
+irene thompson 44 0.30
+quinn robinson 36 2.57
+ulysses ellison 23 2.74
+luke ichabod 47 0.49
+irene ichabod 74 3.55
+victor carson 54 0.36
+tom laertes 48 3.97
+gabriella young 35 3.03
+yuri ichabod 52 0.38
+sarah miller 22 1.63
+rachel carson 57 0.34
+sarah laertes 68 2.18
+nick allen 57 3.71
+nick brown 20 3.65
+quinn polk 62 1.83
+priscilla quirinius 22 1.93
+jessica young 64 3.04
+xavier van buren 22 2.17
+bob van buren 61 1.78
+rachel ichabod 66 3.47
+luke brown 34 0.34
+katie carson 71 3.97
+ulysses white 34 1.15
+zach falkner 33 2.28
+fred miller 74 3.70
+alice hernandez 28 3.51
+tom brown 21 1.57
+tom thompson 65 2.35
+david robinson 43 2.96
+rachel xylophone 45 0.03
+sarah miller 21 0.04
+luke hernandez 29 0.65
+fred carson 43 0.79
+luke nixon 28 0.84
+holly underhill 28 2.83
+tom underhill 58 3.74
+david falkner 71 1.33
+wendy zipper 53 1.01
+bob ichabod 37 1.65
+victor king 54 3.58
+sarah young 50 3.62
+bob thompson 29 3.29
+tom zipper 62 3.53
+katie ovid 42 0.53
+yuri robinson 62 0.25
+tom ellison 60 2.55
+tom van buren 25 3.46
+david ichabod 30 2.68
+wendy hernandez 27 2.38
+alice underhill 74 0.34
+calvin hernandez 73 1.35
+gabriella garcia 29 3.33
+zach falkner 76 1.34
+david thompson 75 0.15
+katie steinbeck 72 2.23
+xavier allen 72 3.76
+priscilla ellison 64 2.32
+yuri hernandez 41 2.68
+ulysses ovid 64 3.25
+oscar carson 51 1.30
+david allen 68 2.06
+mike quirinius 75 1.94
+mike garcia 44 0.58
+rachel laertes 44 3.68
+quinn ellison 70 0.11
+tom quirinius 32 3.39
+yuri polk 37 1.57
+luke johnson 75 3.70
+tom johnson 46 1.81
+irene miller 57 2.85
+holly white 58 1.24
+holly ichabod 55 1.21
+zach thompson 47 0.21
+gabriella davidson 66 1.58
+tom steinbeck 69 2.82
+katie ovid 42 0.45
+david johnson 41 1.25
+priscilla miller 69 2.13
+gabriella polk 22 2.88
+holly ovid 73 3.75
+fred zipper 26 1.50
+ulysses steinbeck 60 2.37
+jessica miller 71 3.17
+sarah polk 53 3.55
+quinn johnson 32 0.65
+priscilla young 63 2.33
+holly laertes 65 3.64
+alice king 57 3.91
+priscilla ovid 21 0.02
+david carson 77 3.27
+holly ichabod 33 0.51
+mike zipper 66 3.32
+david ellison 37 3.48
+ethan ellison 32 3.59
+katie brown 39 2.28
+zach thompson 51 0.02
+ethan davidson 58 1.82
+oscar johnson 25 0.99
+ulysses falkner 71 1.14
+mike hernandez 60 1.66
+gabriella nixon 48 3.10
+ethan steinbeck 22 3.45
+david van buren 32 2.96
+bob zipper 74 2.52
+david garcia 51 3.99
+nick xylophone 29 0.58
+david hernandez 70 3.16
+jessica ichabod 59 0.09
+wendy laertes 70 2.58
+gabriella polk 76 3.84
+tom carson 70 1.68
+katie robinson 35 0.22
+nick miller 38 1.79
+zach xylophone 31 1.02
+bob quirinius 20 2.44
+ethan xylophone 18 0.71
+holly johnson 55 1.67
+gabriella thompson 52 3.71
+tom king 48 0.34
+wendy brown 40 0.46
+fred allen 38 3.96
+mike ichabod 54 3.13
+luke van buren 65 3.31
+yuri zipper 66 2.10
+xavier laertes 73 0.93
+mike robinson 26 2.84
+luke steinbeck 58 2.46
+rachel steinbeck 63 1.20
+alice polk 28 2.14
+luke davidson 52 1.95
+ulysses steinbeck 54 3.80
+oscar quirinius 53 0.04
+gabriella xylophone 72 0.97
+tom ovid 34 3.17
+tom white 38 0.53
+tom ichabod 18 0.94
+irene steinbeck 66 0.49
+holly brown 77 1.46
+quinn young 65 1.56
+tom king 64 3.18
+wendy thompson 51 2.84
+katie xylophone 41 1.47
+zach hernandez 62 1.00
+oscar brown 21 0.56
+sarah nixon 69 3.33
+fred zipper 70 1.69
+rachel xylophone 60 2.46
+jessica ellison 55 1.55
+jessica king 71 3.59
+xavier hernandez 27 3.40
+bob brown 72 0.89
+yuri brown 47 3.07
+david van buren 37 3.04
+fred king 48 0.27
+fred falkner 41 1.74
+victor zipper 34 1.97
+fred quirinius 71 0.91
+rachel johnson 55 0.96
+katie ellison 62 2.13
+david king 67 1.17
+bob laertes 36 2.08
+calvin robinson 34 2.28
+xavier king 63 3.92
+ulysses thompson 27 0.43
+xavier ellison 65 1.00
+fred underhill 46 3.38
+sarah zipper 22 1.39
+oscar hernandez 62 2.87
+fred falkner 49 3.05
+ulysses nixon 66 0.59
+bob underhill 73 3.23
+fred hernandez 49 1.21
+xavier king 25 3.45
+quinn xylophone 69 3.31
+xavier young 76 1.70
+mike king 45 0.75
+calvin brown 50 1.06
+jessica young 30 0.54
+fred brown 38 2.74
+zach ichabod 35 1.05
+rachel zipper 24 1.41
+oscar underhill 37 0.50
+fred underhill 26 3.62
+alice allen 68 3.39
+oscar thompson 68 0.08
+mike underhill 38 2.98
+zach van buren 76 2.15
+sarah laertes 21 0.08
+sarah polk 68 2.80
+david ovid 43 0.77
+oscar polk 54 2.05
+oscar xylophone 63 2.44
+tom quirinius 66 3.52
+tom miller 35 3.80
+ulysses thompson 32 0.20
+xavier ichabod 48 1.73
+sarah davidson 60 2.67
+ulysses quirinius 38 0.48
+ulysses miller 48 2.77
+quinn zipper 32 1.77
+zach garcia 32 0.94
+holly laertes 75 3.72
+holly underhill 21 1.69
+tom underhill 62 3.50
+nick polk 57 0.76
+tom king 34 2.26
+katie garcia 24 0.50
+luke xylophone 60 2.37
+priscilla young 18 0.88
+bob thompson 52 2.59
+ulysses garcia 57 3.16
+david garcia 34 0.59
+jessica ovid 69 2.13
+katie zipper 63 0.28
+bob miller 73 0.68
+mike johnson 28 1.26
+david xylophone 29 3.84
+ulysses king 61 3.99
+ulysses ichabod 47 3.27
+oscar underhill 71 0.47
+yuri garcia 74 2.97
+ulysses falkner 61 0.79
+quinn xylophone 22 2.88
+zach king 47 2.20
+victor van buren 34 1.40
+nick xylophone 74 0.29
+victor xylophone 67 3.73
+priscilla van buren 76 2.32
+mike johnson 65 3.53
+jessica white 57 0.09
+alice steinbeck 64 3.81
+nick brown 40 0.34
+gabriella polk 76 3.49
+nick miller 18 0.06
+bob hernandez 53 2.90
+mike garcia 72 3.35
+david ichabod 72 2.24
+fred young 35 1.94
+holly polk 22 2.03
+jessica quirinius 65 2.74
+katie quirinius 72 3.79
+zach garcia 68 3.87
+ulysses young 63 0.07
+quinn garcia 72 2.04
+xavier king 37 0.87
+nick falkner 48 0.25
+bob robinson 57 2.25
+xavier van buren 73 2.77
+wendy quirinius 26 1.25
+holly robinson 70 2.51
+wendy carson 22 0.34
+wendy quirinius 71 0.23
+bob garcia 77 2.96
+holly allen 67 1.29
+ulysses steinbeck 75 1.30
+bob polk 44 2.45
+mike ichabod 44 3.82
+ulysses ovid 34 3.56
+alice thompson 35 2.14
+gabriella thompson 64 2.12
+fred garcia 68 3.29
+ulysses davidson 74 3.18
+mike xylophone 35 1.73
+priscilla allen 48 0.62
+yuri zipper 37 0.23
+ethan allen 51 0.40
+yuri carson 60 1.41
+tom white 63 0.18
+sarah hernandez 38 2.51
+rachel allen 76 2.85
+fred miller 37 3.05
+gabriella thompson 22 1.06
+bob garcia 21 1.99
+jessica van buren 76 1.55
+oscar ovid 38 2.42
+priscilla ichabod 29 1.94
+ulysses johnson 48 0.96
+calvin zipper 62 0.34
+david xylophone 51 0.61
+gabriella laertes 64 2.41
+bob ellison 29 1.46
+gabriella carson 40 2.45
+wendy johnson 32 1.22
+fred brown 38 3.62
+xavier thompson 58 1.57
+sarah thompson 62 2.51
+katie allen 29 2.15
+david young 34 0.11
+bob underhill 30 0.13
+rachel laertes 68 1.65
+wendy king 48 3.72
+zach van buren 57 0.77
+alice polk 54 3.95
+fred falkner 27 0.16
+holly van buren 71 3.91
+irene king 23 2.10
+david brown 60 0.98
+fred ovid 64 3.27
+holly xylophone 50 0.44
+rachel xylophone 18 1.62
+xavier ovid 37 3.56
+david hernandez 46 0.48
+oscar ellison 20 0.50
+tom van buren 71 0.52
+irene young 50 3.04
+irene ichabod 61 0.03
+mike garcia 66 3.88
+quinn ichabod 50 1.55
+rachel steinbeck 39 2.10
+zach zipper 40 0.98
+tom white 69 2.28
+katie brown 40 1.21
+calvin king 57 0.01
+mike garcia 60 2.50
+oscar steinbeck 18 2.53
+priscilla xylophone 35 0.20
+victor davidson 55 2.98
+tom polk 74 2.72
+alice allen 77 2.52
+xavier zipper 48 1.17
+david carson 73 1.11
+luke laertes 35 0.87
+holly hernandez 60 1.56
+gabriella miller 51 3.72
+yuri laertes 65 1.93
+sarah johnson 37 1.01
+luke quirinius 77 0.04
+victor brown 39 3.86
+calvin zipper 42 0.02
+irene zipper 26 1.04
+david laertes 35 0.04
+mike carson 75 1.67
+ulysses king 38 2.81
+oscar zipper 61 3.42
+holly laertes 34 3.66
+luke carson 37 2.43
+yuri davidson 73 1.96
+holly quirinius 37 1.10
+tom underhill 45 2.97
+katie king 37 1.03
+calvin robinson 74 0.48
+zach nixon 60 1.25
+tom young 73 0.15
+priscilla laertes 75 0.84
+oscar thompson 71 2.92
+rachel hernandez 77 2.23
+wendy brown 23 1.79
+xavier zipper 31 2.93
+rachel allen 56 0.88
+ethan ichabod 73 0.37
+irene white 77 3.91
+quinn nixon 27 2.79
+gabriella xylophone 42 3.06
+victor miller 21 0.01
+oscar thompson 51 1.64
+holly van buren 33 0.38
+alice white 37 0.21
+mike nixon 30 1.43
+ulysses steinbeck 60 2.24
+holly ovid 76 2.69
+alice johnson 69 0.16
+katie robinson 23 2.05
+yuri zipper 42 1.20
+bob thompson 29 2.80
+victor brown 21 1.25
+gabriella brown 18 3.49
+zach nixon 21 2.28
+ethan underhill 64 0.36
+ulysses white 43 1.36
+mike thompson 21 2.04
+katie garcia 54 3.18
+yuri ellison 56 1.57
+nick quirinius 53 0.88
+holly thompson 24 0.99
+fred ellison 43 1.59
+sarah robinson 19 1.75
+wendy xylophone 40 2.86
+xavier brown 54 0.28
+rachel hernandez 76 1.49
+holly ovid 59 2.32
+mike thompson 68 2.61
+katie steinbeck 75 0.19
+yuri ovid 45 0.34
+tom young 43 3.46
+yuri thompson 34 3.93
+mike underhill 18 0.40
+david underhill 36 3.01
+jessica garcia 20 2.21
+calvin underhill 47 0.42
+holly polk 76 0.95
+victor johnson 21 1.17
+fred johnson 59 3.23
+fred xylophone 57 0.93
+ulysses carson 53 0.73
+ulysses garcia 64 1.98
+ethan falkner 28 2.08
+victor nixon 49 3.60
+ethan polk 57 1.89
+quinn underhill 57 2.69
+rachel ichabod 56 2.22
+oscar xylophone 76 3.57
+ethan garcia 19 1.79
+bob white 64 1.12
+david zipper 71 3.47
+calvin miller 20 0.82
+yuri allen 59 2.94
+holly van buren 49 0.60
+ulysses laertes 76 1.28
+luke ovid 74 3.22
+xavier miller 68 0.41
+ulysses davidson 27 1.29
+nick ichabod 18 0.54
+tom johnson 64 1.26
+oscar steinbeck 55 3.37
+mike miller 41 2.48
+bob ovid 37 1.08
+quinn nixon 71 3.46
+nick falkner 22 0.08
+quinn zipper 20 1.85
+holly miller 39 0.91
+tom robinson 20 2.99
+bob white 65 1.06
+ulysses polk 23 2.19
+alice davidson 77 0.07
+priscilla hernandez 69 2.65
+sarah underhill 74 3.74
+zach falkner 67 1.64
+yuri brown 75 1.32
+tom ichabod 40 2.37
+katie zipper 63 3.29
+oscar miller 53 3.92
+tom thompson 77 0.43
+ulysses johnson 49 2.76
+ethan king 59 3.00
+calvin steinbeck 68 2.60
+calvin hernandez 46 0.30
+ulysses steinbeck 56 2.68
+fred allen 71 2.86
+katie johnson 75 1.92
+katie garcia 66 3.82
+katie hernandez 69 2.29
+quinn xylophone 54 1.71
+wendy zipper 36 2.07
+luke zipper 71 1.56
+yuri johnson 56 3.83
+gabriella steinbeck 54 1.60
+holly miller 59 1.77
+gabriella xylophone 40 3.11
+mike nixon 60 3.28
+quinn ovid 75 3.87
+tom ellison 60 3.64
+tom zipper 46 2.45
+david johnson 41 3.29
+zach carson 51 3.30
+jessica zipper 22 1.44
+fred zipper 62 3.91
+bob white 19 2.57
+xavier thompson 35 0.09
+david thompson 57 0.74
+jessica johnson 67 1.18
+quinn ichabod 72 2.05
+nick garcia 59 1.99
+katie quirinius 74 1.85
+wendy thompson 28 2.03
+tom nixon 19 1.07
+jessica miller 56 2.23
+gabriella polk 29 2.81
+oscar white 44 1.00
+alice zipper 43 1.82
+xavier ellison 31 3.65
+victor davidson 34 1.13
+alice robinson 60 2.89
+rachel hernandez 19 1.33
+oscar carson 77 0.63
+holly miller 37 2.12
+mike ichabod 25 1.25
+sarah king 41 3.32
+tom young 32 3.30
+katie king 37 2.54
+jessica underhill 29 0.70
+holly young 37 3.07
+tom thompson 62 1.57
+calvin hernandez 41 3.29
+wendy ellison 25 1.27
+alice xylophone 66 3.71
+david young 49 3.45
+rachel johnson 54 2.01
+alice white 73 3.63
+yuri ovid 44 3.01
+sarah underhill 39 3.24
+calvin polk 37 1.31
+mike young 76 3.28
+rachel miller 33 3.19
+calvin ichabod 74 0.01
+xavier steinbeck 40 1.30
+luke hernandez 50 2.81
+jessica quirinius 61 0.73
+ulysses ellison 25 0.02
+rachel thompson 67 2.37
+fred davidson 24 0.05
+tom carson 38 2.72
+xavier xylophone 67 1.47
+tom falkner 74 0.83
+katie thompson 65 3.29
+sarah brown 28 0.74
+bob xylophone 18 1.19
+zach nixon 55 1.23
+tom robinson 29 1.54
+priscilla ellison 72 2.86
+katie quirinius 31 3.64
+nick brown 47 2.15
+alice xylophone 33 2.80
+nick young 56 0.65
+jessica falkner 51 2.02
+david carson 40 1.21
+luke thompson 26 2.41
+oscar ellison 46 1.82
+victor carson 75 1.62
+fred robinson 66 2.02
+tom steinbeck 26 0.47
+holly nixon 34 3.49
+rachel steinbeck 24 1.27
+sarah zipper 68 2.92
+bob white 24 0.38
+ulysses quirinius 53 2.55
+zach king 52 1.23
+xavier laertes 41 0.19
+gabriella nixon 21 1.42
+bob quirinius 33 0.19
+luke steinbeck 68 3.04
+wendy steinbeck 55 3.55
+holly king 64 3.54
+katie polk 43 3.07
+priscilla van buren 32 3.89
+ethan brown 48 2.73
+david ellison 56 3.22
+nick ichabod 73 0.18
+yuri garcia 31 1.04
+mike xylophone 28 0.80
+xavier young 73 0.53
+calvin carson 27 3.87
+david laertes 33 2.75
+xavier white 64 3.10
+alice young 43 2.30
+bob hernandez 22 0.12
+xavier brown 36 3.26
+bob brown 40 3.51
+jessica zipper 30 3.56
+gabriella xylophone 60 2.48
+alice underhill 64 0.34
+sarah steinbeck 45 2.79
+calvin van buren 44 2.14
+nick zipper 59 0.82
+xavier robinson 40 2.97
+xavier laertes 45 2.87
+ulysses johnson 25 2.43
+xavier garcia 31 1.21
+tom brown 22 1.54
+fred robinson 42 1.64
+wendy johnson 74 1.50
+quinn zipper 68 2.24
+irene garcia 47 2.23
+oscar zipper 43 2.31
+wendy ellison 40 3.33
+priscilla robinson 68 1.93
+calvin garcia 75 3.98
+victor laertes 64 3.77
+yuri robinson 24 0.41
+luke van buren 19 2.65
+tom miller 41 1.65
+tom ellison 61 0.01
+bob davidson 31 2.58
+jessica davidson 39 2.13
+xavier miller 59 3.79
+david steinbeck 33 1.17
+quinn thompson 61 2.12
+gabriella carson 56 3.99
+ethan miller 28 2.63
+wendy zipper 55 2.05
+david xylophone 40 1.27
+yuri falkner 27 1.99
+quinn underhill 32 2.76
+bob white 27 0.93
+oscar brown 67 0.91
+mike brown 19 1.25
+nick johnson 47 3.98
+luke robinson 61 0.23
+tom laertes 65 2.77
+priscilla garcia 66 0.71
+ethan quirinius 70 2.92
+oscar allen 18 2.11
+calvin polk 40 1.82
+katie laertes 39 2.12
+calvin allen 46 2.89
+priscilla xylophone 28 0.47
+ethan johnson 20 3.16
+ethan xylophone 27 3.85
+victor davidson 26 2.47
+alice ellison 75 0.26
+bob ellison 68 2.24
+gabriella young 44 2.68
+fred underhill 24 2.36
+david johnson 75 3.91
+quinn ovid 56 0.50
+jessica robinson 26 3.95
+victor young 32 3.09
+gabriella robinson 62 1.52
+bob xylophone 73 1.41
+quinn robinson 29 1.29
+sarah allen 42 2.93
+zach white 23 2.65
+gabriella steinbeck 68 1.01
+oscar nixon 68 0.17
+ethan zipper 52 2.71
+gabriella allen 48 0.47
+zach hernandez 21 3.80
+holly johnson 74 0.60
+yuri laertes 18 0.89
+xavier hernandez 61 0.06
+victor zipper 41 0.20
+alice underhill 45 3.16
+nick polk 21 3.59
+katie thompson 77 3.52
+jessica hernandez 26 2.40
+tom van buren 55 3.70
+jessica van buren 51 2.83
+ulysses king 24 1.83
+ethan robinson 72 0.79
+fred allen 45 2.98
+victor quirinius 35 1.83
+oscar van buren 64 0.00
+rachel carson 19 3.86
+wendy polk 53 2.41
+priscilla ichabod 50 2.56
+fred thompson 56 3.57
+jessica zipper 24 1.62
+priscilla carson 55 2.29
+yuri brown 48 0.29
+luke white 69 2.86
+sarah steinbeck 28 1.58
+bob hernandez 47 3.72
+holly robinson 76 2.52
+irene underhill 54 1.12
+mike young 49 3.34
+bob johnson 42 0.53
+xavier johnson 58 2.86
+fred ellison 29 1.02
+victor king 26 2.63
+ethan davidson 55 0.37
+luke quirinius 37 1.43
+oscar nixon 28 1.46
+ethan davidson 39 1.11
+alice brown 73 3.36
+calvin robinson 18 0.13
+zach ovid 54 1.37
+bob young 33 2.60
+irene van buren 43 3.53
+david davidson 47 2.48
+yuri thompson 74 0.28
+yuri xylophone 56 2.10
+rachel thompson 46 3.20
+gabriella ichabod 66 0.46
+sarah polk 61 2.42
+katie thompson 18 2.33
+katie brown 45 3.72
+fred miller 77 3.81
+rachel thompson 61 0.16
+calvin ellison 47 1.94
+bob thompson 62 0.84
+jessica polk 67 0.46
+david falkner 33 1.52
+wendy steinbeck 64 0.50
+mike zipper 43 1.75
+holly king 24 1.43
+irene laertes 18 3.46
+katie underhill 65 2.01
+nick brown 32 2.35
+jessica ovid 59 0.84
+sarah king 32 1.84
+quinn van buren 27 0.22
+oscar ovid 71 3.86
+katie davidson 37 2.50
+irene brown 54 1.74
+mike king 76 1.64
+ulysses allen 62 0.97
+luke davidson 21 1.61
+bob falkner 58 0.87
+alice van buren 55 3.31
+wendy thompson 64 3.03
+ethan laertes 60 3.03
+sarah king 75 3.94
+xavier zipper 28 3.25
+nick miller 65 2.24
+quinn king 64 2.75
+bob brown 25 2.66
+oscar nixon 74 2.72
+alice polk 59 0.39
+sarah ichabod 52 3.94
+holly quirinius 53 1.15
+victor xylophone 38 0.27
+ethan quirinius 29 2.21
+victor allen 61 0.21
+sarah ellison 61 3.61
+ulysses underhill 60 2.59
+tom ichabod 26 0.75
+alice johnson 21 3.54
+priscilla ellison 36 1.94
+ulysses miller 19 2.10
+tom nixon 57 0.86
+ulysses young 62 3.35
+jessica miller 24 2.00
+katie steinbeck 63 0.44
+ulysses zipper 37 0.32
+wendy young 21 0.39
+quinn white 48 0.51
+oscar garcia 76 1.79
+yuri davidson 43 0.57
+alice falkner 28 3.72
+xavier ovid 63 1.06
+yuri ichabod 62 1.02
+zach ichabod 77 0.57
+mike laertes 76 0.54
+luke thompson 29 3.30
+katie quirinius 35 0.76
+nick davidson 76 3.84
+mike allen 49 1.24
+luke xylophone 39 3.75
+rachel carson 44 0.26
+tom laertes 71 2.16
+priscilla underhill 74 3.17
+xavier ellison 27 0.35
+katie king 49 0.32
+quinn garcia 63 0.02
+yuri van buren 61 1.82
+oscar nixon 40 1.15
+irene nixon 38 0.95
+victor davidson 29 1.12
+ethan zipper 64 2.63
+zach quirinius 71 3.27
+yuri johnson 40 2.13
+luke ellison 18 3.31
+luke allen 51 0.31
+luke thompson 68 0.52
+alice falkner 75 2.40
+ethan johnson 48 1.06
+wendy xylophone 40 0.83
+irene xylophone 48 3.84
+priscilla zipper 32 3.51
+gabriella ichabod 49 3.83
+jessica hernandez 53 2.87
+gabriella carson 37 2.95
+rachel brown 36 1.97
+gabriella miller 72 3.61
+quinn ovid 18 0.08
+fred xylophone 54 3.28
+ethan ichabod 39 1.67
+ethan zipper 61 1.82
+nick nixon 37 2.74
+ulysses van buren 74 0.71
+victor xylophone 35 2.47
+oscar ichabod 57 2.93
+mike van buren 68 2.30
+david robinson 21 2.09
+tom polk 69 0.83
+luke hernandez 41 3.01
+victor van buren 74 3.37
+zach hernandez 65 0.16
+irene ovid 26 3.48
+irene johnson 61 2.43
+irene polk 44 0.57
+luke ovid 52 0.66
+ethan king 37 0.19
+zach nixon 69 1.14
+david quirinius 24 2.94
+ulysses white 34 0.45
+rachel van buren 60 2.55
+sarah hernandez 24 3.58
+ulysses garcia 56 1.16
+oscar young 48 0.37
+quinn laertes 45 0.88
+ulysses ovid 72 1.62
+quinn ellison 44 2.35
+ethan xylophone 69 1.63
+tom johnson 72 0.26
+zach carson 19 0.84
+fred zipper 28 1.85
+quinn johnson 32 3.43
+mike robinson 76 1.68
+sarah ichabod 19 0.24
+david xylophone 20 2.46
+david ellison 64 0.65
+fred ichabod 20 3.25
+calvin ovid 64 0.83
+rachel xylophone 67 0.75
+luke polk 62 1.35
+jessica johnson 64 1.72
+quinn miller 40 3.20
+holly king 62 0.87
+alice hernandez 29 0.83
+wendy quirinius 31 3.03
+tom van buren 53 3.52
+fred ichabod 31 1.42
+yuri ovid 70 2.02
+fred hernandez 44 0.09
+gabriella carson 28 2.08
+nick ichabod 37 1.95
+victor quirinius 25 1.01
+gabriella ovid 51 0.78
+holly quirinius 22 3.66
+xavier thompson 33 0.43
+tom hernandez 69 1.42
+xavier xylophone 60 2.20
+quinn steinbeck 67 2.12
+yuri johnson 46 2.41
+xavier johnson 21 2.39
+ulysses king 55 0.00
+ethan white 53 2.28
+fred allen 26 3.28
+rachel carson 37 2.03
+victor underhill 49 3.20
+victor ichabod 37 1.53
+wendy underhill 65 2.11
+luke white 32 2.96
+gabriella white 59 1.03
+ulysses ovid 40 2.35
+mike garcia 73 2.73
+yuri king 59 3.85
+rachel van buren 18 2.21
+holly ichabod 45 3.99
+jessica white 41 2.37
+jessica carson 50 1.59
+wendy ellison 22 3.49
+david young 52 2.63
+fred underhill 75 1.55
+irene young 22 2.01
+oscar underhill 74 1.42
+ulysses davidson 44 2.27
+oscar ovid 57 1.62
+irene steinbeck 30 1.75
+sarah hernandez 39 2.60
+gabriella ichabod 65 3.57
+quinn brown 48 1.56
+nick steinbeck 45 1.13
+katie robinson 33 0.11
+wendy laertes 24 3.00
+victor laertes 58 2.45
+wendy xylophone 18 0.87
+ulysses polk 64 1.41
+nick zipper 72 3.43
+nick laertes 46 1.20
+wendy young 48 0.19
+bob nixon 77 1.58
+rachel steinbeck 58 2.60
+sarah allen 52 0.07
+nick zipper 38 3.37
+gabriella polk 37 0.90
+oscar thompson 26 2.52
+quinn young 33 3.38
+irene falkner 53 2.18
+katie steinbeck 29 2.93
+luke young 24 0.31
+rachel nixon 61 2.12
+irene steinbeck 65 3.30
+irene king 30 1.93
+david robinson 43 1.53
+rachel zipper 58 3.76
+jessica miller 44 2.24
+yuri brown 42 1.76
+luke miller 67 2.39
+xavier young 68 1.92
+jessica king 51 2.62
+oscar carson 71 1.20
+oscar garcia 30 0.09
+david xylophone 21 1.42
+mike allen 77 0.60
+bob davidson 22 1.75
+mike underhill 36 3.46
+quinn garcia 72 3.07
+mike white 34 3.15
+rachel thompson 46 2.96
+bob young 36 1.37
+david steinbeck 42 1.94
+wendy steinbeck 73 2.02
+calvin ovid 46 3.13
+wendy van buren 73 2.94
+oscar quirinius 76 1.45
+yuri underhill 73 2.10
+tom ichabod 67 0.71
+ulysses ichabod 37 0.95
+quinn laertes 61 2.85
+xavier underhill 41 3.95
+ethan miller 51 0.03
+alice white 48 2.71
+wendy garcia 66 2.64
+sarah miller 76 1.32
+victor polk 66 2.75
+victor brown 28 1.65
+mike van buren 40 3.42
+luke brown 37 3.33
+sarah van buren 42 2.74
+katie davidson 38 3.83
+holly robinson 46 0.67
+holly ovid 52 0.16
+david king 48 3.63
+priscilla van buren 19 0.70
+luke king 58 0.97
+david allen 20 3.99
+irene falkner 51 2.86
+ethan brown 70 0.44
+wendy ellison 53 0.25
+david davidson 25 1.91
+oscar laertes 40 3.47
+jessica king 46 2.91
+tom falkner 40 3.45
+oscar thompson 46 2.70
+bob brown 74 3.34
+ethan zipper 66 0.81
+quinn nixon 65 0.29
+alice johnson 64 3.99
+wendy allen 42 1.22
+irene allen 23 2.01
+irene steinbeck 56 0.44
+gabriella carson 53 3.91
+rachel thompson 53 3.36
+priscilla quirinius 70 1.51
+zach polk 54 0.87
+tom robinson 39 1.67
+zach king 39 3.75
+rachel falkner 27 0.20
+gabriella ichabod 58 2.60
+rachel thompson 40 1.07
+david white 29 0.19
+david johnson 77 0.72
+alice steinbeck 58 3.63
+mike davidson 33 3.55
+ethan king 64 0.89
+bob brown 67 3.72
+david miller 69 3.42
+alice ellison 33 2.63
+nick nixon 30 1.90
+mike robinson 34 0.68
+sarah laertes 39 3.45
+bob falkner 57 0.54
+holly hernandez 33 0.28
+gabriella white 47 0.68
+david quirinius 64 1.43
+zach white 69 0.65
+victor quirinius 66 0.92
+priscilla polk 26 1.31
+wendy garcia 56 1.62
+priscilla ichabod 67 2.62
+sarah white 45 0.22
+gabriella xylophone 49 2.97
+calvin nixon 40 0.78
+katie johnson 33 0.04
+david laertes 59 2.53
+mike white 47 2.93
+wendy carson 53 2.34
+wendy nixon 56 1.14
+sarah ellison 40 2.32
+jessica zipper 32 2.28
+wendy young 39 0.18
+rachel hernandez 58 2.55
+alice allen 36 2.27
+david ellison 76 0.15
+gabriella brown 65 2.66
+mike garcia 31 2.79
+calvin garcia 30 2.08
+xavier carson 32 2.36
+quinn white 52 0.30
+holly laertes 46 3.22
+holly robinson 29 2.19
+katie thompson 28 0.89
+irene laertes 59 1.75
+holly polk 74 1.26
+tom ellison 60 3.34
+ulysses miller 18 1.30
+jessica garcia 49 3.47
+calvin allen 64 1.64
+katie thompson 59 3.99
+holly xylophone 77 0.52
+irene van buren 63 3.63
+zach van buren 32 0.45
+xavier underhill 19 2.65
+victor ellison 68 2.06
+luke underhill 23 3.85
+mike nixon 37 1.73
+victor garcia 65 1.09
+nick underhill 75 1.19
+quinn nixon 42 3.20
+wendy robinson 20 3.40
+sarah garcia 74 1.51
+ulysses van buren 74 1.05
+wendy thompson 29 1.97
+mike zipper 69 2.24
+alice robinson 35 3.58
+alice ichabod 64 2.25
+nick brown 20 1.79
+quinn quirinius 18 0.81
+nick hernandez 68 0.51
+gabriella brown 26 0.93
+priscilla steinbeck 66 1.61
+wendy brown 74 3.85
+wendy quirinius 24 2.02
+priscilla carson 54 3.84
+irene young 56 1.96
+luke zipper 28 1.24
+irene polk 66 1.34
+fred white 47 1.48
+oscar carson 67 3.14
+ulysses carson 25 2.92
+jessica zipper 74 0.17
+gabriella nixon 65 1.63
+priscilla ovid 48 3.77
+wendy carson 56 0.90
+victor nixon 49 2.71
+nick white 50 2.66
+gabriella van buren 34 3.21
+wendy garcia 54 1.85
+luke steinbeck 19 2.63
+alice davidson 34 2.64
+yuri falkner 55 0.45
+oscar ichabod 51 3.16
+wendy carson 49 0.18
+zach miller 55 1.79
+xavier van buren 43 2.86
+priscilla quirinius 37 2.19
+wendy young 35 3.30
+jessica young 36 0.29
+calvin steinbeck 25 3.43
+holly underhill 24 1.58
+nick ovid 74 3.83
+gabriella white 60 2.90
+gabriella ovid 32 1.97
+fred johnson 41 2.22
+priscilla xylophone 54 0.79
+bob white 23 2.52
+calvin miller 39 2.74
+ulysses quirinius 71 0.74
+holly young 62 3.84
+sarah xylophone 31 0.12
+zach hernandez 60 2.16
+tom van buren 23 1.32
+ethan johnson 43 2.73
+mike ovid 51 3.07
+ethan polk 22 0.88
+irene king 33 0.53
+ethan allen 46 2.02
+rachel thompson 53 1.38
+holly ovid 48 0.05
+wendy robinson 23 0.93
+katie robinson 44 0.89
+rachel young 19 1.93
+priscilla king 31 1.97
+ulysses thompson 20 2.27
+sarah thompson 46 3.12
+victor white 55 2.09
+alice miller 52 0.91
+rachel zipper 40 0.07
+calvin allen 62 3.29
+oscar johnson 28 0.88
+irene white 73 2.40
+jessica laertes 32 2.27
+david van buren 55 0.13
+bob carson 58 3.05
+bob van buren 77 2.69
+gabriella king 36 3.69
+katie x
<TRUNCATED>
[12/32] hive git commit: HIVE-20060 Refactor HiveSchemaTool and
MetastoreSchemaTool (Alan Gates, reviewed by Daniel Dai)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/MetastoreSchemaTool.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/MetastoreSchemaTool.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/MetastoreSchemaTool.java
index 7c8087f..c2018f4 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/MetastoreSchemaTool.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/MetastoreSchemaTool.java
@@ -18,25 +18,15 @@
package org.apache.hadoop.hive.metastore.tools;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.OptionGroup;
-import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.io.output.NullOutputStream;
-import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.metastore.HiveMetaException;
import org.apache.hadoop.hive.metastore.IMetaStoreSchemaInfo;
import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory;
-import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo;
@@ -44,55 +34,44 @@ import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.NestedScriptParse
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.ImmutableMap;
import sqlline.SqlLine;
import java.io.BufferedReader;
-import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileReader;
-import java.io.FileWriter;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
import java.net.URI;
import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
public class MetastoreSchemaTool {
private static final Logger LOG = LoggerFactory.getLogger(MetastoreSchemaTool.class);
private static final String PASSWD_MASK = "[passwd stripped]";
- @VisibleForTesting
- public static String homeDir;
-
- private String userName = null;
- private String passWord = null;
- private boolean dryRun = false;
- private boolean verbose = false;
- private String dbOpts = null;
- private String url = null;
- private String driver = null;
- private URI[] validationServers = null; // The list of servers the database/partition/table can locate on
- private String hiveUser; // Hive username, for use when creating the user, not for connecting
- private String hivePasswd; // Hive password, for use when creating the user, not for connecting
- private String hiveDb; // Hive database, for use when creating the user, not for connecting
- private final Configuration conf;
- private final String dbType;
- private final IMetaStoreSchemaInfo metaStoreSchemaInfo;
- private boolean needsQuotedIdentifier;
+ protected Configuration conf;
+
+ protected String dbOpts = null;
+ protected String dbType;
+ protected String driver = null;
+ protected boolean dryRun = false;
+ protected String hiveDb; // Hive database, for use when creating the user, not for connecting
+ protected String hivePasswd; // Hive password, for use when creating the user, not for connecting
+ protected String hiveUser; // Hive username, for use when creating the user, not for connecting
+ protected String metaDbType;
+ protected IMetaStoreSchemaInfo metaStoreSchemaInfo;
+ protected boolean needsQuotedIdentifier;
+ protected String quoteCharacter;
+ protected String passWord = null;
+ protected String url = null;
+ protected String userName = null;
+ protected URI[] validationServers = null; // The list of servers the database/partition/table can locate on
+ protected boolean verbose = false;
+ protected SchemaToolCommandLine cmdLine;
+
+ private static String homeDir;
private static String findHomeDir() {
// If METASTORE_HOME is set, use it, else use HIVE_HOME for backwards compatibility.
@@ -100,54 +79,129 @@ public class MetastoreSchemaTool {
return homeDir == null ? System.getenv("HIVE_HOME") : homeDir;
}
- private MetastoreSchemaTool(String dbType) throws HiveMetaException {
- this(findHomeDir(), MetastoreConf.newMetastoreConf(), dbType);
+ @VisibleForTesting
+ public static void setHomeDirForTesting() {
+ homeDir = System.getProperty("test.tmp.dir", "target/tmp");
}
- MetastoreSchemaTool(String metastoreHome, Configuration conf, String dbType)
- throws HiveMetaException {
+ @VisibleForTesting
+ public MetastoreSchemaTool() {
+
+ }
+
+ @VisibleForTesting
+ public void init(String metastoreHome, String[] args, OptionGroup additionalOptions,
+ Configuration conf) throws HiveMetaException {
+ try {
+ cmdLine = new SchemaToolCommandLine(args, additionalOptions);
+ } catch (ParseException e) {
+ System.err.println("Failed to parse command line. ");
+ throw new HiveMetaException(e);
+ }
+
if (metastoreHome == null || metastoreHome.isEmpty()) {
throw new HiveMetaException("No Metastore home directory provided");
}
this.conf = conf;
- this.dbType = dbType;
- this.needsQuotedIdentifier = getDbCommandParser(dbType).needsQuotedIdentifier();
+ this.dbType = cmdLine.getDbType();
+ this.metaDbType = cmdLine.getMetaDbType();
+ NestedScriptParser parser = getDbCommandParser(dbType, metaDbType);
+ this.needsQuotedIdentifier = parser.needsQuotedIdentifier();
+ this.quoteCharacter = parser.getQuoteCharacter();
this.metaStoreSchemaInfo = MetaStoreSchemaInfoFactory.get(conf, metastoreHome, dbType);
+
+ if (cmdLine.hasOption("userName")) {
+ setUserName(cmdLine.getOptionValue("userName"));
+ } else {
+ setUserName(getConf().get(MetastoreConf.ConfVars.CONNECTION_USER_NAME.getVarname()));
+ }
+ if (cmdLine.hasOption("passWord")) {
+ setPassWord(cmdLine.getOptionValue("passWord"));
+ } else {
+ try {
+ setPassWord(MetastoreConf.getPassword(getConf(), ConfVars.PWD));
+ } catch (IOException err) {
+ throw new HiveMetaException("Error getting metastore password", err);
+ }
+ }
+ if (cmdLine.hasOption("url")) {
+ setUrl(cmdLine.getOptionValue("url"));
+ }
+ if (cmdLine.hasOption("driver")) {
+ setDriver(cmdLine.getOptionValue("driver"));
+ }
+ if (cmdLine.hasOption("dryRun")) {
+ setDryRun(true);
+ }
+ if (cmdLine.hasOption("verbose")) {
+ setVerbose(true);
+ }
+ if (cmdLine.hasOption("dbOpts")) {
+ setDbOpts(cmdLine.getOptionValue("dbOpts"));
+ }
+ if (cmdLine.hasOption("validate") && cmdLine.hasOption("servers")) {
+ setValidationServers(cmdLine.getOptionValue("servers"));
+ }
+ if (cmdLine.hasOption("hiveUser")) {
+ setHiveUser(cmdLine.getOptionValue("hiveUser"));
+ }
+ if (cmdLine.hasOption("hivePassword")) {
+ setHivePasswd(cmdLine.getOptionValue("hivePassword"));
+ }
+ if (cmdLine.hasOption("hiveDb")) {
+ setHiveDb(cmdLine.getOptionValue("hiveDb"));
+ }
}
- Configuration getConf() {
+ public Configuration getConf() {
return conf;
}
- void setUrl(String url) {
+ protected String getDbType() {
+ return dbType;
+ }
+
+ protected void setUrl(String url) {
this.url = url;
}
- void setDriver(String driver) {
+ protected void setDriver(String driver) {
this.driver = driver;
}
- void setUserName(String userName) {
+ public void setUserName(String userName) {
this.userName = userName;
}
- void setPassWord(String passWord) {
+ public void setPassWord(String passWord) {
this.passWord = passWord;
}
- void setDryRun(boolean dryRun) {
+ protected boolean isDryRun() {
+ return dryRun;
+ }
+
+ protected void setDryRun(boolean dryRun) {
this.dryRun = dryRun;
}
- void setVerbose(boolean verbose) {
+ protected boolean isVerbose() {
+ return verbose;
+ }
+
+ protected void setVerbose(boolean verbose) {
this.verbose = verbose;
}
- private void setDbOpts(String dbOpts) {
+ protected void setDbOpts(String dbOpts) {
this.dbOpts = dbOpts;
}
- private void setValidationServers(String servers) {
+ protected URI[] getValidationServers() {
+ return validationServers;
+ }
+
+ protected void setValidationServers(String servers) {
if(StringUtils.isNotEmpty(servers)) {
String[] strServers = servers.split(",");
this.validationServers = new URI[strServers.length];
@@ -157,340 +211,53 @@ public class MetastoreSchemaTool {
}
}
- private void setHiveUser(String hiveUser) {
- this.hiveUser = hiveUser;
- }
-
- private void setHivePasswd(String hivePasswd) {
- this.hivePasswd = hivePasswd;
+ protected String getHiveUser() {
+ return hiveUser;
}
- private void setHiveDb(String hiveDb) {
- this.hiveDb = hiveDb;
+ protected void setHiveUser(String hiveUser) {
+ this.hiveUser = hiveUser;
}
- private static int usage(Options cmdLineOptions) {
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("schemaTool", cmdLineOptions);
- return 1;
+ protected String getHivePasswd() {
+ return hivePasswd;
}
- Connection getConnectionToMetastore(boolean printInfo)
- throws HiveMetaException {
- return HiveSchemaHelper.getConnectionToMetastore(userName,
- passWord, url, driver, printInfo, conf, null);
+ protected void setHivePasswd(String hivePasswd) {
+ this.hivePasswd = hivePasswd;
}
- private NestedScriptParser getDbCommandParser(String dbType) {
- return HiveSchemaHelper.getDbCommandParser(dbType, dbOpts, userName,
- passWord, conf, null, true);
+ protected String getHiveDb() {
+ return hiveDb;
}
- /***
- * Print Hive version and schema version
- */
- private void showInfo() throws HiveMetaException {
- String hiveVersion = metaStoreSchemaInfo.getHiveSchemaVersion();
- String dbVersion = metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(true));
- System.out.println("Hive distribution version:\t " + hiveVersion);
- System.out.println("Metastore schema version:\t " + dbVersion);
- assertCompatibleVersion(hiveVersion, dbVersion);
- }
-
- boolean validateLocations(Connection conn, URI[] defaultServers) throws HiveMetaException {
- System.out.println("Validating DFS locations");
- boolean rtn;
- rtn = checkMetaStoreDBLocation(conn, defaultServers);
- rtn = checkMetaStoreTableLocation(conn, defaultServers) && rtn;
- rtn = checkMetaStorePartitionLocation(conn, defaultServers) && rtn;
- rtn = checkMetaStoreSkewedColumnsLocation(conn, defaultServers) && rtn;
- System.out.println((rtn ? "Succeeded" : "Failed") + " in DFS location validation.");
- return rtn;
+ protected void setHiveDb(String hiveDb) {
+ this.hiveDb = hiveDb;
}
- private String getNameOrID(ResultSet res, int nameInx, int idInx) throws SQLException {
- String itemName = res.getString(nameInx);
- return (itemName == null || itemName.isEmpty()) ? "ID: " + res.getString(idInx) : "Name: " + itemName;
+ protected SchemaToolCommandLine getCmdLine() {
+ return cmdLine;
}
- private boolean checkMetaStoreDBLocation(Connection conn, URI[] defaultServers)
- throws HiveMetaException {
- String dbLoc;
- boolean isValid = true;
- int numOfInvalid = 0;
- if (needsQuotedIdentifier) {
- dbLoc = "select dbt.\"DB_ID\", dbt.\"NAME\", dbt.\"DB_LOCATION_URI\" from \"DBS\" dbt order by dbt.\"DB_ID\" ";
- } else {
- dbLoc = "select dbt.DB_ID, dbt.NAME, dbt.DB_LOCATION_URI from DBS dbt order by dbt.DB_ID";
- }
-
- try(Statement stmt = conn.createStatement();
- ResultSet res = stmt.executeQuery(dbLoc)) {
- while (res.next()) {
- String locValue = res.getString(3);
- String dbName = getNameOrID(res,2,1);
- if (!checkLocation("Database " + dbName, locValue, defaultServers)) {
- numOfInvalid++;
- }
- }
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to get DB Location Info.", e);
- }
- if (numOfInvalid > 0) {
- isValid = false;
- }
- return isValid;
- }
-
- private boolean checkMetaStoreTableLocation(Connection conn, URI[] defaultServers)
- throws HiveMetaException {
- String tabLoc, tabIDRange;
- boolean isValid = true;
- int numOfInvalid = 0;
- if (needsQuotedIdentifier) {
- tabIDRange = "select max(\"TBL_ID\"), min(\"TBL_ID\") from \"TBLS\" ";
- } else {
- tabIDRange = "select max(TBL_ID), min(TBL_ID) from TBLS";
- }
-
- if (needsQuotedIdentifier) {
- tabLoc = "select tbl.\"TBL_ID\", tbl.\"TBL_NAME\", sd.\"LOCATION\", dbt.\"DB_ID\", dbt.\"NAME\" from \"TBLS\" tbl inner join " +
- "\"SDS\" sd on tbl.\"SD_ID\" = sd.\"SD_ID\" and tbl.\"TBL_TYPE\" != '" + TableType.VIRTUAL_VIEW +
- "' and tbl.\"TBL_ID\" >= ? and tbl.\"TBL_ID\"<= ? " + "inner join \"DBS\" dbt on tbl.\"DB_ID\" = dbt.\"DB_ID\" order by tbl.\"TBL_ID\" ";
- } else {
- tabLoc = "select tbl.TBL_ID, tbl.TBL_NAME, sd.LOCATION, dbt.DB_ID, dbt.NAME from TBLS tbl join SDS sd on tbl.SD_ID = sd.SD_ID and tbl.TBL_TYPE !='"
- + TableType.VIRTUAL_VIEW + "' and tbl.TBL_ID >= ? and tbl.TBL_ID <= ? inner join DBS dbt on tbl.DB_ID = dbt.DB_ID order by tbl.TBL_ID";
- }
-
- long maxID = 0, minID = 0;
- long rtnSize = 2000;
-
- try {
- Statement stmt = conn.createStatement();
- ResultSet res = stmt.executeQuery(tabIDRange);
- if (res.next()) {
- maxID = res.getLong(1);
- minID = res.getLong(2);
- }
- res.close();
- stmt.close();
- PreparedStatement pStmt = conn.prepareStatement(tabLoc);
- while (minID <= maxID) {
- pStmt.setLong(1, minID);
- pStmt.setLong(2, minID + rtnSize);
- res = pStmt.executeQuery();
- while (res.next()) {
- String locValue = res.getString(3);
- String entity = "Database " + getNameOrID(res, 5, 4) +
- ", Table " + getNameOrID(res,2,1);
- if (!checkLocation(entity, locValue, defaultServers)) {
- numOfInvalid++;
- }
- }
- res.close();
- minID += rtnSize + 1;
-
- }
- pStmt.close();
-
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to get Table Location Info.", e);
- }
- if (numOfInvalid > 0) {
- isValid = false;
- }
- return isValid;
- }
-
- private boolean checkMetaStorePartitionLocation(Connection conn, URI[] defaultServers)
- throws HiveMetaException {
- String partLoc, partIDRange;
- boolean isValid = true;
- int numOfInvalid = 0;
- if (needsQuotedIdentifier) {
- partIDRange = "select max(\"PART_ID\"), min(\"PART_ID\") from \"PARTITIONS\" ";
- } else {
- partIDRange = "select max(PART_ID), min(PART_ID) from PARTITIONS";
- }
-
- if (needsQuotedIdentifier) {
- partLoc = "select pt.\"PART_ID\", pt.\"PART_NAME\", sd.\"LOCATION\", tbl.\"TBL_ID\", tbl.\"TBL_NAME\",dbt.\"DB_ID\", dbt.\"NAME\" from \"PARTITIONS\" pt "
- + "inner join \"SDS\" sd on pt.\"SD_ID\" = sd.\"SD_ID\" and pt.\"PART_ID\" >= ? and pt.\"PART_ID\"<= ? "
- + " inner join \"TBLS\" tbl on pt.\"TBL_ID\" = tbl.\"TBL_ID\" inner join "
- + "\"DBS\" dbt on tbl.\"DB_ID\" = dbt.\"DB_ID\" order by tbl.\"TBL_ID\" ";
- } else {
- partLoc = "select pt.PART_ID, pt.PART_NAME, sd.LOCATION, tbl.TBL_ID, tbl.TBL_NAME, dbt.DB_ID, dbt.NAME from PARTITIONS pt "
- + "inner join SDS sd on pt.SD_ID = sd.SD_ID and pt.PART_ID >= ? and pt.PART_ID <= ? "
- + "inner join TBLS tbl on tbl.TBL_ID = pt.TBL_ID inner join DBS dbt on tbl.DB_ID = dbt.DB_ID order by tbl.TBL_ID ";
- }
-
- long maxID = 0, minID = 0;
- long rtnSize = 2000;
-
- try {
- Statement stmt = conn.createStatement();
- ResultSet res = stmt.executeQuery(partIDRange);
- if (res.next()) {
- maxID = res.getLong(1);
- minID = res.getLong(2);
- }
- res.close();
- stmt.close();
- PreparedStatement pStmt = conn.prepareStatement(partLoc);
- while (minID <= maxID) {
- pStmt.setLong(1, minID);
- pStmt.setLong(2, minID + rtnSize);
- res = pStmt.executeQuery();
- while (res.next()) {
- String locValue = res.getString(3);
- String entity = "Database " + getNameOrID(res,7,6) +
- ", Table " + getNameOrID(res,5,4) +
- ", Partition " + getNameOrID(res,2,1);
- if (!checkLocation(entity, locValue, defaultServers)) {
- numOfInvalid++;
- }
- }
- res.close();
- minID += rtnSize + 1;
- }
- pStmt.close();
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to get Partiton Location Info.", e);
- }
- if (numOfInvalid > 0) {
- isValid = false;
- }
- return isValid;
+ public Connection getConnectionToMetastore(boolean printInfo) throws HiveMetaException {
+ return HiveSchemaHelper.getConnectionToMetastore(userName,
+ passWord, url, driver, printInfo, conf, null);
}
- private boolean checkMetaStoreSkewedColumnsLocation(Connection conn, URI[] defaultServers)
- throws HiveMetaException {
- String skewedColLoc, skewedColIDRange;
- boolean isValid = true;
- int numOfInvalid = 0;
- if (needsQuotedIdentifier) {
- skewedColIDRange = "select max(\"STRING_LIST_ID_KID\"), min(\"STRING_LIST_ID_KID\") from \"SKEWED_COL_VALUE_LOC_MAP\" ";
- } else {
- skewedColIDRange = "select max(STRING_LIST_ID_KID), min(STRING_LIST_ID_KID) from SKEWED_COL_VALUE_LOC_MAP";
- }
-
- if (needsQuotedIdentifier) {
- skewedColLoc = "select t.\"TBL_NAME\", t.\"TBL_ID\", sk.\"STRING_LIST_ID_KID\", sk.\"LOCATION\", db.\"NAME\", db.\"DB_ID\" "
- + " from \"TBLS\" t, \"SDS\" s, \"DBS\" db, \"SKEWED_COL_VALUE_LOC_MAP\" sk "
- + "where sk.\"SD_ID\" = s.\"SD_ID\" and s.\"SD_ID\" = t.\"SD_ID\" and t.\"DB_ID\" = db.\"DB_ID\" and "
- + "sk.\"STRING_LIST_ID_KID\" >= ? and sk.\"STRING_LIST_ID_KID\" <= ? order by t.\"TBL_ID\" ";
- } else {
- skewedColLoc = "select t.TBL_NAME, t.TBL_ID, sk.STRING_LIST_ID_KID, sk.LOCATION, db.NAME, db.DB_ID from TBLS t, SDS s, DBS db, SKEWED_COL_VALUE_LOC_MAP sk "
- + "where sk.SD_ID = s.SD_ID and s.SD_ID = t.SD_ID and t.DB_ID = db.DB_ID and sk.STRING_LIST_ID_KID >= ? and sk.STRING_LIST_ID_KID <= ? order by t.TBL_ID ";
- }
-
- long maxID = 0, minID = 0;
- long rtnSize = 2000;
-
- try {
- Statement stmt = conn.createStatement();
- ResultSet res = stmt.executeQuery(skewedColIDRange);
- if (res.next()) {
- maxID = res.getLong(1);
- minID = res.getLong(2);
- }
- res.close();
- stmt.close();
- PreparedStatement pStmt = conn.prepareStatement(skewedColLoc);
- while (minID <= maxID) {
- pStmt.setLong(1, minID);
- pStmt.setLong(2, minID + rtnSize);
- res = pStmt.executeQuery();
- while (res.next()) {
- String locValue = res.getString(4);
- String entity = "Database " + getNameOrID(res,5,6) +
- ", Table " + getNameOrID(res,1,2) +
- ", String list " + res.getString(3);
- if (!checkLocation(entity, locValue, defaultServers)) {
- numOfInvalid++;
- }
- }
- res.close();
- minID += rtnSize + 1;
- }
- pStmt.close();
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to get skewed columns location info.", e);
- }
- if (numOfInvalid > 0) {
- isValid = false;
- }
- return isValid;
+ protected NestedScriptParser getDbCommandParser(String dbType, String metaDbType) {
+ return HiveSchemaHelper.getDbCommandParser(dbType, dbOpts, userName,
+ passWord, conf, null, true);
}
- /**
- * Check if the location is valid for the given entity
- * @param entity the entity to represent a database, partition or table
- * @param entityLocation the location
- * @param defaultServers a list of the servers that the location needs to match.
- * The location host needs to match one of the given servers.
- * If empty, then no check against such list.
- * @return true if the location is valid
- */
- private boolean checkLocation(
- String entity,
- String entityLocation,
- URI[] defaultServers) {
- boolean isValid = true;
- if (entityLocation == null) {
- logAndPrintToError(entity + ", Error: empty location");
- isValid = false;
- } else {
- try {
- URI currentUri = new Path(entityLocation).toUri();
- String scheme = currentUri.getScheme();
- String path = currentUri.getPath();
- if (StringUtils.isEmpty(scheme)) {
- logAndPrintToError(entity + ", Location: "+ entityLocation + ", Error: missing location scheme.");
- isValid = false;
- } else if (StringUtils.isEmpty(path)) {
- logAndPrintToError(entity + ", Location: "+ entityLocation + ", Error: missing location path.");
- isValid = false;
- } else if (ArrayUtils.isNotEmpty(defaultServers) && currentUri.getAuthority() != null) {
- String authority = currentUri.getAuthority();
- boolean matchServer = false;
- for(URI server : defaultServers) {
- if (StringUtils.equalsIgnoreCase(server.getScheme(), scheme) &&
- StringUtils.equalsIgnoreCase(server.getAuthority(), authority)) {
- matchServer = true;
- break;
- }
- }
- if (!matchServer) {
- logAndPrintToError(entity + ", Location: " + entityLocation + ", Error: mismatched server.");
- isValid = false;
- }
- }
-
- // if there is no path element other than "/", report it but not fail
- if (isValid && StringUtils.containsOnly(path, "/")) {
- logAndPrintToError(entity + ", Location: "+ entityLocation + ", Warn: location set to root, not a recommended config.");
- }
- } catch (Exception pe) {
- logAndPrintToError(entity + ", Error: invalid location - " + pe.getMessage());
- isValid =false;
- }
- }
-
- return isValid;
+ protected MetaStoreConnectionInfo getConnectionInfo(boolean printInfo) {
+ return new MetaStoreConnectionInfo(userName, passWord, url, driver, printInfo, conf,
+ dbType, hiveDb);
}
- // test the connection metastore using the config property
- private void testConnectionToMetastore() throws HiveMetaException {
- Connection conn = getConnectionToMetastore(true);
- try {
- conn.close();
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to close metastore connection", e);
- }
+ protected IMetaStoreSchemaInfo getMetaStoreSchemaInfo() {
+ return metaStoreSchemaInfo;
}
-
/**
* check if the current schema version in metastore matches the Hive version
*/
@@ -505,7 +272,7 @@ public class MetastoreSchemaTool {
assertCompatibleVersion(metaStoreSchemaInfo.getHiveSchemaVersion(), newSchemaVersion);
}
- private void assertCompatibleVersion(String hiveSchemaVersion, String dbSchemaVersion)
+ protected void assertCompatibleVersion(String hiveSchemaVersion, String dbSchemaVersion)
throws HiveMetaException {
if (!metaStoreSchemaInfo.isVersionCompatible(hiveSchemaVersion, dbSchemaVersion)) {
throw new HiveMetaException("Metastore schema version is not compatible. Hive Version: "
@@ -513,504 +280,22 @@ public class MetastoreSchemaTool {
}
}
- /**
- * Perform metastore schema upgrade. extract the current schema version from metastore
- */
- void doUpgrade() throws HiveMetaException {
- String fromVersion =
- metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(false));
- if (fromVersion == null || fromVersion.isEmpty()) {
- throw new HiveMetaException("Schema version not stored in the metastore. " +
- "Metastore schema is too old or corrupt. Try specifying the version manually");
- }
- doUpgrade(fromVersion);
- }
-
- private MetaStoreConnectionInfo getConnectionInfo(boolean printInfo) {
- return new MetaStoreConnectionInfo(userName, passWord, url, driver, printInfo, conf,
- dbType, hiveDb);
- }
- /**
- * Perform metastore schema upgrade
- *
- * @param fromSchemaVer
- * Existing version of the metastore. If null, then read from the metastore
- */
- void doUpgrade(String fromSchemaVer) throws HiveMetaException {
- if (metaStoreSchemaInfo.getHiveSchemaVersion().equals(fromSchemaVer)) {
- System.out.println("No schema upgrade required from version " + fromSchemaVer);
- return;
- }
- // Find the list of scripts to execute for this upgrade
- List<String> upgradeScripts =
- metaStoreSchemaInfo.getUpgradeScripts(fromSchemaVer);
- testConnectionToMetastore();
- System.out.println("Starting upgrade metastore schema from version " +
- fromSchemaVer + " to " + metaStoreSchemaInfo.getHiveSchemaVersion());
- String scriptDir = metaStoreSchemaInfo.getMetaStoreScriptDir();
- try {
- for (String scriptFile : upgradeScripts) {
- System.out.println("Upgrade script " + scriptFile);
- if (!dryRun) {
- runPreUpgrade(scriptDir, scriptFile);
- runSqlLine(scriptDir, scriptFile);
- System.out.println("Completed " + scriptFile);
- }
- }
- } catch (IOException eIO) {
- throw new HiveMetaException(
- "Upgrade FAILED! Metastore state would be inconsistent !!", eIO);
- }
-
- // Revalidated the new version after upgrade
- verifySchemaVersion();
- }
-
- /**
- * Initialize the metastore schema to current version
- *
- */
- void doInit() throws HiveMetaException {
- doInit(metaStoreSchemaInfo.getHiveSchemaVersion());
-
- // Revalidated the new version after upgrade
- verifySchemaVersion();
- }
-
- /**
- * Initialize the metastore schema
- *
- * @param toVersion
- * If null then current hive version is used
- */
- void doInit(String toVersion) throws HiveMetaException {
- testConnectionToMetastore();
- System.out.println("Starting metastore schema initialization to " + toVersion);
-
- String initScriptDir = metaStoreSchemaInfo.getMetaStoreScriptDir();
- String initScriptFile = metaStoreSchemaInfo.generateInitFileName(toVersion);
-
- try {
- System.out.println("Initialization script " + initScriptFile);
- if (!dryRun) {
- runSqlLine(initScriptDir, initScriptFile);
- System.out.println("Initialization script completed");
- }
- } catch (IOException e) {
- throw new HiveMetaException("Schema initialization FAILED!" +
- " Metastore state would be inconsistent !!", e);
- }
- }
-
- private void doCreateUser() throws HiveMetaException {
- testConnectionToMetastore();
- System.out.println("Starting user creation");
-
- String scriptDir = metaStoreSchemaInfo.getMetaStoreScriptDir();
- String protoCreateFile = metaStoreSchemaInfo.getCreateUserScript();
-
- try {
- File createFile = subUserAndPassword(scriptDir, protoCreateFile);
- System.out.println("Creation script " + createFile.getAbsolutePath());
- if (!dryRun) {
- if ("oracle".equals(dbType)) oracleCreateUserHack(createFile);
- else runSqlLine(createFile.getParent(), createFile.getName());
- System.out.println("User creation completed");
- }
- } catch (IOException e) {
- throw new HiveMetaException("User creation FAILED!" +
- " Metastore unusable !!", e);
- }
- }
-
- private File subUserAndPassword(String parent, String filename) throws IOException {
- File createFile = File.createTempFile("create-hive-user-" + dbType, ".sql");
- BufferedWriter writer = new BufferedWriter(new FileWriter(createFile));
- File proto = new File(parent, filename);
- BufferedReader reader = new BufferedReader(new FileReader(proto));
- reader.lines()
- .map(s -> s.replace("_REPLACE_WITH_USER_", hiveUser)
- .replace("_REPLACE_WITH_PASSWD_", hivePasswd)
- .replace("_REPLACE_WITH_DB_", hiveDb))
- .forEach(s -> {
- try {
- writer.write(s);
- writer.newLine();
- } catch (IOException e) {
- throw new RuntimeException("Unable to write to tmp file ", e);
- }
- });
- reader.close();
- writer.close();
- return createFile;
- }
-
- private void oracleCreateUserHack(File createFile) throws HiveMetaException {
- LOG.debug("Found oracle, hacking our way through it rather than using SqlLine");
- try (BufferedReader reader = new BufferedReader(new FileReader(createFile))) {
- try (Connection conn = getConnectionToMetastore(false)) {
- try (Statement stmt = conn.createStatement()) {
- reader.lines()
- .forEach(s -> {
- assert s.charAt(s.length() - 1) == ';';
- try {
- stmt.execute(s.substring(0, s.length() - 1));
- } catch (SQLException e) {
- LOG.error("statement <" + s.substring(0, s.length() - 2) + "> failed", e);
- throw new RuntimeException(e);
- }
- });
- }
- }
- } catch (IOException e) {
- LOG.error("Caught IOException trying to read modified create user script " +
- createFile.getAbsolutePath(), e);
- throw new HiveMetaException(e);
- } catch (HiveMetaException e) {
- LOG.error("Failed to connect to RDBMS", e);
- throw e;
- } catch (SQLException e) {
- LOG.error("Got SQLException", e);
- }
- }
-
- private int doValidate() throws HiveMetaException {
- System.out.println("Starting metastore validation\n");
- Connection conn = getConnectionToMetastore(false);
- boolean success = true;
- try {
- if (validateSchemaVersions()) {
- System.out.println("[SUCCESS]\n");
- } else {
- success = false;
- System.out.println("[FAIL]\n");
- }
- if (validateSequences(conn)) {
- System.out.println("[SUCCESS]\n");
- } else {
- success = false;
- System.out.println("[FAIL]\n");
- }
- if (validateSchemaTables(conn)) {
- System.out.println("[SUCCESS]\n");
- } else {
- success = false;
- System.out.println("[FAIL]\n");
- }
- if (validateLocations(conn, this.validationServers)) {
- System.out.println("[SUCCESS]\n");
- } else {
- System.out.println("[WARN]\n");
- }
- if (validateColumnNullValues(conn)) {
- System.out.println("[SUCCESS]\n");
- } else {
- System.out.println("[WARN]\n");
- }
- } finally {
- if (conn != null) {
- try {
- conn.close();
- } catch (SQLException e) {
- // Not a lot you can do here.
- }
- }
- }
-
- System.out.print("Done with metastore validation: ");
- if (!success) {
- System.out.println("[FAIL]");
- return 1;
- } else {
- System.out.println("[SUCCESS]");
- return 0;
- }
- }
-
- boolean validateSequences(Connection conn) throws HiveMetaException {
- Map<String, Pair<String, String>> seqNameToTable =
- new ImmutableMap.Builder<String, Pair<String, String>>()
- .put("MDatabase", Pair.of("DBS", "DB_ID"))
- .put("MRole", Pair.of("ROLES", "ROLE_ID"))
- .put("MGlobalPrivilege", Pair.of("GLOBAL_PRIVS", "USER_GRANT_ID"))
- .put("MTable", Pair.of("TBLS","TBL_ID"))
- .put("MStorageDescriptor", Pair.of("SDS", "SD_ID"))
- .put("MSerDeInfo", Pair.of("SERDES", "SERDE_ID"))
- .put("MColumnDescriptor", Pair.of("CDS", "CD_ID"))
- .put("MTablePrivilege", Pair.of("TBL_PRIVS", "TBL_GRANT_ID"))
- .put("MTableColumnStatistics", Pair.of("TAB_COL_STATS", "CS_ID"))
- .put("MPartition", Pair.of("PARTITIONS", "PART_ID"))
- .put("MPartitionColumnStatistics", Pair.of("PART_COL_STATS", "CS_ID"))
- .put("MFunction", Pair.of("FUNCS", "FUNC_ID"))
- .put("MIndex", Pair.of("IDXS", "INDEX_ID"))
- .put("MStringList", Pair.of("SKEWED_STRING_LIST", "STRING_LIST_ID"))
- .build();
-
- System.out.println("Validating sequence number for SEQUENCE_TABLE");
-
- boolean isValid = true;
- try {
- Statement stmt = conn.createStatement();
- for (String seqName : seqNameToTable.keySet()) {
- String tableName = seqNameToTable.get(seqName).getLeft();
- String tableKey = seqNameToTable.get(seqName).getRight();
- String fullSequenceName = "org.apache.hadoop.hive.metastore.model." + seqName;
- String seqQuery = needsQuotedIdentifier ?
- ("select t.\"NEXT_VAL\" from \"SEQUENCE_TABLE\" t WHERE t.\"SEQUENCE_NAME\"=? order by t.\"SEQUENCE_NAME\" ")
- : ("select t.NEXT_VAL from SEQUENCE_TABLE t WHERE t.SEQUENCE_NAME=? order by t.SEQUENCE_NAME ");
- String maxIdQuery = needsQuotedIdentifier ?
- ("select max(\"" + tableKey + "\") from \"" + tableName + "\"")
- : ("select max(" + tableKey + ") from " + tableName);
-
- ResultSet res = stmt.executeQuery(maxIdQuery);
- if (res.next()) {
- long maxId = res.getLong(1);
- if (maxId > 0) {
- PreparedStatement pStmt = conn.prepareStatement(seqQuery);
- pStmt.setString(1, fullSequenceName);
- ResultSet resSeq = pStmt.executeQuery();
- if (!resSeq.next()) {
- isValid = false;
- logAndPrintToError("Missing SEQUENCE_NAME " + seqName + " from SEQUENCE_TABLE");
- } else if (resSeq.getLong(1) < maxId) {
- isValid = false;
- logAndPrintToError("NEXT_VAL for " + seqName + " in SEQUENCE_TABLE < max(" +
- tableKey + ") in " + tableName);
- }
- }
- }
- }
-
- System.out.println((isValid ? "Succeeded" :"Failed") + " in sequence number validation for SEQUENCE_TABLE.");
- return isValid;
- } catch(SQLException e) {
- throw new HiveMetaException("Failed to validate sequence number for SEQUENCE_TABLE", e);
- }
- }
-
- boolean validateSchemaVersions() throws HiveMetaException {
- System.out.println("Validating schema version");
- try {
- String newSchemaVersion = metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(false));
- assertCompatibleVersion(metaStoreSchemaInfo.getHiveSchemaVersion(), newSchemaVersion);
- } catch (HiveMetaException hme) {
- if (hme.getMessage().contains("Metastore schema version is not compatible")
- || hme.getMessage().contains("Multiple versions were found in metastore")
- || hme.getMessage().contains("Could not find version info in metastore VERSION table")) {
- logAndPrintToError(hme.getMessage());
- System.out.println("Failed in schema version validation.");
- return false;
- } else {
- throw hme;
- }
- }
- System.out.println("Succeeded in schema version validation.");
- return true;
- }
-
- boolean validateSchemaTables(Connection conn) throws HiveMetaException {
- String version;
- ResultSet rs = null;
- DatabaseMetaData metadata;
- List<String> dbTables = new ArrayList<>();
- List<String> schemaTables = new ArrayList<>();
- List<String> subScripts = new ArrayList<>();
- Connection hmsConn;
-
- System.out.println("Validating metastore schema tables");
- try {
- version = metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(false));
- } catch (HiveMetaException he) {
- logAndPrintToError("Failed to determine schema version from Hive Metastore DB. " + he.getMessage());
- System.out.println("Failed in schema table validation.");
- LOG.debug("Failed to determine schema version from Hive Metastore DB," + he.getMessage());
- return false;
- }
-
- // re-open the hms connection
- hmsConn = getConnectionToMetastore(false);
-
- LOG.debug("Validating tables in the schema for version " + version);
- try {
- metadata = conn.getMetaData();
- String[] types = {"TABLE"};
- rs = metadata.getTables(null, hmsConn.getSchema(), "%", types);
- String table;
-
- while (rs.next()) {
- table = rs.getString("TABLE_NAME");
- dbTables.add(table.toLowerCase());
- LOG.debug("Found table " + table + " in HMS dbstore");
- }
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to retrieve schema tables from Hive Metastore DB," + e.getMessage());
- } finally {
- if (rs != null) {
- try {
- rs.close();
- } catch (SQLException e) {
- // Not a lot you can do here.
- }
- }
- }
-
- // parse the schema file to determine the tables that are expected to exist
- // we are using oracle schema because it is simpler to parse, no quotes or backticks etc
- String baseDir = new File(metaStoreSchemaInfo.getMetaStoreScriptDir()).getParent();
- String schemaFile = new File(metaStoreSchemaInfo.getMetaStoreScriptDir(),
- metaStoreSchemaInfo.generateInitFileName(version)).getPath();
- try {
- LOG.debug("Parsing schema script " + schemaFile);
- subScripts.addAll(findCreateTable(schemaFile, schemaTables));
- while (subScripts.size() > 0) {
- schemaFile = baseDir + "/" + dbType + "/" + subScripts.remove(0);
- LOG.debug("Parsing subscript " + schemaFile);
- subScripts.addAll(findCreateTable(schemaFile, schemaTables));
- }
- } catch (Exception e) {
- logAndPrintToError("Exception in parsing schema file. Cause:" + e.getMessage());
- System.out.println("Failed in schema table validation.");
- return false;
- }
-
- LOG.debug("Schema tables:[ " + Arrays.toString(schemaTables.toArray()) + " ]");
- LOG.debug("DB tables:[ " + Arrays.toString(dbTables.toArray()) + " ]");
- // now diff the lists
- schemaTables.removeAll(dbTables);
- if (schemaTables.size() > 0) {
- Collections.sort(schemaTables);
- logAndPrintToError("Table(s) [ " + Arrays.toString(schemaTables.toArray())
- + " ] are missing from the metastore database schema.");
- System.out.println("Failed in schema table validation.");
- return false;
- } else {
- System.out.println("Succeeded in schema table validation.");
- return true;
- }
- }
-
- private List<String> findCreateTable(String path, List<String> tableList)
- throws Exception {
- NestedScriptParser sp = HiveSchemaHelper.getDbCommandParser(dbType, true);
- Matcher matcher;
- Pattern regexp;
- List<String> subs = new ArrayList<>();
- int groupNo = 2;
-
- regexp = Pattern.compile("CREATE TABLE(\\s+IF NOT EXISTS)?\\s+(\\S+).*");
-
- if (!(new File(path)).exists()) {
- throw new Exception(path + " does not exist. Potentially incorrect version in the metastore VERSION table");
- }
-
- try (
- BufferedReader reader = new BufferedReader(new FileReader(path))
- ){
- String line;
- while ((line = reader.readLine()) != null) {
- if (sp.isNestedScript(line)) {
- String subScript;
- subScript = sp.getScriptName(line);
- LOG.debug("Schema subscript " + subScript + " found");
- subs.add(subScript);
- continue;
- }
- line = line.replaceAll("( )+", " "); //suppress multi-spaces
- line = line.replaceAll("\\(", " ");
- line = line.replaceAll("IF NOT EXISTS ", "");
- line = line.replaceAll("`","");
- line = line.replaceAll("'","");
- line = line.replaceAll("\"","");
- matcher = regexp.matcher(line);
-
- if (matcher.find()) {
- String table = matcher.group(groupNo);
- if (dbType.equals("derby"))
- table = table.replaceAll("APP\\.","");
- tableList.add(table.toLowerCase());
- LOG.debug("Found table " + table + " in the schema");
- }
- }
- } catch (IOException ex){
- throw new Exception(ex.getMessage());
- }
-
- return subs;
- }
-
- boolean validateColumnNullValues(Connection conn) throws HiveMetaException {
- System.out.println("Validating columns for incorrect NULL values.");
- boolean isValid = true;
- try {
- Statement stmt = conn.createStatement();
- String tblQuery = needsQuotedIdentifier ?
- ("select t.* from \"TBLS\" t WHERE t.\"SD_ID\" IS NULL and (t.\"TBL_TYPE\"='" + TableType.EXTERNAL_TABLE + "' or t.\"TBL_TYPE\"='" + TableType.MANAGED_TABLE + "') order by t.\"TBL_ID\" ")
- : ("select t.* from TBLS t WHERE t.SD_ID IS NULL and (t.TBL_TYPE='" + TableType.EXTERNAL_TABLE + "' or t.TBL_TYPE='" + TableType.MANAGED_TABLE + "') order by t.TBL_ID ");
-
- ResultSet res = stmt.executeQuery(tblQuery);
- while (res.next()) {
- long tableId = res.getLong("TBL_ID");
- String tableName = res.getString("TBL_NAME");
- String tableType = res.getString("TBL_TYPE");
- isValid = false;
- logAndPrintToError("SD_ID in TBLS should not be NULL for Table Name=" + tableName + ", Table ID=" + tableId + ", Table Type=" + tableType);
- }
-
- System.out.println((isValid ? "Succeeded" : "Failed") + " in column validation for incorrect NULL values.");
- return isValid;
- } catch(SQLException e) {
- throw new HiveMetaException("Failed to validate columns for incorrect NULL values", e);
- }
- }
-
- /**
- * Run pre-upgrade scripts corresponding to a given upgrade script,
- * if any exist. The errors from pre-upgrade are ignored.
- * Pre-upgrade scripts typically contain setup statements which
- * may fail on some database versions and failure is ignorable.
- *
- * @param scriptDir upgrade script directory name
- * @param scriptFile upgrade script file name
- */
- private void runPreUpgrade(String scriptDir, String scriptFile) {
- for (int i = 0;; i++) {
- String preUpgradeScript =
- metaStoreSchemaInfo.getPreUpgradeScriptName(i, scriptFile);
- File preUpgradeScriptFile = new File(scriptDir, preUpgradeScript);
- if (!preUpgradeScriptFile.isFile()) {
- break;
- }
-
- try {
- runSqlLine(scriptDir, preUpgradeScript);
- System.out.println("Completed " + preUpgradeScript);
- } catch (Exception e) {
- // Ignore the pre-upgrade script errors
- logAndPrintToError("Warning in pre-upgrade script " + preUpgradeScript + ": "
- + e.getMessage());
- if (verbose) {
- e.printStackTrace();
- }
- }
- }
- }
-
/***
- * Run beeline with the given metastore script. Flatten the nested scripts
- * into single file.
+ * Execute a given metastore script. This default version uses sqlline to execute the files,
+ * which requires only running one file. Subclasses can use other executors.
+ * @param scriptDir directory script is in
+ * @param scriptFile file in the directory to run
+ * @throws IOException if it cannot read the file or directory
+ * @throws HiveMetaException default implementation never throws this
*/
- private void runSqlLine(String scriptDir, String scriptFile)
- throws IOException, HiveMetaException {
+ protected void execSql(String scriptDir, String scriptFile) throws IOException, HiveMetaException {
- // This no longer does expansions of run commands in the files as it used to. Instead it
- // depends on the developers to have already unrolled those in the files.
- runSqlLine(scriptDir + File.separatorChar + scriptFile);
+ execSql(scriptDir + File.separatorChar + scriptFile);
}
// Generate the beeline args per hive conf and execute the given script
- void runSqlLine(String sqlScriptFile) throws IOException {
- CommandBuilder builder = new CommandBuilder(conf, url, driver,
- userName, passWord, sqlScriptFile);
+ protected void execSql(String sqlScriptFile) throws IOException {
+ CommandBuilder builder = new CommandBuilder(conf, url, driver, userName, passWord, sqlScriptFile);
// run the script using SqlLine
SqlLine sqlLine = new SqlLine();
@@ -1025,11 +310,6 @@ public class MetastoreSchemaTool {
sqlLine.setOutputStream(new PrintStream(out));
System.setProperty("sqlline.silent", "true");
}
- //sqlLine.getOpts().setAllowMultiLineCommand(false);
- //System.setProperty("sqlline.isolation","TRANSACTION_READ_COMMITTED");
- // We can be pretty sure that an entire line can be processed as a single command since
- // we always add a line separator at the end while calling dbCommandParser.buildCommand.
- //sqlLine.getOpts().setEntireLineAsCommand(true);
LOG.info("Going to run command <" + builder.buildToLog() + ">");
SqlLine.Status status = sqlLine.begin(builder.buildToRun(), null, false);
if (LOG.isDebugEnabled() && outputForLog != null) {
@@ -1041,38 +321,55 @@ public class MetastoreSchemaTool {
}
}
- static class CommandBuilder {
- private final Configuration conf;
- private final String userName;
- private final String password;
- private final String sqlScriptFile;
- private final String driver;
- private final String url;
+ // test the connection metastore using the config property
+ protected void testConnectionToMetastore() throws HiveMetaException {
+ Connection conn = getConnectionToMetastore(true);
+ try {
+ conn.close();
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to close metastore connection", e);
+ }
+ }
+
+ // Quote if the database requires it
+ protected String quote(String stmt) {
+ stmt = stmt.replace("<q>", needsQuotedIdentifier ? quoteCharacter : "");
+ stmt = stmt.replace("<qa>", quoteCharacter);
+ return stmt;
+ }
+
+ protected static class CommandBuilder {
+ protected final String userName;
+ protected final String password;
+ protected final String sqlScriptFile;
+ protected final String driver;
+ protected final String url;
- CommandBuilder(Configuration conf, String url, String driver,
- String userName, String password, String sqlScriptFile) {
- this.conf = conf;
+ protected CommandBuilder(Configuration conf, String url, String driver, String userName,
+ String password, String sqlScriptFile) throws IOException {
this.userName = userName;
this.password = password;
- this.url = url;
- this.driver = driver;
+ this.url = url == null ?
+ HiveSchemaHelper.getValidConfVar(MetastoreConf.ConfVars.CONNECT_URL_KEY, conf) : url;
+ this.driver = driver == null ?
+ HiveSchemaHelper.getValidConfVar(MetastoreConf.ConfVars.CONNECTION_DRIVER, conf) : driver;
this.sqlScriptFile = sqlScriptFile;
}
- String[] buildToRun() throws IOException {
+ public String[] buildToRun() throws IOException {
return argsWith(password);
}
- String buildToLog() throws IOException {
+ public String buildToLog() throws IOException {
logScript();
return StringUtils.join(argsWith(PASSWD_MASK), " ");
}
- private String[] argsWith(String password) throws IOException {
+ protected String[] argsWith(String password) throws IOException {
return new String[]
{
- "-u", url == null ? MetastoreConf.getVar(conf, ConfVars.CONNECT_URL_KEY) : url,
- "-d", driver == null ? MetastoreConf.getVar(conf, ConfVars.CONNECTION_DRIVER) : driver,
+ "-u", url,
+ "-d", driver,
"-n", userName,
"-p", password,
"--isolation=TRANSACTION_READ_COMMITTED",
@@ -1094,194 +391,51 @@ public class MetastoreSchemaTool {
}
// Create the required command line options
- @SuppressWarnings("static-access")
- private static void initOptions(Options cmdLineOptions) {
- Option help = new Option("help", "print this message");
- Option upgradeOpt = new Option("upgradeSchema", "Schema upgrade");
- Option upgradeFromOpt = OptionBuilder.withArgName("upgradeFrom").hasArg().
- withDescription("Schema upgrade from a version").
- create("upgradeSchemaFrom");
- Option initOpt = new Option("initSchema", "Schema initialization");
- Option initToOpt = OptionBuilder.withArgName("initTo").hasArg().
- withDescription("Schema initialization to a version").
- create("initSchemaTo");
- Option infoOpt = new Option("info", "Show config and schema details");
- Option validateOpt = new Option("validate", "Validate the database");
- Option createUserOpt = new Option("createUser", "Create the Hive user, use admin user and " +
- "password with this");
-
- OptionGroup optGroup = new OptionGroup();
- optGroup.addOption(upgradeOpt).addOption(initOpt).
- addOption(help).addOption(upgradeFromOpt).addOption(createUserOpt)
- .addOption(initToOpt).addOption(infoOpt).addOption(validateOpt);
- optGroup.setRequired(true);
-
- Option userNameOpt = OptionBuilder.withArgName("user")
- .hasArgs()
- .withDescription("Override config file user name")
- .create("userName");
- Option passwdOpt = OptionBuilder.withArgName("password")
- .hasArgs()
- .withDescription("Override config file password")
- .create("passWord");
- Option hiveUserOpt = OptionBuilder
- .hasArg()
- .withDescription("Hive user (for use with createUser)")
- .create("hiveUser");
- Option hivePasswdOpt = OptionBuilder
- .hasArg()
- .withDescription("Hive password (for use with createUser)")
- .create("hivePassword");
- Option hiveDbOpt = OptionBuilder
- .hasArg()
- .withDescription("Hive database (for use with createUser)")
- .create("hiveDb");
- Option dbTypeOpt = OptionBuilder.withArgName("databaseType")
- .hasArgs().withDescription("Metastore database type")
- .create("dbType");
- Option urlOpt = OptionBuilder.withArgName("url")
- .hasArgs().withDescription("connection url to the database")
- .create("url");
- Option driverOpt = OptionBuilder.withArgName("driver")
- .hasArgs().withDescription("driver name for connection")
- .create("driver");
- Option dbOpts = OptionBuilder.withArgName("databaseOpts")
- .hasArgs().withDescription("Backend DB specific options")
- .create("dbOpts");
- Option dryRunOpt = new Option("dryRun", "list SQL scripts (no execute)");
- Option verboseOpt = new Option("verbose", "only print SQL statements");
- Option serversOpt = OptionBuilder.withArgName("serverList")
- .hasArgs().withDescription("a comma-separated list of servers used in location validation in the format of scheme://authority (e.g. hdfs://localhost:8000)")
- .create("servers");
- cmdLineOptions.addOption(help);
- cmdLineOptions.addOption(dryRunOpt);
- cmdLineOptions.addOption(userNameOpt);
- cmdLineOptions.addOption(passwdOpt);
- cmdLineOptions.addOption(dbTypeOpt);
- cmdLineOptions.addOption(verboseOpt);
- cmdLineOptions.addOption(urlOpt);
- cmdLineOptions.addOption(driverOpt);
- cmdLineOptions.addOption(dbOpts);
- cmdLineOptions.addOption(serversOpt);
- cmdLineOptions.addOption(hiveUserOpt);
- cmdLineOptions.addOption(hivePasswdOpt);
- cmdLineOptions.addOption(hiveDbOpt);
- cmdLineOptions.addOptionGroup(optGroup);
- }
-
private static void logAndPrintToError(String errmsg) {
LOG.error(errmsg);
System.err.println(errmsg);
}
public static void main(String[] args) {
- System.exit(run(args));
+ MetastoreSchemaTool tool = new MetastoreSchemaTool();
+ System.exit(tool.run(args));
}
- public static int run(String[] args) {
- LOG.debug("Going to run command: " + StringUtils.join(args, " "));
- CommandLineParser parser = new GnuParser();
- CommandLine line;
- String dbType;
- String schemaVer;
- Options cmdLineOptions = new Options();
-
- // Argument handling
- initOptions(cmdLineOptions);
- try {
- line = parser.parse(cmdLineOptions, args);
- } catch (ParseException e) {
- logAndPrintToError("HiveSchemaTool:Parsing failed. Reason: " + e.getLocalizedMessage());
- return usage(cmdLineOptions);
- }
-
- assert line != null;
- if (line.hasOption("help")) {
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("schemaTool", cmdLineOptions);
- return 1;
- }
-
- if (line.hasOption("dbType")) {
- dbType = line.getOptionValue("dbType");
- if ((!dbType.equalsIgnoreCase(HiveSchemaHelper.DB_DERBY) &&
- !dbType.equalsIgnoreCase(HiveSchemaHelper.DB_MSSQL) &&
- !dbType.equalsIgnoreCase(HiveSchemaHelper.DB_MYSQL) &&
- !dbType.equalsIgnoreCase(HiveSchemaHelper.DB_POSTGRACE) && !dbType
- .equalsIgnoreCase(HiveSchemaHelper.DB_ORACLE))) {
- logAndPrintToError("Unsupported dbType " + dbType);
- return usage(cmdLineOptions);
- }
- } else {
- logAndPrintToError("no dbType supplied");
- return usage(cmdLineOptions);
- }
+ public int run(String[] args) {
+ return run(findHomeDir(), args, null, MetastoreConf.newMetastoreConf());
+ }
- System.setProperty(ConfVars.SCHEMA_VERIFICATION.toString(), "true");
+ public int run(String metastoreHome, String[] args, OptionGroup additionalOptions,
+ Configuration conf) {
try {
- MetastoreSchemaTool schemaTool = new MetastoreSchemaTool(dbType);
-
- if (line.hasOption("userName")) {
- schemaTool.setUserName(line.getOptionValue("userName"));
- } else {
- schemaTool.setUserName(MetastoreConf.getVar(schemaTool.getConf(), ConfVars.CONNECTION_USER_NAME));
- }
- if (line.hasOption("passWord")) {
- schemaTool.setPassWord(line.getOptionValue("passWord"));
- } else {
- try {
- schemaTool.setPassWord(MetastoreConf.getPassword(schemaTool.getConf(), ConfVars.PWD));
- } catch (IOException err) {
- throw new HiveMetaException("Error getting metastore password", err);
- }
- }
- if (line.hasOption("hiveUser")) {
- schemaTool.setHiveUser(line.getOptionValue("hiveUser"));
- }
- if (line.hasOption("hivePassword")) {
- schemaTool.setHivePasswd(line.getOptionValue("hivePassword"));
- }
- if (line.hasOption("hiveDb")) {
- schemaTool.setHiveDb(line.getOptionValue("hiveDb"));
- }
- if (line.hasOption("url")) {
- schemaTool.setUrl(line.getOptionValue("url"));
- }
- if (line.hasOption("driver")) {
- schemaTool.setDriver(line.getOptionValue("driver"));
- }
- if (line.hasOption("dryRun")) {
- schemaTool.setDryRun(true);
- }
- if (line.hasOption("verbose")) {
- schemaTool.setVerbose(true);
- }
- if (line.hasOption("dbOpts")) {
- schemaTool.setDbOpts(line.getOptionValue("dbOpts"));
- }
- if (line.hasOption("validate") && line.hasOption("servers")) {
- schemaTool.setValidationServers(line.getOptionValue("servers"));
- }
- if (line.hasOption("info")) {
- schemaTool.showInfo();
- } else if (line.hasOption("upgradeSchema")) {
- schemaTool.doUpgrade();
- } else if (line.hasOption("upgradeSchemaFrom")) {
- schemaVer = line.getOptionValue("upgradeSchemaFrom");
- schemaTool.doUpgrade(schemaVer);
- } else if (line.hasOption("initSchema")) {
- schemaTool.doInit();
- } else if (line.hasOption("initSchemaTo")) {
- schemaVer = line.getOptionValue("initSchemaTo");
- schemaTool.doInit(schemaVer);
- } else if (line.hasOption("validate")) {
- return schemaTool.doValidate();
- } else if (line.hasOption("createUser")) {
- schemaTool.doCreateUser();
+ init(metastoreHome, args, additionalOptions, conf);
+ SchemaToolTask task;
+ if (cmdLine.hasOption("info")) {
+ task = new SchemaToolTaskInfo();
+ } else if (cmdLine.hasOption("upgradeSchema") || cmdLine.hasOption("upgradeSchemaFrom")) {
+ task = new SchemaToolTaskUpgrade();
+ } else if (cmdLine.hasOption("initSchema") || cmdLine.hasOption("initSchemaTo")) {
+ task = new SchemaToolTaskInit();
+ } else if (cmdLine.hasOption("validate")) {
+ task = new SchemaToolTaskValidate();
+ } else if (cmdLine.hasOption("createCatalog")) {
+ task = new SchemaToolTaskCreateCatalog();
+ } else if (cmdLine.hasOption("alterCatalog")) {
+ task = new SchemaToolTaskAlterCatalog();
+ } else if (cmdLine.hasOption("moveDatabase")) {
+ task = new SchemaToolTaskMoveDatabase();
+ } else if (cmdLine.hasOption("moveTable")) {
+ task = new SchemaToolTaskMoveTable();
+ } else if (cmdLine.hasOption("createUser")) {
+ task = new SchemaToolTaskCreateUser();
} else {
- logAndPrintToError("no valid option supplied");
- return usage(cmdLineOptions);
+ throw new HiveMetaException("No task defined!");
}
+
+ task.setHiveSchemaTool(this);
+ task.setCommandLineArguments(cmdLine);
+ task.execute();
+ return 0;
} catch (HiveMetaException e) {
logAndPrintToError(e.getMessage());
if (e.getCause() != null) {
@@ -1290,19 +444,17 @@ public class MetastoreSchemaTool {
+ t.getClass().getName() + " : "
+ t.getMessage());
if (e.getCause() instanceof SQLException) {
- logAndPrintToError("SQL Error code: " + ((SQLException)t).getErrorCode());
+ logAndPrintToError("SQL Error code: " + ((SQLException) t).getErrorCode());
}
}
- if (line.hasOption("verbose")) {
+ if (cmdLine.hasOption("verbose")) {
e.printStackTrace();
} else {
logAndPrintToError("Use --verbose for detailed stacktrace.");
}
logAndPrintToError("*** schemaTool failed ***");
return 1;
- }
- System.out.println("schemaTool completed");
- return 0;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolCommandLine.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolCommandLine.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolCommandLine.java
new file mode 100644
index 0000000..7eba2b7
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolCommandLine.java
@@ -0,0 +1,308 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.OptionGroup;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper;
+
+import com.google.common.collect.ImmutableSet;
+
+import java.util.Set;
+
+public class SchemaToolCommandLine {
+ private final Options cmdLineOptions;
+
+ @SuppressWarnings("static-access")
+ private Options createOptions(OptionGroup additionalOptions) {
+ Option help = new Option("help", "print this message");
+ Option infoOpt = new Option("info", "Show config and schema details");
+ Option upgradeOpt = new Option("upgradeSchema", "Schema upgrade");
+ Option upgradeFromOpt = OptionBuilder.withArgName("upgradeFrom").hasArg()
+ .withDescription("Schema upgrade from a version")
+ .create("upgradeSchemaFrom");
+ Option initOpt = new Option("initSchema", "Schema initialization");
+ Option initToOpt = OptionBuilder.withArgName("initTo").hasArg()
+ .withDescription("Schema initialization to a version")
+ .create("initSchemaTo");
+ Option validateOpt = new Option("validate", "Validate the database");
+ Option createCatalog = OptionBuilder
+ .hasArg()
+ .withDescription("Create a catalog, requires --catalogLocation parameter as well")
+ .create("createCatalog");
+ Option alterCatalog = OptionBuilder
+ .hasArg()
+ .withDescription("Alter a catalog, requires --catalogLocation and/or --catalogDescription parameter as well")
+ .create("alterCatalog");
+ Option moveDatabase = OptionBuilder
+ .hasArg()
+ .withDescription("Move a database between catalogs. Argument is the database name. " +
+ "Requires --fromCatalog and --toCatalog parameters as well")
+ .create("moveDatabase");
+ Option moveTable = OptionBuilder
+ .hasArg()
+ .withDescription("Move a table to a different database. Argument is the table name. " +
+ "Requires --fromCatalog, --toCatalog, --fromDatabase, and --toDatabase " +
+ " parameters as well.")
+ .create("moveTable");
+ Option createUserOpt = new Option("createUser", "Create the Hive user, set hiveUser to the db" +
+ " admin user and the hive password to the db admin password with this");
+
+ OptionGroup optGroup = new OptionGroup();
+ optGroup
+ .addOption(help)
+ .addOption(infoOpt)
+ .addOption(upgradeOpt)
+ .addOption(upgradeFromOpt)
+ .addOption(initOpt)
+ .addOption(initToOpt)
+ .addOption(validateOpt)
+ .addOption(createCatalog)
+ .addOption(alterCatalog)
+ .addOption(moveDatabase)
+ .addOption(moveTable)
+ .addOption(createUserOpt);
+ optGroup.setRequired(true);
+
+ Option userNameOpt = OptionBuilder.withArgName("user")
+ .hasArgs()
+ .withDescription("Override config file user name")
+ .create("userName");
+ Option passwdOpt = OptionBuilder.withArgName("password")
+ .hasArgs()
+ .withDescription("Override config file password")
+ .create("passWord");
+ Option dbTypeOpt = OptionBuilder.withArgName("databaseType")
+ .hasArgs().withDescription("Metastore database type").isRequired()
+ .create("dbType");
+ Option hiveUserOpt = OptionBuilder
+ .hasArg()
+ .withDescription("Hive user (for use with createUser)")
+ .create("hiveUser");
+ Option hivePasswdOpt = OptionBuilder
+ .hasArg()
+ .withDescription("Hive password (for use with createUser)")
+ .create("hivePassword");
+ Option hiveDbOpt = OptionBuilder
+ .hasArg()
+ .withDescription("Hive database (for use with createUser)")
+ .create("hiveDb");
+ /*
+ Option metaDbTypeOpt = OptionBuilder.withArgName("metaDatabaseType")
+ .hasArgs().withDescription("Used only if upgrading the system catalog for hive")
+ .create("metaDbType");
+ */
+ Option urlOpt = OptionBuilder.withArgName("url")
+ .hasArgs().withDescription("connection url to the database")
+ .create("url");
+ Option driverOpt = OptionBuilder.withArgName("driver")
+ .hasArgs().withDescription("driver name for connection")
+ .create("driver");
+ Option dbOpts = OptionBuilder.withArgName("databaseOpts")
+ .hasArgs().withDescription("Backend DB specific options")
+ .create("dbOpts");
+ Option dryRunOpt = new Option("dryRun", "list SQL scripts (no execute)");
+ Option verboseOpt = new Option("verbose", "only print SQL statements");
+ Option serversOpt = OptionBuilder.withArgName("serverList")
+ .hasArgs().withDescription("a comma-separated list of servers used in location validation in the format of " +
+ "scheme://authority (e.g. hdfs://localhost:8000)")
+ .create("servers");
+ Option catalogLocation = OptionBuilder
+ .hasArg()
+ .withDescription("Location of new catalog, required when adding a catalog")
+ .create("catalogLocation");
+ Option catalogDescription = OptionBuilder
+ .hasArg()
+ .withDescription("Description of new catalog")
+ .create("catalogDescription");
+ Option ifNotExists = OptionBuilder
+ .withDescription("If passed then it is not an error to create an existing catalog")
+ .create("ifNotExists");
+ Option fromCatalog = OptionBuilder
+ .hasArg()
+ .withDescription("Catalog a moving database or table is coming from. This is " +
+ "required if you are moving a database or table.")
+ .create("fromCatalog");
+ Option toCatalog = OptionBuilder
+ .hasArg()
+ .withDescription("Catalog a moving database or table is going to. This is " +
+ "required if you are moving a database or table.")
+ .create("toCatalog");
+ Option fromDatabase = OptionBuilder
+ .hasArg()
+ .withDescription("Database a moving table is coming from. This is " +
+ "required if you are moving a table.")
+ .create("fromDatabase");
+ Option toDatabase = OptionBuilder
+ .hasArg()
+ .withDescription("Database a moving table is going to. This is " +
+ "required if you are moving a table.")
+ .create("toDatabase");
+
+ Options options = new Options();
+ options.addOption(help);
+ options.addOptionGroup(optGroup);
+ options.addOption(dbTypeOpt);
+ //options.addOption(metaDbTypeOpt);
+ options.addOption(userNameOpt);
+ options.addOption(passwdOpt);
+ options.addOption(urlOpt);
+ options.addOption(driverOpt);
+ options.addOption(dbOpts);
+ options.addOption(dryRunOpt);
+ options.addOption(verboseOpt);
+ options.addOption(serversOpt);
+ options.addOption(catalogLocation);
+ options.addOption(catalogDescription);
+ options.addOption(ifNotExists);
+ options.addOption(fromCatalog);
+ options.addOption(toCatalog);
+ options.addOption(fromDatabase);
+ options.addOption(toDatabase);
+ options.addOption(hiveUserOpt);
+ options.addOption(hivePasswdOpt);
+ options.addOption(hiveDbOpt);
+ if (additionalOptions != null) options.addOptionGroup(additionalOptions);
+
+ return options;
+ }
+
+ private final CommandLine cl;
+ private final String dbType;
+ private final String metaDbType;
+
+ public SchemaToolCommandLine(String[] args, OptionGroup additionalOptions) throws ParseException {
+ cmdLineOptions = createOptions(additionalOptions);
+ cl = getCommandLine(args);
+ if (cl.hasOption("help")) {
+ printAndExit(null);
+ }
+
+ dbType = cl.getOptionValue("dbType");
+ metaDbType = cl.getOptionValue("metaDbType");
+
+ validate();
+ }
+
+ private CommandLine getCommandLine(String[] args) throws ParseException {
+ try {
+ CommandLineParser parser = new GnuParser();
+ return parser.parse(cmdLineOptions, args);
+ } catch (ParseException e) {
+ printAndExit("HiveSchemaTool:Parsing failed. Reason: " + e.getLocalizedMessage());
+ return null;
+ }
+ }
+
+ private static final Set<String> VALID_DB_TYPES = ImmutableSet.of(HiveSchemaHelper.DB_DERBY,
+ HiveSchemaHelper.DB_HIVE, HiveSchemaHelper.DB_MSSQL, HiveSchemaHelper.DB_MYSQL,
+ HiveSchemaHelper.DB_POSTGRACE, HiveSchemaHelper.DB_ORACLE);
+
+ private static final Set<String> VALID_META_DB_TYPES = ImmutableSet.of(HiveSchemaHelper.DB_DERBY,
+ HiveSchemaHelper.DB_MSSQL, HiveSchemaHelper.DB_MYSQL, HiveSchemaHelper.DB_POSTGRACE,
+ HiveSchemaHelper.DB_ORACLE);
+
+ private void validate() throws ParseException {
+ if (!VALID_DB_TYPES.contains(dbType)) {
+ printAndExit("Unsupported dbType " + dbType);
+ }
+
+ if (metaDbType != null) {
+ if (!dbType.equals(HiveSchemaHelper.DB_HIVE)) {
+ printAndExit("metaDbType may only be set if dbType is hive");
+ }
+ if (!VALID_META_DB_TYPES.contains(metaDbType)) {
+ printAndExit("Unsupported metaDbType " + metaDbType);
+ }
+ } else if (dbType.equalsIgnoreCase(HiveSchemaHelper.DB_HIVE)) {
+ System.err.println();
+ printAndExit("metaDbType must be set if dbType is hive");
+ }
+
+ if ((cl.hasOption("createCatalog")) && !cl.hasOption("catalogLocation")) {
+ System.err.println();
+ printAndExit("catalogLocation must be set for createCatalog");
+ }
+
+ if (!cl.hasOption("createCatalog") && !cl.hasOption("alterCatalog") &&
+ (cl.hasOption("catalogLocation") || cl.hasOption("catalogDescription"))) {
+ printAndExit("catalogLocation and catalogDescription may be set only for createCatalog and alterCatalog");
+ }
+
+ if (!cl.hasOption("createCatalog") && cl.hasOption("ifNotExists")) {
+ printAndExit("ifNotExists may be set only for createCatalog");
+ }
+
+ if (cl.hasOption("moveDatabase") &&
+ (!cl.hasOption("fromCatalog") || !cl.hasOption("toCatalog"))) {
+ printAndExit("fromCatalog and toCatalog must be set for moveDatabase");
+ }
+
+ if (cl.hasOption("moveTable") &&
+ (!cl.hasOption("fromCatalog") || !cl.hasOption("toCatalog") ||
+ !cl.hasOption("fromDatabase") || !cl.hasOption("toDatabase"))) {
+ printAndExit("fromCatalog, toCatalog, fromDatabase and toDatabase must be set for moveTable");
+ }
+
+ if ((!cl.hasOption("moveDatabase") && !cl.hasOption("moveTable")) &&
+ (cl.hasOption("fromCatalog") || cl.hasOption("toCatalog"))) {
+ printAndExit("fromCatalog and toCatalog may be set only for moveDatabase and moveTable");
+ }
+
+ if (!cl.hasOption("moveTable") &&
+ (cl.hasOption("fromDatabase") || cl.hasOption("toDatabase"))) {
+ printAndExit("fromDatabase and toDatabase may be set only for moveTable");
+ }
+ }
+
+ private void printAndExit(String reason) throws ParseException {
+ if (reason != null) {
+ System.err.println(reason);
+ }
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("schemaTool", cmdLineOptions);
+ if (reason != null) {
+ throw new ParseException(reason);
+ } else {
+ System.exit(0);
+ }
+ }
+
+ public String getDbType() {
+ return dbType;
+ }
+
+ public String getMetaDbType() {
+ return metaDbType;
+ }
+
+ boolean hasOption(String opt) {
+ return cl.hasOption(opt);
+ }
+
+ String getOptionValue(String opt) {
+ return cl.getOptionValue(opt);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTask.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTask.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTask.java
new file mode 100644
index 0000000..87ef6b5
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTask.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools;
+
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+
+public abstract class SchemaToolTask {
+ protected MetastoreSchemaTool schemaTool;
+
+ void setHiveSchemaTool(MetastoreSchemaTool schemaTool) {
+ this.schemaTool = schemaTool;
+ }
+
+ abstract void setCommandLineArguments(SchemaToolCommandLine cl);
+
+ abstract void execute() throws HiveMetaException;
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskAlterCatalog.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskAlterCatalog.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskAlterCatalog.java
new file mode 100644
index 0000000..4e3b3d1
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskAlterCatalog.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools;
+
+import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Print Hive version and schema version.
+ */
+class SchemaToolTaskAlterCatalog extends SchemaToolTask {
+ private static final Logger LOG = LoggerFactory.getLogger(SchemaToolTaskAlterCatalog.class.getName());
+
+ private String catName;
+ private String location;
+ private String description;
+
+ @Override
+ void setCommandLineArguments(SchemaToolCommandLine cl) {
+ catName = normalizeIdentifier(cl.getOptionValue("alterCatalog"));
+ location = cl.getOptionValue("catalogLocation");
+ description = cl.getOptionValue("catalogDescription");
+ }
+
+ private static final String UPDATE_CATALOG_STMT =
+ "update <q>CTLGS<q> " +
+ " set <q>LOCATION_URI<q> = %s, " +
+ " <qa>DESC<qa> = %s " +
+ " where <q>NAME<q> = '%s'";
+
+ @Override
+ void execute() throws HiveMetaException {
+ if (location == null && description == null) {
+ throw new HiveMetaException("Asked to update catalog " + catName + " but not given any changes to update");
+ }
+ System.out.println("Updating catalog " + catName);
+
+ Connection conn = schemaTool.getConnectionToMetastore(true);
+ boolean success = false;
+ try {
+ conn.setAutoCommit(false);
+ try (Statement stmt = conn.createStatement()) {
+ Object updateLocation = location == null ? schemaTool.quote("<q>LOCATION_URI<q>") : "'" + location + "'";
+ Object updateDescription = description == null ? schemaTool.quote("<qa>DESC<qa>") : "'" + description + "'";
+ String update = String.format(schemaTool.quote(UPDATE_CATALOG_STMT), updateLocation, updateDescription,
+ catName);
+ LOG.debug("Going to run " + update);
+ int count = stmt.executeUpdate(update);
+ if (count != 1) {
+ throw new HiveMetaException("Failed to find catalog " + catName + " to update");
+ }
+ conn.commit();
+ success = true;
+ }
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to update catalog", e);
+ } finally {
+ try {
+ if (!success) {
+ conn.rollback();
+ }
+ } catch (SQLException e) {
+ // Not really much we can do here.
+ LOG.error("Failed to rollback, everything will probably go bad from here.", e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskCreateCatalog.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskCreateCatalog.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskCreateCatalog.java
new file mode 100644
index 0000000..7857970
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskCreateCatalog.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools;
+
+import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Print Hive version and schema version.
+ */
+class SchemaToolTaskCreateCatalog extends SchemaToolTask {
+ private static final Logger LOG = LoggerFactory.getLogger(SchemaToolTaskCreateCatalog.class.getName());
+
+ private String catName;
+ private String location;
+ private String description;
+ private boolean ifNotExists;
+
+ @Override
+ void setCommandLineArguments(SchemaToolCommandLine cl) {
+ catName = normalizeIdentifier(cl.getOptionValue("createCatalog"));
+ location = cl.getOptionValue("catalogLocation");
+ description = cl.getOptionValue("catalogDescription");
+ ifNotExists = cl.hasOption("ifNotExists");
+ }
+
+ @Override
+ void execute() throws HiveMetaException {
+ System.out.println("Create catalog " + catName + " at location " + location);
+
+ Connection conn = schemaTool.getConnectionToMetastore(true);
+ boolean success = false;
+ try {
+ conn.setAutoCommit(false);
+ try (Statement stmt = conn.createStatement()) {
+ // If they set ifNotExists check for existence first, and bail if it exists. This is
+ // more reliable then attempting to parse the error message from the SQLException.
+ if (ifNotExists && catalogExists(stmt)) {
+ return;
+ }
+
+ int catNum = getNextCatalogId(stmt);
+ addCatalog(conn, stmt, catNum);
+ success = true;
+ }
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to add catalog", e);
+ } finally {
+ try {
+ if (!success) {
+ conn.rollback();
+ }
+ } catch (SQLException e) {
+ // Not really much we can do here.
+ LOG.error("Failed to rollback, everything will probably go bad from here.", e);
+ }
+ }
+ }
+
+ private static final String CATALOG_EXISTS_QUERY =
+ "select <q>NAME<q> " +
+ " from <q>CTLGS<q> " +
+ " where <q>NAME<q> = '%s'";
+
+ private boolean catalogExists(Statement stmt) throws SQLException {
+ String query = String.format(schemaTool.quote(CATALOG_EXISTS_QUERY), catName);
+ LOG.debug("Going to run " + query);
+ try (ResultSet rs = stmt.executeQuery(query)) {
+ if (rs.next()) {
+ System.out.println("Catalog " + catName + " already exists");
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ private static final String NEXT_CATALOG_ID_QUERY =
+ "select max(<q>CTLG_ID<q>) " +
+ " from <q>CTLGS<q>";
+
+ private int getNextCatalogId(Statement stmt) throws SQLException, HiveMetaException {
+ String query = schemaTool.quote(NEXT_CATALOG_ID_QUERY);
+ LOG.debug("Going to run " + query);
+ try (ResultSet rs = stmt.executeQuery(query)) {
+ if (!rs.next()) {
+ throw new HiveMetaException("No catalogs found, have you upgraded the database?");
+ }
+ int nextId = rs.getInt(1) + 1;
+ // We need to stay out of the way of any sequences used by the underlying database.
+ // Otherwise the next time the client tries to add a catalog we'll get an error.
+ // There should never be billions of catalogs, so we'll shift our sequence number up
+ // there to avoid clashes.
+ int floor = 1 << 30;
+ return Math.max(nextId, floor);
+ }
+ }
+
+ private static final String ADD_CATALOG_STMT =
+ "insert into <q>CTLGS<q> (<q>CTLG_ID<q>, <q>NAME<q>, <qa>DESC<qa>, <q>LOCATION_URI<q>) " +
+ " values (%d, '%s', '%s', '%s')";
+
+ private void addCatalog(Connection conn, Statement stmt, int catNum) throws SQLException {
+ String update = String.format(schemaTool.quote(ADD_CATALOG_STMT), catNum, catName, description, location);
+ LOG.debug("Going to run " + update);
+ stmt.execute(update);
+ conn.commit();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskCreateUser.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskCreateUser.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskCreateUser.java
new file mode 100644
index 0000000..a9ad005
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskCreateUser.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools;
+
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+public class SchemaToolTaskCreateUser extends SchemaToolTask {
+ private static final Logger LOG = LoggerFactory.getLogger(SchemaToolTaskCreateUser.class);
+
+ @Override
+ void setCommandLineArguments(SchemaToolCommandLine cl) {
+
+ }
+
+ @Override
+ void execute() throws HiveMetaException {
+ schemaTool.testConnectionToMetastore();
+ System.out.println("Starting user creation");
+
+ String scriptDir = schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir();
+ String protoCreateFile = schemaTool.getMetaStoreSchemaInfo().getCreateUserScript();
+
+ try {
+ File createFile = subUserAndPassword(scriptDir, protoCreateFile);
+ System.out.println("Creation script " + createFile.getAbsolutePath());
+ if (!schemaTool.isDryRun()) {
+ if ("oracle".equals(schemaTool.getDbType())) oracleCreateUserHack(createFile);
+ else schemaTool.execSql(createFile.getParent(), createFile.getName());
+ System.out.println("User creation completed");
+ }
+ } catch (IOException e) {
+ throw new HiveMetaException("User creation FAILED!" +
+ " Metastore unusable !!", e);
+ }
+
+ }
+
+ private File subUserAndPassword(String parent, String filename) throws IOException {
+ File createFile = File.createTempFile("create-hive-user-" + schemaTool.getDbType(), ".sql");
+ BufferedWriter writer = new BufferedWriter(new FileWriter(createFile));
+ File proto = new File(parent, filename);
+ BufferedReader reader = new BufferedReader(new FileReader(proto));
+ reader.lines()
+ .map(s -> s.replace("_REPLACE_WITH_USER_", schemaTool.getHiveUser())
+ .replace("_REPLACE_WITH_PASSWD_", schemaTool.getHivePasswd())
+ .replace("_REPLACE_WITH_DB_", schemaTool.getHiveDb()))
+ .forEach(s -> {
+ try {
+ writer.write(s);
+ writer.newLine();
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to write to tmp file ", e);
+ }
+ });
+ reader.close();
+ writer.close();
+ return createFile;
+ }
+
+ private void oracleCreateUserHack(File createFile) throws HiveMetaException {
+ LOG.debug("Found oracle, hacking our way through it rather than using SqlLine");
+ try (BufferedReader reader = new BufferedReader(new FileReader(createFile))) {
+ try (Connection conn = schemaTool.getConnectionToMetastore(false)) {
+ try (Statement stmt = conn.createStatement()) {
+ reader.lines()
+ .forEach(s -> {
+ assert s.charAt(s.length() - 1) == ';';
+ try {
+ stmt.execute(s.substring(0, s.length() - 1));
+ } catch (SQLException e) {
+ LOG.error("statement <" + s.substring(0, s.length() - 2) + "> failed", e);
+ throw new RuntimeException(e);
+ }
+ });
+ }
+ }
+ } catch (IOException e) {
+ LOG.error("Caught IOException trying to read modified create user script " +
+ createFile.getAbsolutePath(), e);
+ throw new HiveMetaException(e);
+ } catch (HiveMetaException e) {
+ LOG.error("Failed to connect to RDBMS", e);
+ throw e;
+ } catch (SQLException e) {
+ LOG.error("Got SQLException", e);
+ }
+ }
+}
[08/32] hive git commit: Revert "HIVE-19765: Add Parquet specific
tests to BlobstoreCliDriver (Sahil Takiar, reviewed by Marta Kuczora)"
Posted by se...@apache.org.
Revert "HIVE-19765: Add Parquet specific tests to BlobstoreCliDriver (Sahil Takiar, reviewed by Marta Kuczora)"
This reverts commit cbcfab9eb1c58e0c389b093af188d6483440beb0.
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9bd040ad
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9bd040ad
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9bd040ad
Branch: refs/heads/master-txnstats
Commit: 9bd040ade233391c0cb62ee07b79168e46caa13a
Parents: 2ca2757
Author: Zoltan Haindrich <ki...@rxd.hu>
Authored: Fri Jul 6 09:11:47 2018 +0200
Committer: Zoltan Haindrich <ki...@rxd.hu>
Committed: Fri Jul 6 09:11:47 2018 +0200
----------------------------------------------------------------------
.../queries/clientpositive/parquet_buckets.q | 31 --
.../clientpositive/parquet_format_nonpart.q | 30 --
.../clientpositive/parquet_format_part.q | 67 ---
.../parquet_nonstd_partitions_loc.q | 100 ----
.../queries/clientpositive/ptf_parquetfile.q | 32 --
.../clientpositive/parquet_buckets.q.out | 183 -------
.../clientpositive/parquet_format_nonpart.q.out | 201 -------
.../clientpositive/parquet_format_part.q.out | 290 ----------
.../parquet_nonstd_partitions_loc.q.out | 537 -------------------
.../clientpositive/ptf_parquetfile.q.out | 104 ----
10 files changed, 1575 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9bd040ad/itests/hive-blobstore/src/test/queries/clientpositive/parquet_buckets.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/parquet_buckets.q b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_buckets.q
deleted file mode 100644
index c5a2cd3..0000000
--- a/itests/hive-blobstore/src/test/queries/clientpositive/parquet_buckets.q
+++ /dev/null
@@ -1,31 +0,0 @@
--- Test simple interaction with partitioned bucketed table with paquet format in blobstore
-
-SET hive.exec.dynamic.partition=true;
-SET hive.exec.reducers.max=10;
-SET hive.exec.dynamic.partition.mode=nonstrict;
-
-DROP TABLE blobstore_source;
-CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE)
-ROW FORMAT DELIMITED
-FIELDS TERMINATED BY ' '
-COLLECTION ITEMS TERMINATED BY '\t'
-LINES TERMINATED BY '\n'
-LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_buckets/blobstore_source/';
-LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source;
-
-DROP TABLE parquet_buckets;
-CREATE TABLE parquet_buckets (a STRING, value DOUBLE)
-PARTITIONED BY (b STRING)
-CLUSTERED BY (a) INTO 10 BUCKETS
-STORED AS PARQUET
-LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_buckets/parquet_buckets';
-
-INSERT OVERWRITE TABLE parquet_buckets
-PARTITION (b)
-SELECT a, c, b FROM blobstore_source;
-SELECT * FROM parquet_buckets;
-
-INSERT INTO TABLE parquet_buckets
-PARTITION (b)
-SELECT a, c, b FROM blobstore_source;
-SELECT * FROM parquet_buckets;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/9bd040ad/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_nonpart.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_nonpart.q b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_nonpart.q
deleted file mode 100644
index a7827e9..0000000
--- a/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_nonpart.q
+++ /dev/null
@@ -1,30 +0,0 @@
--- Test INSERT OVERWRITE and INSERT INTO on parquet table in blobstore
-
-DROP TABLE blobstore_source;
-CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE)
-ROW FORMAT DELIMITED
-FIELDS TERMINATED BY ' '
-COLLECTION ITEMS TERMINATED BY '\t'
-LINES TERMINATED BY '\n'
-LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_format_nonpart/blobstore_source/';
-LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source;
-
-DROP TABLE parquet_table;
-CREATE EXTERNAL TABLE parquet_table (a INT, b STRING, value DOUBLE) STORED AS PARQUET
-LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_format_nonpart/parquet_table';
-
-INSERT OVERWRITE TABLE parquet_table
-SELECT * FROM blobstore_source;
-
-SELECT * FROM parquet_table;
-SELECT a FROM parquet_table GROUP BY a;
-SELECT b FROM parquet_table GROUP BY b;
-SELECT value FROM parquet_table GROUP BY value;
-
-INSERT INTO TABLE parquet_table
-SELECT * FROM blobstore_source;
-
-SELECT * FROM parquet_table;
-SELECT a FROM parquet_table GROUP BY a;
-SELECT b FROM parquet_table GROUP BY b;
-SELECT value FROM parquet_table GROUP BY value;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/9bd040ad/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_part.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_part.q b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_part.q
deleted file mode 100644
index e9924fe..0000000
--- a/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_part.q
+++ /dev/null
@@ -1,67 +0,0 @@
--- Test INSERT INTO and INSERT OVERWRITE on partitioned parquet table in blobstore
-
-DROP TABLE src_events;
-CREATE TABLE src_events
-(
- log_id BIGINT,
- `time` BIGINT,
- uid BIGINT,
- user_id BIGINT,
- type INT,
- event_data STRING,
- session_id STRING,
- full_uid BIGINT,
- run_date STRING,
- game_id INT,
- event_name STRING
-)
-ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_format_part/src_events/';
-LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events;
-
-DROP TABLE parquet_events;
-CREATE TABLE parquet_events
-(
- log_id BIGINT,
- `time` BIGINT,
- uid BIGINT,
- user_id BIGINT,
- type INT,
- event_data STRING,
- session_id STRING,
- full_uid BIGINT
-)
-PARTITIONED BY (run_date STRING, game_id INT, event_name STRING)
-STORED AS PARQUET
-LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_format_part/parquet_events';
-
-SET hive.exec.dynamic.partition=true;
-SET hive.exec.dynamic.partition.mode=nonstrict;
-
-INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
-SELECT * FROM src_events;
-SHOW PARTITIONS parquet_events;
-SELECT COUNT(*) FROM parquet_events;
-SELECT COUNT(*) FROM parquet_events WHERE run_date=20120921;
-SELECT COUNT(*) FROM parquet_events WHERE run_date=20121121;
-
-INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id, event_name)
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events
-WHERE SUBSTR(run_date,1,6)='201211';
-SHOW PARTITIONS parquet_events;
-SELECT COUNT(*) FROM parquet_events;
-
-INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name)
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events
-WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39;
-SELECT COUNT(*) FROM parquet_events;
-
-INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
-WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change';
-SELECT COUNT(*) FROM parquet_events;
-
-INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
-WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change';
-SELECT COUNT(*) FROM parquet_events;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/9bd040ad/itests/hive-blobstore/src/test/queries/clientpositive/parquet_nonstd_partitions_loc.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/parquet_nonstd_partitions_loc.q b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_nonstd_partitions_loc.q
deleted file mode 100644
index 2d20b99..0000000
--- a/itests/hive-blobstore/src/test/queries/clientpositive/parquet_nonstd_partitions_loc.q
+++ /dev/null
@@ -1,100 +0,0 @@
--- Test table in parquet format with non-standard partition locations in blobstore
-
-DROP TABLE src_events;
-CREATE TABLE src_events
-(
- log_id BIGINT,
- `time` BIGINT,
- uid BIGINT,
- user_id BIGINT,
- type INT,
- event_data STRING,
- session_id STRING,
- full_uid BIGINT,
- run_date STRING,
- game_id INT,
- event_name STRING
-)
-ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_nonstd_partitions_loc/src_events/';
-LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events;
-
-DROP TABLE parquet_events;
-CREATE TABLE parquet_events
-(
- log_id BIGINT,
- `time` BIGINT,
- uid BIGINT,
- user_id BIGINT,
- type INT,
- event_data STRING,
- session_id STRING,
- full_uid BIGINT
-)
-PARTITIONED BY (run_date STRING, game_id INT, event_name STRING)
-STORED AS PARQUET
-LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_nonstd_partitions_loc/parquet_events/';
-
-SET hive.exec.dynamic.partition=true;
-SET hive.exec.dynamic.partition.mode=nonstrict;
-
-INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
-SELECT * FROM src_events;
-SHOW PARTITIONS parquet_events;
-SELECT COUNT(*) FROM parquet_events;
-
--- verify INSERT OVERWRITE and INSERT INTO nonstandard partition location
-ALTER TABLE parquet_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change')
-LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-1/';
-INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change')
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
-WHERE SUBSTR(run_date,1,6)='201211';
-SHOW PARTITIONS parquet_events;
-SELECT COUNT(*) FROM parquet_events;
-INSERT INTO TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change')
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
-WHERE SUBSTR(run_date,1,6)='201211';
-SHOW PARTITIONS parquet_events;
-SELECT COUNT(*) FROM parquet_events;
-
-SET hive.merge.mapfiles=false;
-
--- verify INSERT OVERWRITE and INSERT INTO nonstandard partition location with hive.merge.mapfiles false
-ALTER TABLE parquet_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change')
-LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-2/';
-INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
-WHERE SUBSTR(run_date,1,6)='201209';
-INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
-WHERE SUBSTR(run_date,1,6)='201209';
-SHOW PARTITIONS parquet_events;
-SELECT COUNT(*) FROM parquet_events;
-
--- verify dynamic INSERT OVERWRITE over all partitions (standard and nonstandard locations) with hive.merge.mapfiles false
-INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
-SELECT * FROM src_events;
-SHOW PARTITIONS parquet_events;
-SELECT COUNT(*) FROM parquet_events;
-
-SET hive.merge.mapfiles=true;
-
-ALTER TABLE parquet_events ADD PARTITION (run_date=201207, game_id=39, event_name='hq_change')
-LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-3/';
-INSERT INTO TABLE parquet_events PARTITION (run_date=201207, game_id=39, event_name='hq_change')
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
-WHERE SUBSTR(run_date,1,6)='201209';
-SHOW PARTITIONS parquet_events;
-SELECT COUNT(*) FROM parquet_events;
-
--- verify dynamic INSERT OVERWRITE over all partitions (standard and nonstandard locations) with hive.merge.mapfiles true
-INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
-SELECT * FROM src_events;
-SHOW PARTITIONS parquet_events;
-SELECT COUNT(*) FROM parquet_events;
-
-ALTER TABLE parquet_events DROP PARTITION (run_date=201211, game_id=39, event_name='hq_change');
-ALTER TABLE parquet_events DROP PARTITION (run_date=201209, game_id=39, event_name='hq_change');
-ALTER TABLE parquet_events DROP PARTITION (run_date=201207, game_id=39, event_name='hq_change');
-SHOW PARTITIONS parquet_events;
-SELECT COUNT(*) FROM parquet_events;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/9bd040ad/itests/hive-blobstore/src/test/queries/clientpositive/ptf_parquetfile.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/ptf_parquetfile.q b/itests/hive-blobstore/src/test/queries/clientpositive/ptf_parquetfile.q
deleted file mode 100644
index fbf5ff3..0000000
--- a/itests/hive-blobstore/src/test/queries/clientpositive/ptf_parquetfile.q
+++ /dev/null
@@ -1,32 +0,0 @@
--- Test several window functions on a table stored using PARQUET
-DROP TABLE part_parquet;
-CREATE TABLE part_parquet(
- p_partkey int,
- p_name string,
- p_mfgr string,
- p_brand string,
- p_type string,
- p_size int,
- p_container string,
- p_retailprice double,
- p_comment string
-)
-STORED AS PARQUET
-LOCATION '${hiveconf:test.blobstore.path.unique}/ptf_parquetfile/part_parquet';
-
-LOAD DATA LOCAL INPATH '../../data/files/part.parquet' OVERWRITE INTO TABLE part_parquet;
-
--- Test windowing PTFs with several partitions, using PARQUET storage
-SELECT
- p_mfgr, p_name, p_size,
- RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS r,
- DENSE_RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS dr,
- SUM(p_retailprice) OVER (
- PARTITION BY p_mfgr
- ORDER BY p_name ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
- ) AS s1
-FROM NOOP(
- ON part_parquet
- PARTITION BY p_mfgr
- ORDER BY p_name
-);
http://git-wip-us.apache.org/repos/asf/hive/blob/9bd040ad/itests/hive-blobstore/src/test/results/clientpositive/parquet_buckets.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_buckets.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_buckets.q.out
deleted file mode 100644
index 355958d..0000000
--- a/itests/hive-blobstore/src/test/results/clientpositive/parquet_buckets.q.out
+++ /dev/null
@@ -1,183 +0,0 @@
-PREHOOK: query: DROP TABLE blobstore_source
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: DROP TABLE blobstore_source
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE)
-ROW FORMAT DELIMITED
-FIELDS TERMINATED BY ' '
-COLLECTION ITEMS TERMINATED BY '\t'
-LINES TERMINATED BY '\n'
-#### A masked pattern was here ####
-PREHOOK: type: CREATETABLE
-PREHOOK: Input: ### test.blobstore.path ###/parquet_buckets/blobstore_source
-PREHOOK: Output: database:default
-PREHOOK: Output: default@blobstore_source
-POSTHOOK: query: CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE)
-ROW FORMAT DELIMITED
-FIELDS TERMINATED BY ' '
-COLLECTION ITEMS TERMINATED BY '\t'
-LINES TERMINATED BY '\n'
-#### A masked pattern was here ####
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Input: ### test.blobstore.path ###/parquet_buckets/blobstore_source
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@blobstore_source
-PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@blobstore_source
-POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@blobstore_source
-PREHOOK: query: DROP TABLE parquet_buckets
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: DROP TABLE parquet_buckets
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: CREATE TABLE parquet_buckets (a STRING, value DOUBLE)
-PARTITIONED BY (b STRING)
-CLUSTERED BY (a) INTO 10 BUCKETS
-STORED AS PARQUET
-#### A masked pattern was here ####
-PREHOOK: type: CREATETABLE
-PREHOOK: Input: ### test.blobstore.path ###/parquet_buckets/parquet_buckets
-PREHOOK: Output: database:default
-PREHOOK: Output: default@parquet_buckets
-POSTHOOK: query: CREATE TABLE parquet_buckets (a STRING, value DOUBLE)
-PARTITIONED BY (b STRING)
-CLUSTERED BY (a) INTO 10 BUCKETS
-STORED AS PARQUET
-#### A masked pattern was here ####
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Input: ### test.blobstore.path ###/parquet_buckets/parquet_buckets
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@parquet_buckets
-PREHOOK: query: INSERT OVERWRITE TABLE parquet_buckets
-PARTITION (b)
-SELECT a, c, b FROM blobstore_source
-PREHOOK: type: QUERY
-PREHOOK: Input: default@blobstore_source
-PREHOOK: Output: default@parquet_buckets
-POSTHOOK: query: INSERT OVERWRITE TABLE parquet_buckets
-PARTITION (b)
-SELECT a, c, b FROM blobstore_source
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@blobstore_source
-POSTHOOK: Output: default@parquet_buckets@b=abc
-POSTHOOK: Output: default@parquet_buckets@b=ajss
-POSTHOOK: Output: default@parquet_buckets@b=data
-POSTHOOK: Output: default@parquet_buckets@b=def
-POSTHOOK: Output: default@parquet_buckets@b=djns
-POSTHOOK: Output: default@parquet_buckets@b=ne
-POSTHOOK: Output: default@parquet_buckets@b=random
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=abc).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=abc).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=ajss).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=ajss).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=data).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=data).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=def).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=def).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=djns).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=djns).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=ne).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=ne).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=random).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=random).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
-PREHOOK: query: SELECT * FROM parquet_buckets
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet_buckets
-PREHOOK: Input: default@parquet_buckets@b=abc
-PREHOOK: Input: default@parquet_buckets@b=ajss
-PREHOOK: Input: default@parquet_buckets@b=data
-PREHOOK: Input: default@parquet_buckets@b=def
-PREHOOK: Input: default@parquet_buckets@b=djns
-PREHOOK: Input: default@parquet_buckets@b=ne
-PREHOOK: Input: default@parquet_buckets@b=random
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT * FROM parquet_buckets
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet_buckets
-POSTHOOK: Input: default@parquet_buckets@b=abc
-POSTHOOK: Input: default@parquet_buckets@b=ajss
-POSTHOOK: Input: default@parquet_buckets@b=data
-POSTHOOK: Input: default@parquet_buckets@b=def
-POSTHOOK: Input: default@parquet_buckets@b=djns
-POSTHOOK: Input: default@parquet_buckets@b=ne
-POSTHOOK: Input: default@parquet_buckets@b=random
-#### A masked pattern was here ####
-1 10.5 abc
-3 90.23232 ajss
-6 3.002 data
-2 11.5 def
-4 89.02002 djns
-7 71.9084 ne
-5 2.99 random
-PREHOOK: query: INSERT INTO TABLE parquet_buckets
-PARTITION (b)
-SELECT a, c, b FROM blobstore_source
-PREHOOK: type: QUERY
-PREHOOK: Input: default@blobstore_source
-PREHOOK: Output: default@parquet_buckets
-POSTHOOK: query: INSERT INTO TABLE parquet_buckets
-PARTITION (b)
-SELECT a, c, b FROM blobstore_source
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@blobstore_source
-POSTHOOK: Output: default@parquet_buckets@b=abc
-POSTHOOK: Output: default@parquet_buckets@b=ajss
-POSTHOOK: Output: default@parquet_buckets@b=data
-POSTHOOK: Output: default@parquet_buckets@b=def
-POSTHOOK: Output: default@parquet_buckets@b=djns
-POSTHOOK: Output: default@parquet_buckets@b=ne
-POSTHOOK: Output: default@parquet_buckets@b=random
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=abc).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=abc).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=ajss).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=ajss).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=data).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=data).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=def).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=def).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=djns).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=djns).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=ne).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=ne).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=random).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_buckets PARTITION(b=random).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
-PREHOOK: query: SELECT * FROM parquet_buckets
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet_buckets
-PREHOOK: Input: default@parquet_buckets@b=abc
-PREHOOK: Input: default@parquet_buckets@b=ajss
-PREHOOK: Input: default@parquet_buckets@b=data
-PREHOOK: Input: default@parquet_buckets@b=def
-PREHOOK: Input: default@parquet_buckets@b=djns
-PREHOOK: Input: default@parquet_buckets@b=ne
-PREHOOK: Input: default@parquet_buckets@b=random
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT * FROM parquet_buckets
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet_buckets
-POSTHOOK: Input: default@parquet_buckets@b=abc
-POSTHOOK: Input: default@parquet_buckets@b=ajss
-POSTHOOK: Input: default@parquet_buckets@b=data
-POSTHOOK: Input: default@parquet_buckets@b=def
-POSTHOOK: Input: default@parquet_buckets@b=djns
-POSTHOOK: Input: default@parquet_buckets@b=ne
-POSTHOOK: Input: default@parquet_buckets@b=random
-#### A masked pattern was here ####
-1 10.5 abc
-1 10.5 abc
-3 90.23232 ajss
-3 90.23232 ajss
-6 3.002 data
-6 3.002 data
-2 11.5 def
-2 11.5 def
-4 89.02002 djns
-4 89.02002 djns
-7 71.9084 ne
-7 71.9084 ne
-5 2.99 random
-5 2.99 random
http://git-wip-us.apache.org/repos/asf/hive/blob/9bd040ad/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_nonpart.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_nonpart.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_nonpart.q.out
deleted file mode 100644
index cba6b1a..0000000
--- a/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_nonpart.q.out
+++ /dev/null
@@ -1,201 +0,0 @@
-PREHOOK: query: DROP TABLE blobstore_source
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: DROP TABLE blobstore_source
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE)
-ROW FORMAT DELIMITED
-FIELDS TERMINATED BY ' '
-COLLECTION ITEMS TERMINATED BY '\t'
-LINES TERMINATED BY '\n'
-#### A masked pattern was here ####
-PREHOOK: type: CREATETABLE
-PREHOOK: Input: ### test.blobstore.path ###/parquet_format_nonpart/blobstore_source
-PREHOOK: Output: database:default
-PREHOOK: Output: default@blobstore_source
-POSTHOOK: query: CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE)
-ROW FORMAT DELIMITED
-FIELDS TERMINATED BY ' '
-COLLECTION ITEMS TERMINATED BY '\t'
-LINES TERMINATED BY '\n'
-#### A masked pattern was here ####
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Input: ### test.blobstore.path ###/parquet_format_nonpart/blobstore_source
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@blobstore_source
-PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@blobstore_source
-POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@blobstore_source
-PREHOOK: query: DROP TABLE parquet_table
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: DROP TABLE parquet_table
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: CREATE EXTERNAL TABLE parquet_table (a INT, b STRING, value DOUBLE) STORED AS PARQUET
-#### A masked pattern was here ####
-PREHOOK: type: CREATETABLE
-PREHOOK: Input: ### test.blobstore.path ###/parquet_format_nonpart/parquet_table
-PREHOOK: Output: database:default
-PREHOOK: Output: default@parquet_table
-POSTHOOK: query: CREATE EXTERNAL TABLE parquet_table (a INT, b STRING, value DOUBLE) STORED AS PARQUET
-#### A masked pattern was here ####
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Input: ### test.blobstore.path ###/parquet_format_nonpart/parquet_table
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@parquet_table
-PREHOOK: query: INSERT OVERWRITE TABLE parquet_table
-SELECT * FROM blobstore_source
-PREHOOK: type: QUERY
-PREHOOK: Input: default@blobstore_source
-PREHOOK: Output: default@parquet_table
-POSTHOOK: query: INSERT OVERWRITE TABLE parquet_table
-SELECT * FROM blobstore_source
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@blobstore_source
-POSTHOOK: Output: default@parquet_table
-POSTHOOK: Lineage: parquet_table.a EXPRESSION [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_table.b SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:b, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_table.value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
-PREHOOK: query: SELECT * FROM parquet_table
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet_table
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT * FROM parquet_table
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet_table
-#### A masked pattern was here ####
-1 abc 10.5
-2 def 11.5
-3 ajss 90.23232
-4 djns 89.02002
-5 random 2.99
-6 data 3.002
-7 ne 71.9084
-PREHOOK: query: SELECT a FROM parquet_table GROUP BY a
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet_table
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT a FROM parquet_table GROUP BY a
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet_table
-#### A masked pattern was here ####
-1
-2
-3
-4
-5
-6
-7
-PREHOOK: query: SELECT b FROM parquet_table GROUP BY b
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet_table
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT b FROM parquet_table GROUP BY b
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet_table
-#### A masked pattern was here ####
-abc
-ajss
-data
-def
-djns
-ne
-random
-PREHOOK: query: SELECT value FROM parquet_table GROUP BY value
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet_table
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT value FROM parquet_table GROUP BY value
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet_table
-#### A masked pattern was here ####
-2.99
-3.002
-10.5
-11.5
-71.9084
-89.02002
-90.23232
-PREHOOK: query: INSERT INTO TABLE parquet_table
-SELECT * FROM blobstore_source
-PREHOOK: type: QUERY
-PREHOOK: Input: default@blobstore_source
-PREHOOK: Output: default@parquet_table
-POSTHOOK: query: INSERT INTO TABLE parquet_table
-SELECT * FROM blobstore_source
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@blobstore_source
-POSTHOOK: Output: default@parquet_table
-POSTHOOK: Lineage: parquet_table.a EXPRESSION [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_table.b SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:b, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_table.value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
-PREHOOK: query: SELECT * FROM parquet_table
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet_table
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT * FROM parquet_table
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet_table
-#### A masked pattern was here ####
-1 abc 10.5
-2 def 11.5
-3 ajss 90.23232
-4 djns 89.02002
-5 random 2.99
-6 data 3.002
-7 ne 71.9084
-1 abc 10.5
-2 def 11.5
-3 ajss 90.23232
-4 djns 89.02002
-5 random 2.99
-6 data 3.002
-7 ne 71.9084
-PREHOOK: query: SELECT a FROM parquet_table GROUP BY a
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet_table
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT a FROM parquet_table GROUP BY a
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet_table
-#### A masked pattern was here ####
-1
-2
-3
-4
-5
-6
-7
-PREHOOK: query: SELECT b FROM parquet_table GROUP BY b
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet_table
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT b FROM parquet_table GROUP BY b
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet_table
-#### A masked pattern was here ####
-abc
-ajss
-data
-def
-djns
-ne
-random
-PREHOOK: query: SELECT value FROM parquet_table GROUP BY value
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet_table
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT value FROM parquet_table GROUP BY value
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet_table
-#### A masked pattern was here ####
-2.99
-3.002
-10.5
-11.5
-71.9084
-89.02002
-90.23232
http://git-wip-us.apache.org/repos/asf/hive/blob/9bd040ad/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out
deleted file mode 100644
index 0931e3d..0000000
--- a/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out
+++ /dev/null
@@ -1,290 +0,0 @@
-PREHOOK: query: DROP TABLE src_events
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: DROP TABLE src_events
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: CREATE TABLE src_events
-(
- log_id BIGINT,
- `time` BIGINT,
- uid BIGINT,
- user_id BIGINT,
- type INT,
- event_data STRING,
- session_id STRING,
- full_uid BIGINT,
- run_date STRING,
- game_id INT,
- event_name STRING
-)
-ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-#### A masked pattern was here ####
-PREHOOK: type: CREATETABLE
-PREHOOK: Input: ### test.blobstore.path ###/parquet_format_part/src_events
-PREHOOK: Output: database:default
-PREHOOK: Output: default@src_events
-POSTHOOK: query: CREATE TABLE src_events
-(
- log_id BIGINT,
- `time` BIGINT,
- uid BIGINT,
- user_id BIGINT,
- type INT,
- event_data STRING,
- session_id STRING,
- full_uid BIGINT,
- run_date STRING,
- game_id INT,
- event_name STRING
-)
-ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-#### A masked pattern was here ####
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Input: ### test.blobstore.path ###/parquet_format_part/src_events
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@src_events
-PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@src_events
-POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@src_events
-PREHOOK: query: DROP TABLE parquet_events
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: DROP TABLE parquet_events
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: CREATE TABLE parquet_events
-(
- log_id BIGINT,
- `time` BIGINT,
- uid BIGINT,
- user_id BIGINT,
- type INT,
- event_data STRING,
- session_id STRING,
- full_uid BIGINT
-)
-PARTITIONED BY (run_date STRING, game_id INT, event_name STRING)
-STORED AS PARQUET
-#### A masked pattern was here ####
-PREHOOK: type: CREATETABLE
-PREHOOK: Input: ### test.blobstore.path ###/parquet_format_part/parquet_events
-PREHOOK: Output: database:default
-PREHOOK: Output: default@parquet_events
-POSTHOOK: query: CREATE TABLE parquet_events
-(
- log_id BIGINT,
- `time` BIGINT,
- uid BIGINT,
- user_id BIGINT,
- type INT,
- event_data STRING,
- session_id STRING,
- full_uid BIGINT
-)
-PARTITIONED BY (run_date STRING, game_id INT, event_name STRING)
-STORED AS PARQUET
-#### A masked pattern was here ####
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Input: ### test.blobstore.path ###/parquet_format_part/parquet_events
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@parquet_events
-PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
-SELECT * FROM src_events
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src_events
-PREHOOK: Output: default@parquet_events
-POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
-SELECT * FROM src_events
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src_events
-POSTHOOK: Output: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change
-POSTHOOK: Output: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change
-POSTHOOK: Output: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
-PREHOOK: query: SHOW PARTITIONS parquet_events
-PREHOOK: type: SHOWPARTITIONS
-PREHOOK: Input: default@parquet_events
-POSTHOOK: query: SHOW PARTITIONS parquet_events
-POSTHOOK: type: SHOWPARTITIONS
-POSTHOOK: Input: default@parquet_events
-run_date=20120921/game_id=39/event_name=hq_change
-run_date=20121021/game_id=39/event_name=hq_change
-run_date=20121121/game_id=39/event_name=hq_change
-PREHOOK: query: SELECT COUNT(*) FROM parquet_events
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-200
-PREHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20120921
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20120921
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-50
-PREHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20121121
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20121121
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-100
-PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id, event_name)
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events
-WHERE SUBSTR(run_date,1,6)='201211'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src_events
-PREHOOK: Output: default@parquet_events@run_date=201211
-POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id, event_name)
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events
-WHERE SUBSTR(run_date,1,6)='201211'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src_events
-POSTHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
-PREHOOK: query: SHOW PARTITIONS parquet_events
-PREHOOK: type: SHOWPARTITIONS
-PREHOOK: Input: default@parquet_events
-POSTHOOK: query: SHOW PARTITIONS parquet_events
-POSTHOOK: type: SHOWPARTITIONS
-POSTHOOK: Input: default@parquet_events
-run_date=20120921/game_id=39/event_name=hq_change
-run_date=20121021/game_id=39/event_name=hq_change
-run_date=201211/game_id=39/event_name=hq_change
-run_date=20121121/game_id=39/event_name=hq_change
-PREHOOK: query: SELECT COUNT(*) FROM parquet_events
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-300
-PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name)
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events
-WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src_events
-PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39
-POSTHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name)
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events
-WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src_events
-POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
-PREHOOK: query: SELECT COUNT(*) FROM parquet_events
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-350
-PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
-WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src_events
-PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
-POSTHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
-WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src_events
-POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
-PREHOOK: query: SELECT COUNT(*) FROM parquet_events
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-400
-PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
-WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src_events
-PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
-POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
-WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src_events
-POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
-PREHOOK: query: SELECT COUNT(*) FROM parquet_events
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-350
[16/32] hive git commit: HIVE-20085 : Druid-Hive (managed) table
creation fails with strict managed table checks: Table is marked as a managed
table but is not transactional (Nishant Bangarwa via Ashutosh Chauhan)
Posted by se...@apache.org.
HIVE-20085 : Druid-Hive (managed) table creation fails with strict managed table checks: Table is marked as a managed table but is not transactional (Nishant Bangarwa via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/406cde98
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/406cde98
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/406cde98
Branch: refs/heads/master-txnstats
Commit: 406cde9883a0a9f2d740c8c9e0104b5c06a4fbff
Parents: 8494522
Author: Nishant Bangarwa <ni...@gmail.com>
Authored: Sat Jul 7 09:22:17 2018 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Sat Jul 7 09:22:17 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 2 +
.../hadoop/hive/druid/DruidStorageHandler.java | 71 ++++++--------------
.../hadoop/hive/druid/serde/DruidSerDe.java | 3 +-
.../hive/druid/TestDruidStorageHandler.java | 4 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 2 +-
ql/src/test/queries/clientnegative/ctas.q | 2 +-
.../queries/clientnegative/druid_datasource2.q | 1 +
.../queries/clientpositive/druid_timestamptz.q | 5 +-
.../queries/clientpositive/druid_timestamptz2.q | 14 ++--
.../clientpositive/druidkafkamini_basic.q | 2 +-
.../druidmini_dynamic_partition.q | 10 +--
.../clientpositive/druidmini_expressions.q | 3 +-
.../clientpositive/druidmini_extractTime.q | 6 +-
.../clientpositive/druidmini_floorTime.q | 4 +-
.../queries/clientpositive/druidmini_joins.q | 5 +-
.../queries/clientpositive/druidmini_masking.q | 4 +-
.../queries/clientpositive/druidmini_test1.q | 5 +-
.../clientpositive/druidmini_test_alter.q | 5 +-
.../clientpositive/druidmini_test_insert.q | 8 ++-
.../clientnegative/druid_datasource2.q.out | 2 +-
.../druid/druid_timestamptz.q.out | 4 +-
.../druid/druid_timestamptz2.q.out | 20 +++---
.../druid/druidkafkamini_basic.q.out | 6 +-
.../druid/druidmini_dynamic_partition.q.out | 18 ++---
.../druid/druidmini_expressions.q.out | 6 +-
.../druid/druidmini_extractTime.q.out | 8 +--
.../druid/druidmini_floorTime.q.out | 4 +-
.../clientpositive/druid/druidmini_joins.q.out | 4 +-
.../druid/druidmini_masking.q.out | 4 +-
.../clientpositive/druid/druidmini_test1.q.out | 4 +-
.../druid/druidmini_test_alter.q.out | 4 +-
.../druid/druidmini_test_insert.q.out | 12 ++--
.../hive/metastore/utils/MetaStoreUtils.java | 19 ++++++
33 files changed, 143 insertions(+), 128 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 2da9086..d8346bb 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2981,6 +2981,8 @@ public class HiveConf extends Configuration {
HIVE_INSERT_INTO_MULTILEVEL_DIRS("hive.insert.into.multilevel.dirs", false,
"Where to insert into multilevel directories like\n" +
"\"insert directory '/HIVEFT25686/chinna/' from table\""),
+ HIVE_CTAS_EXTERNAL_TABLES("hive.ctas.external.tables", true,
+ "whether CTAS for external tables is allowed"),
HIVE_INSERT_INTO_EXTERNAL_TABLES("hive.insert.into.external.tables", true,
"whether insert into external tables is allowed"),
HIVE_TEMPORARY_TABLE_STORAGE(
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java
----------------------------------------------------------------------
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java
index 57e4800..53d93e1 100644
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java
+++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java
@@ -215,12 +215,10 @@ public class DruidStorageHandler extends DefaultHiveMetaHook implements HiveStor
@Override
public void preCreateTable(Table table) throws MetaException {
- // Do safety checks
- if (MetaStoreUtils.isExternalTable(table) && !StringUtils
- .isEmpty(table.getSd().getLocation())) {
+ if(!StringUtils
+ .isEmpty(table.getSd().getLocation())) {
throw new MetaException("LOCATION may not be specified for Druid");
}
-
if (table.getPartitionKeysSize() != 0) {
throw new MetaException("PARTITIONED BY may not be specified for Druid");
}
@@ -228,25 +226,17 @@ public class DruidStorageHandler extends DefaultHiveMetaHook implements HiveStor
throw new MetaException("CLUSTERED BY may not be specified for Druid");
}
String dataSourceName = table.getParameters().get(Constants.DRUID_DATA_SOURCE);
- if (MetaStoreUtils.isExternalTable(table)) {
- if (dataSourceName == null) {
- throw new MetaException(
- String.format("Datasource name should be specified using [%s] for external tables "
- + "using Druid", Constants.DRUID_DATA_SOURCE));
- }
- // If it is an external table, we are done
+ if(dataSourceName != null){
+ // Already Existing datasource in Druid.
return;
}
- // It is not an external table
- // We need to check that datasource was not specified by user
- if (dataSourceName != null) {
- throw new MetaException(
- String.format("Datasource name cannot be specified using [%s] for managed tables "
- + "using Druid", Constants.DRUID_DATA_SOURCE));
- }
- // We need to check the Druid metadata
+
+ // create dataSourceName based on Hive Table name
dataSourceName = Warehouse.getQualifiedName(table);
try {
+ // NOTE: This just created druid_segments table in Druid metastore.
+ // This is needed for the case when hive is started before any of druid services
+ // and druid_segments table has not been created yet.
getConnector().createSegmentTable();
} catch (Exception e) {
LOG.error("Exception while trying to create druid segments table", e);
@@ -255,6 +245,7 @@ public class DruidStorageHandler extends DefaultHiveMetaHook implements HiveStor
Collection<String> existingDataSources = DruidStorageHandlerUtils
.getAllDataSourceNames(getConnector(), getDruidMetadataStorageTablesConfig());
LOG.debug("pre-create data source with name {}", dataSourceName);
+ // Check for existence of for the datasource we are going to create in druid_segments table.
if (existingDataSources.contains(dataSourceName)) {
throw new MetaException(String.format("Data source [%s] already existing", dataSourceName));
}
@@ -263,38 +254,17 @@ public class DruidStorageHandler extends DefaultHiveMetaHook implements HiveStor
@Override
public void rollbackCreateTable(Table table) {
- if (MetaStoreUtils.isExternalTable(table)) {
- return;
- }
- final Path segmentDescriptorDir = getSegmentDescriptorDir();
- try {
- List<DataSegment> dataSegmentList = DruidStorageHandlerUtils
- .getCreatedSegments(segmentDescriptorDir, getConf());
- for (DataSegment dataSegment : dataSegmentList) {
- try {
- deleteSegment(dataSegment);
- } catch (SegmentLoadingException e) {
- LOG.error(String.format("Error while trying to clean the segment [%s]", dataSegment), e);
- }
- }
- } catch (IOException e) {
- LOG.error("Exception while rollback", e);
- throw Throwables.propagate(e);
- } finally {
- cleanWorkingDir();
- }
+ cleanWorkingDir();
}
@Override
public void commitCreateTable(Table table) throws MetaException {
- if (MetaStoreUtils.isExternalTable(table)) {
- // For external tables, we do not need to do anything else
- return;
- }
if(isKafkaStreamingTable(table)){
updateKafkaIngestion(table);
}
- this.commitInsertTable(table, true);
+ // For CTAS queries when user has explicitly specified the datasource.
+ // We will append the data to existing druid datasource.
+ this.commitInsertTable(table, false);
}
private void updateKafkaIngestion(Table table){
@@ -762,9 +732,6 @@ public class DruidStorageHandler extends DefaultHiveMetaHook implements HiveStor
@Override
public void commitDropTable(Table table, boolean deleteData) {
- if (MetaStoreUtils.isExternalTable(table)) {
- return;
- }
if(isKafkaStreamingTable(table)) {
// Stop Kafka Ingestion first
final String overlordAddress = Preconditions.checkNotNull(HiveConf
@@ -775,12 +742,15 @@ public class DruidStorageHandler extends DefaultHiveMetaHook implements HiveStor
"Druid Datasource name is null");
stopKafkaIngestion(overlordAddress, dataSourceName);
}
+
String dataSourceName = Preconditions
.checkNotNull(table.getParameters().get(Constants.DRUID_DATA_SOURCE),
"DataSource name is null !"
);
-
- if (deleteData == true) {
+ // TODO: Move MetaStoreUtils.isExternalTablePurge(table) calls to a common place for all StorageHandlers
+ // deleteData flag passed down to StorageHandler should be true only if
+ // MetaStoreUtils.isExternalTablePurge(table) returns true.
+ if (deleteData == true && MetaStoreUtils.isExternalTablePurge(table)) {
LOG.info("Dropping with purge all the data for data source {}", dataSourceName);
List<DataSegment> dataSegmentList = DruidStorageHandlerUtils
.getDataSegmentList(getConnector(), getDruidMetadataStorageTablesConfig(), dataSourceName);
@@ -806,9 +776,6 @@ public class DruidStorageHandler extends DefaultHiveMetaHook implements HiveStor
public void commitInsertTable(Table table, boolean overwrite) throws MetaException {
LOG.debug("commit insert into table {} overwrite {}", table.getTableName(),
overwrite);
- if (MetaStoreUtils.isExternalTable(table)) {
- throw new MetaException("Cannot insert data into external table backed by Druid");
- }
try {
// Check if there segments to load
final Path segmentDescriptorDir = getSegmentDescriptorDir();
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java
----------------------------------------------------------------------
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java
index f0e12a2..df9049e 100644
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java
+++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java
@@ -483,7 +483,6 @@ import static org.joda.time.format.ISODateTimeFormat.dateOptionalTimeParser;
}
@Override public boolean shouldStoreFieldsInMetastore(Map<String, String> tableParams) {
- // If Druid table is not an external table store the schema in metadata store.
- return !MetaStoreUtils.isExternal(tableParams);
+ return true;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidStorageHandler.java
----------------------------------------------------------------------
diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidStorageHandler.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidStorageHandler.java
index b96a13f..510330d 100644
--- a/druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidStorageHandler.java
+++ b/druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidStorageHandler.java
@@ -106,7 +106,9 @@ public class TestDruidStorageHandler {
public void before() throws Throwable {
tableWorkingPath = temporaryFolder.newFolder().getAbsolutePath();
segmentsTable = derbyConnectorRule.metadataTablesConfigSupplier().get().getSegmentsTable();
- Mockito.when(tableMock.getParameters()).thenReturn(new HashMap<>());
+ HashMap<String, String> params = new HashMap<>();
+ params.put("external.table.purge", "TRUE");
+ Mockito.when(tableMock.getParameters()).thenReturn(params);
Mockito.when(tableMock.getPartitionKeysSize()).thenReturn(0);
StorageDescriptor storageDes = Mockito.mock(StorageDescriptor.class);
Mockito.when(storageDes.getBucketColsSize()).thenReturn(0);
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 1271799..12a915c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -13049,7 +13049,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
throw new SemanticException(ErrorMsg.CTAS_PARCOL_COEXISTENCE.getMsg());
}
}
- if (isExt) {
+ if (!conf.getBoolVar(ConfVars.HIVE_CTAS_EXTERNAL_TABLES) && isExt) {
throw new SemanticException(ErrorMsg.CTAS_EXTTBL_COEXISTENCE.getMsg());
}
command_type = CTAS;
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/queries/clientnegative/ctas.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/ctas.q b/ql/src/test/queries/clientnegative/ctas.q
index 548ad05..27c1c0e 100644
--- a/ql/src/test/queries/clientnegative/ctas.q
+++ b/ql/src/test/queries/clientnegative/ctas.q
@@ -1,6 +1,6 @@
--! qt:dataset:src
-
+SET hive.ctas.external.tables=false;
create external table nzhang_ctas4 as select key, value from src;
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/queries/clientnegative/druid_datasource2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/druid_datasource2.q b/ql/src/test/queries/clientnegative/druid_datasource2.q
index cc20931..2e7400f 100644
--- a/ql/src/test/queries/clientnegative/druid_datasource2.q
+++ b/ql/src/test/queries/clientnegative/druid_datasource2.q
@@ -1,3 +1,4 @@
+SET metastore.strict.managed.tables=true;
CREATE TABLE druid_table_1
STORED BY 'org.apache.hadoop.hive.druid.QTestDruidStorageHandler'
TBLPROPERTIES ("property" = "localhost", "druid.datasource" = "mydatasource");
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/queries/clientpositive/druid_timestamptz.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/druid_timestamptz.q b/ql/src/test/queries/clientpositive/druid_timestamptz.q
index 4830044..605d240 100644
--- a/ql/src/test/queries/clientpositive/druid_timestamptz.q
+++ b/ql/src/test/queries/clientpositive/druid_timestamptz.q
@@ -1,9 +1,10 @@
set hive.fetch.task.conversion=more;
-
+SET hive.ctas.external.tables=true;
+SET hive.external.table.purge.default = true;
drop table tstz1_n0;
-create table tstz1_n0(`__time` timestamp with local time zone, n string, v integer)
+create external table tstz1_n0(`__time` timestamp with local time zone, n string, v integer)
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR");
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/queries/clientpositive/druid_timestamptz2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/druid_timestamptz2.q b/ql/src/test/queries/clientpositive/druid_timestamptz2.q
index 8f573c8..8b2c092 100644
--- a/ql/src/test/queries/clientpositive/druid_timestamptz2.q
+++ b/ql/src/test/queries/clientpositive/druid_timestamptz2.q
@@ -1,9 +1,9 @@
+SET hive.ctas.external.tables=true;
+SET hive.external.table.purge.default = true;
CREATE database druid_test_dst;
use druid_test_dst;
-
-
-create table test_base_table(`timecolumn` timestamp, `interval_marker` string, `num_l` double);
+create external table test_base_table(`timecolumn` timestamp, `interval_marker` string, `num_l` double);
insert into test_base_table values ('2015-03-08 00:00:00', 'i1-start', 4);
insert into test_base_table values ('2015-03-08 23:59:59', 'i1-end', 1);
insert into test_base_table values ('2015-03-09 00:00:00', 'i2-start', 4);
@@ -11,7 +11,7 @@ insert into test_base_table values ('2015-03-09 23:59:59', 'i2-end', 1);
insert into test_base_table values ('2015-03-10 00:00:00', 'i3-start', 2);
insert into test_base_table values ('2015-03-10 23:59:59', 'i3-end', 2);
-CREATE TABLE druid_test_table_1
+CREATE EXTERNAL TABLE druid_test_table_1
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "DAY")
AS
@@ -20,7 +20,7 @@ FROM druid_test_dst.test_base_table;
select * FROM druid_test_table_1;
-CREATE TABLE druid_test_table_2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double)
+CREATE EXTERNAL TABLE druid_test_table_2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double)
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "DAY");
@@ -36,7 +36,7 @@ select * FROM druid_test_table_2;
SET TIME ZONE UTC;
-CREATE TABLE druid_test_table_utc
+CREATE EXTERNAL TABLE druid_test_table_utc
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "DAY")
AS
@@ -45,7 +45,7 @@ FROM druid_test_dst.test_base_table;
select * FROM druid_test_table_utc;
-CREATE TABLE druid_test_table_utc2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double)
+CREATE EXTERNAL TABLE druid_test_table_utc2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double)
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "DAY");
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/queries/clientpositive/druidkafkamini_basic.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/druidkafkamini_basic.q b/ql/src/test/queries/clientpositive/druidkafkamini_basic.q
index 2eb9dbc..b9f817e 100644
--- a/ql/src/test/queries/clientpositive/druidkafkamini_basic.q
+++ b/ql/src/test/queries/clientpositive/druidkafkamini_basic.q
@@ -1,5 +1,5 @@
SET hive.vectorized.execution.enabled=false;
-CREATE TABLE druid_kafka_test(`__time` timestamp, page string, `user` string, language string, added int, deleted int)
+CREATE EXTERNAL TABLE druid_kafka_test(`__time` timestamp, page string, `user` string, language string, added int, deleted int)
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES (
"druid.segment.granularity" = "MONTH",
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/queries/clientpositive/druidmini_dynamic_partition.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/druidmini_dynamic_partition.q b/ql/src/test/queries/clientpositive/druidmini_dynamic_partition.q
index 71d1c56..1b20862 100644
--- a/ql/src/test/queries/clientpositive/druidmini_dynamic_partition.q
+++ b/ql/src/test/queries/clientpositive/druidmini_dynamic_partition.q
@@ -1,6 +1,8 @@
--! qt:dataset:alltypesorc
SET hive.vectorized.execution.enabled=false;
-CREATE TABLE druid_partitioned_table_0
+SET hive.ctas.external.tables=true;
+SET hive.external.table.purge.default = true;
+CREATE EXTERNAL TABLE druid_partitioned_table_0
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES (
"druid.segment.granularity" = "HOUR",
@@ -21,7 +23,7 @@ CREATE TABLE druid_partitioned_table_0
cboolean2
FROM alltypesorc where ctimestamp1 IS NOT NULL;
-EXPLAIN CREATE TABLE druid_partitioned_table
+EXPLAIN CREATE EXTERNAL TABLE druid_partitioned_table
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES (
"druid.segment.granularity" = "HOUR",
@@ -44,7 +46,7 @@ EXPLAIN CREATE TABLE druid_partitioned_table
-CREATE TABLE druid_partitioned_table
+CREATE EXTERNAL TABLE druid_partitioned_table
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES (
"druid.segment.granularity" = "HOUR",
@@ -155,7 +157,7 @@ SELECT sum(cint), sum(cbigint) FROM druid_partitioned_table ;
set hive.druid.indexer.partition.size.max=10;
-CREATE TABLE druid_max_size_partition
+CREATE EXTERNAL TABLE druid_max_size_partition
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES (
"druid.segment.granularity" = "HOUR",
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/queries/clientpositive/druidmini_expressions.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/druidmini_expressions.q b/ql/src/test/queries/clientpositive/druidmini_expressions.q
index fad8f73..7857973 100644
--- a/ql/src/test/queries/clientpositive/druidmini_expressions.q
+++ b/ql/src/test/queries/clientpositive/druidmini_expressions.q
@@ -1,7 +1,8 @@
--! qt:dataset:alltypesorc
+SET hive.ctas.external.tables=true;
SET hive.vectorized.execution.enabled=false;
-CREATE TABLE druid_table_n0
+CREATE EXTERNAL TABLE druid_table_n0
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE")
AS
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/queries/clientpositive/druidmini_extractTime.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/druidmini_extractTime.q b/ql/src/test/queries/clientpositive/druidmini_extractTime.q
index 9541361..0dbcd25 100644
--- a/ql/src/test/queries/clientpositive/druidmini_extractTime.q
+++ b/ql/src/test/queries/clientpositive/druidmini_extractTime.q
@@ -1,7 +1,9 @@
--! qt:dataset:alltypesorc
SET hive.vectorized.execution.enabled=false;
-CREATE TABLE druid_table
+SET hive.ctas.external.tables=true;
+SET hive.external.table.purge.default = true;
+CREATE EXTERNAL TABLE druid_table
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE")
AS
@@ -181,7 +183,7 @@ SELECT CAST(`__time` AS DATE) AS `x_date` FROM druid_table ORDER BY `x_date` LIM
create table test_extract_from_string_base_table(`timecolumn` timestamp, `date_c` string, `timestamp_c` string, `metric_c` double);
insert into test_extract_from_string_base_table values ('2015-03-08 00:00:00', '2015-03-10', '2015-03-08 05:30:20', 5.0);
-CREATE TABLE druid_test_extract_from_string_table
+CREATE EXTERNAL TABLE druid_test_extract_from_string_table
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "DAY")
AS select
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/queries/clientpositive/druidmini_floorTime.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/druidmini_floorTime.q b/ql/src/test/queries/clientpositive/druidmini_floorTime.q
index a526413..3b5334f 100644
--- a/ql/src/test/queries/clientpositive/druidmini_floorTime.q
+++ b/ql/src/test/queries/clientpositive/druidmini_floorTime.q
@@ -1,7 +1,9 @@
--! qt:dataset:alltypesorc
SET hive.vectorized.execution.enabled=false;
-CREATE TABLE druid_table_n2
+SET hive.ctas.external.tables=true;
+SET hive.external.table.purge.default = true;
+CREATE EXTERNAL TABLE druid_table_n2
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE")
AS
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/queries/clientpositive/druidmini_joins.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/druidmini_joins.q b/ql/src/test/queries/clientpositive/druidmini_joins.q
index 720127e..1f92a0d 100644
--- a/ql/src/test/queries/clientpositive/druidmini_joins.q
+++ b/ql/src/test/queries/clientpositive/druidmini_joins.q
@@ -1,11 +1,12 @@
SET hive.vectorized.execution.enabled=false;
SET hive.explain.user=false;
-
+SET hive.ctas.external.tables=true;
+SET hive.external.table.purge.default = true;
--SET hive.execution.mode=llap;
DROP TABLE druid_table_with_nulls;
-CREATE TABLE druid_table_with_nulls
+CREATE EXTERNAL TABLE druid_table_with_nulls
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR")
AS
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/queries/clientpositive/druidmini_masking.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/druidmini_masking.q b/ql/src/test/queries/clientpositive/druidmini_masking.q
index f0f2c0c..824611f 100644
--- a/ql/src/test/queries/clientpositive/druidmini_masking.q
+++ b/ql/src/test/queries/clientpositive/druidmini_masking.q
@@ -1,7 +1,9 @@
set hive.mapred.mode=nonstrict;
set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest;
+SET hive.ctas.external.tables=true;
+SET hive.external.table.purge.default = true;
-CREATE TABLE masking_test_druid
+CREATE EXTERNAL TABLE masking_test_druid
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR")
AS
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/queries/clientpositive/druidmini_test1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/druidmini_test1.q b/ql/src/test/queries/clientpositive/druidmini_test1.q
index f93665e..30abf3c 100644
--- a/ql/src/test/queries/clientpositive/druidmini_test1.q
+++ b/ql/src/test/queries/clientpositive/druidmini_test1.q
@@ -1,5 +1,8 @@
--! qt:dataset:alltypesorc
-CREATE TABLE druid_table_n3
+SET hive.ctas.external.tables=true;
+SET hive.external.table.purge.default = true;
+
+CREATE EXTERNAL TABLE druid_table_n3
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE")
AS
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/queries/clientpositive/druidmini_test_alter.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/druidmini_test_alter.q b/ql/src/test/queries/clientpositive/druidmini_test_alter.q
index e19a006..0d8623d 100644
--- a/ql/src/test/queries/clientpositive/druidmini_test_alter.q
+++ b/ql/src/test/queries/clientpositive/druidmini_test_alter.q
@@ -1,4 +1,7 @@
-CREATE TABLE druid_alltypesorc_n0
+--! qt:dataset:alltypesorc
+SET hive.ctas.external.tables=true;
+SET hive.external.table.purge.default = true;
+CREATE EXTERNAL TABLE druid_alltypesorc_n0
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE")
AS
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/queries/clientpositive/druidmini_test_insert.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/druidmini_test_insert.q b/ql/src/test/queries/clientpositive/druidmini_test_insert.q
index 03657fd..09e4a19 100644
--- a/ql/src/test/queries/clientpositive/druidmini_test_insert.q
+++ b/ql/src/test/queries/clientpositive/druidmini_test_insert.q
@@ -1,5 +1,7 @@
--! qt:dataset:alltypesorc
-CREATE TABLE druid_alltypesorc
+SET hive.ctas.external.tables=true;
+SET hive.external.table.purge.default = true;
+CREATE EXTERNAL TABLE druid_alltypesorc
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE")
AS
@@ -63,7 +65,7 @@ DROP TABLE druid_alltypesorc;
insert into test_table_n9 values ('2015-01-08 00:00:00', 'i1-start', 4);
insert into test_table_n9 values ('2015-01-08 23:59:59', 'i1-end', 1);
- CREATE TABLE druid_table_n1 (`__time` timestamp with local time zone, `userid` string, `num_l` float)
+ CREATE EXTERNAL TABLE druid_table_n1 (`__time` timestamp with local time zone, `userid` string, `num_l` float)
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "DAY");
@@ -90,7 +92,7 @@ insert into test_base_table values ('2015-03-09 23:59:59', 'i2-end', 1);
insert into test_base_table values ('2015-03-10 00:00:00', 'i3-start', 2);
insert into test_base_table values ('2015-03-10 23:59:59', 'i3-end', 2);
-CREATE TABLE druid_test_table_n9
+CREATE EXTERNAL TABLE druid_test_table_n9
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "DAY")
AS
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/results/clientnegative/druid_datasource2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/druid_datasource2.q.out b/ql/src/test/results/clientnegative/druid_datasource2.q.out
index 1b74b06..2f783fe 100644
--- a/ql/src/test/results/clientnegative/druid_datasource2.q.out
+++ b/ql/src/test/results/clientnegative/druid_datasource2.q.out
@@ -4,4 +4,4 @@ TBLPROPERTIES ("property" = "localhost", "druid.datasource" = "mydatasource")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@druid_table_1
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Datasource name cannot be specified using [druid.datasource] for managed tables using Druid)
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Table default.druid_table_1 failed strict managed table checks due to the following reason: Table is marked as a managed table but is not transactional.)
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/results/clientpositive/druid/druid_timestamptz.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid/druid_timestamptz.q.out b/ql/src/test/results/clientpositive/druid/druid_timestamptz.q.out
index ed1a6ea..fa9583a 100644
--- a/ql/src/test/results/clientpositive/druid/druid_timestamptz.q.out
+++ b/ql/src/test/results/clientpositive/druid/druid_timestamptz.q.out
@@ -2,13 +2,13 @@ PREHOOK: query: drop table tstz1_n0
PREHOOK: type: DROPTABLE
POSTHOOK: query: drop table tstz1_n0
POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table tstz1_n0(`__time` timestamp with local time zone, n string, v integer)
+PREHOOK: query: create external table tstz1_n0(`__time` timestamp with local time zone, n string, v integer)
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@tstz1_n0
-POSTHOOK: query: create table tstz1_n0(`__time` timestamp with local time zone, n string, v integer)
+POSTHOOK: query: create external table tstz1_n0(`__time` timestamp with local time zone, n string, v integer)
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR")
POSTHOOK: type: CREATETABLE
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/results/clientpositive/druid/druid_timestamptz2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid/druid_timestamptz2.q.out b/ql/src/test/results/clientpositive/druid/druid_timestamptz2.q.out
index c71a435..b62095a 100644
--- a/ql/src/test/results/clientpositive/druid/druid_timestamptz2.q.out
+++ b/ql/src/test/results/clientpositive/druid/druid_timestamptz2.q.out
@@ -10,11 +10,11 @@ PREHOOK: Input: database:druid_test_dst
POSTHOOK: query: use druid_test_dst
POSTHOOK: type: SWITCHDATABASE
POSTHOOK: Input: database:druid_test_dst
-PREHOOK: query: create table test_base_table(`timecolumn` timestamp, `interval_marker` string, `num_l` double)
+PREHOOK: query: create external table test_base_table(`timecolumn` timestamp, `interval_marker` string, `num_l` double)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:druid_test_dst
PREHOOK: Output: druid_test_dst@test_base_table
-POSTHOOK: query: create table test_base_table(`timecolumn` timestamp, `interval_marker` string, `num_l` double)
+POSTHOOK: query: create external table test_base_table(`timecolumn` timestamp, `interval_marker` string, `num_l` double)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:druid_test_dst
POSTHOOK: Output: druid_test_dst@test_base_table
@@ -84,7 +84,7 @@ POSTHOOK: Output: druid_test_dst@test_base_table
POSTHOOK: Lineage: test_base_table.interval_marker SCRIPT []
POSTHOOK: Lineage: test_base_table.num_l SCRIPT []
POSTHOOK: Lineage: test_base_table.timecolumn SCRIPT []
-PREHOOK: query: CREATE TABLE druid_test_table_1
+PREHOOK: query: CREATE EXTERNAL TABLE druid_test_table_1
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "DAY")
AS
@@ -94,7 +94,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: druid_test_dst@test_base_table
PREHOOK: Output: database:druid_test_dst
PREHOOK: Output: druid_test_dst@druid_test_table_1
-POSTHOOK: query: CREATE TABLE druid_test_table_1
+POSTHOOK: query: CREATE EXTERNAL TABLE druid_test_table_1
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "DAY")
AS
@@ -121,13 +121,13 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
2015-03-09 23:59:59.0 US/Pacific i2-end 1.0
2015-03-10 00:00:00.0 US/Pacific i3-start 2.0
2015-03-10 23:59:59.0 US/Pacific i3-end 2.0
-PREHOOK: query: CREATE TABLE druid_test_table_2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double)
+PREHOOK: query: CREATE EXTERNAL TABLE druid_test_table_2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double)
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "DAY")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:druid_test_dst
PREHOOK: Output: druid_test_dst@druid_test_table_2
-POSTHOOK: query: CREATE TABLE druid_test_table_2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double)
+POSTHOOK: query: CREATE EXTERNAL TABLE druid_test_table_2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double)
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "DAY")
POSTHOOK: type: CREATETABLE
@@ -195,7 +195,7 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
2015-03-09 23:59:59.0 US/Pacific i2-end 1.0
2015-03-10 00:00:00.0 US/Pacific i3-start 2.0
2015-03-10 23:59:59.0 US/Pacific i3-end 2.0
-PREHOOK: query: CREATE TABLE druid_test_table_utc
+PREHOOK: query: CREATE EXTERNAL TABLE druid_test_table_utc
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "DAY")
AS
@@ -205,7 +205,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: druid_test_dst@test_base_table
PREHOOK: Output: database:druid_test_dst
PREHOOK: Output: druid_test_dst@druid_test_table_utc
-POSTHOOK: query: CREATE TABLE druid_test_table_utc
+POSTHOOK: query: CREATE EXTERNAL TABLE druid_test_table_utc
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "DAY")
AS
@@ -232,13 +232,13 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
2015-03-09 23:59:59.0 UTC i2-end 1.0
2015-03-10 00:00:00.0 UTC i3-start 2.0
2015-03-10 23:59:59.0 UTC i3-end 2.0
-PREHOOK: query: CREATE TABLE druid_test_table_utc2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double)
+PREHOOK: query: CREATE EXTERNAL TABLE druid_test_table_utc2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double)
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "DAY")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:druid_test_dst
PREHOOK: Output: druid_test_dst@druid_test_table_utc2
-POSTHOOK: query: CREATE TABLE druid_test_table_utc2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double)
+POSTHOOK: query: CREATE EXTERNAL TABLE druid_test_table_utc2 (`__time` timestamp with local time zone, `interval_marker` string, `num_l` double)
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "DAY")
POSTHOOK: type: CREATETABLE
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out b/ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out
index 774ed54..f76a36d 100644
--- a/ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out
+++ b/ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out
@@ -1,4 +1,4 @@
-PREHOOK: query: CREATE TABLE druid_kafka_test(`__time` timestamp, page string, `user` string, language string, added int, deleted int)
+PREHOOK: query: CREATE EXTERNAL TABLE druid_kafka_test(`__time` timestamp, page string, `user` string, language string, added int, deleted int)
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES (
"druid.segment.granularity" = "MONTH",
@@ -15,7 +15,7 @@ PREHOOK: query: CREATE TABLE druid_kafka_test(`__time` timestamp, page string, `
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@druid_kafka_test
-POSTHOOK: query: CREATE TABLE druid_kafka_test(`__time` timestamp, page string, `user` string, language string, added int, deleted int)
+POSTHOOK: query: CREATE EXTERNAL TABLE druid_kafka_test(`__time` timestamp, page string, `user` string, language string, added int, deleted int)
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES (
"druid.segment.granularity" = "MONTH",
@@ -340,6 +340,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"__time":"true","added":"true","deleted":"true","language":"true","page":"true","user":"true"}}
+ EXTERNAL TRUE
bucket_count -1
bucketing_version 2
column.name.delimiter ,
@@ -380,6 +381,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"__time":"true","added":"true","deleted":"true","language":"true","page":"true","user":"true"}}
+ EXTERNAL TRUE
bucket_count -1
bucketing_version 2
column.name.delimiter ,
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out b/ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out
index 1278a57..bd56df5 100644
--- a/ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out
+++ b/ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out
@@ -1,4 +1,4 @@
-PREHOOK: query: CREATE TABLE druid_partitioned_table_0
+PREHOOK: query: CREATE EXTERNAL TABLE druid_partitioned_table_0
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES (
"druid.segment.granularity" = "HOUR",
@@ -22,7 +22,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@druid_partitioned_table_0
-POSTHOOK: query: CREATE TABLE druid_partitioned_table_0
+POSTHOOK: query: CREATE EXTERNAL TABLE druid_partitioned_table_0
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES (
"druid.segment.granularity" = "HOUR",
@@ -57,7 +57,7 @@ POSTHOOK: Lineage: druid_partitioned_table_0.csmallint SIMPLE [(alltypesorc)allt
POSTHOOK: Lineage: druid_partitioned_table_0.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ]
POSTHOOK: Lineage: druid_partitioned_table_0.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ]
POSTHOOK: Lineage: druid_partitioned_table_0.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ]
-PREHOOK: query: EXPLAIN CREATE TABLE druid_partitioned_table
+PREHOOK: query: EXPLAIN CREATE EXTERNAL TABLE druid_partitioned_table
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES (
"druid.segment.granularity" = "HOUR",
@@ -78,7 +78,7 @@ PREHOOK: query: EXPLAIN CREATE TABLE druid_partitioned_table
cboolean2
FROM alltypesorc where ctimestamp1 IS NOT NULL
PREHOOK: type: CREATETABLE_AS_SELECT
-POSTHOOK: query: EXPLAIN CREATE TABLE druid_partitioned_table
+POSTHOOK: query: EXPLAIN CREATE EXTERNAL TABLE druid_partitioned_table
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES (
"druid.segment.granularity" = "HOUR",
@@ -166,6 +166,8 @@ STAGE PLANS:
druid.query.granularity MINUTE
druid.segment.granularity HOUR
druid.segment.targetShardsPerGranularity 6
+ external.table.purge true
+ isExternal: true
Stage: Stage-3
Stats Work
@@ -177,7 +179,7 @@ STAGE PLANS:
hdfs directory: true
destination: hdfs://### HDFS PATH ###
-PREHOOK: query: CREATE TABLE druid_partitioned_table
+PREHOOK: query: CREATE EXTERNAL TABLE druid_partitioned_table
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES (
"druid.segment.granularity" = "HOUR",
@@ -201,7 +203,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@druid_partitioned_table
-POSTHOOK: query: CREATE TABLE druid_partitioned_table
+POSTHOOK: query: CREATE EXTERNAL TABLE druid_partitioned_table
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES (
"druid.segment.granularity" = "HOUR",
@@ -570,7 +572,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@druid_partitioned_table
POSTHOOK: Output: hdfs://### HDFS PATH ###
1408069801800 10992545287
-PREHOOK: query: CREATE TABLE druid_max_size_partition
+PREHOOK: query: CREATE EXTERNAL TABLE druid_max_size_partition
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES (
"druid.segment.granularity" = "HOUR",
@@ -593,7 +595,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@druid_max_size_partition
-POSTHOOK: query: CREATE TABLE druid_max_size_partition
+POSTHOOK: query: CREATE EXTERNAL TABLE druid_max_size_partition
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES (
"druid.segment.granularity" = "HOUR",
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out b/ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out
index fd77a91..e322d8f 100644
--- a/ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out
+++ b/ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out
@@ -1,4 +1,4 @@
-PREHOOK: query: CREATE TABLE druid_table_n0
+PREHOOK: query: CREATE EXTERNAL TABLE druid_table_n0
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE")
AS
@@ -18,7 +18,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@druid_table_n0
-POSTHOOK: query: CREATE TABLE druid_table_n0
+POSTHOOK: query: CREATE EXTERNAL TABLE druid_table_n0
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE")
AS
@@ -270,6 +270,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ EXTERNAL TRUE
bucket_count -1
bucketing_version 2
column.name.delimiter ,
@@ -301,6 +302,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ EXTERNAL TRUE
bucket_count -1
bucketing_version 2
column.name.delimiter ,
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out b/ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out
index 30e273b..116c326 100644
--- a/ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out
+++ b/ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out
@@ -1,4 +1,4 @@
-PREHOOK: query: CREATE TABLE druid_table
+PREHOOK: query: CREATE EXTERNAL TABLE druid_table
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE")
AS
@@ -18,7 +18,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@druid_table
-POSTHOOK: query: CREATE TABLE druid_table
+POSTHOOK: query: CREATE EXTERNAL TABLE druid_table
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE")
AS
@@ -776,7 +776,7 @@ POSTHOOK: Lineage: test_extract_from_string_base_table.date_c SCRIPT []
POSTHOOK: Lineage: test_extract_from_string_base_table.metric_c SCRIPT []
POSTHOOK: Lineage: test_extract_from_string_base_table.timecolumn SCRIPT []
POSTHOOK: Lineage: test_extract_from_string_base_table.timestamp_c SCRIPT []
-PREHOOK: query: CREATE TABLE druid_test_extract_from_string_table
+PREHOOK: query: CREATE EXTERNAL TABLE druid_test_extract_from_string_table
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "DAY")
AS select
@@ -786,7 +786,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@test_extract_from_string_base_table
PREHOOK: Output: database:default
PREHOOK: Output: default@druid_test_extract_from_string_table
-POSTHOOK: query: CREATE TABLE druid_test_extract_from_string_table
+POSTHOOK: query: CREATE EXTERNAL TABLE druid_test_extract_from_string_table
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "DAY")
AS select
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/results/clientpositive/druid/druidmini_floorTime.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid/druidmini_floorTime.q.out b/ql/src/test/results/clientpositive/druid/druidmini_floorTime.q.out
index c54fd93..1c9e9c6 100644
--- a/ql/src/test/results/clientpositive/druid/druidmini_floorTime.q.out
+++ b/ql/src/test/results/clientpositive/druid/druidmini_floorTime.q.out
@@ -1,4 +1,4 @@
-PREHOOK: query: CREATE TABLE druid_table_n2
+PREHOOK: query: CREATE EXTERNAL TABLE druid_table_n2
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE")
AS
@@ -18,7 +18,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@druid_table_n2
-POSTHOOK: query: CREATE TABLE druid_table_n2
+POSTHOOK: query: CREATE EXTERNAL TABLE druid_table_n2
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE")
AS
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/results/clientpositive/druid/druidmini_joins.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid/druidmini_joins.q.out b/ql/src/test/results/clientpositive/druid/druidmini_joins.q.out
index 73a3c9f..014c7b5 100644
--- a/ql/src/test/results/clientpositive/druid/druidmini_joins.q.out
+++ b/ql/src/test/results/clientpositive/druid/druidmini_joins.q.out
@@ -2,7 +2,7 @@ PREHOOK: query: DROP TABLE druid_table_with_nulls
PREHOOK: type: DROPTABLE
POSTHOOK: query: DROP TABLE druid_table_with_nulls
POSTHOOK: type: DROPTABLE
-PREHOOK: query: CREATE TABLE druid_table_with_nulls
+PREHOOK: query: CREATE EXTERNAL TABLE druid_table_with_nulls
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR")
AS
@@ -22,7 +22,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: database:default
PREHOOK: Output: default@druid_table_with_nulls
-POSTHOOK: query: CREATE TABLE druid_table_with_nulls
+POSTHOOK: query: CREATE EXTERNAL TABLE druid_table_with_nulls
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR")
AS
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/results/clientpositive/druid/druidmini_masking.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid/druidmini_masking.q.out b/ql/src/test/results/clientpositive/druid/druidmini_masking.q.out
index 1aad967..e3b50a4 100644
--- a/ql/src/test/results/clientpositive/druid/druidmini_masking.q.out
+++ b/ql/src/test/results/clientpositive/druid/druidmini_masking.q.out
@@ -1,4 +1,4 @@
-PREHOOK: query: CREATE TABLE masking_test_druid
+PREHOOK: query: CREATE EXTERNAL TABLE masking_test_druid
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR")
AS
@@ -17,7 +17,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: database:default
PREHOOK: Output: default@masking_test_druid
-POSTHOOK: query: CREATE TABLE masking_test_druid
+POSTHOOK: query: CREATE EXTERNAL TABLE masking_test_druid
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR")
AS
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/results/clientpositive/druid/druidmini_test1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid/druidmini_test1.q.out b/ql/src/test/results/clientpositive/druid/druidmini_test1.q.out
index 4e078aa..7f6c6b0 100644
--- a/ql/src/test/results/clientpositive/druid/druidmini_test1.q.out
+++ b/ql/src/test/results/clientpositive/druid/druidmini_test1.q.out
@@ -1,4 +1,4 @@
-PREHOOK: query: CREATE TABLE druid_table_n3
+PREHOOK: query: CREATE EXTERNAL TABLE druid_table_n3
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE")
AS
@@ -18,7 +18,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@druid_table_n3
-POSTHOOK: query: CREATE TABLE druid_table_n3
+POSTHOOK: query: CREATE EXTERNAL TABLE druid_table_n3
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE")
AS
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/results/clientpositive/druid/druidmini_test_alter.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid/druidmini_test_alter.q.out b/ql/src/test/results/clientpositive/druid/druidmini_test_alter.q.out
index 439dc9c..a175f76 100644
--- a/ql/src/test/results/clientpositive/druid/druidmini_test_alter.q.out
+++ b/ql/src/test/results/clientpositive/druid/druidmini_test_alter.q.out
@@ -1,4 +1,4 @@
-PREHOOK: query: CREATE TABLE druid_alltypesorc_n0
+PREHOOK: query: CREATE EXTERNAL TABLE druid_alltypesorc_n0
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE")
AS
@@ -16,7 +16,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@druid_alltypesorc_n0
-POSTHOOK: query: CREATE TABLE druid_alltypesorc_n0
+POSTHOOK: query: CREATE EXTERNAL TABLE druid_alltypesorc_n0
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE")
AS
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out b/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out
index c675aeb..0da7610 100644
--- a/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out
+++ b/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out
@@ -1,4 +1,4 @@
-PREHOOK: query: CREATE TABLE druid_alltypesorc
+PREHOOK: query: CREATE EXTERNAL TABLE druid_alltypesorc
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE")
AS
@@ -18,7 +18,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@druid_alltypesorc
-POSTHOOK: query: CREATE TABLE druid_alltypesorc
+POSTHOOK: query: CREATE EXTERNAL TABLE druid_alltypesorc
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE")
AS
@@ -194,13 +194,13 @@ POSTHOOK: Output: druid_test_create_then_insert@test_table_n9
POSTHOOK: Lineage: test_table_n9.num_l SCRIPT []
POSTHOOK: Lineage: test_table_n9.timecolumn SCRIPT []
POSTHOOK: Lineage: test_table_n9.userid SCRIPT []
-PREHOOK: query: CREATE TABLE druid_table_n1 (`__time` timestamp with local time zone, `userid` string, `num_l` float)
+PREHOOK: query: CREATE EXTERNAL TABLE druid_table_n1 (`__time` timestamp with local time zone, `userid` string, `num_l` float)
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "DAY")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:druid_test_create_then_insert
PREHOOK: Output: druid_test_create_then_insert@druid_table_n1
-POSTHOOK: query: CREATE TABLE druid_table_n1 (`__time` timestamp with local time zone, `userid` string, `num_l` float)
+POSTHOOK: query: CREATE EXTERNAL TABLE druid_table_n1 (`__time` timestamp with local time zone, `userid` string, `num_l` float)
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "DAY")
POSTHOOK: type: CREATETABLE
@@ -335,7 +335,7 @@ POSTHOOK: Output: druid_test_dst@test_base_table
POSTHOOK: Lineage: test_base_table.num_l SCRIPT []
POSTHOOK: Lineage: test_base_table.timecolumn SCRIPT []
POSTHOOK: Lineage: test_base_table.userid SCRIPT []
-PREHOOK: query: CREATE TABLE druid_test_table_n9
+PREHOOK: query: CREATE EXTERNAL TABLE druid_test_table_n9
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "DAY")
AS
@@ -344,7 +344,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: druid_test_dst@test_base_table
PREHOOK: Output: database:druid_test_dst
PREHOOK: Output: druid_test_dst@druid_test_table_n9
-POSTHOOK: query: CREATE TABLE druid_test_table_n9
+POSTHOOK: query: CREATE EXTERNAL TABLE druid_test_table_n9
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.segment.granularity" = "DAY")
AS
http://git-wip-us.apache.org/repos/asf/hive/blob/406cde98/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
index 73924ee..7cdcd62 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
@@ -567,6 +567,25 @@ public class MetaStoreUtils {
return isExternal(params);
}
+ /**
+ * Determines whether an table needs to be purged or not.
+ *
+ * @param table table of interest
+ *
+ * @return true if external table needs to be purged
+ */
+ public static boolean isExternalTablePurge(Table table) {
+ if (table == null) {
+ return false;
+ }
+ Map<String, String> params = table.getParameters();
+ if (params == null) {
+ return false;
+ }
+
+ return isPropertyTrue(params, EXTERNAL_TABLE_PURGE);
+ }
+
public static boolean isExternal(Map<String, String> tableParams){
return isPropertyTrue(tableParams, "EXTERNAL");
}
[29/32] hive git commit: HIVE-20112: Accumulo-Hive (managed) table
creation fails with strict managed table checks: Table is marked as a managed
table but is not transactional (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by se...@apache.org.
HIVE-20112: Accumulo-Hive (managed) table creation fails with strict managed table checks: Table is marked as a managed table but is not transactional (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c4c55d2a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c4c55d2a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c4c55d2a
Branch: refs/heads/master-txnstats
Commit: c4c55d2abf2f21d776a7f45559084a838bac42eb
Parents: 1105ef3
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Tue Jul 10 09:03:49 2018 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Tue Jul 10 09:03:49 2018 -0700
----------------------------------------------------------------------
.../hive/accumulo/AccumuloStorageHandler.java | 19 ++------
.../accumulo/TestAccumuloStorageHandler.java | 44 +++++++++----------
.../test/queries/positive/accumulo_custom_key.q | 5 ++-
.../queries/positive/accumulo_custom_key2.q | 5 ++-
.../src/test/queries/positive/accumulo_index.q | 5 ++-
.../src/test/queries/positive/accumulo_joins.q | 25 ++++++-----
.../positive/accumulo_predicate_pushdown.q | 5 ++-
.../test/queries/positive/accumulo_queries.q | 35 +++++++++------
.../accumulo_single_sourced_multi_insert.q | 5 ++-
.../results/positive/accumulo_custom_key.q.out | 6 ++-
.../results/positive/accumulo_custom_key2.q.out | 6 ++-
.../test/results/positive/accumulo_index.q.out | 6 ++-
.../test/results/positive/accumulo_joins.q.out | 30 ++++++++-----
.../positive/accumulo_predicate_pushdown.q.out | 6 ++-
.../results/positive/accumulo_queries.q.out | 46 +++++++++++++-------
.../accumulo_single_sourced_multi_insert.q.out | 6 ++-
16 files changed, 147 insertions(+), 107 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c4c55d2a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/AccumuloStorageHandler.java
----------------------------------------------------------------------
diff --git a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/AccumuloStorageHandler.java b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/AccumuloStorageHandler.java
index 1a6e728..596eaa9 100644
--- a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/AccumuloStorageHandler.java
+++ b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/AccumuloStorageHandler.java
@@ -338,7 +338,6 @@ public class AccumuloStorageHandler extends DefaultStorageHandler implements Hiv
@Override
public void preCreateTable(Table table) throws MetaException {
- boolean isExternal = isExternalTable(table);
if (table.getSd().getLocation() != null) {
throw new MetaException("Location can't be specified for Accumulo");
}
@@ -357,17 +356,7 @@ public class AccumuloStorageHandler extends DefaultStorageHandler implements Hiv
// Attempt to create the table, taking EXTERNAL into consideration
if (!tableOpts.exists(tblName)) {
- if (!isExternal) {
- tableOpts.create(tblName);
- } else {
- throw new MetaException("Accumulo table " + tblName
- + " doesn't exist even though declared external");
- }
- } else {
- if (!isExternal) {
- throw new MetaException("Table " + tblName
- + " already exists in Accumulo. Use CREATE EXTERNAL TABLE to register with Hive.");
- }
+ tableOpts.create(tblName);
}
String idxTable = getIndexTableName(table);
@@ -388,8 +377,8 @@ public class AccumuloStorageHandler extends DefaultStorageHandler implements Hiv
}
}
- protected boolean isExternalTable(Table table) {
- return MetaStoreUtils.isExternalTable(table);
+ protected boolean isPurge(Table table) {
+ return !MetaStoreUtils.isExternalTable(table) || MetaStoreUtils.isExternalTablePurge(table);
}
@Override
@@ -406,7 +395,7 @@ public class AccumuloStorageHandler extends DefaultStorageHandler implements Hiv
@Override
public void commitDropTable(Table table, boolean deleteData) throws MetaException {
String tblName = getTableName(table);
- if (!isExternalTable(table)) {
+ if (isPurge(table)) {
try {
if (deleteData) {
TableOperations tblOpts = connectionParams.getConnector().tableOperations();
http://git-wip-us.apache.org/repos/asf/hive/blob/c4c55d2a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestAccumuloStorageHandler.java
----------------------------------------------------------------------
diff --git a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestAccumuloStorageHandler.java b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestAccumuloStorageHandler.java
index 58bf4a6..7b12362 100644
--- a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestAccumuloStorageHandler.java
+++ b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestAccumuloStorageHandler.java
@@ -200,8 +200,8 @@ public class TestAccumuloStorageHandler {
// Return our known table name
Mockito.when(storageHandler.getTableName(table)).thenReturn(tableName);
- // Not an EXTERNAL table
- Mockito.when(storageHandler.isExternalTable(table)).thenReturn(false);
+ // Is marked for purge
+ Mockito.when(storageHandler.isPurge(table)).thenReturn(true);
// Return the mocked StorageDescriptor
Mockito.when(table.getSd()).thenReturn(sd);
@@ -248,8 +248,8 @@ public class TestAccumuloStorageHandler {
// Return our known table name
Mockito.when(storageHandler.getTableName(table)).thenReturn(tableName);
- // Not an EXTERNAL table
- Mockito.when(storageHandler.isExternalTable(table)).thenReturn(false);
+ // Is marked for purge
+ Mockito.when(storageHandler.isPurge(table)).thenReturn(true);
// Return the mocked StorageDescriptor
Mockito.when(table.getSd()).thenReturn(sd);
@@ -294,8 +294,8 @@ public class TestAccumuloStorageHandler {
// Return our known table name
Mockito.when(storageHandler.getTableName(table)).thenReturn(tableName);
- // Not an EXTERNAL table
- Mockito.when(storageHandler.isExternalTable(table)).thenReturn(false);
+ // Is marked for purge
+ Mockito.when(storageHandler.isPurge(table)).thenReturn(true);
// Return the mocked StorageDescriptor
Mockito.when(table.getSd()).thenReturn(sd);
@@ -317,8 +317,8 @@ public class TestAccumuloStorageHandler {
storageHandler.preCreateTable(table);
}
- @Test(expected = MetaException.class)
- public void testExternalNonExistentTableFails() throws Exception {
+ @Test
+ public void testExternalNonExistentTable() throws Exception {
MockInstance inst = new MockInstance(test.getMethodName());
Connector conn = inst.getConnector("root", new PasswordToken(""));
String tableName = "table";
@@ -340,8 +340,8 @@ public class TestAccumuloStorageHandler {
// Return our known table name
Mockito.when(storageHandler.getTableName(table)).thenReturn(tableName);
- // Is an EXTERNAL table
- Mockito.when(storageHandler.isExternalTable(table)).thenReturn(true);
+ // Is not marked for purge
+ Mockito.when(storageHandler.isPurge(table)).thenReturn(false);
// Return the mocked StorageDescriptor
Mockito.when(table.getSd()).thenReturn(sd);
@@ -363,8 +363,8 @@ public class TestAccumuloStorageHandler {
storageHandler.preCreateTable(table);
}
- @Test(expected = MetaException.class)
- public void testNonExternalExistentTable() throws Exception {
+ @Test
+ public void testExternalExistentTable() throws Exception {
MockInstance inst = new MockInstance(test.getMethodName());
Connector conn = inst.getConnector("root", new PasswordToken(""));
String tableName = "table";
@@ -389,8 +389,8 @@ public class TestAccumuloStorageHandler {
// Return our known table name
Mockito.when(storageHandler.getTableName(table)).thenReturn(tableName);
- // Is not an EXTERNAL table
- Mockito.when(storageHandler.isExternalTable(table)).thenReturn(false);
+ // Is marked for purge
+ Mockito.when(storageHandler.isPurge(table)).thenReturn(true);
// Return the mocked StorageDescriptor
Mockito.when(table.getSd()).thenReturn(sd);
@@ -429,8 +429,8 @@ public class TestAccumuloStorageHandler {
// Return our known table name
Mockito.when(storageHandler.getTableName(table)).thenReturn(tableName);
- // Is not an EXTERNAL table
- Mockito.when(storageHandler.isExternalTable(table)).thenReturn(false);
+ // Is marked for purge
+ Mockito.when(storageHandler.isPurge(table)).thenReturn(true);
// Return the MockInstance's Connector
Mockito.when(connectionParams.getConnector()).thenReturn(conn);
@@ -461,8 +461,8 @@ public class TestAccumuloStorageHandler {
// Return our known table name
Mockito.when(storageHandler.getTableName(table)).thenReturn(tableName);
- // Is not an EXTERNAL table
- Mockito.when(storageHandler.isExternalTable(table)).thenReturn(false);
+ // Is marked for purge
+ Mockito.when(storageHandler.isPurge(table)).thenReturn(true);
// Return the MockInstance's Connector
Mockito.when(connectionParams.getConnector()).thenReturn(conn);
@@ -495,8 +495,8 @@ public class TestAccumuloStorageHandler {
// Return our known table name
Mockito.when(storageHandler.getTableName(table)).thenReturn(tableName);
- // Is not an EXTERNAL table
- Mockito.when(storageHandler.isExternalTable(table)).thenReturn(true);
+ // Is not marked for purge
+ Mockito.when(storageHandler.isPurge(table)).thenReturn(false);
// Return the MockInstance's Connector
Mockito.when(connectionParams.getConnector()).thenReturn(conn);
@@ -528,8 +528,8 @@ public class TestAccumuloStorageHandler {
// Return our known table name
Mockito.when(storageHandler.getTableName(table)).thenReturn(tableName);
- // Is not an EXTERNAL table
- Mockito.when(storageHandler.isExternalTable(table)).thenReturn(false);
+ // Is marked for purge
+ Mockito.when(storageHandler.isPurge(table)).thenReturn(true);
// Return the MockInstance's Connector
Mockito.when(connectionParams.getConnector()).thenReturn(conn);
http://git-wip-us.apache.org/repos/asf/hive/blob/c4c55d2a/accumulo-handler/src/test/queries/positive/accumulo_custom_key.q
----------------------------------------------------------------------
diff --git a/accumulo-handler/src/test/queries/positive/accumulo_custom_key.q b/accumulo-handler/src/test/queries/positive/accumulo_custom_key.q
index ca7e6c7..d5e18d6 100644
--- a/accumulo-handler/src/test/queries/positive/accumulo_custom_key.q
+++ b/accumulo-handler/src/test/queries/positive/accumulo_custom_key.q
@@ -1,11 +1,12 @@
--! qt:dataset:src
-CREATE TABLE accumulo_ck_1(key struct<col1:string,col2:string,col3:string>, value string)
+CREATE EXTERNAL TABLE accumulo_ck_1(key struct<col1:string,col2:string,col3:string>, value string)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.table.name" = "accumulo_custom",
"accumulo.columns.mapping" = ":rowid,cf:string",
"accumulo.composite.rowid.factory"="org.apache.hadoop.hive.accumulo.serde.DelimitedAccumuloRowIdFactory",
- "accumulo.composite.delimiter" = "$");
+ "accumulo.composite.delimiter" = "$")
+TBLPROPERTIES ("external.table.purge" = "true");
CREATE EXTERNAL TABLE accumulo_ck_2(key string, value string)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
http://git-wip-us.apache.org/repos/asf/hive/blob/c4c55d2a/accumulo-handler/src/test/queries/positive/accumulo_custom_key2.q
----------------------------------------------------------------------
diff --git a/accumulo-handler/src/test/queries/positive/accumulo_custom_key2.q b/accumulo-handler/src/test/queries/positive/accumulo_custom_key2.q
index 615e5f8..104ca16 100644
--- a/accumulo-handler/src/test/queries/positive/accumulo_custom_key2.q
+++ b/accumulo-handler/src/test/queries/positive/accumulo_custom_key2.q
@@ -1,10 +1,11 @@
--! qt:dataset:src
-CREATE TABLE accumulo_ck_3(key struct<col1:string,col2:string,col3:string>, value string)
+CREATE EXTERNAL TABLE accumulo_ck_3(key struct<col1:string,col2:string,col3:string>, value string)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.table.name" = "accumulo_custom2",
"accumulo.columns.mapping" = ":rowid,cf:string",
- "accumulo.composite.rowid"="org.apache.hadoop.hive.accumulo.serde.FirstCharAccumuloCompositeRowId");
+ "accumulo.composite.rowid"="org.apache.hadoop.hive.accumulo.serde.FirstCharAccumuloCompositeRowId")
+TBLPROPERTIES ("external.table.purge" = "true");
insert overwrite table accumulo_ck_3 select struct('abcd','mnop','wxyz'),'value'
from src where key = 100;
http://git-wip-us.apache.org/repos/asf/hive/blob/c4c55d2a/accumulo-handler/src/test/queries/positive/accumulo_index.q
----------------------------------------------------------------------
diff --git a/accumulo-handler/src/test/queries/positive/accumulo_index.q b/accumulo-handler/src/test/queries/positive/accumulo_index.q
index 52a33af..445cc74 100644
--- a/accumulo-handler/src/test/queries/positive/accumulo_index.q
+++ b/accumulo-handler/src/test/queries/positive/accumulo_index.q
@@ -1,6 +1,6 @@
DROP TABLE accumulo_index_test;
-CREATE TABLE accumulo_index_test (
+CREATE EXTERNAL TABLE accumulo_index_test (
rowid string,
active boolean,
num_offices tinyint,
@@ -21,7 +21,8 @@ WITH SERDEPROPERTIES (
"accumulo.table.name"="accumulo_index_test",
"accumulo.indexed.columns"="*",
"accumulo.indextable.name"="accumulo_index_idx"
- );
+ )
+TBLPROPERTIES ("external.table.purge" = "true");
insert into accumulo_index_test values( "row1", true, 55, 107, 555555, 1223232332,
http://git-wip-us.apache.org/repos/asf/hive/blob/c4c55d2a/accumulo-handler/src/test/queries/positive/accumulo_joins.q
----------------------------------------------------------------------
diff --git a/accumulo-handler/src/test/queries/positive/accumulo_joins.q b/accumulo-handler/src/test/queries/positive/accumulo_joins.q
index a6b09ab..9d93029 100644
--- a/accumulo-handler/src/test/queries/positive/accumulo_joins.q
+++ b/accumulo-handler/src/test/queries/positive/accumulo_joins.q
@@ -6,23 +6,26 @@ DROP TABLE users_level;
-- From HIVE-1257
-CREATE TABLE users(key string, state string, country string, country_id int)
+CREATE EXTERNAL TABLE users(key string, state string, country string, country_id int)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.columns.mapping" = ":rowID,info:state,info:country,info:country_id"
-);
+)
+TBLPROPERTIES ("external.table.purge" = "true");
-CREATE TABLE states(key string, name string)
+CREATE EXTERNAL TABLE states(key string, name string)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.columns.mapping" = ":rowID,state:name"
-);
+)
+TBLPROPERTIES ("external.table.purge" = "true");
-CREATE TABLE countries(key string, name string, country string, country_id int)
+CREATE EXTERNAL TABLE countries(key string, name string, country string, country_id int)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.columns.mapping" = ":rowID,info:name,info:country,info:country_id"
-);
+)
+TBLPROPERTIES ("external.table.purge" = "true");
INSERT OVERWRITE TABLE users SELECT 'user1', 'IA', 'USA', 0
FROM src WHERE key=100;
@@ -65,13 +68,15 @@ DROP TABLE users;
DROP TABLE states;
DROP TABLE countries;
-CREATE TABLE users(key int, userid int, username string, created int)
+CREATE EXTERNAL TABLE users(key int, userid int, username string, created int)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
-WITH SERDEPROPERTIES ("accumulo.columns.mapping" = ":rowID,f:userid,f:nickname,f:created");
+WITH SERDEPROPERTIES ("accumulo.columns.mapping" = ":rowID,f:userid,f:nickname,f:created")
+TBLPROPERTIES ("external.table.purge" = "true");
-CREATE TABLE users_level(key int, userid int, level int)
+CREATE EXTERNAL TABLE users_level(key int, userid int, level int)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
-WITH SERDEPROPERTIES ("accumulo.columns.mapping" = ":rowID,f:userid,f:level");
+WITH SERDEPROPERTIES ("accumulo.columns.mapping" = ":rowID,f:userid,f:level")
+TBLPROPERTIES ("external.table.purge" = "true");
-- HIVE-1903: the problem fixed here showed up even without any data,
-- so no need to load any to test it
http://git-wip-us.apache.org/repos/asf/hive/blob/c4c55d2a/accumulo-handler/src/test/queries/positive/accumulo_predicate_pushdown.q
----------------------------------------------------------------------
diff --git a/accumulo-handler/src/test/queries/positive/accumulo_predicate_pushdown.q b/accumulo-handler/src/test/queries/positive/accumulo_predicate_pushdown.q
index 4cb93b5..60a2869 100644
--- a/accumulo-handler/src/test/queries/positive/accumulo_predicate_pushdown.q
+++ b/accumulo-handler/src/test/queries/positive/accumulo_predicate_pushdown.q
@@ -1,8 +1,9 @@
--! qt:dataset:src
--! qt:dataset:part
-CREATE TABLE accumulo_pushdown(key string, value string)
+CREATE EXTERNAL TABLE accumulo_pushdown(key string, value string)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
-WITH SERDEPROPERTIES ("accumulo.columns.mapping" = ":rowid,cf:string");
+WITH SERDEPROPERTIES ("accumulo.columns.mapping" = ":rowid,cf:string")
+TBLPROPERTIES ("external.table.purge" = "true");
INSERT OVERWRITE TABLE accumulo_pushdown
SELECT cast(key as string), value
http://git-wip-us.apache.org/repos/asf/hive/blob/c4c55d2a/accumulo-handler/src/test/queries/positive/accumulo_queries.q
----------------------------------------------------------------------
diff --git a/accumulo-handler/src/test/queries/positive/accumulo_queries.q b/accumulo-handler/src/test/queries/positive/accumulo_queries.q
index bc93d8a..ed2fce3 100644
--- a/accumulo-handler/src/test/queries/positive/accumulo_queries.q
+++ b/accumulo-handler/src/test/queries/positive/accumulo_queries.q
@@ -6,10 +6,11 @@ set hive.optimize.ppd=false;
DROP TABLE accumulo_table_1;
-CREATE TABLE accumulo_table_1(key int, value string)
+CREATE EXTERNAL TABLE accumulo_table_1(key int, value string)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES ("accumulo.columns.mapping" = ":rowID,cf:string")
-TBLPROPERTIES ("accumulo.table.name" = "accumulo_table_0");
+TBLPROPERTIES ("accumulo.table.name" = "accumulo_table_0",
+ "external.table.purge" = "true");
DESCRIBE EXTENDED accumulo_table_1;
@@ -59,9 +60,10 @@ ON (x.key = Y.key)
ORDER BY key,value;
DROP TABLE empty_accumulo_table;
-CREATE TABLE empty_accumulo_table(key int, value string)
+CREATE EXTERNAL TABLE empty_accumulo_table(key int, value string)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
-WITH SERDEPROPERTIES ("accumulo.columns.mapping" = ":rowID,cf:string");
+WITH SERDEPROPERTIES ("accumulo.columns.mapping" = ":rowID,cf:string")
+TBLPROPERTIES ("external.table.purge" = "true");
DROP TABLE empty_normal_table;
CREATE TABLE empty_normal_table(key int, value string);
@@ -71,11 +73,12 @@ select * from (select count(1) c from empty_normal_table union all select count(
select * from (select count(1) c from src union all select count(1) as c from empty_accumulo_table) x order by c;
select * from (select count(1) c from src union all select count(1) as c from accumulo_table_1) x order by c;
-CREATE TABLE accumulo_table_3(key int, value string, count int)
+CREATE EXTERNAL TABLE accumulo_table_3(key int, value string, count int)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.columns.mapping" = ":rowID,cf:val,cf2:count"
-);
+)
+TBLPROPERTIES ("external.table.purge" = "true");
EXPLAIN
INSERT OVERWRITE TABLE accumulo_table_3
@@ -99,11 +102,12 @@ select * from accumulo_table_3 order by key, value limit 5;
select key, count from accumulo_table_3 order by key, count desc limit 5;
DROP TABLE accumulo_table_4;
-CREATE TABLE accumulo_table_4(key int, value1 string, value2 int, value3 int)
+CREATE EXTERNAL TABLE accumulo_table_4(key int, value1 string, value2 int, value3 int)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.columns.mapping" = ":rowID,a:b,a:c,d:e"
-);
+)
+TBLPROPERTIES ("external.table.purge" = "true");
INSERT OVERWRITE TABLE accumulo_table_4 SELECT key, value, key+1, key+2
FROM src WHERE key=98 OR key=100;
@@ -119,22 +123,24 @@ TBLPROPERTIES ("accumulo.table.name" = "accumulo_table_4");
SELECT * FROM accumulo_table_5 ORDER BY key;
DROP TABLE accumulo_table_6;
-CREATE TABLE accumulo_table_6(key int, value map<string,string>)
+CREATE EXTERNAL TABLE accumulo_table_6(key int, value map<string,string>)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.columns.mapping" = ":rowID,cf:*"
-);
+)
+TBLPROPERTIES ("external.table.purge" = "true");
INSERT OVERWRITE TABLE accumulo_table_6 SELECT key, map(value, key) FROM src
WHERE key=98 OR key=100;
SELECT * FROM accumulo_table_6 ORDER BY key;
DROP TABLE accumulo_table_7;
-CREATE TABLE accumulo_table_7(value map<string,string>, key int)
+CREATE EXTERNAL TABLE accumulo_table_7(value map<string,string>, key int)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.columns.mapping" = "cf:*,:rowID"
-);
+)
+TBLPROPERTIES ("external.table.purge" = "true");
INSERT OVERWRITE TABLE accumulo_table_7
SELECT map(value, key, upper(value), key+1), key FROM src
WHERE key=98 OR key=100;
@@ -142,11 +148,12 @@ WHERE key=98 OR key=100;
SELECT * FROM accumulo_table_7 ORDER BY key;
DROP TABLE accumulo_table_8;
-CREATE TABLE accumulo_table_8(key int, value1 string, value2 int, value3 int)
+CREATE EXTERNAL TABLE accumulo_table_8(key int, value1 string, value2 int, value3 int)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.columns.mapping" = ":rowID,a:b,a:c,d:e"
-);
+)
+TBLPROPERTIES ("external.table.purge" = "true");
INSERT OVERWRITE TABLE accumulo_table_8 SELECT key, value, key+1, key+2
FROM src WHERE key=98 OR key=100;
http://git-wip-us.apache.org/repos/asf/hive/blob/c4c55d2a/accumulo-handler/src/test/queries/positive/accumulo_single_sourced_multi_insert.q
----------------------------------------------------------------------
diff --git a/accumulo-handler/src/test/queries/positive/accumulo_single_sourced_multi_insert.q b/accumulo-handler/src/test/queries/positive/accumulo_single_sourced_multi_insert.q
index c12f962..56468a1 100644
--- a/accumulo-handler/src/test/queries/positive/accumulo_single_sourced_multi_insert.q
+++ b/accumulo-handler/src/test/queries/positive/accumulo_single_sourced_multi_insert.q
@@ -1,9 +1,10 @@
--! qt:dataset:src
-- HIVE-4375 Single sourced multi insert consists of native and non-native table mixed throws NPE
CREATE TABLE src_x1(key string, value string);
-CREATE TABLE src_x2(key string, value string)
+CREATE EXTERNAL TABLE src_x2(key string, value string)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
-WITH SERDEPROPERTIES ("accumulo.columns.mapping" = ":rowid, cf:value");
+WITH SERDEPROPERTIES ("accumulo.columns.mapping" = ":rowid, cf:value")
+TBLPROPERTIES ("external.table.purge" = "true");
explain
from src a
http://git-wip-us.apache.org/repos/asf/hive/blob/c4c55d2a/accumulo-handler/src/test/results/positive/accumulo_custom_key.q.out
----------------------------------------------------------------------
diff --git a/accumulo-handler/src/test/results/positive/accumulo_custom_key.q.out b/accumulo-handler/src/test/results/positive/accumulo_custom_key.q.out
index ca5f37b..328ac7d 100644
--- a/accumulo-handler/src/test/results/positive/accumulo_custom_key.q.out
+++ b/accumulo-handler/src/test/results/positive/accumulo_custom_key.q.out
@@ -1,20 +1,22 @@
-PREHOOK: query: CREATE TABLE accumulo_ck_1(key struct<col1:string,col2:string,col3:string>, value string)
+PREHOOK: query: CREATE EXTERNAL TABLE accumulo_ck_1(key struct<col1:string,col2:string,col3:string>, value string)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.table.name" = "accumulo_custom",
"accumulo.columns.mapping" = ":rowid,cf:string",
"accumulo.composite.rowid.factory"="org.apache.hadoop.hive.accumulo.serde.DelimitedAccumuloRowIdFactory",
"accumulo.composite.delimiter" = "$")
+TBLPROPERTIES ("external.table.purge" = "true")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@accumulo_ck_1
-POSTHOOK: query: CREATE TABLE accumulo_ck_1(key struct<col1:string,col2:string,col3:string>, value string)
+POSTHOOK: query: CREATE EXTERNAL TABLE accumulo_ck_1(key struct<col1:string,col2:string,col3:string>, value string)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.table.name" = "accumulo_custom",
"accumulo.columns.mapping" = ":rowid,cf:string",
"accumulo.composite.rowid.factory"="org.apache.hadoop.hive.accumulo.serde.DelimitedAccumuloRowIdFactory",
"accumulo.composite.delimiter" = "$")
+TBLPROPERTIES ("external.table.purge" = "true")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@accumulo_ck_1
http://git-wip-us.apache.org/repos/asf/hive/blob/c4c55d2a/accumulo-handler/src/test/results/positive/accumulo_custom_key2.q.out
----------------------------------------------------------------------
diff --git a/accumulo-handler/src/test/results/positive/accumulo_custom_key2.q.out b/accumulo-handler/src/test/results/positive/accumulo_custom_key2.q.out
index b9e1009..29d89df 100644
--- a/accumulo-handler/src/test/results/positive/accumulo_custom_key2.q.out
+++ b/accumulo-handler/src/test/results/positive/accumulo_custom_key2.q.out
@@ -1,18 +1,20 @@
-PREHOOK: query: CREATE TABLE accumulo_ck_3(key struct<col1:string,col2:string,col3:string>, value string)
+PREHOOK: query: CREATE EXTERNAL TABLE accumulo_ck_3(key struct<col1:string,col2:string,col3:string>, value string)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.table.name" = "accumulo_custom2",
"accumulo.columns.mapping" = ":rowid,cf:string",
"accumulo.composite.rowid"="org.apache.hadoop.hive.accumulo.serde.FirstCharAccumuloCompositeRowId")
+TBLPROPERTIES ("external.table.purge" = "true")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@accumulo_ck_3
-POSTHOOK: query: CREATE TABLE accumulo_ck_3(key struct<col1:string,col2:string,col3:string>, value string)
+POSTHOOK: query: CREATE EXTERNAL TABLE accumulo_ck_3(key struct<col1:string,col2:string,col3:string>, value string)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.table.name" = "accumulo_custom2",
"accumulo.columns.mapping" = ":rowid,cf:string",
"accumulo.composite.rowid"="org.apache.hadoop.hive.accumulo.serde.FirstCharAccumuloCompositeRowId")
+TBLPROPERTIES ("external.table.purge" = "true")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@accumulo_ck_3
http://git-wip-us.apache.org/repos/asf/hive/blob/c4c55d2a/accumulo-handler/src/test/results/positive/accumulo_index.q.out
----------------------------------------------------------------------
diff --git a/accumulo-handler/src/test/results/positive/accumulo_index.q.out b/accumulo-handler/src/test/results/positive/accumulo_index.q.out
index a7c66a7..65328e2 100644
--- a/accumulo-handler/src/test/results/positive/accumulo_index.q.out
+++ b/accumulo-handler/src/test/results/positive/accumulo_index.q.out
@@ -2,7 +2,7 @@ PREHOOK: query: DROP TABLE accumulo_index_test
PREHOOK: type: DROPTABLE
POSTHOOK: query: DROP TABLE accumulo_index_test
POSTHOOK: type: DROPTABLE
-PREHOOK: query: CREATE TABLE accumulo_index_test (
+PREHOOK: query: CREATE EXTERNAL TABLE accumulo_index_test (
rowid string,
active boolean,
num_offices tinyint,
@@ -24,10 +24,11 @@ WITH SERDEPROPERTIES (
"accumulo.indexed.columns"="*",
"accumulo.indextable.name"="accumulo_index_idx"
)
+TBLPROPERTIES ("external.table.purge" = "true")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@accumulo_index_test
-POSTHOOK: query: CREATE TABLE accumulo_index_test (
+POSTHOOK: query: CREATE EXTERNAL TABLE accumulo_index_test (
rowid string,
active boolean,
num_offices tinyint,
@@ -49,6 +50,7 @@ WITH SERDEPROPERTIES (
"accumulo.indexed.columns"="*",
"accumulo.indextable.name"="accumulo_index_idx"
)
+TBLPROPERTIES ("external.table.purge" = "true")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@accumulo_index_test
http://git-wip-us.apache.org/repos/asf/hive/blob/c4c55d2a/accumulo-handler/src/test/results/positive/accumulo_joins.q.out
----------------------------------------------------------------------
diff --git a/accumulo-handler/src/test/results/positive/accumulo_joins.q.out b/accumulo-handler/src/test/results/positive/accumulo_joins.q.out
index 5f11f94..ed65e08 100644
--- a/accumulo-handler/src/test/results/positive/accumulo_joins.q.out
+++ b/accumulo-handler/src/test/results/positive/accumulo_joins.q.out
@@ -14,51 +14,57 @@ PREHOOK: query: DROP TABLE users_level
PREHOOK: type: DROPTABLE
POSTHOOK: query: DROP TABLE users_level
POSTHOOK: type: DROPTABLE
-PREHOOK: query: CREATE TABLE users(key string, state string, country string, country_id int)
+PREHOOK: query: CREATE EXTERNAL TABLE users(key string, state string, country string, country_id int)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.columns.mapping" = ":rowID,info:state,info:country,info:country_id"
)
+TBLPROPERTIES ("external.table.purge" = "true")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@users
-POSTHOOK: query: CREATE TABLE users(key string, state string, country string, country_id int)
+POSTHOOK: query: CREATE EXTERNAL TABLE users(key string, state string, country string, country_id int)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.columns.mapping" = ":rowID,info:state,info:country,info:country_id"
)
+TBLPROPERTIES ("external.table.purge" = "true")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@users
-PREHOOK: query: CREATE TABLE states(key string, name string)
+PREHOOK: query: CREATE EXTERNAL TABLE states(key string, name string)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.columns.mapping" = ":rowID,state:name"
)
+TBLPROPERTIES ("external.table.purge" = "true")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@states
-POSTHOOK: query: CREATE TABLE states(key string, name string)
+POSTHOOK: query: CREATE EXTERNAL TABLE states(key string, name string)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.columns.mapping" = ":rowID,state:name"
)
+TBLPROPERTIES ("external.table.purge" = "true")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@states
-PREHOOK: query: CREATE TABLE countries(key string, name string, country string, country_id int)
+PREHOOK: query: CREATE EXTERNAL TABLE countries(key string, name string, country string, country_id int)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.columns.mapping" = ":rowID,info:name,info:country,info:country_id"
)
+TBLPROPERTIES ("external.table.purge" = "true")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@countries
-POSTHOOK: query: CREATE TABLE countries(key string, name string, country string, country_id int)
+POSTHOOK: query: CREATE EXTERNAL TABLE countries(key string, name string, country string, country_id int)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.columns.mapping" = ":rowID,info:name,info:country,info:country_id"
)
+TBLPROPERTIES ("external.table.purge" = "true")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@countries
@@ -218,27 +224,31 @@ POSTHOOK: query: DROP TABLE countries
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@countries
POSTHOOK: Output: default@countries
-PREHOOK: query: CREATE TABLE users(key int, userid int, username string, created int)
+PREHOOK: query: CREATE EXTERNAL TABLE users(key int, userid int, username string, created int)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES ("accumulo.columns.mapping" = ":rowID,f:userid,f:nickname,f:created")
+TBLPROPERTIES ("external.table.purge" = "true")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@users
-POSTHOOK: query: CREATE TABLE users(key int, userid int, username string, created int)
+POSTHOOK: query: CREATE EXTERNAL TABLE users(key int, userid int, username string, created int)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES ("accumulo.columns.mapping" = ":rowID,f:userid,f:nickname,f:created")
+TBLPROPERTIES ("external.table.purge" = "true")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@users
-PREHOOK: query: CREATE TABLE users_level(key int, userid int, level int)
+PREHOOK: query: CREATE EXTERNAL TABLE users_level(key int, userid int, level int)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES ("accumulo.columns.mapping" = ":rowID,f:userid,f:level")
+TBLPROPERTIES ("external.table.purge" = "true")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@users_level
-POSTHOOK: query: CREATE TABLE users_level(key int, userid int, level int)
+POSTHOOK: query: CREATE EXTERNAL TABLE users_level(key int, userid int, level int)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES ("accumulo.columns.mapping" = ":rowID,f:userid,f:level")
+TBLPROPERTIES ("external.table.purge" = "true")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@users_level
http://git-wip-us.apache.org/repos/asf/hive/blob/c4c55d2a/accumulo-handler/src/test/results/positive/accumulo_predicate_pushdown.q.out
----------------------------------------------------------------------
diff --git a/accumulo-handler/src/test/results/positive/accumulo_predicate_pushdown.q.out b/accumulo-handler/src/test/results/positive/accumulo_predicate_pushdown.q.out
index 6db7f5a..0389362 100644
--- a/accumulo-handler/src/test/results/positive/accumulo_predicate_pushdown.q.out
+++ b/accumulo-handler/src/test/results/positive/accumulo_predicate_pushdown.q.out
@@ -1,12 +1,14 @@
-PREHOOK: query: CREATE TABLE accumulo_pushdown(key string, value string)
+PREHOOK: query: CREATE EXTERNAL TABLE accumulo_pushdown(key string, value string)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES ("accumulo.columns.mapping" = ":rowid,cf:string")
+TBLPROPERTIES ("external.table.purge" = "true")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@accumulo_pushdown
-POSTHOOK: query: CREATE TABLE accumulo_pushdown(key string, value string)
+POSTHOOK: query: CREATE EXTERNAL TABLE accumulo_pushdown(key string, value string)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES ("accumulo.columns.mapping" = ":rowid,cf:string")
+TBLPROPERTIES ("external.table.purge" = "true")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@accumulo_pushdown
http://git-wip-us.apache.org/repos/asf/hive/blob/c4c55d2a/accumulo-handler/src/test/results/positive/accumulo_queries.q.out
----------------------------------------------------------------------
diff --git a/accumulo-handler/src/test/results/positive/accumulo_queries.q.out b/accumulo-handler/src/test/results/positive/accumulo_queries.q.out
index f7e7699..d0fdae4 100644
--- a/accumulo-handler/src/test/results/positive/accumulo_queries.q.out
+++ b/accumulo-handler/src/test/results/positive/accumulo_queries.q.out
@@ -2,17 +2,19 @@ PREHOOK: query: DROP TABLE accumulo_table_1
PREHOOK: type: DROPTABLE
POSTHOOK: query: DROP TABLE accumulo_table_1
POSTHOOK: type: DROPTABLE
-PREHOOK: query: CREATE TABLE accumulo_table_1(key int, value string)
+PREHOOK: query: CREATE EXTERNAL TABLE accumulo_table_1(key int, value string)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES ("accumulo.columns.mapping" = ":rowID,cf:string")
-TBLPROPERTIES ("accumulo.table.name" = "accumulo_table_0")
+TBLPROPERTIES ("accumulo.table.name" = "accumulo_table_0",
+ "external.table.purge" = "true")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@accumulo_table_1
-POSTHOOK: query: CREATE TABLE accumulo_table_1(key int, value string)
+POSTHOOK: query: CREATE EXTERNAL TABLE accumulo_table_1(key int, value string)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES ("accumulo.columns.mapping" = ":rowID,cf:string")
-TBLPROPERTIES ("accumulo.table.name" = "accumulo_table_0")
+TBLPROPERTIES ("accumulo.table.name" = "accumulo_table_0",
+ "external.table.purge" = "true")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@accumulo_table_1
@@ -393,15 +395,17 @@ PREHOOK: query: DROP TABLE empty_accumulo_table
PREHOOK: type: DROPTABLE
POSTHOOK: query: DROP TABLE empty_accumulo_table
POSTHOOK: type: DROPTABLE
-PREHOOK: query: CREATE TABLE empty_accumulo_table(key int, value string)
+PREHOOK: query: CREATE EXTERNAL TABLE empty_accumulo_table(key int, value string)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES ("accumulo.columns.mapping" = ":rowID,cf:string")
+TBLPROPERTIES ("external.table.purge" = "true")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@empty_accumulo_table
-POSTHOOK: query: CREATE TABLE empty_accumulo_table(key int, value string)
+POSTHOOK: query: CREATE EXTERNAL TABLE empty_accumulo_table(key int, value string)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES ("accumulo.columns.mapping" = ":rowID,cf:string")
+TBLPROPERTIES ("external.table.purge" = "true")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@empty_accumulo_table
@@ -465,19 +469,21 @@ POSTHOOK: Input: default@src
#### A masked pattern was here ####
155
500
-PREHOOK: query: CREATE TABLE accumulo_table_3(key int, value string, count int)
+PREHOOK: query: CREATE EXTERNAL TABLE accumulo_table_3(key int, value string, count int)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.columns.mapping" = ":rowID,cf:val,cf2:count"
)
+TBLPROPERTIES ("external.table.purge" = "true")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@accumulo_table_3
-POSTHOOK: query: CREATE TABLE accumulo_table_3(key int, value string, count int)
+POSTHOOK: query: CREATE EXTERNAL TABLE accumulo_table_3(key int, value string, count int)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.columns.mapping" = ":rowID,cf:val,cf2:count"
)
+TBLPROPERTIES ("external.table.purge" = "true")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@accumulo_table_3
@@ -663,19 +669,21 @@ PREHOOK: query: DROP TABLE accumulo_table_4
PREHOOK: type: DROPTABLE
POSTHOOK: query: DROP TABLE accumulo_table_4
POSTHOOK: type: DROPTABLE
-PREHOOK: query: CREATE TABLE accumulo_table_4(key int, value1 string, value2 int, value3 int)
+PREHOOK: query: CREATE EXTERNAL TABLE accumulo_table_4(key int, value1 string, value2 int, value3 int)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.columns.mapping" = ":rowID,a:b,a:c,d:e"
)
+TBLPROPERTIES ("external.table.purge" = "true")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@accumulo_table_4
-POSTHOOK: query: CREATE TABLE accumulo_table_4(key int, value1 string, value2 int, value3 int)
+POSTHOOK: query: CREATE EXTERNAL TABLE accumulo_table_4(key int, value1 string, value2 int, value3 int)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.columns.mapping" = ":rowID,a:b,a:c,d:e"
)
+TBLPROPERTIES ("external.table.purge" = "true")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@accumulo_table_4
@@ -731,19 +739,21 @@ PREHOOK: query: DROP TABLE accumulo_table_6
PREHOOK: type: DROPTABLE
POSTHOOK: query: DROP TABLE accumulo_table_6
POSTHOOK: type: DROPTABLE
-PREHOOK: query: CREATE TABLE accumulo_table_6(key int, value map<string,string>)
+PREHOOK: query: CREATE EXTERNAL TABLE accumulo_table_6(key int, value map<string,string>)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.columns.mapping" = ":rowID,cf:*"
)
+TBLPROPERTIES ("external.table.purge" = "true")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@accumulo_table_6
-POSTHOOK: query: CREATE TABLE accumulo_table_6(key int, value map<string,string>)
+POSTHOOK: query: CREATE EXTERNAL TABLE accumulo_table_6(key int, value map<string,string>)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.columns.mapping" = ":rowID,cf:*"
)
+TBLPROPERTIES ("external.table.purge" = "true")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@accumulo_table_6
@@ -771,19 +781,21 @@ PREHOOK: query: DROP TABLE accumulo_table_7
PREHOOK: type: DROPTABLE
POSTHOOK: query: DROP TABLE accumulo_table_7
POSTHOOK: type: DROPTABLE
-PREHOOK: query: CREATE TABLE accumulo_table_7(value map<string,string>, key int)
+PREHOOK: query: CREATE EXTERNAL TABLE accumulo_table_7(value map<string,string>, key int)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.columns.mapping" = "cf:*,:rowID"
)
+TBLPROPERTIES ("external.table.purge" = "true")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@accumulo_table_7
-POSTHOOK: query: CREATE TABLE accumulo_table_7(value map<string,string>, key int)
+POSTHOOK: query: CREATE EXTERNAL TABLE accumulo_table_7(value map<string,string>, key int)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.columns.mapping" = "cf:*,:rowID"
)
+TBLPROPERTIES ("external.table.purge" = "true")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@accumulo_table_7
@@ -813,19 +825,21 @@ PREHOOK: query: DROP TABLE accumulo_table_8
PREHOOK: type: DROPTABLE
POSTHOOK: query: DROP TABLE accumulo_table_8
POSTHOOK: type: DROPTABLE
-PREHOOK: query: CREATE TABLE accumulo_table_8(key int, value1 string, value2 int, value3 int)
+PREHOOK: query: CREATE EXTERNAL TABLE accumulo_table_8(key int, value1 string, value2 int, value3 int)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.columns.mapping" = ":rowID,a:b,a:c,d:e"
)
+TBLPROPERTIES ("external.table.purge" = "true")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@accumulo_table_8
-POSTHOOK: query: CREATE TABLE accumulo_table_8(key int, value1 string, value2 int, value3 int)
+POSTHOOK: query: CREATE EXTERNAL TABLE accumulo_table_8(key int, value1 string, value2 int, value3 int)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
"accumulo.columns.mapping" = ":rowID,a:b,a:c,d:e"
)
+TBLPROPERTIES ("external.table.purge" = "true")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@accumulo_table_8
http://git-wip-us.apache.org/repos/asf/hive/blob/c4c55d2a/accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out
----------------------------------------------------------------------
diff --git a/accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out b/accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out
index 5905ecd..7e36206 100644
--- a/accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out
+++ b/accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out
@@ -6,15 +6,17 @@ POSTHOOK: query: CREATE TABLE src_x1(key string, value string)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@src_x1
-PREHOOK: query: CREATE TABLE src_x2(key string, value string)
+PREHOOK: query: CREATE EXTERNAL TABLE src_x2(key string, value string)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES ("accumulo.columns.mapping" = ":rowid, cf:value")
+TBLPROPERTIES ("external.table.purge" = "true")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@src_x2
-POSTHOOK: query: CREATE TABLE src_x2(key string, value string)
+POSTHOOK: query: CREATE EXTERNAL TABLE src_x2(key string, value string)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES ("accumulo.columns.mapping" = ":rowid, cf:value")
+TBLPROPERTIES ("external.table.purge" = "true")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@src_x2
[24/32] hive git commit: HIVE-20126: OrcInputFormat does not pass
conf to orc reader options (Prasanth Jayachandran reviewed by Sergey
Shelukhin)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/18fb1b3f/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 2795e3d..9e012ce 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -194,6 +194,7 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\
orc_ppd_schema_evol_1b.q,\
orc_ppd_schema_evol_2a.q,\
orc_ppd_schema_evol_2b.q,\
+ orc_schema_evol_quoted.q,\
parallel.q,\
partition_pruning.q,\
ptf.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/18fb1b3f/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
index 3c11847..f34f393 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
@@ -325,7 +325,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
*/
TypeDescription schema = getDesiredRowTypeDescr(conf, false, Integer.MAX_VALUE);
- Reader.Options options = new Reader.Options().range(offset, length);
+ Reader.Options options = new Reader.Options(conf).range(offset, length);
options.schema(schema);
boolean isOriginal = isOriginal(file);
if (schema == null) {
@@ -2148,7 +2148,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
*/
TypeDescription schema =
OrcInputFormat.getDesiredRowTypeDescr(conf, true, Integer.MAX_VALUE);
- Reader.Options readerOptions = new Reader.Options().schema(schema);
+ Reader.Options readerOptions = new Reader.Options(conf).schema(schema);
// TODO: Convert genIncludedColumns and setSearchArgument to use TypeDescription.
final List<OrcProto.Type> schemaTypes = OrcUtils.getOrcTypes(schema);
readerOptions.include(OrcInputFormat.genIncludedColumns(schema, conf));
@@ -2397,7 +2397,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
mergerOptions.rootPath(deltaDirectory[0].getParent());
}
return new OrcRawRecordMerger(conf, collapseEvents, null, isOriginal,
- bucket, validWriteIdList, new Reader.Options(), deltaDirectory, mergerOptions);
+ bucket, validWriteIdList, new Reader.Options(conf), deltaDirectory, mergerOptions);
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/18fb1b3f/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java
index 892fcc0..8cb9af7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java
@@ -92,7 +92,7 @@ public class VectorizedOrcInputFormat extends FileInputFormat<NullWritable, Vect
}
}
List<OrcProto.Type> types = OrcUtils.getOrcTypes(schema);
- Reader.Options options = new Reader.Options().schema(schema);
+ Reader.Options options = new Reader.Options(conf).schema(schema);
this.offset = fileSplit.getStart();
this.length = fileSplit.getLength();
http://git-wip-us.apache.org/repos/asf/hive/blob/18fb1b3f/ql/src/test/queries/clientpositive/orc_schema_evol_quoted.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/orc_schema_evol_quoted.q b/ql/src/test/queries/clientpositive/orc_schema_evol_quoted.q
new file mode 100644
index 0000000..25461d2
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/orc_schema_evol_quoted.q
@@ -0,0 +1,42 @@
+set hive.support.quoted.identifiers=column;
+set orc.force.positional.evolution=true;
+set hive.vectorized.execution.enabled=false;
+
+CREATE TABLE studenttab10k(
+ name string,
+ age int,
+ gpa double
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
+STORED AS TEXTFILE;
+
+LOAD DATA LOCAL INPATH '../../data/files/studenttab10k' OVERWRITE INTO TABLE studenttab10k;
+
+create table temptable1(
+ `!@#$%^&*()_name` string,
+ age int,
+ gpa double
+) stored as orc;
+
+insert overwrite table temptable1 select * from studenttab10k;
+
+alter table temptable1 change age `!@#$%^&*()_age` int;
+alter table temptable1 change gpa `!@#$%^&*()_gpa` double;
+
+select `!@#$%^&*()_age`, count(*) from temptable1 group by `!@#$%^&*()_age` order by `!@#$%^&*()_age`;
+
+drop table temptable1;
+set hive.vectorized.execution.enabled=true;
+
+create table temptable1(
+ `!@#$%^&*()_name` string,
+ age int,
+ gpa double
+) stored as orc;
+
+insert overwrite table temptable1 select * from studenttab10k;
+
+alter table temptable1 change age `!@#$%^&*()_age` int;
+alter table temptable1 change gpa `!@#$%^&*()_gpa` double;
+
+select `!@#$%^&*()_age`, count(*) from temptable1 group by `!@#$%^&*()_age` order by `!@#$%^&*()_age`;
http://git-wip-us.apache.org/repos/asf/hive/blob/18fb1b3f/ql/src/test/results/clientpositive/llap/orc_schema_evol_quoted.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/orc_schema_evol_quoted.q.out b/ql/src/test/results/clientpositive/llap/orc_schema_evol_quoted.q.out
new file mode 100644
index 0000000..7e65fd8
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/orc_schema_evol_quoted.q.out
@@ -0,0 +1,258 @@
+PREHOOK: query: CREATE TABLE studenttab10k(
+ name string,
+ age int,
+ gpa double
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
+STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@studenttab10k
+POSTHOOK: query: CREATE TABLE studenttab10k(
+ name string,
+ age int,
+ gpa double
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
+STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@studenttab10k
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/studenttab10k' OVERWRITE INTO TABLE studenttab10k
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@studenttab10k
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/studenttab10k' OVERWRITE INTO TABLE studenttab10k
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@studenttab10k
+PREHOOK: query: create table temptable1(
+ `!@#$%^&*()_name` string,
+ age int,
+ gpa double
+) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@temptable1
+POSTHOOK: query: create table temptable1(
+ `!@#$%^&*()_name` string,
+ age int,
+ gpa double
+) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@temptable1
+PREHOOK: query: insert overwrite table temptable1 select * from studenttab10k
+PREHOOK: type: QUERY
+PREHOOK: Input: default@studenttab10k
+PREHOOK: Output: default@temptable1
+POSTHOOK: query: insert overwrite table temptable1 select * from studenttab10k
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@studenttab10k
+POSTHOOK: Output: default@temptable1
+POSTHOOK: Lineage: temptable1.!@#$%^&*()_name SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:name, type:string, comment:null), ]
+POSTHOOK: Lineage: temptable1.age SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:age, type:int, comment:null), ]
+POSTHOOK: Lineage: temptable1.gpa SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:gpa, type:double, comment:null), ]
+PREHOOK: query: alter table temptable1 change age `!@#$%^&*()_age` int
+PREHOOK: type: ALTERTABLE_RENAMECOL
+PREHOOK: Input: default@temptable1
+PREHOOK: Output: default@temptable1
+POSTHOOK: query: alter table temptable1 change age `!@#$%^&*()_age` int
+POSTHOOK: type: ALTERTABLE_RENAMECOL
+POSTHOOK: Input: default@temptable1
+POSTHOOK: Output: default@temptable1
+PREHOOK: query: alter table temptable1 change gpa `!@#$%^&*()_gpa` double
+PREHOOK: type: ALTERTABLE_RENAMECOL
+PREHOOK: Input: default@temptable1
+PREHOOK: Output: default@temptable1
+POSTHOOK: query: alter table temptable1 change gpa `!@#$%^&*()_gpa` double
+POSTHOOK: type: ALTERTABLE_RENAMECOL
+POSTHOOK: Input: default@temptable1
+POSTHOOK: Output: default@temptable1
+PREHOOK: query: select `!@#$%^&*()_age`, count(*) from temptable1 group by `!@#$%^&*()_age` order by `!@#$%^&*()_age`
+PREHOOK: type: QUERY
+PREHOOK: Input: default@temptable1
+#### A masked pattern was here ####
+POSTHOOK: query: select `!@#$%^&*()_age`, count(*) from temptable1 group by `!@#$%^&*()_age` order by `!@#$%^&*()_age`
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@temptable1
+#### A masked pattern was here ####
+18 160
+19 164
+20 166
+21 165
+22 184
+23 148
+24 166
+25 170
+26 174
+27 173
+28 181
+29 173
+30 159
+31 147
+32 176
+33 160
+34 165
+35 163
+36 150
+37 177
+38 163
+39 162
+40 164
+41 156
+42 141
+43 156
+44 144
+45 142
+46 174
+47 178
+48 169
+49 158
+50 162
+51 174
+52 175
+53 171
+54 179
+55 166
+56 175
+57 184
+58 176
+59 162
+60 168
+61 175
+62 185
+63 175
+64 181
+65 166
+66 151
+67 165
+68 191
+69 177
+70 142
+71 175
+72 182
+73 168
+74 181
+75 149
+76 162
+77 155
+PREHOOK: query: drop table temptable1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@temptable1
+PREHOOK: Output: default@temptable1
+POSTHOOK: query: drop table temptable1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@temptable1
+POSTHOOK: Output: default@temptable1
+PREHOOK: query: create table temptable1(
+ `!@#$%^&*()_name` string,
+ age int,
+ gpa double
+) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@temptable1
+POSTHOOK: query: create table temptable1(
+ `!@#$%^&*()_name` string,
+ age int,
+ gpa double
+) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@temptable1
+PREHOOK: query: insert overwrite table temptable1 select * from studenttab10k
+PREHOOK: type: QUERY
+PREHOOK: Input: default@studenttab10k
+PREHOOK: Output: default@temptable1
+POSTHOOK: query: insert overwrite table temptable1 select * from studenttab10k
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@studenttab10k
+POSTHOOK: Output: default@temptable1
+POSTHOOK: Lineage: temptable1.!@#$%^&*()_name SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:name, type:string, comment:null), ]
+POSTHOOK: Lineage: temptable1.age SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:age, type:int, comment:null), ]
+POSTHOOK: Lineage: temptable1.gpa SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:gpa, type:double, comment:null), ]
+PREHOOK: query: alter table temptable1 change age `!@#$%^&*()_age` int
+PREHOOK: type: ALTERTABLE_RENAMECOL
+PREHOOK: Input: default@temptable1
+PREHOOK: Output: default@temptable1
+POSTHOOK: query: alter table temptable1 change age `!@#$%^&*()_age` int
+POSTHOOK: type: ALTERTABLE_RENAMECOL
+POSTHOOK: Input: default@temptable1
+POSTHOOK: Output: default@temptable1
+PREHOOK: query: alter table temptable1 change gpa `!@#$%^&*()_gpa` double
+PREHOOK: type: ALTERTABLE_RENAMECOL
+PREHOOK: Input: default@temptable1
+PREHOOK: Output: default@temptable1
+POSTHOOK: query: alter table temptable1 change gpa `!@#$%^&*()_gpa` double
+POSTHOOK: type: ALTERTABLE_RENAMECOL
+POSTHOOK: Input: default@temptable1
+POSTHOOK: Output: default@temptable1
+PREHOOK: query: select `!@#$%^&*()_age`, count(*) from temptable1 group by `!@#$%^&*()_age` order by `!@#$%^&*()_age`
+PREHOOK: type: QUERY
+PREHOOK: Input: default@temptable1
+#### A masked pattern was here ####
+POSTHOOK: query: select `!@#$%^&*()_age`, count(*) from temptable1 group by `!@#$%^&*()_age` order by `!@#$%^&*()_age`
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@temptable1
+#### A masked pattern was here ####
+18 160
+19 164
+20 166
+21 165
+22 184
+23 148
+24 166
+25 170
+26 174
+27 173
+28 181
+29 173
+30 159
+31 147
+32 176
+33 160
+34 165
+35 163
+36 150
+37 177
+38 163
+39 162
+40 164
+41 156
+42 141
+43 156
+44 144
+45 142
+46 174
+47 178
+48 169
+49 158
+50 162
+51 174
+52 175
+53 171
+54 179
+55 166
+56 175
+57 184
+58 176
+59 162
+60 168
+61 175
+62 185
+63 175
+64 181
+65 166
+66 151
+67 165
+68 191
+69 177
+70 142
+71 175
+72 182
+73 168
+74 181
+75 149
+76 162
+77 155
http://git-wip-us.apache.org/repos/asf/hive/blob/18fb1b3f/ql/src/test/results/clientpositive/orc_schema_evol_quoted.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/orc_schema_evol_quoted.q.out b/ql/src/test/results/clientpositive/orc_schema_evol_quoted.q.out
new file mode 100644
index 0000000..7e65fd8
--- /dev/null
+++ b/ql/src/test/results/clientpositive/orc_schema_evol_quoted.q.out
@@ -0,0 +1,258 @@
+PREHOOK: query: CREATE TABLE studenttab10k(
+ name string,
+ age int,
+ gpa double
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
+STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@studenttab10k
+POSTHOOK: query: CREATE TABLE studenttab10k(
+ name string,
+ age int,
+ gpa double
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
+STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@studenttab10k
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/studenttab10k' OVERWRITE INTO TABLE studenttab10k
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@studenttab10k
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/studenttab10k' OVERWRITE INTO TABLE studenttab10k
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@studenttab10k
+PREHOOK: query: create table temptable1(
+ `!@#$%^&*()_name` string,
+ age int,
+ gpa double
+) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@temptable1
+POSTHOOK: query: create table temptable1(
+ `!@#$%^&*()_name` string,
+ age int,
+ gpa double
+) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@temptable1
+PREHOOK: query: insert overwrite table temptable1 select * from studenttab10k
+PREHOOK: type: QUERY
+PREHOOK: Input: default@studenttab10k
+PREHOOK: Output: default@temptable1
+POSTHOOK: query: insert overwrite table temptable1 select * from studenttab10k
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@studenttab10k
+POSTHOOK: Output: default@temptable1
+POSTHOOK: Lineage: temptable1.!@#$%^&*()_name SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:name, type:string, comment:null), ]
+POSTHOOK: Lineage: temptable1.age SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:age, type:int, comment:null), ]
+POSTHOOK: Lineage: temptable1.gpa SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:gpa, type:double, comment:null), ]
+PREHOOK: query: alter table temptable1 change age `!@#$%^&*()_age` int
+PREHOOK: type: ALTERTABLE_RENAMECOL
+PREHOOK: Input: default@temptable1
+PREHOOK: Output: default@temptable1
+POSTHOOK: query: alter table temptable1 change age `!@#$%^&*()_age` int
+POSTHOOK: type: ALTERTABLE_RENAMECOL
+POSTHOOK: Input: default@temptable1
+POSTHOOK: Output: default@temptable1
+PREHOOK: query: alter table temptable1 change gpa `!@#$%^&*()_gpa` double
+PREHOOK: type: ALTERTABLE_RENAMECOL
+PREHOOK: Input: default@temptable1
+PREHOOK: Output: default@temptable1
+POSTHOOK: query: alter table temptable1 change gpa `!@#$%^&*()_gpa` double
+POSTHOOK: type: ALTERTABLE_RENAMECOL
+POSTHOOK: Input: default@temptable1
+POSTHOOK: Output: default@temptable1
+PREHOOK: query: select `!@#$%^&*()_age`, count(*) from temptable1 group by `!@#$%^&*()_age` order by `!@#$%^&*()_age`
+PREHOOK: type: QUERY
+PREHOOK: Input: default@temptable1
+#### A masked pattern was here ####
+POSTHOOK: query: select `!@#$%^&*()_age`, count(*) from temptable1 group by `!@#$%^&*()_age` order by `!@#$%^&*()_age`
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@temptable1
+#### A masked pattern was here ####
+18 160
+19 164
+20 166
+21 165
+22 184
+23 148
+24 166
+25 170
+26 174
+27 173
+28 181
+29 173
+30 159
+31 147
+32 176
+33 160
+34 165
+35 163
+36 150
+37 177
+38 163
+39 162
+40 164
+41 156
+42 141
+43 156
+44 144
+45 142
+46 174
+47 178
+48 169
+49 158
+50 162
+51 174
+52 175
+53 171
+54 179
+55 166
+56 175
+57 184
+58 176
+59 162
+60 168
+61 175
+62 185
+63 175
+64 181
+65 166
+66 151
+67 165
+68 191
+69 177
+70 142
+71 175
+72 182
+73 168
+74 181
+75 149
+76 162
+77 155
+PREHOOK: query: drop table temptable1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@temptable1
+PREHOOK: Output: default@temptable1
+POSTHOOK: query: drop table temptable1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@temptable1
+POSTHOOK: Output: default@temptable1
+PREHOOK: query: create table temptable1(
+ `!@#$%^&*()_name` string,
+ age int,
+ gpa double
+) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@temptable1
+POSTHOOK: query: create table temptable1(
+ `!@#$%^&*()_name` string,
+ age int,
+ gpa double
+) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@temptable1
+PREHOOK: query: insert overwrite table temptable1 select * from studenttab10k
+PREHOOK: type: QUERY
+PREHOOK: Input: default@studenttab10k
+PREHOOK: Output: default@temptable1
+POSTHOOK: query: insert overwrite table temptable1 select * from studenttab10k
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@studenttab10k
+POSTHOOK: Output: default@temptable1
+POSTHOOK: Lineage: temptable1.!@#$%^&*()_name SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:name, type:string, comment:null), ]
+POSTHOOK: Lineage: temptable1.age SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:age, type:int, comment:null), ]
+POSTHOOK: Lineage: temptable1.gpa SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:gpa, type:double, comment:null), ]
+PREHOOK: query: alter table temptable1 change age `!@#$%^&*()_age` int
+PREHOOK: type: ALTERTABLE_RENAMECOL
+PREHOOK: Input: default@temptable1
+PREHOOK: Output: default@temptable1
+POSTHOOK: query: alter table temptable1 change age `!@#$%^&*()_age` int
+POSTHOOK: type: ALTERTABLE_RENAMECOL
+POSTHOOK: Input: default@temptable1
+POSTHOOK: Output: default@temptable1
+PREHOOK: query: alter table temptable1 change gpa `!@#$%^&*()_gpa` double
+PREHOOK: type: ALTERTABLE_RENAMECOL
+PREHOOK: Input: default@temptable1
+PREHOOK: Output: default@temptable1
+POSTHOOK: query: alter table temptable1 change gpa `!@#$%^&*()_gpa` double
+POSTHOOK: type: ALTERTABLE_RENAMECOL
+POSTHOOK: Input: default@temptable1
+POSTHOOK: Output: default@temptable1
+PREHOOK: query: select `!@#$%^&*()_age`, count(*) from temptable1 group by `!@#$%^&*()_age` order by `!@#$%^&*()_age`
+PREHOOK: type: QUERY
+PREHOOK: Input: default@temptable1
+#### A masked pattern was here ####
+POSTHOOK: query: select `!@#$%^&*()_age`, count(*) from temptable1 group by `!@#$%^&*()_age` order by `!@#$%^&*()_age`
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@temptable1
+#### A masked pattern was here ####
+18 160
+19 164
+20 166
+21 165
+22 184
+23 148
+24 166
+25 170
+26 174
+27 173
+28 181
+29 173
+30 159
+31 147
+32 176
+33 160
+34 165
+35 163
+36 150
+37 177
+38 163
+39 162
+40 164
+41 156
+42 141
+43 156
+44 144
+45 142
+46 174
+47 178
+48 169
+49 158
+50 162
+51 174
+52 175
+53 171
+54 179
+55 166
+56 175
+57 184
+58 176
+59 162
+60 168
+61 175
+62 185
+63 175
+64 181
+65 166
+66 151
+67 165
+68 191
+69 177
+70 142
+71 175
+72 182
+73 168
+74 181
+75 149
+76 162
+77 155
[13/32] hive git commit: HIVE-20060 Refactor HiveSchemaTool and
MetastoreSchemaTool (Alan Gates, reviewed by Daniel Dai)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/beeline/src/test/org/apache/hive/beeline/schematool/TestHiveSchemaTool.java
----------------------------------------------------------------------
diff --git a/beeline/src/test/org/apache/hive/beeline/schematool/TestHiveSchemaTool.java b/beeline/src/test/org/apache/hive/beeline/schematool/TestHiveSchemaTool.java
index 8b477bd..8514dc8 100644
--- a/beeline/src/test/org/apache/hive/beeline/schematool/TestHiveSchemaTool.java
+++ b/beeline/src/test/org/apache/hive/beeline/schematool/TestHiveSchemaTool.java
@@ -43,13 +43,13 @@ import static org.powermock.api.mockito.PowerMockito.verifyStatic;
@RunWith(PowerMockRunner.class)
@PowerMockIgnore("javax.management.*")
-@PrepareForTest({ HiveSchemaHelper.class, HiveSchemaTool.CommandBuilder.class })
+@PrepareForTest({ HiveSchemaHelper.class, HiveSchemaTool.HiveSchemaToolCommandBuilder.class })
public class TestHiveSchemaTool {
String scriptFile = System.getProperty("java.io.tmpdir") + File.separator + "someScript.sql";
@Mock
private HiveConf hiveConf;
- private HiveSchemaTool.CommandBuilder builder;
+ private HiveSchemaTool.HiveSchemaToolCommandBuilder builder;
private String pasword = "reallySimplePassword";
@Before
@@ -66,7 +66,7 @@ public class TestHiveSchemaTool {
if (!file.exists()) {
file.createNewFile();
}
- builder = new HiveSchemaTool.CommandBuilder(hiveConf, null, null, "testUser", pasword, scriptFile);
+ builder = new HiveSchemaTool.HiveSchemaToolCommandBuilder(hiveConf, null, null, "testUser", pasword, scriptFile);
}
@After
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolCatalogOps.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolCatalogOps.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolCatalogOps.java
new file mode 100644
index 0000000..42eb979
--- /dev/null
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolCatalogOps.java
@@ -0,0 +1,479 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.text.StrTokenizer;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.Catalog;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.Function;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.client.builder.CatalogBuilder;
+import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
+import org.apache.hadoop.hive.metastore.client.builder.FunctionBuilder;
+import org.apache.hadoop.hive.metastore.client.builder.PartitionBuilder;
+import org.apache.hadoop.hive.metastore.client.builder.TableBuilder;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.thrift.TException;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME;
+import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME;
+
+public class TestSchemaToolCatalogOps {
+ private static MetastoreSchemaTool schemaTool;
+ private static HiveConf conf;
+ private IMetaStoreClient client;
+ private static String testMetastoreDB;
+ private static String argsBase;
+
+ @BeforeClass
+ public static void initDb() throws HiveMetaException, IOException {
+ conf = new HiveConf();
+ MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.AUTO_CREATE_ALL, false);
+ MetastoreConf.setLongVar(conf, MetastoreConf.ConfVars.HMS_HANDLER_ATTEMPTS, 1);
+ MetastoreConf.setLongVar(conf, MetastoreConf.ConfVars.THRIFT_CONNECTION_RETRIES, 1);
+ testMetastoreDB = System.getProperty("java.io.tmpdir") +
+ File.separator + "testschematoolcatopsdb";
+ MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CONNECT_URL_KEY,
+ "jdbc:derby:" + testMetastoreDB + ";create=true");
+ schemaTool = new MetastoreSchemaTool();
+ schemaTool.init(System.getProperty("test.tmp.dir", "target/tmp"),
+ new String[]{"-dbType", "derby", "--info"}, null, conf);
+
+ String userName = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.CONNECTION_USER_NAME);
+ String passWord = MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.PWD);
+ schemaTool.setUserName(userName);
+ schemaTool.setPassWord(passWord);
+
+ argsBase = "-dbType derby -userName " + userName + " -passWord " + passWord + " ";
+ execute(new SchemaToolTaskInit(), "-initSchema"); // Pre-install the database so all the tables are there.
+ }
+
+ @AfterClass
+ public static void removeDb() throws Exception {
+ File metaStoreDir = new File(testMetastoreDB);
+ if (metaStoreDir.exists()) {
+ FileUtils.forceDeleteOnExit(metaStoreDir);
+ }
+ }
+
+ @Before
+ public void createClient() throws MetaException {
+ client = new HiveMetaStoreClient(conf);
+ }
+
+ @Test
+ public void createCatalog() throws HiveMetaException, TException {
+ String catName = "my_test_catalog";
+ String location = "file:///tmp/my_test_catalog";
+ String description = "very descriptive";
+ String argsCreate = String.format("-createCatalog %s -catalogLocation \"%s\" -catalogDescription \"%s\"",
+ catName, location, description);
+ execute(new SchemaToolTaskCreateCatalog(), argsCreate);
+
+ Catalog cat = client.getCatalog(catName);
+ Assert.assertEquals(location, cat.getLocationUri());
+ Assert.assertEquals(description, cat.getDescription());
+ }
+
+ @Test(expected = HiveMetaException.class)
+ public void createExistingCatalog() throws HiveMetaException {
+ String catName = "hive";
+ String location = "somewhere";
+ String argsCreate = String.format("-createCatalog %s -catalogLocation \"%s\"",
+ catName, location);
+ execute(new SchemaToolTaskCreateCatalog(), argsCreate);
+ }
+
+ @Test
+ public void createExistingCatalogWithIfNotExists() throws HiveMetaException {
+ String catName = "my_existing_test_catalog";
+ String location = "file:///tmp/my_test_catalog";
+ String description = "very descriptive";
+ String argsCreate1 = String.format("-createCatalog %s -catalogLocation \"%s\" -catalogDescription \"%s\"",
+ catName, location, description);
+ execute(new SchemaToolTaskCreateCatalog(), argsCreate1);
+
+ String argsCreate2 =
+ String.format("-createCatalog %s -catalogLocation \"%s\" -catalogDescription \"%s\" -ifNotExists",
+ catName, location, description);
+ execute(new SchemaToolTaskCreateCatalog(), argsCreate2);
+ }
+
+ @Test
+ public void alterCatalog() throws HiveMetaException, TException {
+ String catName = "an_alterable_catalog";
+ String location = "file:///tmp/an_alterable_catalog";
+ String description = "description";
+ String argsCreate = String.format("-createCatalog %s -catalogLocation \"%s\" -catalogDescription \"%s\"",
+ catName, location, description);
+ execute(new SchemaToolTaskCreateCatalog(), argsCreate);
+
+ location = "file:///tmp/somewhere_else";
+ String argsAlter1 = String.format("-alterCatalog %s -catalogLocation \"%s\"",
+ catName, location);
+ execute(new SchemaToolTaskAlterCatalog(), argsAlter1);
+ Catalog cat = client.getCatalog(catName);
+ Assert.assertEquals(location, cat.getLocationUri());
+ Assert.assertEquals(description, cat.getDescription());
+
+ description = "a better description";
+ String argsAlter2 = String.format("-alterCatalog %s -catalogDescription \"%s\"",
+ catName, description);
+ execute(new SchemaToolTaskAlterCatalog(), argsAlter2);
+ cat = client.getCatalog(catName);
+ Assert.assertEquals(location, cat.getLocationUri());
+ Assert.assertEquals(description, cat.getDescription());
+
+ location = "file:///tmp/a_third_location";
+ description = "best description yet";
+ String argsAlter3 = String.format("-alterCatalog %s -catalogLocation \"%s\" -catalogDescription \"%s\"",
+ catName, location, description);
+ execute(new SchemaToolTaskAlterCatalog(), argsAlter3);
+ cat = client.getCatalog(catName);
+ Assert.assertEquals(location, cat.getLocationUri());
+ Assert.assertEquals(description, cat.getDescription());
+ }
+
+ @Test(expected = HiveMetaException.class)
+ public void alterBogusCatalog() throws HiveMetaException {
+ String catName = "nosuch";
+ String location = "file:///tmp/somewhere";
+ String description = "whatever";
+ String argsAlter = String.format("-alterCatalog %s -catalogLocation \"%s\" -catalogDescription \"%s\"",
+ catName, location, description);
+ execute(new SchemaToolTaskAlterCatalog(), argsAlter);
+ }
+
+ @Test(expected = HiveMetaException.class)
+ public void alterCatalogNoChange() throws HiveMetaException {
+ String catName = "alter_cat_no_change";
+ String location = "file:///tmp/alter_cat_no_change";
+ String description = "description";
+ String argsCreate = String.format("-createCatalog %s -catalogLocation \"%s\" -catalogDescription \"%s\"",
+ catName, location, description);
+ execute(new SchemaToolTaskCreateCatalog(), argsCreate);
+
+ String argsAlter = String.format("-alterCatalog %s", catName);
+ execute(new SchemaToolTaskAlterCatalog(), argsAlter);
+ }
+
+ @Test
+ public void moveDatabase() throws HiveMetaException, TException {
+ String toCatName = "moveDbCat";
+ String dbName = "moveDbDb";
+ String tableName = "moveDbTable";
+ String funcName = "movedbfunc";
+ String partVal = "moveDbKey";
+
+ new CatalogBuilder()
+ .setName(toCatName)
+ .setLocation("file:///tmp")
+ .create(client);
+
+ Database db = new DatabaseBuilder()
+ .setCatalogName(DEFAULT_CATALOG_NAME)
+ .setName(dbName)
+ .create(client, conf);
+
+ new FunctionBuilder()
+ .inDb(db)
+ .setName(funcName)
+ .setClass("org.apache.hive.myudf")
+ .create(client, conf);
+
+ Table table = new TableBuilder()
+ .inDb(db)
+ .setTableName(tableName)
+ .addCol("a", "int")
+ .addPartCol("p", "string")
+ .create(client, conf);
+
+ new PartitionBuilder()
+ .inTable(table)
+ .addValue(partVal)
+ .addToTable(client, conf);
+
+ String argsMoveDB = String.format("-moveDatabase %s -fromCatalog %s -toCatalog %s", dbName,
+ DEFAULT_CATALOG_NAME, toCatName);
+ execute(new SchemaToolTaskMoveDatabase(), argsMoveDB);
+
+ Database fetchedDb = client.getDatabase(toCatName, dbName);
+ Assert.assertNotNull(fetchedDb);
+ Assert.assertEquals(toCatName.toLowerCase(), fetchedDb.getCatalogName());
+
+ Function fetchedFunction = client.getFunction(toCatName, dbName, funcName);
+ Assert.assertNotNull(fetchedFunction);
+ Assert.assertEquals(toCatName.toLowerCase(), fetchedFunction.getCatName());
+ Assert.assertEquals(dbName.toLowerCase(), fetchedFunction.getDbName());
+
+ Table fetchedTable = client.getTable(toCatName, dbName, tableName);
+ Assert.assertNotNull(fetchedTable);
+ Assert.assertEquals(toCatName.toLowerCase(), fetchedTable.getCatName());
+ Assert.assertEquals(dbName.toLowerCase(), fetchedTable.getDbName());
+
+ Partition fetchedPart =
+ client.getPartition(toCatName, dbName, tableName, Collections.singletonList(partVal));
+ Assert.assertNotNull(fetchedPart);
+ Assert.assertEquals(toCatName.toLowerCase(), fetchedPart.getCatName());
+ Assert.assertEquals(dbName.toLowerCase(), fetchedPart.getDbName());
+ Assert.assertEquals(tableName.toLowerCase(), fetchedPart.getTableName());
+ }
+
+ @Test
+ public void moveDatabaseWithExistingDbOfSameNameAlreadyInTargetCatalog()
+ throws TException, HiveMetaException {
+ String catName = "clobberCatalog";
+ new CatalogBuilder()
+ .setName(catName)
+ .setLocation("file:///tmp")
+ .create(client);
+ try {
+ String argsMoveDB = String.format("-moveDatabase %s -fromCatalog %s -toCatalog %s",
+ DEFAULT_DATABASE_NAME, catName, DEFAULT_CATALOG_NAME);
+ execute(new SchemaToolTaskMoveDatabase(), argsMoveDB);
+ Assert.fail("Attempt to move default database should have failed.");
+ } catch (HiveMetaException e) {
+ // good
+ }
+
+ // Make sure nothing really moved
+ Set<String> dbNames = new HashSet<>(client.getAllDatabases(DEFAULT_CATALOG_NAME));
+ Assert.assertTrue(dbNames.contains(DEFAULT_DATABASE_NAME));
+ }
+
+ @Test(expected = HiveMetaException.class)
+ public void moveNonExistentDatabase() throws TException, HiveMetaException {
+ String catName = "moveNonExistentDb";
+ new CatalogBuilder()
+ .setName(catName)
+ .setLocation("file:///tmp")
+ .create(client);
+ String argsMoveDB = String.format("-moveDatabase nosuch -fromCatalog %s -toCatalog %s",
+ catName, DEFAULT_CATALOG_NAME);
+ execute(new SchemaToolTaskMoveDatabase(), argsMoveDB);
+ }
+
+ @Test
+ public void moveDbToNonExistentCatalog() throws TException, HiveMetaException {
+ String dbName = "doomedToHomelessness";
+ new DatabaseBuilder()
+ .setName(dbName)
+ .create(client, conf);
+ try {
+ String argsMoveDB = String.format("-moveDatabase %s -fromCatalog %s -toCatalog nosuch",
+ dbName, DEFAULT_CATALOG_NAME);
+ execute(new SchemaToolTaskMoveDatabase(), argsMoveDB);
+ Assert.fail("Attempt to move database to non-existent catalog should have failed.");
+ } catch (HiveMetaException e) {
+ // good
+ }
+
+ // Make sure nothing really moved
+ Set<String> dbNames = new HashSet<>(client.getAllDatabases(DEFAULT_CATALOG_NAME));
+ Assert.assertTrue(dbNames.contains(dbName.toLowerCase()));
+ }
+
+ @Test
+ public void moveTable() throws TException, HiveMetaException {
+ String toCatName = "moveTableCat";
+ String toDbName = "moveTableDb";
+ String tableName = "moveTableTable";
+ String partVal = "moveTableKey";
+
+ new CatalogBuilder()
+ .setName(toCatName)
+ .setLocation("file:///tmp")
+ .create(client);
+
+ new DatabaseBuilder()
+ .setCatalogName(toCatName)
+ .setName(toDbName)
+ .create(client, conf);
+
+ Table table = new TableBuilder()
+ .setTableName(tableName)
+ .addCol("a", "int")
+ .addPartCol("p", "string")
+ .create(client, conf);
+
+ new PartitionBuilder()
+ .inTable(table)
+ .addValue(partVal)
+ .addToTable(client, conf);
+
+ String argsMoveTable = String.format("-moveTable %s -fromCatalog %s -toCatalog %s -fromDatabase %s -toDatabase %s",
+ tableName, DEFAULT_CATALOG_NAME, toCatName, DEFAULT_DATABASE_NAME, toDbName);
+ execute(new SchemaToolTaskMoveTable(), argsMoveTable);
+
+ Table fetchedTable = client.getTable(toCatName, toDbName, tableName);
+ Assert.assertNotNull(fetchedTable);
+ Assert.assertEquals(toCatName.toLowerCase(), fetchedTable.getCatName());
+ Assert.assertEquals(toDbName.toLowerCase(), fetchedTable.getDbName());
+
+ Partition fetchedPart =
+ client.getPartition(toCatName, toDbName, tableName, Collections.singletonList(partVal));
+ Assert.assertNotNull(fetchedPart);
+ Assert.assertEquals(toCatName.toLowerCase(), fetchedPart.getCatName());
+ Assert.assertEquals(toDbName.toLowerCase(), fetchedPart.getDbName());
+ Assert.assertEquals(tableName.toLowerCase(), fetchedPart.getTableName());
+ }
+
+ @Test
+ public void moveTableWithinCatalog() throws TException, HiveMetaException {
+ String toDbName = "moveTableWithinCatalogDb";
+ String tableName = "moveTableWithinCatalogTable";
+ String partVal = "moveTableWithinCatalogKey";
+
+ new DatabaseBuilder()
+ .setName(toDbName)
+ .create(client, conf);
+
+ Table table = new TableBuilder()
+ .setTableName(tableName)
+ .addCol("a", "int")
+ .addPartCol("p", "string")
+ .create(client, conf);
+
+ new PartitionBuilder()
+ .inTable(table)
+ .addValue(partVal)
+ .addToTable(client, conf);
+
+ String argsMoveTable = String.format("-moveTable %s -fromCatalog %s -toCatalog %s -fromDatabase %s -toDatabase %s",
+ tableName, DEFAULT_CATALOG_NAME, DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME, toDbName);
+ execute(new SchemaToolTaskMoveTable(), argsMoveTable);
+
+ Table fetchedTable = client.getTable(DEFAULT_CATALOG_NAME, toDbName, tableName);
+ Assert.assertNotNull(fetchedTable);
+ Assert.assertEquals(DEFAULT_CATALOG_NAME, fetchedTable.getCatName());
+ Assert.assertEquals(toDbName.toLowerCase(), fetchedTable.getDbName());
+
+ Partition fetchedPart =
+ client.getPartition(DEFAULT_CATALOG_NAME, toDbName, tableName, Collections.singletonList(partVal));
+ Assert.assertNotNull(fetchedPart);
+ Assert.assertEquals(DEFAULT_CATALOG_NAME, fetchedPart.getCatName());
+ Assert.assertEquals(toDbName.toLowerCase(), fetchedPart.getDbName());
+ Assert.assertEquals(tableName.toLowerCase(), fetchedPart.getTableName());
+ }
+
+ @Test
+ public void moveTableWithExistingTableOfSameNameAlreadyInTargetDatabase()
+ throws TException, HiveMetaException {
+ String toDbName = "clobberTableDb";
+ String tableName = "clobberTableTable";
+
+ Database toDb = new DatabaseBuilder()
+ .setName(toDbName)
+ .create(client, conf);
+
+ new TableBuilder()
+ .setTableName(tableName)
+ .addCol("a", "int")
+ .create(client, conf);
+
+ new TableBuilder()
+ .inDb(toDb)
+ .setTableName(tableName)
+ .addCol("b", "varchar(32)")
+ .create(client, conf);
+
+ try {
+ String argsMoveTable =
+ String.format("-moveTable %s -fromCatalog %s -toCatalog %s -fromDatabase %s -toDatabase %s",
+ tableName, DEFAULT_CATALOG_NAME, DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME, toDbName);
+ execute(new SchemaToolTaskMoveTable(), argsMoveTable);
+ Assert.fail("Attempt to move table should have failed.");
+ } catch (HiveMetaException e) {
+ // good
+ }
+
+ // Make sure nothing really moved
+ Set<String> tableNames = new HashSet<>(client.getAllTables(DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME));
+ Assert.assertTrue(tableNames.contains(tableName.toLowerCase()));
+
+ // Make sure the table in the target database didn't get clobbered
+ Table fetchedTable = client.getTable(DEFAULT_CATALOG_NAME, toDbName, tableName);
+ Assert.assertEquals("b", fetchedTable.getSd().getCols().get(0).getName());
+ }
+
+ @Test(expected = HiveMetaException.class)
+ public void moveNonExistentTable() throws TException, HiveMetaException {
+ String toDbName = "moveNonExistentTable";
+ new DatabaseBuilder()
+ .setName(toDbName)
+ .create(client, conf);
+ String argsMoveTable =
+ String.format("-moveTable nosuch -fromCatalog %s -toCatalog %s -fromDatabase %s -toDatabase %s",
+ DEFAULT_CATALOG_NAME, DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME, toDbName);
+ execute(new SchemaToolTaskMoveTable(), argsMoveTable);
+ }
+
+ @Test
+ public void moveTableToNonExistentDb() throws TException, HiveMetaException {
+ String tableName = "doomedToWander";
+ new TableBuilder()
+ .setTableName(tableName)
+ .addCol("a", "int")
+ .create(client, conf);
+
+ try {
+ String argsMoveTable =
+ String.format("-moveTable %s -fromCatalog %s -toCatalog %s -fromDatabase %s -toDatabase nosuch",
+ tableName, DEFAULT_CATALOG_NAME, DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME);
+ execute(new SchemaToolTaskMoveTable(), argsMoveTable);
+ Assert.fail("Attempt to move table to non-existent table should have failed.");
+ } catch (HiveMetaException e) {
+ // good
+ }
+
+ // Make sure nothing really moved
+ Set<String> tableNames = new HashSet<>(client.getAllTables(DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME));
+ Assert.assertTrue(tableNames.contains(tableName.toLowerCase()));
+ }
+
+ private static void execute(SchemaToolTask task, String taskArgs) throws HiveMetaException {
+ try {
+ StrTokenizer tokenizer = new StrTokenizer(argsBase + taskArgs, ' ', '\"');
+ SchemaToolCommandLine cl = new SchemaToolCommandLine(tokenizer.getTokenArray(), null);
+ task.setCommandLineArguments(cl);
+ } catch (Exception e) {
+ throw new IllegalStateException("Could not parse comman line \n" + argsBase + taskArgs, e);
+ }
+
+ task.setHiveSchemaTool(schemaTool);
+ task.execute();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/itests/hive-unit/src/test/java/org/apache/hive/beeline/schematool/TestSchemaTool.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/schematool/TestSchemaTool.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/schematool/TestSchemaTool.java
index 314aa60..0132a0b 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/beeline/schematool/TestSchemaTool.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/schematool/TestSchemaTool.java
@@ -19,340 +19,24 @@
package org.apache.hive.beeline.schematool;
import java.io.BufferedWriter;
-import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
-import java.io.OutputStream;
-import java.io.PrintStream;
-import java.net.URI;
-import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.SQLException;
-import java.util.Random;
-import junit.framework.TestCase;
-import org.apache.commons.dbcp.DelegatingConnection;
-import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang3.text.StrTokenizer;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaException;
-import org.apache.hadoop.hive.metastore.IMetaStoreSchemaInfo;
-import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper;
import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.NestedScriptParser;
import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.PostgresCommandParser;
-import org.apache.hadoop.hive.shims.ShimLoader;
+import org.junit.Assert;
+import org.junit.Test;
-public class TestSchemaTool extends TestCase {
- private static HiveSchemaTool schemaTool;
- private Connection conn;
- private HiveConf hiveConf;
- private String testMetastoreDB;
- private PrintStream errStream;
- private PrintStream outStream;
- private String argsBase;
- private HiveSchemaToolTaskValidate validator;
-
- @Override
- protected void setUp() throws Exception {
- super.setUp();
- testMetastoreDB = System.getProperty("java.io.tmpdir") +
- File.separator + "test_metastore-" + new Random().nextInt();
- System.setProperty(MetastoreConf.ConfVars.CONNECT_URL_KEY.getVarname(),
- "jdbc:derby:" + testMetastoreDB + ";create=true");
- hiveConf = new HiveConf(this.getClass());
- schemaTool = new HiveSchemaTool(
- System.getProperty("test.tmp.dir", "target/tmp"), hiveConf, "derby", null);
-
- String userName = hiveConf.get(MetastoreConf.ConfVars.CONNECTION_USER_NAME.getVarname());
- String passWord = ShimLoader.getHadoopShims().getPassword(schemaTool.getHiveConf(),
- MetastoreConf.ConfVars.PWD.getVarname());
- schemaTool.setUserName(userName);
- schemaTool.setPassWord(passWord);
-
- argsBase = "-dbType derby -userName " + userName + " -passWord " + passWord + " ";
-
- System.setProperty("beeLine.system.exit", "true");
- errStream = System.err;
- outStream = System.out;
- conn = schemaTool.getConnectionToMetastore(false);
-
- validator = new HiveSchemaToolTaskValidate();
- validator.setHiveSchemaTool(schemaTool);
- }
-
- @Override
- protected void tearDown() throws Exception {
- File metaStoreDir = new File(testMetastoreDB);
- if (metaStoreDir.exists()) {
- FileUtils.forceDeleteOnExit(metaStoreDir);
- }
- System.setOut(outStream);
- System.setErr(errStream);
- if (conn != null) {
- conn.close();
- }
- }
-
- /**
- * Test the sequence validation functionality
- * @throws Exception
- */
- public void testValidateSequences() throws Exception {
- execute(new HiveSchemaToolTaskInit(), "-initSchema");
-
- // Test empty database
- boolean isValid = validator.validateSequences(conn);
- assertTrue(isValid);
-
- // Test valid case
- String[] scripts = new String[] {
- "insert into CTLGS values(99, 'test_cat_1', 'description', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb')",
- "insert into SEQUENCE_TABLE values('org.apache.hadoop.hive.metastore.model.MDatabase', 100)",
- "insert into DBS values(99, 'test db1', 'hdfs:///tmp', 'db1', 'test', 'test', 'test_cat_1')"
- };
- File scriptFile = generateTestScript(scripts);
- schemaTool.runBeeLine(scriptFile.getPath());
- isValid = validator.validateSequences(conn);
- assertTrue(isValid);
-
- // Test invalid case
- scripts = new String[] {
- "delete from SEQUENCE_TABLE",
- "delete from DBS",
- "insert into SEQUENCE_TABLE values('org.apache.hadoop.hive.metastore.model.MDatabase', 100)",
- "insert into DBS values(102, 'test db1', 'hdfs:///tmp', 'db1', 'test', 'test', 'test_cat_1')"
- };
- scriptFile = generateTestScript(scripts);
- schemaTool.runBeeLine(scriptFile.getPath());
- isValid = validator.validateSequences(conn);
- assertFalse(isValid);
- }
-
- /**
- * Test to validate that all tables exist in the HMS metastore.
- * @throws Exception
- */
- public void testValidateSchemaTables() throws Exception {
- execute(new HiveSchemaToolTaskInit(), "-initSchemaTo 2.0.0");
-
- boolean isValid = (boolean)validator.validateSchemaTables(conn);
- assertTrue(isValid);
-
- // upgrade from 2.0.0 schema and re-validate
- execute(new HiveSchemaToolTaskUpgrade(), "-upgradeSchemaFrom 2.0.0");
- isValid = (boolean)validator.validateSchemaTables(conn);
- assertTrue(isValid);
-
- // Simulate a missing table scenario by renaming a couple of tables
- String[] scripts = new String[] {
- "RENAME TABLE SEQUENCE_TABLE to SEQUENCE_TABLE_RENAMED",
- "RENAME TABLE NUCLEUS_TABLES to NUCLEUS_TABLES_RENAMED"
- };
-
- File scriptFile = generateTestScript(scripts);
- schemaTool.runBeeLine(scriptFile.getPath());
- isValid = validator.validateSchemaTables(conn);
- assertFalse(isValid);
-
- // Restored the renamed tables
- scripts = new String[] {
- "RENAME TABLE SEQUENCE_TABLE_RENAMED to SEQUENCE_TABLE",
- "RENAME TABLE NUCLEUS_TABLES_RENAMED to NUCLEUS_TABLES"
- };
-
- scriptFile = generateTestScript(scripts);
- schemaTool.runBeeLine(scriptFile.getPath());
- isValid = validator.validateSchemaTables(conn);
- assertTrue(isValid);
-
- // Check that an exception from getMetaData() is reported correctly
- try {
- // Make a Connection object that will throw an exception
- BadMetaDataConnection bad = new BadMetaDataConnection(conn);
- validator.validateSchemaTables(bad);
- fail("did not get expected exception");
- } catch (HiveMetaException hme) {
- String message = hme.getMessage();
- assertTrue("Bad HiveMetaException message :" + message,
- message.contains("Failed to retrieve schema tables from Hive Metastore DB"));
- Throwable cause = hme.getCause();
- assertNotNull("HiveMetaException did not contain a cause", cause);
- String causeMessage = cause.getMessage();
- assertTrue("Bad SQLException message: " + causeMessage, causeMessage.contains(
- BadMetaDataConnection.FAILURE_TEXT));
- }
- }
-
- /*
- * Test the validation of incorrect NULL values in the tables
- * @throws Exception
- */
- public void testValidateNullValues() throws Exception {
- execute(new HiveSchemaToolTaskInit(), "-initSchema");
-
- // Test empty database
- boolean isValid = validator.validateColumnNullValues(conn);
- assertTrue(isValid);
-
- // Test valid case
- createTestHiveTableSchemas();
- isValid = validator.validateColumnNullValues(conn);
-
- // Test invalid case
- String[] scripts = new String[] {
- "update TBLS set SD_ID=null"
- };
- File scriptFile = generateTestScript(scripts);
- schemaTool.runBeeLine(scriptFile.getPath());
- isValid = validator.validateColumnNullValues(conn);
- assertFalse(isValid);
- }
-
- /**
- * Test dryrun of schema initialization
- * @throws Exception
- */
- public void testSchemaInitDryRun() throws Exception {
- schemaTool.setDryRun(true);
- execute(new HiveSchemaToolTaskInit(), "-initSchemaTo 0.7.0");
- schemaTool.setDryRun(false);
- try {
- schemaTool.verifySchemaVersion();
- } catch (HiveMetaException e) {
- // The connection should fail since it the dry run
- return;
- }
- fail("Dry run shouldn't create actual metastore");
- }
-
- /**
- * Test dryrun of schema upgrade
- * @throws Exception
- */
- public void testSchemaUpgradeDryRun() throws Exception {
- execute(new HiveSchemaToolTaskInit(), "-initSchemaTo 0.7.0");
-
- schemaTool.setDryRun(true);
- execute(new HiveSchemaToolTaskUpgrade(), "-upgradeSchemaFrom 0.7.0");
- schemaTool.setDryRun(false);
- try {
- schemaTool.verifySchemaVersion();
- } catch (HiveMetaException e) {
- // The connection should fail since it the dry run
- return;
- }
- fail("Dry run shouldn't upgrade metastore schema");
- }
-
- /**
- * Test schema initialization
- * @throws Exception
- */
- public void testSchemaInit() throws Exception {
- IMetaStoreSchemaInfo metastoreSchemaInfo = MetaStoreSchemaInfoFactory.get(hiveConf,
- System.getProperty("test.tmp.dir", "target/tmp"), "derby");
- execute(new HiveSchemaToolTaskInit(), "-initSchemaTo " + metastoreSchemaInfo.getHiveSchemaVersion());
- schemaTool.verifySchemaVersion();
- }
-
- /**
- * Test validation for schema versions
- * @throws Exception
- */
- public void testValidateSchemaVersions() throws Exception {
- execute(new HiveSchemaToolTaskInit(), "-initSchema");
- boolean isValid = validator.validateSchemaVersions();
- // Test an invalid case with multiple versions
- String[] scripts = new String[] {
- "insert into VERSION values(100, '2.2.0', 'Hive release version 2.2.0')"
- };
- File scriptFile = generateTestScript(scripts);
- schemaTool.runBeeLine(scriptFile.getPath());
- isValid = validator.validateSchemaVersions();
- assertFalse(isValid);
-
- scripts = new String[] {
- "delete from VERSION where VER_ID = 100"
- };
- scriptFile = generateTestScript(scripts);
- schemaTool.runBeeLine(scriptFile.getPath());
- isValid = validator.validateSchemaVersions();
- assertTrue(isValid);
-
- // Test an invalid case without version
- scripts = new String[] {
- "delete from VERSION"
- };
- scriptFile = generateTestScript(scripts);
- schemaTool.runBeeLine(scriptFile.getPath());
- isValid = validator.validateSchemaVersions();
- assertFalse(isValid);
- }
-
- /**
- * Test schema upgrade
- * @throws Exception
- */
- public void testSchemaUpgrade() throws Exception {
- boolean foundException = false;
- // Initialize 0.7.0 schema
- execute(new HiveSchemaToolTaskInit(), "-initSchemaTo 0.7.0");
- // verify that driver fails due to older version schema
- try {
- schemaTool.verifySchemaVersion();
- } catch (HiveMetaException e) {
- // Expected to fail due to old schema
- foundException = true;
- }
- if (!foundException) {
- throw new Exception(
- "Hive operations shouldn't pass with older version schema");
- }
-
- // Generate dummy pre-upgrade script with errors
- String invalidPreUpgradeScript = writeDummyPreUpgradeScript(
- 0, "upgrade-0.11.0-to-0.12.0.derby.sql", "foo bar;");
- // Generate dummy pre-upgrade scripts with valid SQL
- String validPreUpgradeScript0 = writeDummyPreUpgradeScript(
- 0, "upgrade-0.12.0-to-0.13.0.derby.sql",
- "CREATE TABLE schema_test0 (id integer);");
- String validPreUpgradeScript1 = writeDummyPreUpgradeScript(
- 1, "upgrade-0.12.0-to-0.13.0.derby.sql",
- "CREATE TABLE schema_test1 (id integer);");
-
- // Capture system out and err
- schemaTool.setVerbose(true);
- OutputStream stderr = new ByteArrayOutputStream();
- PrintStream errPrintStream = new PrintStream(stderr);
- System.setErr(errPrintStream);
- OutputStream stdout = new ByteArrayOutputStream();
- PrintStream outPrintStream = new PrintStream(stdout);
- System.setOut(outPrintStream);
-
- // Upgrade schema from 0.7.0 to latest
- execute(new HiveSchemaToolTaskUpgrade(), "-upgradeSchemaFrom 0.7.0");
-
- // Verify that the schemaTool ran pre-upgrade scripts and ignored errors
- assertTrue(stderr.toString().contains(invalidPreUpgradeScript));
- assertTrue(stderr.toString().contains("foo"));
- assertFalse(stderr.toString().contains(validPreUpgradeScript0));
- assertFalse(stderr.toString().contains(validPreUpgradeScript1));
- assertTrue(stdout.toString().contains(validPreUpgradeScript0));
- assertTrue(stdout.toString().contains(validPreUpgradeScript1));
-
- // Verify that driver works fine with latest schema
- schemaTool.verifySchemaVersion();
- }
+public class TestSchemaTool {
/**
* Test script formatting
- * @throws Exception
*/
+ @Test
public void testScripts() throws Exception {
String testScript[] = {
"-- this is a comment",
@@ -375,13 +59,13 @@ public class TestSchemaTool extends TestCase {
.buildCommand(testScriptFile.getParentFile().getPath(),
testScriptFile.getName());
- assertEquals(expectedSQL, flattenedSql);
+ Assert.assertEquals(expectedSQL, flattenedSql);
}
/**
* Test nested script formatting
- * @throws Exception
*/
+ @Test
public void testNestedScriptsForDerby() throws Exception {
String childTab1 = "childTab1";
String childTab2 = "childTab2";
@@ -416,17 +100,17 @@ public class TestSchemaTool extends TestCase {
String flattenedSql = HiveSchemaHelper.getDbCommandParser("derby", false)
.buildCommand(testScriptFile.getParentFile().getPath(),
testScriptFile.getName());
- assertFalse(flattenedSql.contains("RUN"));
- assertFalse(flattenedSql.contains("comment"));
- assertTrue(flattenedSql.contains(childTab1));
- assertTrue(flattenedSql.contains(childTab2));
- assertTrue(flattenedSql.contains(parentTab));
+ Assert.assertFalse(flattenedSql.contains("RUN"));
+ Assert.assertFalse(flattenedSql.contains("comment"));
+ Assert.assertTrue(flattenedSql.contains(childTab1));
+ Assert.assertTrue(flattenedSql.contains(childTab2));
+ Assert.assertTrue(flattenedSql.contains(parentTab));
}
/**
* Test nested script formatting
- * @throws Exception
*/
+ @Test
public void testNestedScriptsForMySQL() throws Exception {
String childTab1 = "childTab1";
String childTab2 = "childTab2";
@@ -461,17 +145,17 @@ public class TestSchemaTool extends TestCase {
String flattenedSql = HiveSchemaHelper.getDbCommandParser("mysql", false)
.buildCommand(testScriptFile.getParentFile().getPath(),
testScriptFile.getName());
- assertFalse(flattenedSql.contains("RUN"));
- assertFalse(flattenedSql.contains("comment"));
- assertTrue(flattenedSql.contains(childTab1));
- assertTrue(flattenedSql.contains(childTab2));
- assertTrue(flattenedSql.contains(parentTab));
+ Assert.assertFalse(flattenedSql.contains("RUN"));
+ Assert.assertFalse(flattenedSql.contains("comment"));
+ Assert.assertTrue(flattenedSql.contains(childTab1));
+ Assert.assertTrue(flattenedSql.contains(childTab2));
+ Assert.assertTrue(flattenedSql.contains(parentTab));
}
/**
* Test script formatting
- * @throws Exception
*/
+ @Test
public void testScriptWithDelimiter() throws Exception {
String testScript[] = {
"-- this is a comment",
@@ -504,13 +188,13 @@ public class TestSchemaTool extends TestCase {
String flattenedSql = testDbParser.buildCommand(testScriptFile.getParentFile().getPath(),
testScriptFile.getName());
- assertEquals(expectedSQL, flattenedSql);
+ Assert.assertEquals(expectedSQL, flattenedSql);
}
/**
* Test script formatting
- * @throws Exception
*/
+ @Test
public void testScriptMultiRowComment() throws Exception {
String testScript[] = {
"-- this is a comment",
@@ -539,13 +223,13 @@ public class TestSchemaTool extends TestCase {
String flattenedSql = testDbParser.buildCommand(testScriptFile.getParentFile().getPath(),
testScriptFile.getName());
- assertEquals(expectedSQL, flattenedSql);
+ Assert.assertEquals(expectedSQL, flattenedSql);
}
/**
* Test nested script formatting
- * @throws Exception
*/
+ @Test
public void testNestedScriptsForOracle() throws Exception {
String childTab1 = "childTab1";
String childTab2 = "childTab2";
@@ -580,17 +264,17 @@ public class TestSchemaTool extends TestCase {
String flattenedSql = HiveSchemaHelper.getDbCommandParser("oracle", false)
.buildCommand(testScriptFile.getParentFile().getPath(),
testScriptFile.getName());
- assertFalse(flattenedSql.contains("@"));
- assertFalse(flattenedSql.contains("comment"));
- assertTrue(flattenedSql.contains(childTab1));
- assertTrue(flattenedSql.contains(childTab2));
- assertTrue(flattenedSql.contains(parentTab));
+ Assert.assertFalse(flattenedSql.contains("@"));
+ Assert.assertFalse(flattenedSql.contains("comment"));
+ Assert.assertTrue(flattenedSql.contains(childTab1));
+ Assert.assertTrue(flattenedSql.contains(childTab2));
+ Assert.assertTrue(flattenedSql.contains(parentTab));
}
/**
* Test script formatting
- * @throws Exception
*/
+ @Test
public void testPostgresFilter() throws Exception {
String testScript[] = {
"-- this is a comment",
@@ -616,7 +300,7 @@ public class TestSchemaTool extends TestCase {
File testScriptFile = generateTestScript(testScript);
String flattenedSql = noDbOptParser.buildCommand(
testScriptFile.getParentFile().getPath(), testScriptFile.getName());
- assertEquals(expectedSQL, flattenedSql);
+ Assert.assertEquals(expectedSQL, flattenedSql);
String expectedScriptWithOptionAbsent[] = {
"DROP TABLE IF EXISTS fooTab",
@@ -634,101 +318,7 @@ public class TestSchemaTool extends TestCase {
testScriptFile = generateTestScript(testScript);
flattenedSql = dbOptParser.buildCommand(
testScriptFile.getParentFile().getPath(), testScriptFile.getName());
- assertEquals(expectedSQL, flattenedSql);
- }
-
- /**
- * Test validate uri of locations
- * @throws Exception
- */
- public void testValidateLocations() throws Exception {
- execute(new HiveSchemaToolTaskInit(), "-initSchema");
- URI defaultRoot = new URI("hdfs://myhost.com:8020");
- URI defaultRoot2 = new URI("s3://myhost2.com:8888");
- //check empty DB
- boolean isValid = validator.validateLocations(conn, null);
- assertTrue(isValid);
- isValid = validator.validateLocations(conn, new URI[] {defaultRoot, defaultRoot2});
- assertTrue(isValid);
-
- // Test valid case
- String[] scripts = new String[] {
- "insert into CTLGS values(3, 'test_cat_2', 'description', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb')",
- "insert into DBS values(2, 'my db', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'test_cat_2')",
- "insert into DBS values(7, 'db with bad port', 'hdfs://myhost.com:8020/', 'haDB', 'public', 'role', 'test_cat_2')",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3,null,'org.apache.hadoop.mapred.TextInputFormat','N','N',null,-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n')",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (3 ,1435255431,2,0 ,'hive',0,3,'myView','VIRTUAL_VIEW','select a.col1,a.col2 from foo','select * from foo','n')",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4012 ,1435255431,7,0 ,'hive',0,4000,'mytal4012','MANAGED_TABLE',NULL,NULL,'n')",
- "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(1, 1441402388,0, 'd1=1/d2=1',2,2)",
- "insert into SKEWED_STRING_LIST values(1)",
- "insert into SKEWED_STRING_LIST values(2)",
- "insert into SKEWED_COL_VALUE_LOC_MAP values(1,1,'hdfs://myhost.com:8020/user/hive/warehouse/mytal/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/')",
- "insert into SKEWED_COL_VALUE_LOC_MAP values(2,2,'s3://myhost.com:8020/user/hive/warehouse/mytal/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/')"
- };
- File scriptFile = generateTestScript(scripts);
- schemaTool.runBeeLine(scriptFile.getPath());
- isValid = validator.validateLocations(conn, null);
- assertTrue(isValid);
- isValid = validator.validateLocations(conn, new URI[] {defaultRoot, defaultRoot2});
- assertTrue(isValid);
- scripts = new String[] {
- "delete from SKEWED_COL_VALUE_LOC_MAP",
- "delete from SKEWED_STRING_LIST",
- "delete from PARTITIONS",
- "delete from TBLS",
- "delete from SDS",
- "delete from DBS",
- "insert into DBS values(2, 'my db', '/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'test_cat_2')",
- "insert into DBS values(4, 'my db2', 'hdfs://myhost.com:8020', '', 'public', 'role', 'test_cat_2')",
- "insert into DBS values(6, 'db with bad port', 'hdfs://myhost.com:8020:', 'zDB', 'public', 'role', 'test_cat_2')",
- "insert into DBS values(7, 'db with bad port', 'hdfs://mynameservice.com/', 'haDB', 'public', 'role', 'test_cat_2')",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://yourhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','file:///user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n')",
- "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(1, 1441402388,0, 'd1=1/d2=1',2,2)",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','yourhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4001,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4003,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4004,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4002,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (5000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','file:///user/admin/2016_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (3000 ,1435255431,2,0 ,'hive',0,3000,'mytal3000','MANAGED_TABLE',NULL,NULL,'n')",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4011 ,1435255431,4,0 ,'hive',0,4001,'mytal4011','MANAGED_TABLE',NULL,NULL,'n')",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4012 ,1435255431,4,0 ,'hive',0,4002,'','MANAGED_TABLE',NULL,NULL,'n')",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4013 ,1435255431,4,0 ,'hive',0,4003,'mytal4013','MANAGED_TABLE',NULL,NULL,'n')",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4014 ,1435255431,2,0 ,'hive',0,4003,'','MANAGED_TABLE',NULL,NULL,'n')",
- "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(4001, 1441402388,0, 'd1=1/d2=4001',4001,4011)",
- "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(4002, 1441402388,0, 'd1=1/d2=4002',4002,4012)",
- "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(4003, 1441402388,0, 'd1=1/d2=4003',4003,4013)",
- "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(4004, 1441402388,0, 'd1=1/d2=4004',4004,4014)",
- "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(5000, 1441402388,0, 'd1=1/d2=5000',5000,2)",
- "insert into SKEWED_STRING_LIST values(1)",
- "insert into SKEWED_STRING_LIST values(2)",
- "insert into SKEWED_COL_VALUE_LOC_MAP values(1,1,'hdfs://yourhost.com:8020/user/hive/warehouse/mytal/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/')",
- "insert into SKEWED_COL_VALUE_LOC_MAP values(2,2,'file:///user/admin/warehouse/mytal/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/')"
- };
- scriptFile = generateTestScript(scripts);
- schemaTool.runBeeLine(scriptFile.getPath());
- isValid = validator.validateLocations(conn, null);
- assertFalse(isValid);
- isValid = validator.validateLocations(conn, new URI[] {defaultRoot, defaultRoot2});
- assertFalse(isValid);
- }
-
- public void testHiveMetastoreDbPropertiesTable() throws HiveMetaException, IOException {
- execute(new HiveSchemaToolTaskInit(), "-initSchemaTo 3.0.0");
- validateMetastoreDbPropertiesTable();
- }
-
- public void testMetastoreDbPropertiesAfterUpgrade() throws HiveMetaException, IOException {
- execute(new HiveSchemaToolTaskInit(), "-initSchemaTo 2.0.0");
- execute(new HiveSchemaToolTaskUpgrade(), "-upgradeSchema");
- validateMetastoreDbPropertiesTable();
+ Assert.assertEquals(expectedSQL, flattenedSql);
}
private File generateTestScript(String [] stmts) throws IOException {
@@ -743,84 +333,4 @@ public class TestSchemaTool extends TestCase {
out.close();
return testScriptFile;
}
-
- private void validateMetastoreDbPropertiesTable() throws HiveMetaException, IOException {
- boolean isValid = (boolean) validator.validateSchemaTables(conn);
- assertTrue(isValid);
- // adding same property key twice should throw unique key constraint violation exception
- String[] scripts = new String[] {
- "insert into METASTORE_DB_PROPERTIES values ('guid', 'test-uuid-1', 'dummy uuid 1')",
- "insert into METASTORE_DB_PROPERTIES values ('guid', 'test-uuid-2', 'dummy uuid 2')", };
- File scriptFile = generateTestScript(scripts);
- Exception ex = null;
- try {
- schemaTool.runBeeLine(scriptFile.getPath());
- } catch (Exception iox) {
- ex = iox;
- }
- assertTrue(ex != null && ex instanceof IOException);
- }
- /**
- * Write out a dummy pre-upgrade script with given SQL statement.
- */
- private String writeDummyPreUpgradeScript(int index, String upgradeScriptName,
- String sql) throws Exception {
- String preUpgradeScript = "pre-" + index + "-" + upgradeScriptName;
- String dummyPreScriptPath = System.getProperty("test.tmp.dir", "target/tmp") +
- File.separatorChar + "scripts" + File.separatorChar + "metastore" +
- File.separatorChar + "upgrade" + File.separatorChar + "derby" +
- File.separatorChar + preUpgradeScript;
- FileWriter fstream = new FileWriter(dummyPreScriptPath);
- BufferedWriter out = new BufferedWriter(fstream);
- out.write(sql + System.getProperty("line.separator") + ";");
- out.close();
- return preUpgradeScript;
- }
-
- /**
- * Insert the records in DB to simulate a hive table
- * @throws IOException
- */
- private void createTestHiveTableSchemas() throws IOException {
- String[] scripts = new String[] {
- "insert into CTLGS values(2, 'my_catalog', 'description', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb')",
- "insert into DBS values(2, 'my db', 'hdfs://myhost.com:8021/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'my_catalog')",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n')",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (3 ,1435255431,2,0 ,'hive',0,2,'aTable','MANAGED_TABLE',NULL,NULL,'n')",
- "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(1, 1441402388,0, 'd1=1/d2=1',2,2)"
- };
- File scriptFile = generateTestScript(scripts);
- schemaTool.runBeeLine(scriptFile.getPath());
- }
-
- /**
- * A mock Connection class that throws an exception out of getMetaData().
- */
- class BadMetaDataConnection extends DelegatingConnection {
- static final String FAILURE_TEXT = "fault injected";
-
- BadMetaDataConnection(Connection connection) {
- super(connection);
- }
-
- @Override
- public DatabaseMetaData getMetaData() throws SQLException {
- throw new SQLException(FAILURE_TEXT);
- }
- }
-
- private void execute(HiveSchemaToolTask task, String taskArgs) throws HiveMetaException {
- try {
- StrTokenizer tokenizer = new StrTokenizer(argsBase + taskArgs, ' ', '\"');
- HiveSchemaToolCommandLine cl = new HiveSchemaToolCommandLine(tokenizer.getTokenArray());
- task.setCommandLineArguments(cl);
- } catch (Exception e) {
- throw new IllegalStateException("Could not parse comman line \n" + argsBase + taskArgs, e);
- }
-
- task.setHiveSchemaTool(schemaTool);
- task.execute();
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/itests/hive-unit/src/test/java/org/apache/hive/beeline/schematool/TestSchemaToolCatalogOps.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/schematool/TestSchemaToolCatalogOps.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/schematool/TestSchemaToolCatalogOps.java
deleted file mode 100644
index a13603d..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hive/beeline/schematool/TestSchemaToolCatalogOps.java
+++ /dev/null
@@ -1,478 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.beeline.schematool;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.text.StrTokenizer;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaException;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
-import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hadoop.hive.metastore.api.Catalog;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.Function;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.metastore.client.builder.CatalogBuilder;
-import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
-import org.apache.hadoop.hive.metastore.client.builder.FunctionBuilder;
-import org.apache.hadoop.hive.metastore.client.builder.PartitionBuilder;
-import org.apache.hadoop.hive.metastore.client.builder.TableBuilder;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
-import org.apache.thrift.TException;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
-
-import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME;
-import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME;
-
-public class TestSchemaToolCatalogOps {
- private static HiveSchemaTool schemaTool;
- private static HiveConf conf;
- private IMetaStoreClient client;
- private static String testMetastoreDB;
- private static String argsBase;
-
- @BeforeClass
- public static void initDb() throws HiveMetaException, IOException {
- conf = new HiveConf();
- MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.AUTO_CREATE_ALL, false);
- MetastoreConf.setLongVar(conf, MetastoreConf.ConfVars.HMS_HANDLER_ATTEMPTS, 1);
- MetastoreConf.setLongVar(conf, MetastoreConf.ConfVars.THRIFT_CONNECTION_RETRIES, 1);
- testMetastoreDB = System.getProperty("java.io.tmpdir") +
- File.separator + "testschematoolcatopsdb";
- MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CONNECT_URL_KEY,
- "jdbc:derby:" + testMetastoreDB + ";create=true");
- schemaTool = new HiveSchemaTool(
- System.getProperty("test.tmp.dir", "target/tmp"), conf, "derby", null);
-
- String userName = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.CONNECTION_USER_NAME);
- String passWord = MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.PWD);
- schemaTool.setUserName(userName);
- schemaTool.setPassWord(passWord);
-
- argsBase = "-dbType derby -userName " + userName + " -passWord " + passWord + " ";
- execute(new HiveSchemaToolTaskInit(), "-initSchema"); // Pre-install the database so all the tables are there.
- }
-
- @AfterClass
- public static void removeDb() throws Exception {
- File metaStoreDir = new File(testMetastoreDB);
- if (metaStoreDir.exists()) {
- FileUtils.forceDeleteOnExit(metaStoreDir);
- }
- }
-
- @Before
- public void createClient() throws MetaException {
- client = new HiveMetaStoreClient(conf);
- }
-
- @Test
- public void createCatalog() throws HiveMetaException, TException {
- String catName = "my_test_catalog";
- String location = "file:///tmp/my_test_catalog";
- String description = "very descriptive";
- String argsCreate = String.format("-createCatalog %s -catalogLocation \"%s\" -catalogDescription \"%s\"",
- catName, location, description);
- execute(new HiveSchemaToolTaskCreateCatalog(), argsCreate);
-
- Catalog cat = client.getCatalog(catName);
- Assert.assertEquals(location, cat.getLocationUri());
- Assert.assertEquals(description, cat.getDescription());
- }
-
- @Test(expected = HiveMetaException.class)
- public void createExistingCatalog() throws HiveMetaException {
- String catName = "hive";
- String location = "somewhere";
- String argsCreate = String.format("-createCatalog %s -catalogLocation \"%s\"",
- catName, location);
- execute(new HiveSchemaToolTaskCreateCatalog(), argsCreate);
- }
-
- @Test
- public void createExistingCatalogWithIfNotExists() throws HiveMetaException {
- String catName = "my_existing_test_catalog";
- String location = "file:///tmp/my_test_catalog";
- String description = "very descriptive";
- String argsCreate1 = String.format("-createCatalog %s -catalogLocation \"%s\" -catalogDescription \"%s\"",
- catName, location, description);
- execute(new HiveSchemaToolTaskCreateCatalog(), argsCreate1);
-
- String argsCreate2 =
- String.format("-createCatalog %s -catalogLocation \"%s\" -catalogDescription \"%s\" -ifNotExists",
- catName, location, description);
- execute(new HiveSchemaToolTaskCreateCatalog(), argsCreate2);
- }
-
- @Test
- public void alterCatalog() throws HiveMetaException, TException {
- String catName = "an_alterable_catalog";
- String location = "file:///tmp/an_alterable_catalog";
- String description = "description";
- String argsCreate = String.format("-createCatalog %s -catalogLocation \"%s\" -catalogDescription \"%s\"",
- catName, location, description);
- execute(new HiveSchemaToolTaskCreateCatalog(), argsCreate);
-
- location = "file:///tmp/somewhere_else";
- String argsAlter1 = String.format("-alterCatalog %s -catalogLocation \"%s\"",
- catName, location);
- execute(new HiveSchemaToolTaskAlterCatalog(), argsAlter1);
- Catalog cat = client.getCatalog(catName);
- Assert.assertEquals(location, cat.getLocationUri());
- Assert.assertEquals(description, cat.getDescription());
-
- description = "a better description";
- String argsAlter2 = String.format("-alterCatalog %s -catalogDescription \"%s\"",
- catName, description);
- execute(new HiveSchemaToolTaskAlterCatalog(), argsAlter2);
- cat = client.getCatalog(catName);
- Assert.assertEquals(location, cat.getLocationUri());
- Assert.assertEquals(description, cat.getDescription());
-
- location = "file:///tmp/a_third_location";
- description = "best description yet";
- String argsAlter3 = String.format("-alterCatalog %s -catalogLocation \"%s\" -catalogDescription \"%s\"",
- catName, location, description);
- execute(new HiveSchemaToolTaskAlterCatalog(), argsAlter3);
- cat = client.getCatalog(catName);
- Assert.assertEquals(location, cat.getLocationUri());
- Assert.assertEquals(description, cat.getDescription());
- }
-
- @Test(expected = HiveMetaException.class)
- public void alterBogusCatalog() throws HiveMetaException {
- String catName = "nosuch";
- String location = "file:///tmp/somewhere";
- String description = "whatever";
- String argsAlter = String.format("-alterCatalog %s -catalogLocation \"%s\" -catalogDescription \"%s\"",
- catName, location, description);
- execute(new HiveSchemaToolTaskAlterCatalog(), argsAlter);
- }
-
- @Test(expected = HiveMetaException.class)
- public void alterCatalogNoChange() throws HiveMetaException {
- String catName = "alter_cat_no_change";
- String location = "file:///tmp/alter_cat_no_change";
- String description = "description";
- String argsCreate = String.format("-createCatalog %s -catalogLocation \"%s\" -catalogDescription \"%s\"",
- catName, location, description);
- execute(new HiveSchemaToolTaskCreateCatalog(), argsCreate);
-
- String argsAlter = String.format("-alterCatalog %s", catName);
- execute(new HiveSchemaToolTaskAlterCatalog(), argsAlter);
- }
-
- @Test
- public void moveDatabase() throws HiveMetaException, TException {
- String toCatName = "moveDbCat";
- String dbName = "moveDbDb";
- String tableName = "moveDbTable";
- String funcName = "movedbfunc";
- String partVal = "moveDbKey";
-
- new CatalogBuilder()
- .setName(toCatName)
- .setLocation("file:///tmp")
- .create(client);
-
- Database db = new DatabaseBuilder()
- .setCatalogName(DEFAULT_CATALOG_NAME)
- .setName(dbName)
- .create(client, conf);
-
- new FunctionBuilder()
- .inDb(db)
- .setName(funcName)
- .setClass("org.apache.hive.myudf")
- .create(client, conf);
-
- Table table = new TableBuilder()
- .inDb(db)
- .setTableName(tableName)
- .addCol("a", "int")
- .addPartCol("p", "string")
- .create(client, conf);
-
- new PartitionBuilder()
- .inTable(table)
- .addValue(partVal)
- .addToTable(client, conf);
-
- String argsMoveDB = String.format("-moveDatabase %s -fromCatalog %s -toCatalog %s", dbName,
- DEFAULT_CATALOG_NAME, toCatName);
- execute(new HiveSchemaToolTaskMoveDatabase(), argsMoveDB);
-
- Database fetchedDb = client.getDatabase(toCatName, dbName);
- Assert.assertNotNull(fetchedDb);
- Assert.assertEquals(toCatName.toLowerCase(), fetchedDb.getCatalogName());
-
- Function fetchedFunction = client.getFunction(toCatName, dbName, funcName);
- Assert.assertNotNull(fetchedFunction);
- Assert.assertEquals(toCatName.toLowerCase(), fetchedFunction.getCatName());
- Assert.assertEquals(dbName.toLowerCase(), fetchedFunction.getDbName());
-
- Table fetchedTable = client.getTable(toCatName, dbName, tableName);
- Assert.assertNotNull(fetchedTable);
- Assert.assertEquals(toCatName.toLowerCase(), fetchedTable.getCatName());
- Assert.assertEquals(dbName.toLowerCase(), fetchedTable.getDbName());
-
- Partition fetchedPart =
- client.getPartition(toCatName, dbName, tableName, Collections.singletonList(partVal));
- Assert.assertNotNull(fetchedPart);
- Assert.assertEquals(toCatName.toLowerCase(), fetchedPart.getCatName());
- Assert.assertEquals(dbName.toLowerCase(), fetchedPart.getDbName());
- Assert.assertEquals(tableName.toLowerCase(), fetchedPart.getTableName());
- }
-
- @Test
- public void moveDatabaseWithExistingDbOfSameNameAlreadyInTargetCatalog()
- throws TException, HiveMetaException {
- String catName = "clobberCatalog";
- new CatalogBuilder()
- .setName(catName)
- .setLocation("file:///tmp")
- .create(client);
- try {
- String argsMoveDB = String.format("-moveDatabase %s -fromCatalog %s -toCatalog %s",
- DEFAULT_DATABASE_NAME, catName, DEFAULT_CATALOG_NAME);
- execute(new HiveSchemaToolTaskMoveDatabase(), argsMoveDB);
- Assert.fail("Attempt to move default database should have failed.");
- } catch (HiveMetaException e) {
- // good
- }
-
- // Make sure nothing really moved
- Set<String> dbNames = new HashSet<>(client.getAllDatabases(DEFAULT_CATALOG_NAME));
- Assert.assertTrue(dbNames.contains(DEFAULT_DATABASE_NAME));
- }
-
- @Test(expected = HiveMetaException.class)
- public void moveNonExistentDatabase() throws TException, HiveMetaException {
- String catName = "moveNonExistentDb";
- new CatalogBuilder()
- .setName(catName)
- .setLocation("file:///tmp")
- .create(client);
- String argsMoveDB = String.format("-moveDatabase nosuch -fromCatalog %s -toCatalog %s",
- catName, DEFAULT_CATALOG_NAME);
- execute(new HiveSchemaToolTaskMoveDatabase(), argsMoveDB);
- }
-
- @Test
- public void moveDbToNonExistentCatalog() throws TException, HiveMetaException {
- String dbName = "doomedToHomelessness";
- new DatabaseBuilder()
- .setName(dbName)
- .create(client, conf);
- try {
- String argsMoveDB = String.format("-moveDatabase %s -fromCatalog %s -toCatalog nosuch",
- dbName, DEFAULT_CATALOG_NAME);
- execute(new HiveSchemaToolTaskMoveDatabase(), argsMoveDB);
- Assert.fail("Attempt to move database to non-existent catalog should have failed.");
- } catch (HiveMetaException e) {
- // good
- }
-
- // Make sure nothing really moved
- Set<String> dbNames = new HashSet<>(client.getAllDatabases(DEFAULT_CATALOG_NAME));
- Assert.assertTrue(dbNames.contains(dbName.toLowerCase()));
- }
-
- @Test
- public void moveTable() throws TException, HiveMetaException {
- String toCatName = "moveTableCat";
- String toDbName = "moveTableDb";
- String tableName = "moveTableTable";
- String partVal = "moveTableKey";
-
- new CatalogBuilder()
- .setName(toCatName)
- .setLocation("file:///tmp")
- .create(client);
-
- new DatabaseBuilder()
- .setCatalogName(toCatName)
- .setName(toDbName)
- .create(client, conf);
-
- Table table = new TableBuilder()
- .setTableName(tableName)
- .addCol("a", "int")
- .addPartCol("p", "string")
- .create(client, conf);
-
- new PartitionBuilder()
- .inTable(table)
- .addValue(partVal)
- .addToTable(client, conf);
-
- String argsMoveTable = String.format("-moveTable %s -fromCatalog %s -toCatalog %s -fromDatabase %s -toDatabase %s",
- tableName, DEFAULT_CATALOG_NAME, toCatName, DEFAULT_DATABASE_NAME, toDbName);
- execute(new HiveSchemaToolTaskMoveTable(), argsMoveTable);
-
- Table fetchedTable = client.getTable(toCatName, toDbName, tableName);
- Assert.assertNotNull(fetchedTable);
- Assert.assertEquals(toCatName.toLowerCase(), fetchedTable.getCatName());
- Assert.assertEquals(toDbName.toLowerCase(), fetchedTable.getDbName());
-
- Partition fetchedPart =
- client.getPartition(toCatName, toDbName, tableName, Collections.singletonList(partVal));
- Assert.assertNotNull(fetchedPart);
- Assert.assertEquals(toCatName.toLowerCase(), fetchedPart.getCatName());
- Assert.assertEquals(toDbName.toLowerCase(), fetchedPart.getDbName());
- Assert.assertEquals(tableName.toLowerCase(), fetchedPart.getTableName());
- }
-
- @Test
- public void moveTableWithinCatalog() throws TException, HiveMetaException {
- String toDbName = "moveTableWithinCatalogDb";
- String tableName = "moveTableWithinCatalogTable";
- String partVal = "moveTableWithinCatalogKey";
-
- new DatabaseBuilder()
- .setName(toDbName)
- .create(client, conf);
-
- Table table = new TableBuilder()
- .setTableName(tableName)
- .addCol("a", "int")
- .addPartCol("p", "string")
- .create(client, conf);
-
- new PartitionBuilder()
- .inTable(table)
- .addValue(partVal)
- .addToTable(client, conf);
-
- String argsMoveTable = String.format("-moveTable %s -fromCatalog %s -toCatalog %s -fromDatabase %s -toDatabase %s",
- tableName, DEFAULT_CATALOG_NAME, DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME, toDbName);
- execute(new HiveSchemaToolTaskMoveTable(), argsMoveTable);
-
- Table fetchedTable = client.getTable(DEFAULT_CATALOG_NAME, toDbName, tableName);
- Assert.assertNotNull(fetchedTable);
- Assert.assertEquals(DEFAULT_CATALOG_NAME, fetchedTable.getCatName());
- Assert.assertEquals(toDbName.toLowerCase(), fetchedTable.getDbName());
-
- Partition fetchedPart =
- client.getPartition(DEFAULT_CATALOG_NAME, toDbName, tableName, Collections.singletonList(partVal));
- Assert.assertNotNull(fetchedPart);
- Assert.assertEquals(DEFAULT_CATALOG_NAME, fetchedPart.getCatName());
- Assert.assertEquals(toDbName.toLowerCase(), fetchedPart.getDbName());
- Assert.assertEquals(tableName.toLowerCase(), fetchedPart.getTableName());
- }
-
- @Test
- public void moveTableWithExistingTableOfSameNameAlreadyInTargetDatabase()
- throws TException, HiveMetaException {
- String toDbName = "clobberTableDb";
- String tableName = "clobberTableTable";
-
- Database toDb = new DatabaseBuilder()
- .setName(toDbName)
- .create(client, conf);
-
- new TableBuilder()
- .setTableName(tableName)
- .addCol("a", "int")
- .create(client, conf);
-
- new TableBuilder()
- .inDb(toDb)
- .setTableName(tableName)
- .addCol("b", "varchar(32)")
- .create(client, conf);
-
- try {
- String argsMoveTable =
- String.format("-moveTable %s -fromCatalog %s -toCatalog %s -fromDatabase %s -toDatabase %s",
- tableName, DEFAULT_CATALOG_NAME, DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME, toDbName);
- execute(new HiveSchemaToolTaskMoveTable(), argsMoveTable);
- Assert.fail("Attempt to move table should have failed.");
- } catch (HiveMetaException e) {
- // good
- }
-
- // Make sure nothing really moved
- Set<String> tableNames = new HashSet<>(client.getAllTables(DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME));
- Assert.assertTrue(tableNames.contains(tableName.toLowerCase()));
-
- // Make sure the table in the target database didn't get clobbered
- Table fetchedTable = client.getTable(DEFAULT_CATALOG_NAME, toDbName, tableName);
- Assert.assertEquals("b", fetchedTable.getSd().getCols().get(0).getName());
- }
-
- @Test(expected = HiveMetaException.class)
- public void moveNonExistentTable() throws TException, HiveMetaException {
- String toDbName = "moveNonExistentTable";
- new DatabaseBuilder()
- .setName(toDbName)
- .create(client, conf);
- String argsMoveTable =
- String.format("-moveTable nosuch -fromCatalog %s -toCatalog %s -fromDatabase %s -toDatabase %s",
- DEFAULT_CATALOG_NAME, DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME, toDbName);
- execute(new HiveSchemaToolTaskMoveTable(), argsMoveTable);
- }
-
- @Test
- public void moveTableToNonExistentDb() throws TException, HiveMetaException {
- String tableName = "doomedToWander";
- new TableBuilder()
- .setTableName(tableName)
- .addCol("a", "int")
- .create(client, conf);
-
- try {
- String argsMoveTable =
- String.format("-moveTable %s -fromCatalog %s -toCatalog %s -fromDatabase %s -toDatabase nosuch",
- tableName, DEFAULT_CATALOG_NAME, DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME);
- execute(new HiveSchemaToolTaskMoveTable(), argsMoveTable);
- Assert.fail("Attempt to move table to non-existent table should have failed.");
- } catch (HiveMetaException e) {
- // good
- }
-
- // Make sure nothing really moved
- Set<String> tableNames = new HashSet<>(client.getAllTables(DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME));
- Assert.assertTrue(tableNames.contains(tableName.toLowerCase()));
- }
-
- private static void execute(HiveSchemaToolTask task, String taskArgs) throws HiveMetaException {
- try {
- StrTokenizer tokenizer = new StrTokenizer(argsBase + taskArgs, ' ', '\"');
- HiveSchemaToolCommandLine cl = new HiveSchemaToolCommandLine(tokenizer.getTokenArray());
- task.setCommandLineArguments(cl);
- } catch (Exception e) {
- throw new IllegalStateException("Could not parse comman line \n" + argsBase + taskArgs, e);
- }
-
- task.setHiveSchemaTool(schemaTool);
- task.execute();
- }
-}
[06/32] hive git commit: HIVE-20073: Additional tests for
to_utc_timestamp function based on HIVE-20068 (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by se...@apache.org.
HIVE-20073: Additional tests for to_utc_timestamp function based on HIVE-20068 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2ca27576
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2ca27576
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2ca27576
Branch: refs/heads/master-txnstats
Commit: 2ca275766ae76d2863aa09e6fdde1ab5d62a415f
Parents: ef53d69
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Fri Jul 6 00:04:28 2018 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Fri Jul 6 00:04:44 2018 -0700
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 3 +-
ql/src/test/queries/clientpositive/dst.q | 12 +++
.../test/results/clientpositive/llap/dst.q.out | 108 +++++++++++++++++++
3 files changed, 122 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/2ca27576/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index fedb77b..a27ce5e 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -898,7 +898,8 @@ minillaplocal.query.files=\
vectorized_join46.q,\
vectorized_multi_output_select.q,\
partialdhj.q,\
- stats_date.q
+ stats_date.q,\
+ dst.q
encrypted.query.files=encryption_join_unencrypted_tbl.q,\
encryption_insert_partition_static.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/2ca27576/ql/src/test/queries/clientpositive/dst.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/dst.q b/ql/src/test/queries/clientpositive/dst.q
new file mode 100644
index 0000000..4938324
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/dst.q
@@ -0,0 +1,12 @@
+SELECT '2017-03-11 18:00:00', to_utc_timestamp(timestamp '2017-03-11 18:00:00','US/Eastern');
+SELECT '2017-03-11 19:00:00', to_utc_timestamp(timestamp '2017-03-11 19:00:00','US/Eastern');
+SELECT '2017-03-11 20:00:00', to_utc_timestamp(timestamp '2017-03-11 20:00:00','US/Eastern');
+SELECT '2017-03-11 21:00:00', to_utc_timestamp(timestamp '2017-03-11 21:00:00','US/Eastern');
+SELECT '2017-03-11 22:00:00', to_utc_timestamp(timestamp '2017-03-11 22:00:00','US/Eastern');
+SELECT '2017-03-11 23:00:00', to_utc_timestamp(timestamp '2017-03-11 23:00:00','US/Eastern');
+SELECT '2017-03-12 00:00:00', to_utc_timestamp(timestamp '2017-03-12 00:00:00','US/Eastern');
+SELECT '2017-03-12 01:00:00', to_utc_timestamp(timestamp '2017-03-12 01:00:00','US/Eastern');
+SELECT '2017-03-12 02:00:00', to_utc_timestamp(timestamp '2017-03-12 02:00:00','US/Eastern');
+SELECT '2017-03-12 03:00:00', to_utc_timestamp(timestamp '2017-03-12 03:00:00','US/Eastern');
+SELECT '2017-03-12 04:00:00', to_utc_timestamp(timestamp '2017-03-12 04:00:00','US/Eastern');
+SELECT '2017-03-12 05:00:00', to_utc_timestamp(timestamp '2017-03-12 05:00:00','US/Eastern');
http://git-wip-us.apache.org/repos/asf/hive/blob/2ca27576/ql/src/test/results/clientpositive/llap/dst.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dst.q.out b/ql/src/test/results/clientpositive/llap/dst.q.out
new file mode 100644
index 0000000..c912f88
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/dst.q.out
@@ -0,0 +1,108 @@
+PREHOOK: query: SELECT '2017-03-11 18:00:00', to_utc_timestamp(timestamp '2017-03-11 18:00:00','US/Eastern')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT '2017-03-11 18:00:00', to_utc_timestamp(timestamp '2017-03-11 18:00:00','US/Eastern')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+2017-03-11 18:00:00 2017-03-11 23:00:00
+PREHOOK: query: SELECT '2017-03-11 19:00:00', to_utc_timestamp(timestamp '2017-03-11 19:00:00','US/Eastern')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT '2017-03-11 19:00:00', to_utc_timestamp(timestamp '2017-03-11 19:00:00','US/Eastern')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+2017-03-11 19:00:00 2017-03-12 00:00:00
+PREHOOK: query: SELECT '2017-03-11 20:00:00', to_utc_timestamp(timestamp '2017-03-11 20:00:00','US/Eastern')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT '2017-03-11 20:00:00', to_utc_timestamp(timestamp '2017-03-11 20:00:00','US/Eastern')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+2017-03-11 20:00:00 2017-03-12 01:00:00
+PREHOOK: query: SELECT '2017-03-11 21:00:00', to_utc_timestamp(timestamp '2017-03-11 21:00:00','US/Eastern')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT '2017-03-11 21:00:00', to_utc_timestamp(timestamp '2017-03-11 21:00:00','US/Eastern')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+2017-03-11 21:00:00 2017-03-12 02:00:00
+PREHOOK: query: SELECT '2017-03-11 22:00:00', to_utc_timestamp(timestamp '2017-03-11 22:00:00','US/Eastern')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT '2017-03-11 22:00:00', to_utc_timestamp(timestamp '2017-03-11 22:00:00','US/Eastern')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+2017-03-11 22:00:00 2017-03-12 03:00:00
+PREHOOK: query: SELECT '2017-03-11 23:00:00', to_utc_timestamp(timestamp '2017-03-11 23:00:00','US/Eastern')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT '2017-03-11 23:00:00', to_utc_timestamp(timestamp '2017-03-11 23:00:00','US/Eastern')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+2017-03-11 23:00:00 2017-03-12 04:00:00
+PREHOOK: query: SELECT '2017-03-12 00:00:00', to_utc_timestamp(timestamp '2017-03-12 00:00:00','US/Eastern')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT '2017-03-12 00:00:00', to_utc_timestamp(timestamp '2017-03-12 00:00:00','US/Eastern')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+2017-03-12 00:00:00 2017-03-12 05:00:00
+PREHOOK: query: SELECT '2017-03-12 01:00:00', to_utc_timestamp(timestamp '2017-03-12 01:00:00','US/Eastern')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT '2017-03-12 01:00:00', to_utc_timestamp(timestamp '2017-03-12 01:00:00','US/Eastern')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+2017-03-12 01:00:00 2017-03-12 06:00:00
+PREHOOK: query: SELECT '2017-03-12 02:00:00', to_utc_timestamp(timestamp '2017-03-12 02:00:00','US/Eastern')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT '2017-03-12 02:00:00', to_utc_timestamp(timestamp '2017-03-12 02:00:00','US/Eastern')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+2017-03-12 02:00:00 2017-03-12 07:00:00
+PREHOOK: query: SELECT '2017-03-12 03:00:00', to_utc_timestamp(timestamp '2017-03-12 03:00:00','US/Eastern')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT '2017-03-12 03:00:00', to_utc_timestamp(timestamp '2017-03-12 03:00:00','US/Eastern')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+2017-03-12 03:00:00 2017-03-12 07:00:00
+PREHOOK: query: SELECT '2017-03-12 04:00:00', to_utc_timestamp(timestamp '2017-03-12 04:00:00','US/Eastern')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT '2017-03-12 04:00:00', to_utc_timestamp(timestamp '2017-03-12 04:00:00','US/Eastern')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+2017-03-12 04:00:00 2017-03-12 08:00:00
+PREHOOK: query: SELECT '2017-03-12 05:00:00', to_utc_timestamp(timestamp '2017-03-12 05:00:00','US/Eastern')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT '2017-03-12 05:00:00', to_utc_timestamp(timestamp '2017-03-12 05:00:00','US/Eastern')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+2017-03-12 05:00:00 2017-03-12 09:00:00
[09/32] hive git commit: HIVE-20065: metastore should not rely on
jackson 1.x (Zoltan Haindrich reviewed by Sankar Hariappan)
Posted by se...@apache.org.
HIVE-20065: metastore should not rely on jackson 1.x (Zoltan Haindrich reviewed by Sankar Hariappan)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/403c1ea9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/403c1ea9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/403c1ea9
Branch: refs/heads/master-txnstats
Commit: 403c1ea9c98d0aadf446b75b9b5c2859f07569a9
Parents: 9bd040a
Author: Zoltan Haindrich <ki...@rxd.hu>
Authored: Fri Jul 6 13:16:20 2018 +0200
Committer: Zoltan Haindrich <ki...@rxd.hu>
Committed: Fri Jul 6 13:16:20 2018 +0200
----------------------------------------------------------------------
.../metastore/messaging/PartitionFiles.java | 2 +-
.../messaging/json/JSONAbortTxnMessage.java | 3 ++-
.../messaging/json/JSONAcidWriteMessage.java | 2 +-
.../json/JSONAddForeignKeyMessage.java | 3 ++-
.../json/JSONAddNotNullConstraintMessage.java | 3 ++-
.../messaging/json/JSONAddPartitionMessage.java | 26 ++++++++++--------
.../json/JSONAddPrimaryKeyMessage.java | 3 ++-
.../json/JSONAddUniqueConstraintMessage.java | 3 ++-
.../messaging/json/JSONAllocWriteIdMessage.java | 8 +++---
.../json/JSONAlterDatabaseMessage.java | 3 ++-
.../json/JSONAlterPartitionMessage.java | 11 +++++---
.../messaging/json/JSONAlterTableMessage.java | 9 +++++--
.../messaging/json/JSONCommitTxnMessage.java | 3 ++-
.../json/JSONCreateCatalogMessage.java | 3 ++-
.../json/JSONCreateDatabaseMessage.java | 4 ++-
.../json/JSONCreateFunctionMessage.java | 3 ++-
.../messaging/json/JSONCreateTableMessage.java | 8 ++++--
.../messaging/json/JSONDropCatalogMessage.java | 3 ++-
.../json/JSONDropConstraintMessage.java | 3 ++-
.../messaging/json/JSONDropDatabaseMessage.java | 3 ++-
.../messaging/json/JSONDropFunctionMessage.java | 3 ++-
.../json/JSONDropPartitionMessage.java | 13 ++++++---
.../messaging/json/JSONDropTableMessage.java | 9 +++++--
.../messaging/json/JSONInsertMessage.java | 14 ++++++----
.../messaging/json/JSONMessageDeserializer.java | 22 ++++++++-------
.../messaging/json/JSONMessageFactory.java | 21 +++++++--------
.../messaging/json/JSONOpenTxnMessage.java | 6 +++--
.../json/TestJSONMessageDeserializer.java | 28 ++++++++++++--------
28 files changed, 140 insertions(+), 82 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/PartitionFiles.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/PartitionFiles.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/PartitionFiles.java
index 308eaf1..56a53e0 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/PartitionFiles.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/PartitionFiles.java
@@ -21,8 +21,8 @@ package org.apache.hadoop.hive.metastore.messaging;
import java.util.Iterator;
import java.util.List;
+import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.Lists;
-import org.codehaus.jackson.annotate.JsonProperty;
public class PartitionFiles {
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAbortTxnMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAbortTxnMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAbortTxnMessage.java
index f169fc0..5da816b 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAbortTxnMessage.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAbortTxnMessage.java
@@ -19,7 +19,8 @@
package org.apache.hadoop.hive.metastore.messaging.json;
import org.apache.hadoop.hive.metastore.messaging.AbortTxnMessage;
-import org.codehaus.jackson.annotate.JsonProperty;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
/**
* JSON implementation of AbortTxnMessage
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAcidWriteMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAcidWriteMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAcidWriteMessage.java
index 515a2cb..cc528ee 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAcidWriteMessage.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAcidWriteMessage.java
@@ -19,13 +19,13 @@
package org.apache.hadoop.hive.metastore.messaging.json;
+import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.Lists;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.events.AcidWriteEvent;
import org.apache.hadoop.hive.metastore.messaging.AcidWriteMessage;
import org.apache.thrift.TException;
-import org.codehaus.jackson.annotate.JsonProperty;
import java.util.Iterator;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddForeignKeyMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddForeignKeyMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddForeignKeyMessage.java
index 9c912bf..d4a0bc2 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddForeignKeyMessage.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddForeignKeyMessage.java
@@ -24,7 +24,8 @@ import java.util.List;
import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
import org.apache.hadoop.hive.metastore.messaging.AddForeignKeyMessage;
import org.apache.thrift.TException;
-import org.codehaus.jackson.annotate.JsonProperty;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
/**
* JSON implementation of AddForeignKeyMessage
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddNotNullConstraintMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddNotNullConstraintMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddNotNullConstraintMessage.java
index 79312e4..1c3e8a8 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddNotNullConstraintMessage.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddNotNullConstraintMessage.java
@@ -23,7 +23,8 @@ import java.util.List;
import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
import org.apache.hadoop.hive.metastore.messaging.AddNotNullConstraintMessage;
import org.apache.thrift.TException;
-import org.codehaus.jackson.annotate.JsonProperty;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
public class JSONAddNotNullConstraintMessage extends AddNotNullConstraintMessage {
@JsonProperty
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java
index c232cce..11921f2 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java
@@ -19,23 +19,23 @@
package org.apache.hadoop.hive.metastore.messaging.json;
-import com.google.common.base.Function;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Nullable;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.messaging.AddPartitionMessage;
import org.apache.hadoop.hive.metastore.messaging.PartitionFiles;
import org.apache.thrift.TException;
-import org.codehaus.jackson.annotate.JsonProperty;
-import javax.annotation.Nullable;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
/**
* JSON implementation of AddPartitionMessage.
@@ -114,7 +114,11 @@ public class JSONAddPartitionMessage extends AddPartitionMessage {
@Override
public String getTableType() {
- if (tableType != null) return tableType; else return "";
+ if (tableType != null) {
+ return tableType;
+ } else {
+ return "";
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPrimaryKeyMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPrimaryKeyMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPrimaryKeyMessage.java
index f4f4d5d..3a18be8 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPrimaryKeyMessage.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPrimaryKeyMessage.java
@@ -24,7 +24,8 @@ import java.util.List;
import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
import org.apache.hadoop.hive.metastore.messaging.AddPrimaryKeyMessage;
import org.apache.thrift.TException;
-import org.codehaus.jackson.annotate.JsonProperty;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
/**
* JSON implementation of AddPrimaryKeyMessage
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddUniqueConstraintMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddUniqueConstraintMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddUniqueConstraintMessage.java
index 5a3cf96..3c4d5e0 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddUniqueConstraintMessage.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddUniqueConstraintMessage.java
@@ -25,7 +25,8 @@ import java.util.List;
import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
import org.apache.hadoop.hive.metastore.messaging.AddUniqueConstraintMessage;
import org.apache.thrift.TException;
-import org.codehaus.jackson.annotate.JsonProperty;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
public class JSONAddUniqueConstraintMessage extends AddUniqueConstraintMessage {
@JsonProperty
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAllocWriteIdMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAllocWriteIdMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAllocWriteIdMessage.java
index d4aba11..33b24fe 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAllocWriteIdMessage.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAllocWriteIdMessage.java
@@ -19,12 +19,14 @@
package org.apache.hadoop.hive.metastore.messaging.json;
-import org.apache.hadoop.hive.metastore.api.TxnToWriteId;
-import org.apache.hadoop.hive.metastore.messaging.AllocWriteIdMessage;
-import org.codehaus.jackson.annotate.JsonProperty;
import java.util.ArrayList;
import java.util.List;
+import org.apache.hadoop.hive.metastore.api.TxnToWriteId;
+import org.apache.hadoop.hive.metastore.messaging.AllocWriteIdMessage;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
/**
* JSON implementation of AllocWriteId
*/
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterDatabaseMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterDatabaseMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterDatabaseMessage.java
index d223480..7b316d5 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterDatabaseMessage.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterDatabaseMessage.java
@@ -21,7 +21,8 @@ package org.apache.hadoop.hive.metastore.messaging.json;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.messaging.AlterDatabaseMessage;
import org.apache.thrift.TException;
-import org.codehaus.jackson.annotate.JsonProperty;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
/**
* JSON alter database message.
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java
index 68cbd95..2ddec51 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java
@@ -18,13 +18,14 @@
*/
package org.apache.hadoop.hive.metastore.messaging.json;
+import java.util.Map;
+
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.messaging.AlterPartitionMessage;
import org.apache.thrift.TException;
-import org.codehaus.jackson.annotate.JsonProperty;
-import java.util.Map;
+import com.fasterxml.jackson.annotation.JsonProperty;
/**
* JSON alter table message
@@ -99,7 +100,11 @@ public class JSONAlterPartitionMessage extends AlterPartitionMessage {
@Override
public String getTableType() {
- if (tableType != null) return tableType; else return "";
+ if (tableType != null) {
+ return tableType;
+ } else {
+ return "";
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java
index ce8c0fa..d398fa0 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java
@@ -21,7 +21,8 @@ package org.apache.hadoop.hive.metastore.messaging.json;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage;
import org.apache.thrift.TException;
-import org.codehaus.jackson.annotate.JsonProperty;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
/**
* JSON alter table message
@@ -88,7 +89,11 @@ public class JSONAlterTableMessage extends AlterTableMessage {
@Override
public String getTableType() {
- if (tableType != null) return tableType; else return "";
+ if (tableType != null) {
+ return tableType;
+ } else {
+ return "";
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCommitTxnMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCommitTxnMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCommitTxnMessage.java
index 6082b8e..2c4940b 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCommitTxnMessage.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCommitTxnMessage.java
@@ -23,7 +23,8 @@ import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.WriteEventInfo;
import org.apache.hadoop.hive.metastore.messaging.CommitTxnMessage;
-import org.codehaus.jackson.annotate.JsonProperty;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateCatalogMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateCatalogMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateCatalogMessage.java
index 8a26764..e9038d0 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateCatalogMessage.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateCatalogMessage.java
@@ -18,7 +18,8 @@
package org.apache.hadoop.hive.metastore.messaging.json;
import org.apache.hadoop.hive.metastore.messaging.CreateCatalogMessage;
-import org.codehaus.jackson.annotate.JsonProperty;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
public class JSONCreateCatalogMessage extends CreateCatalogMessage {
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java
index 371bc8c..761ff99 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java
@@ -22,7 +22,8 @@ package org.apache.hadoop.hive.metastore.messaging.json;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.messaging.CreateDatabaseMessage;
import org.apache.thrift.TException;
-import org.codehaus.jackson.annotate.JsonProperty;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
/**
* JSON Implementation of CreateDatabaseMessage.
@@ -54,6 +55,7 @@ public class JSONCreateDatabaseMessage extends CreateDatabaseMessage {
checkValid();
}
+ @Override
public Database getDatabaseObject() throws Exception {
return (Database) JSONMessageFactory.getTObj(dbJson, Database.class);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateFunctionMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateFunctionMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateFunctionMessage.java
index 81f5d25..f7287df 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateFunctionMessage.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateFunctionMessage.java
@@ -22,7 +22,8 @@ package org.apache.hadoop.hive.metastore.messaging.json;
import org.apache.hadoop.hive.metastore.api.Function;
import org.apache.hadoop.hive.metastore.messaging.CreateFunctionMessage;
import org.apache.thrift.TException;
-import org.codehaus.jackson.annotate.JsonProperty;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
/**
* JSON Implementation of CreateFunctionMessage.
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java
index d894af9..d0f4b86 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java
@@ -25,8 +25,8 @@ import java.util.List;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.messaging.CreateTableMessage;
import org.apache.thrift.TException;
-import org.codehaus.jackson.annotate.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.Lists;
/**
@@ -102,7 +102,11 @@ public class JSONCreateTableMessage extends CreateTableMessage {
@Override
public String getTableType() {
- if (tableType != null) return tableType; else return "";
+ if (tableType != null) {
+ return tableType;
+ } else {
+ return "";
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropCatalogMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropCatalogMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropCatalogMessage.java
index 58e95f4..3576351 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropCatalogMessage.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropCatalogMessage.java
@@ -18,7 +18,8 @@
package org.apache.hadoop.hive.metastore.messaging.json;
import org.apache.hadoop.hive.metastore.messaging.DropCatalogMessage;
-import org.codehaus.jackson.annotate.JsonProperty;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
public class JSONDropCatalogMessage extends DropCatalogMessage {
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java
index 9e5d582..8dcfd8b 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java
@@ -20,7 +20,8 @@
package org.apache.hadoop.hive.metastore.messaging.json;
import org.apache.hadoop.hive.metastore.messaging.DropConstraintMessage;
-import org.codehaus.jackson.annotate.JsonProperty;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
/**
* JSON implementation of DropConstraintMessage
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java
index cd8bcdc..d2a75bf 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java
@@ -20,7 +20,8 @@
package org.apache.hadoop.hive.metastore.messaging.json;
import org.apache.hadoop.hive.metastore.messaging.DropDatabaseMessage;
-import org.codehaus.jackson.annotate.JsonProperty;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
/**
* JSON implementation of DropDatabaseMessage.
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropFunctionMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropFunctionMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropFunctionMessage.java
index ca32d16..a7a7a1f 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropFunctionMessage.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropFunctionMessage.java
@@ -21,7 +21,8 @@ package org.apache.hadoop.hive.metastore.messaging.json;
import org.apache.hadoop.hive.metastore.api.Function;
import org.apache.hadoop.hive.metastore.messaging.DropFunctionMessage;
-import org.codehaus.jackson.annotate.JsonProperty;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
/**
* JSON Implementation of CreateDatabaseMessage.
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java
index 4689711..957d595 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java
@@ -19,13 +19,14 @@
package org.apache.hadoop.hive.metastore.messaging.json;
+import java.util.List;
+import java.util.Map;
+
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.messaging.DropPartitionMessage;
import org.apache.thrift.TException;
-import org.codehaus.jackson.annotate.JsonProperty;
-import java.util.List;
-import java.util.Map;
+import com.fasterxml.jackson.annotation.JsonProperty;
/**
* JSON implementation of DropPartitionMessage.
@@ -97,7 +98,11 @@ public class JSONDropPartitionMessage extends DropPartitionMessage {
@Override
public String getTableType() {
- if (tableType != null) return tableType; else return "";
+ if (tableType != null) {
+ return tableType;
+ } else {
+ return "";
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java
index 591c9dd..88374ec 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java
@@ -22,7 +22,8 @@ package org.apache.hadoop.hive.metastore.messaging.json;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.messaging.DropTableMessage;
import org.apache.thrift.TException;
-import org.codehaus.jackson.annotate.JsonProperty;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
/**
* JSON implementation of DropTableMessage.
@@ -76,7 +77,11 @@ public class JSONDropTableMessage extends DropTableMessage {
@Override
public String getTableType() {
- if (tableType != null) return tableType; else return "";
+ if (tableType != null) {
+ return tableType;
+ } else {
+ return "";
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java
index 39372bd..2318a67 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java
@@ -19,17 +19,17 @@
package org.apache.hadoop.hive.metastore.messaging.json;
+import java.util.Iterator;
+import java.util.List;
+
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.messaging.InsertMessage;
import org.apache.thrift.TException;
-import org.codehaus.jackson.annotate.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.Lists;
-import java.util.Iterator;
-import java.util.List;
-
/**
* JSON implementation of InsertMessage
*/
@@ -91,7 +91,11 @@ public class JSONInsertMessage extends InsertMessage {
@Override
public String getTableType() {
- if (tableType != null) return tableType; else return "";
+ if (tableType != null) {
+ return tableType;
+ } else {
+ return "";
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java
index be6b751..3a480aa 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java
@@ -19,14 +19,17 @@
package org.apache.hadoop.hive.metastore.messaging.json;
+import org.apache.hadoop.hive.metastore.messaging.AbortTxnMessage;
import org.apache.hadoop.hive.metastore.messaging.AddForeignKeyMessage;
import org.apache.hadoop.hive.metastore.messaging.AddNotNullConstraintMessage;
import org.apache.hadoop.hive.metastore.messaging.AddPartitionMessage;
import org.apache.hadoop.hive.metastore.messaging.AddPrimaryKeyMessage;
import org.apache.hadoop.hive.metastore.messaging.AddUniqueConstraintMessage;
+import org.apache.hadoop.hive.metastore.messaging.AllocWriteIdMessage;
import org.apache.hadoop.hive.metastore.messaging.AlterDatabaseMessage;
import org.apache.hadoop.hive.metastore.messaging.AlterPartitionMessage;
import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage;
+import org.apache.hadoop.hive.metastore.messaging.CommitTxnMessage;
import org.apache.hadoop.hive.metastore.messaging.CreateDatabaseMessage;
import org.apache.hadoop.hive.metastore.messaging.CreateFunctionMessage;
import org.apache.hadoop.hive.metastore.messaging.CreateTableMessage;
@@ -38,13 +41,10 @@ import org.apache.hadoop.hive.metastore.messaging.DropTableMessage;
import org.apache.hadoop.hive.metastore.messaging.InsertMessage;
import org.apache.hadoop.hive.metastore.messaging.MessageDeserializer;
import org.apache.hadoop.hive.metastore.messaging.OpenTxnMessage;
-import org.apache.hadoop.hive.metastore.messaging.CommitTxnMessage;
-import org.apache.hadoop.hive.metastore.messaging.AbortTxnMessage;
-import org.apache.hadoop.hive.metastore.messaging.AllocWriteIdMessage;
import org.apache.hadoop.hive.metastore.messaging.AcidWriteMessage;
-import org.codehaus.jackson.map.DeserializationConfig;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.SerializationConfig;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.MapperFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
/**
* MessageDeserializer implementation, for deserializing from JSON strings.
@@ -54,10 +54,10 @@ public class JSONMessageDeserializer extends MessageDeserializer {
static ObjectMapper mapper = new ObjectMapper(); // Thread-safe.
static {
- mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- mapper.configure(SerializationConfig.Feature.AUTO_DETECT_GETTERS, false);
- mapper.configure(SerializationConfig.Feature.AUTO_DETECT_IS_GETTERS, false);
- mapper.configure(SerializationConfig.Feature.AUTO_DETECT_FIELDS, false);
+ mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ mapper.configure(MapperFeature.AUTO_DETECT_GETTERS, false);
+ mapper.configure(MapperFeature.AUTO_DETECT_IS_GETTERS, false);
+ mapper.configure(MapperFeature.AUTO_DETECT_FIELDS, false);
}
@Override
@@ -253,6 +253,7 @@ public class JSONMessageDeserializer extends MessageDeserializer {
}
}
+ @Override
public AllocWriteIdMessage getAllocWriteIdMessage(String messageBody) {
try {
return mapper.readValue(messageBody, JSONAllocWriteIdMessage.class);
@@ -261,6 +262,7 @@ public class JSONMessageDeserializer extends MessageDeserializer {
}
}
+ @Override
public AcidWriteMessage getAcidWriteMessage(String messageBody) {
try {
return mapper.readValue(messageBody, JSONAcidWriteMessage.class);
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java
index 07f51f0..6412674 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java
@@ -26,8 +26,6 @@ import java.util.Map;
import javax.annotation.Nullable;
-import com.google.common.collect.Iterables;
-
import org.apache.hadoop.hive.metastore.api.Catalog;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.Function;
@@ -40,15 +38,18 @@ import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.TxnToWriteId;
import org.apache.hadoop.hive.metastore.events.AcidWriteEvent;
+import org.apache.hadoop.hive.metastore.messaging.AbortTxnMessage;
import org.apache.hadoop.hive.metastore.messaging.AddForeignKeyMessage;
import org.apache.hadoop.hive.metastore.messaging.AddNotNullConstraintMessage;
import org.apache.hadoop.hive.metastore.messaging.AddPartitionMessage;
import org.apache.hadoop.hive.metastore.messaging.AddPrimaryKeyMessage;
import org.apache.hadoop.hive.metastore.messaging.AddUniqueConstraintMessage;
+import org.apache.hadoop.hive.metastore.messaging.AllocWriteIdMessage;
import org.apache.hadoop.hive.metastore.messaging.AlterCatalogMessage;
import org.apache.hadoop.hive.metastore.messaging.AlterDatabaseMessage;
import org.apache.hadoop.hive.metastore.messaging.AlterPartitionMessage;
import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage;
+import org.apache.hadoop.hive.metastore.messaging.CommitTxnMessage;
import org.apache.hadoop.hive.metastore.messaging.CreateCatalogMessage;
import org.apache.hadoop.hive.metastore.messaging.CreateDatabaseMessage;
import org.apache.hadoop.hive.metastore.messaging.CreateFunctionMessage;
@@ -62,25 +63,23 @@ import org.apache.hadoop.hive.metastore.messaging.DropTableMessage;
import org.apache.hadoop.hive.metastore.messaging.InsertMessage;
import org.apache.hadoop.hive.metastore.messaging.MessageDeserializer;
import org.apache.hadoop.hive.metastore.messaging.MessageFactory;
-import org.apache.hadoop.hive.metastore.messaging.PartitionFiles;
import org.apache.hadoop.hive.metastore.messaging.OpenTxnMessage;
-import org.apache.hadoop.hive.metastore.messaging.CommitTxnMessage;
-import org.apache.hadoop.hive.metastore.messaging.AbortTxnMessage;
-import org.apache.hadoop.hive.metastore.messaging.AllocWriteIdMessage;
import org.apache.hadoop.hive.metastore.messaging.AcidWriteMessage;
+import org.apache.hadoop.hive.metastore.messaging.PartitionFiles;
import org.apache.thrift.TBase;
import org.apache.thrift.TDeserializer;
import org.apache.thrift.TException;
import org.apache.thrift.TSerializer;
import org.apache.thrift.protocol.TJSONProtocol;
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.JsonNode;
-import org.codehaus.jackson.JsonParser;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.node.ObjectNode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONOpenTxnMessage.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONOpenTxnMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONOpenTxnMessage.java
index 0888920..0c4615b 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONOpenTxnMessage.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONOpenTxnMessage.java
@@ -18,12 +18,14 @@
*/
package org.apache.hadoop.hive.metastore.messaging.json;
-import org.apache.hadoop.hive.metastore.messaging.OpenTxnMessage;
-import org.codehaus.jackson.annotate.JsonProperty;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.LongStream;
+import org.apache.hadoop.hive.metastore.messaging.OpenTxnMessage;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
/**
* JSON implementation of OpenTxnMessage
*/
http://git-wip-us.apache.org/repos/asf/hive/blob/403c1ea9/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/messaging/json/TestJSONMessageDeserializer.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/messaging/json/TestJSONMessageDeserializer.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/messaging/json/TestJSONMessageDeserializer.java
index 21dd2f0..4ae19be 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/messaging/json/TestJSONMessageDeserializer.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/messaging/json/TestJSONMessageDeserializer.java
@@ -17,18 +17,19 @@
*/
package org.apache.hadoop.hive.metastore.messaging.json;
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
-import org.codehaus.jackson.annotate.JsonProperty;
import org.json.JSONException;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.skyscreamer.jsonassert.JSONAssert;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.Assert.*;
+import com.fasterxml.jackson.annotation.JsonProperty;
@Category(MetastoreUnitTest.class)
public class TestJSONMessageDeserializer {
@@ -58,19 +59,24 @@ public class TestJSONMessageDeserializer {
@Override
public boolean equals(Object o) {
- if (this == o)
+ if (this == o) {
return true;
- if (o == null || getClass() != o.getClass())
+ }
+ if (o == null || getClass() != o.getClass()) {
return false;
+ }
MyClass myClass = (MyClass) o;
- if (a != myClass.a)
+ if (a != myClass.a) {
return false;
- if (l != myClass.l)
+ }
+ if (l != myClass.l) {
return false;
- if (!map.equals(myClass.map))
+ }
+ if (!map.equals(myClass.map)) {
return false;
+ }
return shouldNotSerialize.equals(myClass.shouldNotSerialize);
}
[15/32] hive git commit: HIVE-20098 : Statistics: NPE when getting
Date column partition statistics (Andrii via Gopal V)
Posted by se...@apache.org.
HIVE-20098 : Statistics: NPE when getting Date column partition statistics (Andrii via Gopal V)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8494522d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8494522d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8494522d
Branch: refs/heads/master-txnstats
Commit: 8494522d810b8a31764729a8dd814f1e6ceab10e
Parents: eae5225
Author: Andrii Rosa <an...@fb.com>
Authored: Fri Jul 6 16:02:57 2018 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Fri Jul 6 16:02:57 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/metastore/StatObjectConverter.java | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/8494522d/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java
index ef2d670..7a0b21b 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java
@@ -466,8 +466,14 @@ public class StatObjectConverter {
} else if (colType.equals("date")) {
DateColumnStatsDataInspector dateStats = new DateColumnStatsDataInspector();
dateStats.setNumNulls(mStatsObj.getNumNulls());
- dateStats.setHighValue(new Date(mStatsObj.getLongHighValue()));
- dateStats.setLowValue(new Date(mStatsObj.getLongLowValue()));
+ Long highValue = mStatsObj.getLongHighValue();
+ if (highValue != null) {
+ dateStats.setHighValue(new Date(highValue));
+ }
+ Long lowValue = mStatsObj.getLongLowValue();
+ if (lowValue != null) {
+ dateStats.setLowValue(new Date(lowValue));
+ }
dateStats.setNumDVs(mStatsObj.getNumDVs());
dateStats.setBitVectors((mStatsObj.getBitVector()==null||!enableBitVector)? null : mStatsObj.getBitVector());
colStatsData.setDateStats(dateStats);
[18/32] hive git commit: HIVE-20103: WM: Only Aggregate DAG counters
if at least one is used (Gopal V, reviewed by Prasanth Jayachandran)
Posted by se...@apache.org.
HIVE-20103: WM: Only Aggregate DAG counters if at least one is used (Gopal V, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d3bba716
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d3bba716
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d3bba716
Branch: refs/heads/master-txnstats
Commit: d3bba71666c41f13e8efb007d7cd20accde0b296
Parents: 9fb4292
Author: Gopal V <go...@apache.org>
Authored: Mon Jul 9 09:08:21 2018 -0700
Committer: Gopal V <go...@apache.org>
Committed: Mon Jul 9 09:08:21 2018 -0700
----------------------------------------------------------------------
.../ql/exec/tez/monitoring/TezJobMonitor.java | 19 ++++++++++++++-----
1 file changed, 14 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d3bba716/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java
index 98048c8..d3fe190 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java
@@ -166,17 +166,26 @@ public class TezJobMonitor {
context.checkHeartbeaterLockException();
}
- status = dagClient.getDAGStatus(EnumSet.of(StatusGetOpts.GET_COUNTERS), checkInterval);
- TezCounters dagCounters = status.getDAGCounters();
- vertexProgressMap = status.getVertexProgress();
wmContext = context.getWmContext();
+ EnumSet<StatusGetOpts> opts = null;
+ if (wmContext != null) {
+ Set<String> desiredCounters = wmContext.getSubscribedCounters();
+ if (desiredCounters != null && !desiredCounters.isEmpty()) {
+ opts = EnumSet.of(StatusGetOpts.GET_COUNTERS);
+ }
+ }
+
+ status = dagClient.getDAGStatus(opts, checkInterval);
+
+ vertexProgressMap = status.getVertexProgress();
List<String> vertexNames = vertexProgressMap.keySet()
.stream()
.map(k -> k.replaceAll(" ", "_"))
.collect(Collectors.toList());
- if (dagCounters != null && wmContext != null) {
+ if (wmContext != null) {
Set<String> desiredCounters = wmContext.getSubscribedCounters();
- if (desiredCounters != null && !desiredCounters.isEmpty()) {
+ TezCounters dagCounters = status.getDAGCounters();
+ if (dagCounters != null && desiredCounters != null && !desiredCounters.isEmpty()) {
Map<String, Long> currentCounters = getCounterValues(dagCounters, vertexNames, vertexProgressMap,
desiredCounters, done);
if (LOG.isDebugEnabled()) {
[27/32] hive git commit: HIVE-18545: Add UDF to parse complex types
from json (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/1105ef39/serde/src/java/org/apache/hadoop/hive/serde2/JsonSerDe.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/JsonSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/JsonSerDe.java
index 2bb4a0f..f1c8477 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/JsonSerDe.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/JsonSerDe.java
@@ -20,30 +20,21 @@ package org.apache.hadoop.hive.serde2;
import java.io.ByteArrayInputStream;
import java.io.IOException;
-import java.nio.charset.CharacterCodingException;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.type.Date;
-import org.apache.hadoop.hive.common.type.HiveChar;
-import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.common.type.Timestamp;
-import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.json.HiveJsonStructReader;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector;
@@ -61,10 +52,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspect
import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
-import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
@@ -73,42 +60,32 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hive.common.util.HiveStringUtils;
import org.apache.hive.common.util.TimestampParser;
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.JsonParseException;
-import org.codehaus.jackson.JsonParser;
-import org.codehaus.jackson.JsonToken;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@SerDeSpec(schemaProps = {serdeConstants.LIST_COLUMNS,
- serdeConstants.LIST_COLUMN_TYPES,
- serdeConstants.TIMESTAMP_FORMATS})
+ serdeConstants.LIST_COLUMN_TYPES,
+ serdeConstants.TIMESTAMP_FORMATS })
-// FIXME: move TestJsonSerDe from hcat to serde2
public class JsonSerDe extends AbstractSerDe {
private static final Logger LOG = LoggerFactory.getLogger(JsonSerDe.class);
private List<String> columnNames;
- private StructTypeInfo schema;
- private JsonFactory jsonFactory = null;
-
- private StandardStructObjectInspector cachedObjectInspector;
- private TimestampParser tsParser;
+ private HiveJsonStructReader structReader;
+ private StructTypeInfo rowTypeInfo;
@Override
public void initialize(Configuration conf, Properties tbl)
throws SerDeException {
List<TypeInfo> columnTypes;
- StructTypeInfo rowTypeInfo;
-
LOG.debug("Initializing JsonSerDe: {}", tbl.entrySet());
- // Get column names and types
+ // Get column names
String columnNameProperty = tbl.getProperty(serdeConstants.LIST_COLUMNS);
- String columnTypeProperty = tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES);
final String columnNameDelimiter = tbl.containsKey(serdeConstants.COLUMN_NAME_DELIMITER) ? tbl
- .getProperty(serdeConstants.COLUMN_NAME_DELIMITER) : String.valueOf(SerDeUtils.COMMA);
+ .getProperty(serdeConstants.COLUMN_NAME_DELIMITER)
+ : String.valueOf(SerDeUtils.COMMA);
// all table column names
if (columnNameProperty.isEmpty()) {
columnNames = Collections.emptyList();
@@ -117,6 +94,7 @@ public class JsonSerDe extends AbstractSerDe {
}
// all column types
+ String columnTypeProperty = tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES);
if (columnTypeProperty.isEmpty()) {
columnTypes = Collections.emptyList();
} else {
@@ -129,298 +107,34 @@ public class JsonSerDe extends AbstractSerDe {
assert (columnNames.size() == columnTypes.size());
rowTypeInfo = (StructTypeInfo) TypeInfoFactory.getStructTypeInfo(columnNames, columnTypes);
- schema = rowTypeInfo;
- LOG.debug("schema : {}", schema);
- cachedObjectInspector = (StandardStructObjectInspector) TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(rowTypeInfo);
- jsonFactory = new JsonFactory();
- tsParser = new TimestampParser(
- HiveStringUtils.splitAndUnEscape(tbl.getProperty(serdeConstants.TIMESTAMP_FORMATS)));
+ TimestampParser tsParser = new TimestampParser(
+ HiveStringUtils.splitAndUnEscape(tbl.getProperty(serdeConstants.TIMESTAMP_FORMATS)));
+ structReader = new HiveJsonStructReader(rowTypeInfo, tsParser);
+ structReader.setIgnoreUnknownFields(true);
+ structReader.enableHiveColIndexParsing(true);
+ structReader.setWritablesUsage(true);
}
/**
* Takes JSON string in Text form, and has to return an object representation above
* it that's readable by the corresponding object inspector.
+ *
* For this implementation, since we're using the jackson parser, we can construct
* our own object implementation, and we use HCatRecord for it
*/
@Override
public Object deserialize(Writable blob) throws SerDeException {
+ Object row;
Text t = (Text) blob;
- JsonParser p;
- List<Object> r = new ArrayList<>(Collections.nCopies(columnNames.size(), null));
try {
- p = jsonFactory.createJsonParser(new ByteArrayInputStream((t.getBytes())));
- if (p.nextToken() != JsonToken.START_OBJECT) {
- throw new IOException("Start token not found where expected");
- }
- JsonToken token;
- while (((token = p.nextToken()) != JsonToken.END_OBJECT) && (token != null)) {
- // iterate through each token, and create appropriate object here.
- populateRecord(r, token, p, schema);
- }
- } catch (JsonParseException e) {
- LOG.warn("Error [{}] parsing json text [{}].", e, t);
- throw new SerDeException(e);
- } catch (IOException e) {
+ row = structReader.parseStruct(new ByteArrayInputStream((t.getBytes()), 0, t.getLength()));
+ return row;
+ } catch (Exception e) {
LOG.warn("Error [{}] parsing json text [{}].", e, t);
throw new SerDeException(e);
}
-
- return r;
- }
-
- private void populateRecord(List<Object> r, JsonToken token, JsonParser p, StructTypeInfo s) throws IOException {
- if (token != JsonToken.FIELD_NAME) {
- throw new IOException("Field name expected");
- }
- String fieldName = p.getText().toLowerCase();
- int fpos = s.getAllStructFieldNames().indexOf(fieldName);
- if (fpos == -1) {
- fpos = getPositionFromHiveInternalColumnName(fieldName);
- LOG.debug("NPE finding position for field [{}] in schema [{}],"
- + " attempting to check if it is an internal column name like _col0", fieldName, s);
- if (fpos == -1) {
- skipValue(p);
- return; // unknown field, we return. We'll continue from the next field onwards.
- }
- // If we get past this, then the column name did match the hive pattern for an internal
- // column name, such as _col0, etc, so it *MUST* match the schema for the appropriate column.
- // This means people can't use arbitrary column names such as _col0, and expect us to ignore it
- // if we find it.
- if (!fieldName.equalsIgnoreCase(getHiveInternalColumnName(fpos))) {
- LOG.error("Hive internal column name {} and position "
- + "encoding {} for the column name are at odds", fieldName, fpos);
- throw new IOException("Hive internal column name (" + fieldName
- + ") and position encoding (" + fpos
- + ") for the column name are at odds");
- }
- // If we reached here, then we were successful at finding an alternate internal
- // column mapping, and we're about to proceed.
- }
- Object currField = extractCurrentField(p, s.getStructFieldTypeInfo(fieldName), false);
- r.set(fpos, currField);
- }
-
- public String getHiveInternalColumnName(int fpos) {
- return HiveConf.getColumnInternalName(fpos);
- }
-
- public int getPositionFromHiveInternalColumnName(String internalName) {
- // return HiveConf.getPositionFromInternalName(fieldName);
- // The above line should have been all the implementation that
- // we need, but due to a bug in that impl which recognizes
- // only single-digit columns, we need another impl here.
- Pattern internalPattern = Pattern.compile("_col([0-9]+)");
- Matcher m = internalPattern.matcher(internalName);
- if (!m.matches()) {
- return -1;
- } else {
- return Integer.parseInt(m.group(1));
- }
- }
-
- /**
- * Utility method to extract (and forget) the next value token from the JsonParser,
- * as a whole. The reason this function gets called is to yank out the next value altogether,
- * because it corresponds to a field name that we do not recognize, and thus, do not have
- * a schema/type for. Thus, this field is to be ignored.
- *
- * @throws IOException
- * @throws JsonParseException
- */
- private void skipValue(JsonParser p) throws JsonParseException, IOException {
- JsonToken valueToken = p.nextToken();
-
- if ((valueToken == JsonToken.START_ARRAY) || (valueToken == JsonToken.START_OBJECT)) {
- // if the currently read token is a beginning of an array or object, move stream forward
- // skipping any child tokens till we're at the corresponding END_ARRAY or END_OBJECT token
- p.skipChildren();
- }
- // At the end of this function, the stream should be pointing to the last token that
- // corresponds to the value being skipped. This way, the next call to nextToken
- // will advance it to the next field name.
- }
-
- /**
- * Utility method to extract current expected field from given JsonParser
- * isTokenCurrent is a boolean variable also passed in, which determines
- * if the JsonParser is already at the token we expect to read next, or
- * needs advancing to the next before we read.
- */
- private Object extractCurrentField(JsonParser p, TypeInfo fieldTypeInfo,
- boolean isTokenCurrent) throws IOException {
- Object val = null;
- JsonToken valueToken;
- if (isTokenCurrent) {
- valueToken = p.getCurrentToken();
- } else {
- valueToken = p.nextToken();
- }
-
- switch (fieldTypeInfo.getCategory()) {
- case PRIMITIVE:
- PrimitiveObjectInspector.PrimitiveCategory primitiveCategory = PrimitiveObjectInspector.PrimitiveCategory.UNKNOWN;
- if (fieldTypeInfo instanceof PrimitiveTypeInfo) {
- primitiveCategory = ((PrimitiveTypeInfo) fieldTypeInfo).getPrimitiveCategory();
- }
- switch (primitiveCategory) {
- case INT:
- val = (valueToken == JsonToken.VALUE_NULL) ? null : p.getIntValue();
- break;
- case BYTE:
- val = (valueToken == JsonToken.VALUE_NULL) ? null : p.getByteValue();
- break;
- case SHORT:
- val = (valueToken == JsonToken.VALUE_NULL) ? null : p.getShortValue();
- break;
- case LONG:
- val = (valueToken == JsonToken.VALUE_NULL) ? null : p.getLongValue();
- break;
- case BOOLEAN:
- String bval = (valueToken == JsonToken.VALUE_NULL) ? null : p.getText();
- if (bval != null) {
- val = Boolean.valueOf(bval);
- } else {
- val = null;
- }
- break;
- case FLOAT:
- val = (valueToken == JsonToken.VALUE_NULL) ? null : p.getFloatValue();
- break;
- case DOUBLE:
- val = (valueToken == JsonToken.VALUE_NULL) ? null : p.getDoubleValue();
- break;
- case STRING:
- val = (valueToken == JsonToken.VALUE_NULL) ? null : p.getText();
- break;
- case BINARY:
- String b = (valueToken == JsonToken.VALUE_NULL) ? null : p.getText();
- if (b != null) {
- try {
- String t = Text.decode(b.getBytes(), 0, b.getBytes().length);
- return t.getBytes();
- } catch (CharacterCodingException e) {
- LOG.warn("Error generating json binary type from object.", e);
- return null;
- }
- } else {
- val = null;
- }
- break;
- case DATE:
- val = (valueToken == JsonToken.VALUE_NULL) ? null : Date.valueOf(p.getText());
- break;
- case TIMESTAMP:
- val = (valueToken == JsonToken.VALUE_NULL) ? null : tsParser.parseTimestamp(p.getText());
- break;
- case DECIMAL:
- val = (valueToken == JsonToken.VALUE_NULL) ? null : HiveDecimal.create(p.getText());
- break;
- case VARCHAR:
- int vLen = ((BaseCharTypeInfo) fieldTypeInfo).getLength();
- val = (valueToken == JsonToken.VALUE_NULL) ? null : new HiveVarchar(p.getText(), vLen);
- break;
- case CHAR:
- int cLen = ((BaseCharTypeInfo) fieldTypeInfo).getLength();
- val = (valueToken == JsonToken.VALUE_NULL) ? null : new HiveChar(p.getText(), cLen);
- break;
- }
- break;
- case LIST:
- if (valueToken == JsonToken.VALUE_NULL) {
- val = null;
- break;
- }
- if (valueToken != JsonToken.START_ARRAY) {
- throw new IOException("Start of Array expected");
- }
- List<Object> arr = new ArrayList<Object>();
- while ((valueToken = p.nextToken()) != JsonToken.END_ARRAY) {
- arr.add(extractCurrentField(p, ((ListTypeInfo)fieldTypeInfo).getListElementTypeInfo(), true));
- }
- val = arr;
- break;
- case MAP:
- if (valueToken == JsonToken.VALUE_NULL) {
- val = null;
- break;
- }
- if (valueToken != JsonToken.START_OBJECT) {
- throw new IOException("Start of Object expected");
- }
- Map<Object, Object> map = new LinkedHashMap<Object, Object>();
- while ((valueToken = p.nextToken()) != JsonToken.END_OBJECT) {
- Object k = getObjectOfCorrespondingPrimitiveType(p.getCurrentName(),
- (PrimitiveTypeInfo) ((MapTypeInfo)fieldTypeInfo).getMapKeyTypeInfo());
- Object v = extractCurrentField(p, ((MapTypeInfo) fieldTypeInfo).getMapValueTypeInfo(), false);
- map.put(k, v);
- }
- val = map;
- break;
- case STRUCT:
- if (valueToken == JsonToken.VALUE_NULL) {
- val = null;
- break;
- }
- if (valueToken != JsonToken.START_OBJECT) {
- throw new IOException("Start of Object expected");
- }
- ArrayList<TypeInfo> subSchema = ((StructTypeInfo)fieldTypeInfo).getAllStructFieldTypeInfos();
- int sz = subSchema.size();
- List<Object> struct = new ArrayList<Object>(Collections.nCopies(sz, null));
- while ((valueToken = p.nextToken()) != JsonToken.END_OBJECT) {
- populateRecord(struct, valueToken, p, ((StructTypeInfo) fieldTypeInfo));
- }
- val = struct;
- break;
- default:
- LOG.error("Unknown type found: " + fieldTypeInfo);
- return null;
- }
- return val;
- }
-
- private Object getObjectOfCorrespondingPrimitiveType(String s, PrimitiveTypeInfo mapKeyType)
- throws IOException {
- switch (mapKeyType.getPrimitiveCategory()) {
- case INT:
- return Integer.valueOf(s);
- case BYTE:
- return Byte.valueOf(s);
- case SHORT:
- return Short.valueOf(s);
- case LONG:
- return Long.valueOf(s);
- case BOOLEAN:
- return (s.equalsIgnoreCase("true"));
- case FLOAT:
- return Float.valueOf(s);
- case DOUBLE:
- return Double.valueOf(s);
- case STRING:
- return s;
- case BINARY:
- try {
- String t = Text.decode(s.getBytes(), 0, s.getBytes().length);
- return t.getBytes();
- } catch (CharacterCodingException e) {
- LOG.warn("Error generating json binary type from object.", e);
- return null;
- }
- case DATE:
- return Date.valueOf(s);
- case TIMESTAMP:
- return Timestamp.valueOf(s);
- case DECIMAL:
- return HiveDecimal.create(s);
- case VARCHAR:
- return new HiveVarchar(s, ((BaseCharTypeInfo) mapKeyType).getLength());
- case CHAR:
- return new HiveChar(s, ((BaseCharTypeInfo) mapKeyType).getLength());
- }
- throw new IOException("Could not convert from string to map type " + mapKeyType.getTypeName());
}
/**
@@ -462,7 +176,6 @@ public class JsonSerDe extends AbstractSerDe {
private static StringBuilder appendWithQuotes(StringBuilder sb, String value) {
return sb == null ? null : sb.append(SerDeUtils.QUOTE).append(value).append(SerDeUtils.QUOTE);
}
-
// TODO : code section copied over from SerDeUtils because of non-standard json production there
// should use quotes for all field names. We should fix this there, and then remove this copy.
// See http://jackson.codehaus.org/1.7.3/javadoc/org/codehaus/jackson/JsonParser.Feature.html#ALLOW_UNQUOTED_FIELD_NAMES
@@ -472,184 +185,184 @@ public class JsonSerDe extends AbstractSerDe {
private static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi) throws IOException {
switch (oi.getCategory()) {
- case PRIMITIVE: {
- PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
- if (o == null) {
- sb.append("null");
- } else {
- switch (poi.getPrimitiveCategory()) {
- case BOOLEAN: {
- boolean b = ((BooleanObjectInspector) poi).get(o);
- sb.append(b ? "true" : "false");
- break;
- }
- case BYTE: {
- sb.append(((ByteObjectInspector) poi).get(o));
- break;
- }
- case SHORT: {
- sb.append(((ShortObjectInspector) poi).get(o));
- break;
- }
- case INT: {
- sb.append(((IntObjectInspector) poi).get(o));
- break;
- }
- case LONG: {
- sb.append(((LongObjectInspector) poi).get(o));
- break;
- }
- case FLOAT: {
- sb.append(((FloatObjectInspector) poi).get(o));
- break;
- }
- case DOUBLE: {
- sb.append(((DoubleObjectInspector) poi).get(o));
- break;
- }
- case STRING: {
- String s =
- SerDeUtils.escapeString(((StringObjectInspector) poi).getPrimitiveJavaObject(o));
- appendWithQuotes(sb, s);
- break;
- }
- case BINARY:
- byte[] b = ((BinaryObjectInspector) oi).getPrimitiveJavaObject(o);
- Text txt = new Text();
- txt.set(b, 0, b.length);
- appendWithQuotes(sb, SerDeUtils.escapeString(txt.toString()));
- break;
- case DATE:
- Date d = ((DateObjectInspector) poi).getPrimitiveJavaObject(o);
- appendWithQuotes(sb, d.toString());
- break;
- case TIMESTAMP: {
- Timestamp t = ((TimestampObjectInspector) poi).getPrimitiveJavaObject(o);
- appendWithQuotes(sb, t.toString());
- break;
- }
- case DECIMAL:
- sb.append(((HiveDecimalObjectInspector) poi).getPrimitiveJavaObject(o));
- break;
- case VARCHAR: {
- String s = SerDeUtils.escapeString(
- ((HiveVarcharObjectInspector) poi).getPrimitiveJavaObject(o).toString());
- appendWithQuotes(sb, s);
- break;
- }
- case CHAR: {
- //this should use HiveChar.getPaddedValue() but it's protected; currently (v0.13)
- // HiveChar.toString() returns getPaddedValue()
- String s = SerDeUtils.escapeString(
- ((HiveCharObjectInspector) poi).getPrimitiveJavaObject(o).toString());
- appendWithQuotes(sb, s);
- break;
- }
- default:
- throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory());
- }
+ case PRIMITIVE: {
+ PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
+ if (o == null) {
+ sb.append("null");
+ } else {
+ switch (poi.getPrimitiveCategory()) {
+ case BOOLEAN: {
+ boolean b = ((BooleanObjectInspector) poi).get(o);
+ sb.append(b ? "true" : "false");
+ break;
+ }
+ case BYTE: {
+ sb.append(((ByteObjectInspector) poi).get(o));
+ break;
+ }
+ case SHORT: {
+ sb.append(((ShortObjectInspector) poi).get(o));
+ break;
+ }
+ case INT: {
+ sb.append(((IntObjectInspector) poi).get(o));
+ break;
+ }
+ case LONG: {
+ sb.append(((LongObjectInspector) poi).get(o));
+ break;
+ }
+ case FLOAT: {
+ sb.append(((FloatObjectInspector) poi).get(o));
+ break;
+ }
+ case DOUBLE: {
+ sb.append(((DoubleObjectInspector) poi).get(o));
+ break;
+ }
+ case STRING: {
+ String s =
+ SerDeUtils.escapeString(((StringObjectInspector) poi).getPrimitiveJavaObject(o));
+ appendWithQuotes(sb, s);
+ break;
+ }
+ case BINARY:
+ byte[] b = ((BinaryObjectInspector) oi).getPrimitiveJavaObject(o);
+ Text txt = new Text();
+ txt.set(b, 0, b.length);
+ appendWithQuotes(sb, SerDeUtils.escapeString(txt.toString()));
+ break;
+ case DATE:
+ Date d = ((DateObjectInspector) poi).getPrimitiveJavaObject(o);
+ appendWithQuotes(sb, d.toString());
+ break;
+ case TIMESTAMP: {
+ Timestamp t = ((TimestampObjectInspector) poi).getPrimitiveJavaObject(o);
+ appendWithQuotes(sb, t.toString());
+ break;
+ }
+ case DECIMAL:
+ sb.append(((HiveDecimalObjectInspector) poi).getPrimitiveJavaObject(o));
+ break;
+ case VARCHAR: {
+ String s = SerDeUtils.escapeString(
+ ((HiveVarcharObjectInspector) poi).getPrimitiveJavaObject(o).toString());
+ appendWithQuotes(sb, s);
+ break;
+ }
+ case CHAR: {
+ //this should use HiveChar.getPaddedValue() but it's protected; currently (v0.13)
+ // HiveChar.toString() returns getPaddedValue()
+ String s = SerDeUtils.escapeString(
+ ((HiveCharObjectInspector) poi).getPrimitiveJavaObject(o).toString());
+ appendWithQuotes(sb, s);
+ break;
+ }
+ default:
+ throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory());
}
- break;
}
- case LIST: {
- ListObjectInspector loi = (ListObjectInspector) oi;
- ObjectInspector listElementObjectInspector = loi
+ break;
+ }
+ case LIST: {
+ ListObjectInspector loi = (ListObjectInspector) oi;
+ ObjectInspector listElementObjectInspector = loi
.getListElementObjectInspector();
- List<?> olist = loi.getList(o);
- if (olist == null) {
- sb.append("null");
- } else {
- sb.append(SerDeUtils.LBRACKET);
- for (int i = 0; i < olist.size(); i++) {
- if (i > 0) {
- sb.append(SerDeUtils.COMMA);
- }
- buildJSONString(sb, olist.get(i), listElementObjectInspector);
+ List<?> olist = loi.getList(o);
+ if (olist == null) {
+ sb.append("null");
+ } else {
+ sb.append(SerDeUtils.LBRACKET);
+ for (int i = 0; i < olist.size(); i++) {
+ if (i > 0) {
+ sb.append(SerDeUtils.COMMA);
}
- sb.append(SerDeUtils.RBRACKET);
+ buildJSONString(sb, olist.get(i), listElementObjectInspector);
}
- break;
+ sb.append(SerDeUtils.RBRACKET);
}
- case MAP: {
- MapObjectInspector moi = (MapObjectInspector) oi;
- ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
- ObjectInspector mapValueObjectInspector = moi
+ break;
+ }
+ case MAP: {
+ MapObjectInspector moi = (MapObjectInspector) oi;
+ ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
+ ObjectInspector mapValueObjectInspector = moi
.getMapValueObjectInspector();
- Map<?, ?> omap = moi.getMap(o);
- if (omap == null) {
- sb.append("null");
- } else {
- sb.append(SerDeUtils.LBRACE);
- boolean first = true;
- for (Object entry : omap.entrySet()) {
- if (first) {
- first = false;
- } else {
- sb.append(SerDeUtils.COMMA);
- }
- Map.Entry<?, ?> e = (Map.Entry<?, ?>) entry;
- StringBuilder keyBuilder = new StringBuilder();
- buildJSONString(keyBuilder, e.getKey(), mapKeyObjectInspector);
- String keyString = keyBuilder.toString().trim();
- if ((!keyString.isEmpty()) && (keyString.charAt(0) != SerDeUtils.QUOTE)) {
- appendWithQuotes(sb, keyString);
- } else {
- sb.append(keyString);
- }
- sb.append(SerDeUtils.COLON);
- buildJSONString(sb, e.getValue(), mapValueObjectInspector);
+ Map<?, ?> omap = moi.getMap(o);
+ if (omap == null) {
+ sb.append("null");
+ } else {
+ sb.append(SerDeUtils.LBRACE);
+ boolean first = true;
+ for (Object entry : omap.entrySet()) {
+ if (first) {
+ first = false;
+ } else {
+ sb.append(SerDeUtils.COMMA);
+ }
+ Map.Entry<?, ?> e = (Map.Entry<?, ?>) entry;
+ StringBuilder keyBuilder = new StringBuilder();
+ buildJSONString(keyBuilder, e.getKey(), mapKeyObjectInspector);
+ String keyString = keyBuilder.toString().trim();
+ if ((!keyString.isEmpty()) && (keyString.charAt(0) != SerDeUtils.QUOTE)) {
+ appendWithQuotes(sb, keyString);
+ } else {
+ sb.append(keyString);
}
- sb.append(SerDeUtils.RBRACE);
+ sb.append(SerDeUtils.COLON);
+ buildJSONString(sb, e.getValue(), mapValueObjectInspector);
}
- break;
+ sb.append(SerDeUtils.RBRACE);
}
- case STRUCT: {
- StructObjectInspector soi = (StructObjectInspector) oi;
- List<? extends StructField> structFields = soi.getAllStructFieldRefs();
- if (o == null) {
- sb.append("null");
- } else {
- sb.append(SerDeUtils.LBRACE);
- for (int i = 0; i < structFields.size(); i++) {
- if (i > 0) {
- sb.append(SerDeUtils.COMMA);
- }
- appendWithQuotes(sb, structFields.get(i).getFieldName());
- sb.append(SerDeUtils.COLON);
- buildJSONString(sb, soi.getStructFieldData(o, structFields.get(i)),
- structFields.get(i).getFieldObjectInspector());
+ break;
+ }
+ case STRUCT: {
+ StructObjectInspector soi = (StructObjectInspector) oi;
+ List<? extends StructField> structFields = soi.getAllStructFieldRefs();
+ if (o == null) {
+ sb.append("null");
+ } else {
+ sb.append(SerDeUtils.LBRACE);
+ for (int i = 0; i < structFields.size(); i++) {
+ if (i > 0) {
+ sb.append(SerDeUtils.COMMA);
}
- sb.append(SerDeUtils.RBRACE);
+ appendWithQuotes(sb, structFields.get(i).getFieldName());
+ sb.append(SerDeUtils.COLON);
+ buildJSONString(sb, soi.getStructFieldData(o, structFields.get(i)),
+ structFields.get(i).getFieldObjectInspector());
}
- break;
+ sb.append(SerDeUtils.RBRACE);
}
- case UNION: {
- UnionObjectInspector uoi = (UnionObjectInspector) oi;
- if (o == null) {
- sb.append("null");
- } else {
- sb.append(SerDeUtils.LBRACE);
- sb.append(uoi.getTag(o));
- sb.append(SerDeUtils.COLON);
- buildJSONString(sb, uoi.getField(o),
+ break;
+ }
+ case UNION: {
+ UnionObjectInspector uoi = (UnionObjectInspector) oi;
+ if (o == null) {
+ sb.append("null");
+ } else {
+ sb.append(SerDeUtils.LBRACE);
+ sb.append(uoi.getTag(o));
+ sb.append(SerDeUtils.COLON);
+ buildJSONString(sb, uoi.getField(o),
uoi.getObjectInspectors().get(uoi.getTag(o)));
- sb.append(SerDeUtils.RBRACE);
- }
- break;
+ sb.append(SerDeUtils.RBRACE);
}
- default:
- throw new RuntimeException("Unknown type in ObjectInspector!");
+ break;
+ }
+ default:
+ throw new RuntimeException("Unknown type in ObjectInspector!");
}
}
/**
- * Returns an object inspector for the specified schema that
- * is capable of reading in the object representation of the JSON string
+ * Returns an object inspector for the specified schema that
+ * is capable of reading in the object representation of the JSON string
*/
@Override
public ObjectInspector getObjectInspector() throws SerDeException {
- return cachedObjectInspector;
+ return structReader.getObjectInspector();
}
@Override
@@ -663,4 +376,12 @@ public class JsonSerDe extends AbstractSerDe {
return null;
}
+ public StructTypeInfo getTypeInfo() {
+ return rowTypeInfo;
+ }
+
+ public void setWriteablesUsage(boolean b) {
+ structReader.setWritablesUsage(b);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1105ef39/serde/src/java/org/apache/hadoop/hive/serde2/json/HiveJsonStructReader.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/json/HiveJsonStructReader.java b/serde/src/java/org/apache/hadoop/hive/serde2/json/HiveJsonStructReader.java
new file mode 100644
index 0000000..ec4efad
--- /dev/null
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/json/HiveJsonStructReader.java
@@ -0,0 +1,402 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.json;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.CharacterCodingException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.hive.common.type.Date;
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.io.Text;
+import org.apache.hive.common.util.TimestampParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+
+public class HiveJsonStructReader {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HiveJsonStructReader.class);
+
+ private ObjectInspector oi;
+ private JsonFactory factory;
+
+
+ Set<String> reportedUnknownFieldNames = new HashSet<>();
+
+ private static boolean ignoreUnknownFields;
+ private static boolean hiveColIndexParsing;
+ private boolean writeablePrimitives;
+
+ private TimestampParser tsParser;
+
+ public HiveJsonStructReader(TypeInfo t) {
+ this(t, new TimestampParser());
+ }
+
+ public HiveJsonStructReader(TypeInfo t, TimestampParser tsParser) {
+ this.tsParser = tsParser;
+ oi = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(t);
+ factory = new JsonFactory();
+ }
+
+ public Object parseStruct(String text) throws JsonParseException, IOException, SerDeException {
+ JsonParser parser = factory.createParser(text);
+ return parseInternal(parser);
+ }
+
+ public Object parseStruct(InputStream is) throws JsonParseException, IOException, SerDeException {
+ JsonParser parser = factory.createParser(is);
+ return parseInternal(parser);
+ }
+
+ private Object parseInternal(JsonParser parser) throws SerDeException {
+ try {
+ parser.nextToken();
+ Object res = parseDispatcher(parser, oi);
+ return res;
+ } catch (Exception e) {
+ String locationStr = parser.getCurrentLocation().getLineNr() + "," + parser.getCurrentLocation().getColumnNr();
+ throw new SerDeException("at[" + locationStr + "]: " + e.getMessage(), e);
+ }
+ }
+
+ private Object parseDispatcher(JsonParser parser, ObjectInspector oi)
+ throws JsonParseException, IOException, SerDeException {
+
+ switch (oi.getCategory()) {
+ case PRIMITIVE:
+ return parsePrimitive(parser, (PrimitiveObjectInspector) oi);
+ case LIST:
+ return parseList(parser, (ListObjectInspector) oi);
+ case STRUCT:
+ return parseStruct(parser, (StructObjectInspector) oi);
+ case MAP:
+ return parseMap(parser, (MapObjectInspector) oi);
+ default:
+ throw new SerDeException("parsing of: " + oi.getCategory() + " is not handled");
+ }
+ }
+
+ private Object parseMap(JsonParser parser, MapObjectInspector oi) throws IOException, SerDeException {
+
+ if (parser.getCurrentToken() == JsonToken.VALUE_NULL) {
+ parser.nextToken();
+ return null;
+ }
+
+ Map<Object, Object> ret = new LinkedHashMap<>();
+
+ if (parser.getCurrentToken() != JsonToken.START_OBJECT) {
+ throw new SerDeException("struct expected");
+ }
+
+ if (!(oi.getMapKeyObjectInspector() instanceof PrimitiveObjectInspector)) {
+ throw new SerDeException("map key must be a primitive");
+ }
+ PrimitiveObjectInspector keyOI = (PrimitiveObjectInspector) oi.getMapKeyObjectInspector();
+ ObjectInspector valOI = oi.getMapValueObjectInspector();
+
+ JsonToken currentToken = parser.nextToken();
+ while (currentToken != null && currentToken != JsonToken.END_OBJECT) {
+
+ if (currentToken != JsonToken.FIELD_NAME) {
+ throw new SerDeException("unexpected token: " + currentToken);
+ }
+
+ Object key = parseMapKey(parser, keyOI);
+ Object val = parseDispatcher(parser, valOI);
+ ret.put(key, val);
+
+ currentToken = parser.getCurrentToken();
+ }
+ if (currentToken != null) {
+ parser.nextToken();
+ }
+ return ret;
+
+ }
+
+ private Object parseStruct(JsonParser parser, StructObjectInspector oi)
+ throws JsonParseException, IOException, SerDeException {
+
+ Object[] ret = new Object[oi.getAllStructFieldRefs().size()];
+
+ if (parser.getCurrentToken() == JsonToken.VALUE_NULL) {
+ parser.nextToken();
+ return null;
+ }
+ if (parser.getCurrentToken() != JsonToken.START_OBJECT) {
+ throw new SerDeException("struct expected");
+ }
+ JsonToken currentToken = parser.nextToken();
+ while (currentToken != null && currentToken != JsonToken.END_OBJECT) {
+
+ switch (currentToken) {
+ case FIELD_NAME:
+ String name = parser.getCurrentName();
+ try {
+ StructField field = null;
+ try {
+ field = getStructField(oi, name);
+ } catch (RuntimeException e) {
+ if (ignoreUnknownFields) {
+ if (!reportedUnknownFieldNames.contains(name)) {
+ LOG.warn("ignoring field:" + name);
+ reportedUnknownFieldNames.add(name);
+ }
+ parser.nextToken();
+ skipValue(parser);
+ break;
+ }
+ }
+ if (field == null) {
+ throw new SerDeException("undeclared field");
+ }
+ parser.nextToken();
+ ret[field.getFieldID()] = parseDispatcher(parser, field.getFieldObjectInspector());
+ } catch (Exception e) {
+ throw new SerDeException("struct field " + name + ": " + e.getMessage(), e);
+ }
+ break;
+ default:
+ throw new SerDeException("unexpected token: " + currentToken);
+ }
+ currentToken = parser.getCurrentToken();
+ }
+ if (currentToken != null) {
+ parser.nextToken();
+ }
+ return ret;
+ }
+
+ private StructField getStructField(StructObjectInspector oi, String name) {
+ if (hiveColIndexParsing) {
+ int colIndex = getColIndex(name);
+ if (colIndex >= 0) {
+ return oi.getAllStructFieldRefs().get(colIndex);
+ }
+ }
+ // FIXME: linear scan inside the below method...get a map here or something..
+ return oi.getStructFieldRef(name);
+ }
+
+ Pattern internalPattern = Pattern.compile("^_col([0-9]+)$");
+
+ private int getColIndex(String internalName) {
+ // The above line should have been all the implementation that
+ // we need, but due to a bug in that impl which recognizes
+ // only single-digit columns, we need another impl here.
+ Matcher m = internalPattern.matcher(internalName);
+ if (!m.matches()) {
+ return -1;
+ } else {
+ return Integer.parseInt(m.group(1));
+ }
+ }
+
+ private static void skipValue(JsonParser parser) throws JsonParseException, IOException {
+
+ int array = 0;
+ int object = 0;
+ do {
+ JsonToken currentToken = parser.getCurrentToken();
+ if(currentToken == JsonToken.START_ARRAY) {
+ array++;
+ }
+ if (currentToken == JsonToken.END_ARRAY) {
+ array--;
+ }
+ if (currentToken == JsonToken.START_OBJECT) {
+ object++;
+ }
+ if (currentToken == JsonToken.END_OBJECT) {
+ object--;
+ }
+
+ parser.nextToken();
+
+ } while (array > 0 || object > 0);
+
+ }
+
+ private Object parseList(JsonParser parser, ListObjectInspector oi)
+ throws JsonParseException, IOException, SerDeException {
+ List<Object> ret = new ArrayList<>();
+
+ if (parser.getCurrentToken() == JsonToken.VALUE_NULL) {
+ parser.nextToken();
+ return null;
+ }
+
+ if (parser.getCurrentToken() != JsonToken.START_ARRAY) {
+ throw new SerDeException("array expected");
+ }
+ ObjectInspector eOI = oi.getListElementObjectInspector();
+ JsonToken currentToken = parser.nextToken();
+ try {
+ while (currentToken != null && currentToken != JsonToken.END_ARRAY) {
+ ret.add(parseDispatcher(parser, eOI));
+ currentToken = parser.getCurrentToken();
+ }
+ } catch (Exception e) {
+ throw new SerDeException("array: " + e.getMessage(), e);
+ }
+
+ currentToken = parser.nextToken();
+
+ return ret;
+ }
+
+ private Object parsePrimitive(JsonParser parser, PrimitiveObjectInspector oi)
+ throws SerDeException, IOException {
+ JsonToken currentToken = parser.getCurrentToken();
+ if (currentToken == null) {
+ return null;
+ }
+ try {
+ switch (parser.getCurrentToken()) {
+ case VALUE_FALSE:
+ case VALUE_TRUE:
+ case VALUE_NUMBER_INT:
+ case VALUE_NUMBER_FLOAT:
+ case VALUE_STRING:
+ return getObjectOfCorrespondingPrimitiveType(parser.getValueAsString(), oi);
+ case VALUE_NULL:
+ return null;
+ default:
+ throw new SerDeException("unexpected token type: " + currentToken);
+ }
+ } finally {
+ parser.nextToken();
+
+ }
+ }
+
+ private Object getObjectOfCorrespondingPrimitiveType(String s, PrimitiveObjectInspector oi)
+ throws IOException {
+ PrimitiveTypeInfo typeInfo = oi.getTypeInfo();
+ if (writeablePrimitives) {
+ Converter c = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaStringObjectInspector, oi);
+ return c.convert(s);
+ }
+
+ switch (typeInfo.getPrimitiveCategory()) {
+ case INT:
+ return Integer.valueOf(s);
+ case BYTE:
+ return Byte.valueOf(s);
+ case SHORT:
+ return Short.valueOf(s);
+ case LONG:
+ return Long.valueOf(s);
+ case BOOLEAN:
+ return (s.equalsIgnoreCase("true"));
+ case FLOAT:
+ return Float.valueOf(s);
+ case DOUBLE:
+ return Double.valueOf(s);
+ case STRING:
+ return s;
+ case BINARY:
+ try {
+ String t = Text.decode(s.getBytes(), 0, s.getBytes().length);
+ return t.getBytes();
+ } catch (CharacterCodingException e) {
+ LOG.warn("Error generating json binary type from object.", e);
+ return null;
+ }
+ case DATE:
+ return Date.valueOf(s);
+ case TIMESTAMP:
+ return tsParser.parseTimestamp(s);
+ case DECIMAL:
+ return HiveDecimal.create(s);
+ case VARCHAR:
+ return new HiveVarchar(s, ((BaseCharTypeInfo) typeInfo).getLength());
+ case CHAR:
+ return new HiveChar(s, ((BaseCharTypeInfo) typeInfo).getLength());
+ }
+ throw new IOException("Could not convert from string to map type " + typeInfo.getTypeName());
+ }
+
+ private Object parseMapKey(JsonParser parser, PrimitiveObjectInspector oi) throws SerDeException, IOException {
+ JsonToken currentToken = parser.getCurrentToken();
+ if (currentToken == null) {
+ return null;
+ }
+ try {
+ switch (parser.getCurrentToken()) {
+ case FIELD_NAME:
+ return getObjectOfCorrespondingPrimitiveType(parser.getValueAsString(), oi);
+ case VALUE_NULL:
+ return null;
+ default:
+ throw new SerDeException("unexpected token type: " + currentToken);
+ }
+ } finally {
+ parser.nextToken();
+
+ }
+ }
+
+ public void setIgnoreUnknownFields(boolean b) {
+ ignoreUnknownFields = b;
+ }
+
+ public void enableHiveColIndexParsing(boolean b) {
+ hiveColIndexParsing = b;
+ }
+
+ public void setWritablesUsage(boolean b) {
+ writeablePrimitives = b;
+ }
+
+ public ObjectInspector getObjectInspector() {
+ return oi;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/1105ef39/streaming/src/java/org/apache/hive/streaming/StrictJsonWriter.java
----------------------------------------------------------------------
diff --git a/streaming/src/java/org/apache/hive/streaming/StrictJsonWriter.java b/streaming/src/java/org/apache/hive/streaming/StrictJsonWriter.java
index cabb64c..416bd67 100644
--- a/streaming/src/java/org/apache/hive/streaming/StrictJsonWriter.java
+++ b/streaming/src/java/org/apache/hive/streaming/StrictJsonWriter.java
@@ -30,7 +30,7 @@ import com.google.common.base.Joiner;
/**
* Streaming Writer handles utf8 encoded Json (Strict syntax).
- * Uses org.apache.hadoop.hive.serde2.JsonSerDe to process Json input
+ * Uses {@link JsonSerDe} to process Json input
*
* NOTE: This record writer is NOT thread-safe. Use one record writer per streaming connection.
*/
[10/32] hive git commit: HIVE-20077: hcat command should follow same
pattern as hive cli for getting HBase jars (Sean Busbey, reviewed by Aihua Xu,
Sahil Takiar)
Posted by se...@apache.org.
HIVE-20077: hcat command should follow same pattern as hive cli for getting HBase jars (Sean Busbey, reviewed by Aihua Xu, Sahil Takiar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/02a038e2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/02a038e2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/02a038e2
Branch: refs/heads/master-txnstats
Commit: 02a038e230e38685d8050a09b1f578f259d578c8
Parents: 403c1ea
Author: Sean Busbey <bu...@apache.org>
Authored: Fri Jul 6 11:09:57 2018 -0500
Committer: Sahil Takiar <st...@cloudera.com>
Committed: Fri Jul 6 11:09:57 2018 -0500
----------------------------------------------------------------------
hcatalog/bin/hcat | 41 ++++++++++++++++++++++++++++++-----------
1 file changed, 30 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/02a038e2/hcatalog/bin/hcat
----------------------------------------------------------------------
diff --git a/hcatalog/bin/hcat b/hcatalog/bin/hcat
index 7889263..2211407 100644
--- a/hcatalog/bin/hcat
+++ b/hcatalog/bin/hcat
@@ -145,18 +145,37 @@ fi
# Put external jars, hcat jar, and config file in the classpath
HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:${HCAT_CLASSPATH}:${HCAT_JAR}:${HIVE_CONF_DIR}
-# Look for HBase in a BigTop-compatible way. Avoid thrift version
-# conflict with modern versions of HBase.
+# HBase detection. Need bin/hbase and a conf dir for building classpath entries.
+# Start with BigTop defaults for HBASE_HOME and HBASE_CONF_DIR.
HBASE_HOME=${HBASE_HOME:-"/usr/lib/hbase"}
-HBASE_CONF_DIR=${HBASE_CONF_DIR:-"${HBASE_HOME}/conf"}
-if [ -d ${HBASE_HOME} ] ; then
- for jar in $(find $HBASE_HOME -name '*.jar' -not -name '*thrift*'); do
- HBASE_CLASSPATH=$HBASE_CLASSPATH:${jar}
- done
- export HADOOP_CLASSPATH="${HADOOP_CLASSPATH}:${HBASE_CLASSPATH}"
-fi
-if [ -d $HBASE_CONF_DIR ] ; then
- HADOOP_CLASSPATH="${HADOOP_CLASSPATH}:${HBASE_CONF_DIR}"
+HBASE_CONF_DIR=${HBASE_CONF_DIR:-"/etc/hbase/conf"}
+if [[ ! -d $HBASE_CONF_DIR ]] ; then
+ # not explicitly set, nor in BigTop location. Try looking in HBASE_HOME.
+ HBASE_CONF_DIR="$HBASE_HOME/conf"
+fi
+
+# perhaps we've located the HBase config. if so, include it on classpath.
+if [[ -d $HBASE_CONF_DIR ]] ; then
+ export HADOOP_CLASSPATH="${HADOOP_CLASSPATH}:${HBASE_CONF_DIR}"
+fi
+
+# look for the hbase script. First check HBASE_HOME and then ask PATH.
+if [[ -e $HBASE_HOME/bin/hbase ]] ; then
+ HBASE_BIN="$HBASE_HOME/bin/hbase"
+fi
+HBASE_BIN=${HBASE_BIN:-"$(which hbase)"}
+
+# perhaps we've located HBase. If so, include its details on the classpath
+if [[ -n $HBASE_BIN ]] ; then
+ # exclude ZK, PB, and Guava (See HIVE-2055)
+ # depends on HBASE-8438 (hbase-0.94.14+, hbase-0.96.1+) for `hbase mapredcp` command
+ for x in $($HBASE_BIN mapredcp 2>&2 | tr ':' '\n') ; do
+ if [[ $x == *zookeeper* || $x == *protobuf-java* || $x == *guava* ]] ; then
+ continue
+ fi
+ # TODO: should these should be added to AUX_PARAM as well?
+ export HADOOP_CLASSPATH="${HADOOP_CLASSPATH}:${x}"
+ done
fi
export HADOOP_CLASSPATH=$HADOOP_CLASSPATH
[17/32] hive git commit: HIVE-20121: investigate issues with
TestReplicationScenariosAcidTables (Comment out Test for now while we
Investigate)(Mahesh Kumar Behera, reviewed by Anishek Agarwal)
Posted by se...@apache.org.
HIVE-20121: investigate issues with TestReplicationScenariosAcidTables (Comment out Test for now while we Investigate)(Mahesh Kumar Behera, reviewed by Anishek Agarwal)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9fb42927
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9fb42927
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9fb42927
Branch: refs/heads/master-txnstats
Commit: 9fb42927aa14b4afe0c35bf86be4c2a3ff16adbc
Parents: 406cde9
Author: Anishek Agarwal <an...@gmail.com>
Authored: Mon Jul 9 14:32:04 2018 +0530
Committer: Anishek Agarwal <an...@gmail.com>
Committed: Mon Jul 9 14:32:04 2018 +0530
----------------------------------------------------------------------
.../hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9fb42927/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java
index 8c683cf..6949606 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java
@@ -56,10 +56,12 @@ import java.util.List;
import javax.annotation.Nullable;
import java.util.Collections;
import com.google.common.collect.Lists;
+import org.junit.Ignore;
/**
* TestReplicationScenariosAcidTables - test replication for ACID tables
*/
+@Ignore("this is ignored as its taking more time. Need to analyze further to reduce the time")
public class TestReplicationScenariosAcidTables {
@Rule
public final TestName testName = new TestName();
[30/32] hive git commit: HIVE-19765: Add Parquet specific tests to
BlobstoreCliDriver (Sahil Takiar, reviewed by Marta Kuczora)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/b574bcdd/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out
new file mode 100644
index 0000000..15ae3d9
--- /dev/null
+++ b/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out
@@ -0,0 +1,537 @@
+PREHOOK: query: DROP TABLE src_events
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE src_events
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE src_events
+(
+ log_id BIGINT,
+ `time` BIGINT,
+ uid BIGINT,
+ user_id BIGINT,
+ type INT,
+ event_data STRING,
+ session_id STRING,
+ full_uid BIGINT,
+ run_date STRING,
+ game_id INT,
+ event_name STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/src_events
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_events
+POSTHOOK: query: CREATE TABLE src_events
+(
+ log_id BIGINT,
+ `time` BIGINT,
+ uid BIGINT,
+ user_id BIGINT,
+ type INT,
+ event_data STRING,
+ session_id STRING,
+ full_uid BIGINT,
+ run_date STRING,
+ game_id INT,
+ event_name STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/src_events
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_events
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@src_events
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@src_events
+PREHOOK: query: DROP TABLE parquet_events
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE parquet_events
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE parquet_events
+(
+ log_id BIGINT,
+ `time` BIGINT,
+ uid BIGINT,
+ user_id BIGINT,
+ type INT,
+ event_data STRING,
+ session_id STRING,
+ full_uid BIGINT
+)
+PARTITIONED BY (run_date STRING, game_id INT, event_name STRING)
+STORED AS PARQUET
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_events
+PREHOOK: Output: database:default
+PREHOOK: Output: default@parquet_events
+POSTHOOK: query: CREATE TABLE parquet_events
+(
+ log_id BIGINT,
+ `time` BIGINT,
+ uid BIGINT,
+ user_id BIGINT,
+ type INT,
+ event_data STRING,
+ session_id STRING,
+ full_uid BIGINT
+)
+PARTITIONED BY (run_date STRING, game_id INT, event_name STRING)
+STORED AS PARQUET
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_events
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@parquet_events
+PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
+SELECT * FROM src_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events
+POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
+SELECT * FROM src_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change
+POSTHOOK: Output: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change
+POSTHOOK: Output: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: SHOW PARTITIONS parquet_events
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@parquet_events
+POSTHOOK: query: SHOW PARTITIONS parquet_events
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@parquet_events
+run_date=20120921/game_id=39/event_name=hq_change
+run_date=20121021/game_id=39/event_name=hq_change
+run_date=20121121/game_id=39/event_name=hq_change
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+200
+PREHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change')
+#### A masked pattern was here ####
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-1
+PREHOOK: Output: default@parquet_events
+POSTHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change')
+#### A masked pattern was here ####
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-1
+POSTHOOK: Output: default@parquet_events
+POSTHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
+PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201211'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
+POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201211'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: SHOW PARTITIONS parquet_events
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@parquet_events
+POSTHOOK: query: SHOW PARTITIONS parquet_events
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@parquet_events
+run_date=20120921/game_id=39/event_name=hq_change
+run_date=20121021/game_id=39/event_name=hq_change
+run_date=201211/game_id=39/event_name=hq_change
+run_date=20121121/game_id=39/event_name=hq_change
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+300
+PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201211'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
+POSTHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201211'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: SHOW PARTITIONS parquet_events
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@parquet_events
+POSTHOOK: query: SHOW PARTITIONS parquet_events
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@parquet_events
+run_date=20120921/game_id=39/event_name=hq_change
+run_date=20121021/game_id=39/event_name=hq_change
+run_date=201211/game_id=39/event_name=hq_change
+run_date=20121121/game_id=39/event_name=hq_change
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+400
+PREHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+#### A masked pattern was here ####
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-2
+PREHOOK: Output: default@parquet_events
+POSTHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+#### A masked pattern was here ####
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-2
+POSTHOOK: Output: default@parquet_events
+POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+POSTHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: SHOW PARTITIONS parquet_events
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@parquet_events
+POSTHOOK: query: SHOW PARTITIONS parquet_events
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@parquet_events
+run_date=201209/game_id=39/event_name=hq_change
+run_date=20120921/game_id=39/event_name=hq_change
+run_date=20121021/game_id=39/event_name=hq_change
+run_date=201211/game_id=39/event_name=hq_change
+run_date=20121121/game_id=39/event_name=hq_change
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+500
+PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
+SELECT * FROM src_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events
+POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
+SELECT * FROM src_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change
+POSTHOOK: Output: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change
+POSTHOOK: Output: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: SHOW PARTITIONS parquet_events
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@parquet_events
+POSTHOOK: query: SHOW PARTITIONS parquet_events
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@parquet_events
+run_date=201209/game_id=39/event_name=hq_change
+run_date=20120921/game_id=39/event_name=hq_change
+run_date=20121021/game_id=39/event_name=hq_change
+run_date=201211/game_id=39/event_name=hq_change
+run_date=20121121/game_id=39/event_name=hq_change
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+500
+PREHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201207, game_id=39, event_name='hq_change')
+#### A masked pattern was here ####
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-3
+PREHOOK: Output: default@parquet_events
+POSTHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201207, game_id=39, event_name='hq_change')
+#### A masked pattern was here ####
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-3
+POSTHOOK: Output: default@parquet_events
+POSTHOOK: Output: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
+PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201207, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
+POSTHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201207, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: SHOW PARTITIONS parquet_events
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@parquet_events
+POSTHOOK: query: SHOW PARTITIONS parquet_events
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@parquet_events
+run_date=201207/game_id=39/event_name=hq_change
+run_date=201209/game_id=39/event_name=hq_change
+run_date=20120921/game_id=39/event_name=hq_change
+run_date=20121021/game_id=39/event_name=hq_change
+run_date=201211/game_id=39/event_name=hq_change
+run_date=20121121/game_id=39/event_name=hq_change
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+PREHOOK: Input: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
+PREHOOK: Input: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+PREHOOK: Input: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change
+PREHOOK: Input: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change
+PREHOOK: Input: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
+PREHOOK: Input: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+POSTHOOK: Input: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
+POSTHOOK: Input: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+POSTHOOK: Input: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change
+POSTHOOK: Input: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change
+POSTHOOK: Input: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
+POSTHOOK: Input: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change
+#### A masked pattern was here ####
+550
+PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
+SELECT * FROM src_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events
+POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
+SELECT * FROM src_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change
+POSTHOOK: Output: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change
+POSTHOOK: Output: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: SHOW PARTITIONS parquet_events
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@parquet_events
+POSTHOOK: query: SHOW PARTITIONS parquet_events
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@parquet_events
+run_date=201207/game_id=39/event_name=hq_change
+run_date=201209/game_id=39/event_name=hq_change
+run_date=20120921/game_id=39/event_name=hq_change
+run_date=20121021/game_id=39/event_name=hq_change
+run_date=201211/game_id=39/event_name=hq_change
+run_date=20121121/game_id=39/event_name=hq_change
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+PREHOOK: Input: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
+PREHOOK: Input: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+PREHOOK: Input: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change
+PREHOOK: Input: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change
+PREHOOK: Input: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
+PREHOOK: Input: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+POSTHOOK: Input: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
+POSTHOOK: Input: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+POSTHOOK: Input: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change
+POSTHOOK: Input: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change
+POSTHOOK: Input: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
+POSTHOOK: Input: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change
+#### A masked pattern was here ####
+550
+PREHOOK: query: ALTER TABLE parquet_events DROP PARTITION (run_date=201211, game_id=39, event_name='hq_change')
+PREHOOK: type: ALTERTABLE_DROPPARTS
+PREHOOK: Input: default@parquet_events
+PREHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
+POSTHOOK: query: ALTER TABLE parquet_events DROP PARTITION (run_date=201211, game_id=39, event_name='hq_change')
+POSTHOOK: type: ALTERTABLE_DROPPARTS
+POSTHOOK: Input: default@parquet_events
+POSTHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
+PREHOOK: query: ALTER TABLE parquet_events DROP PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+PREHOOK: type: ALTERTABLE_DROPPARTS
+PREHOOK: Input: default@parquet_events
+PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+POSTHOOK: query: ALTER TABLE parquet_events DROP PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+POSTHOOK: type: ALTERTABLE_DROPPARTS
+POSTHOOK: Input: default@parquet_events
+POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+PREHOOK: query: ALTER TABLE parquet_events DROP PARTITION (run_date=201207, game_id=39, event_name='hq_change')
+PREHOOK: type: ALTERTABLE_DROPPARTS
+PREHOOK: Input: default@parquet_events
+PREHOOK: Output: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
+POSTHOOK: query: ALTER TABLE parquet_events DROP PARTITION (run_date=201207, game_id=39, event_name='hq_change')
+POSTHOOK: type: ALTERTABLE_DROPPARTS
+POSTHOOK: Input: default@parquet_events
+POSTHOOK: Output: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
+PREHOOK: query: SHOW PARTITIONS parquet_events
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@parquet_events
+POSTHOOK: query: SHOW PARTITIONS parquet_events
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@parquet_events
+run_date=20120921/game_id=39/event_name=hq_change
+run_date=20121021/game_id=39/event_name=hq_change
+run_date=20121121/game_id=39/event_name=hq_change
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+200
http://git-wip-us.apache.org/repos/asf/hive/blob/b574bcdd/itests/hive-blobstore/src/test/results/clientpositive/ptf_parquetfile.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/ptf_parquetfile.q.out b/itests/hive-blobstore/src/test/results/clientpositive/ptf_parquetfile.q.out
new file mode 100644
index 0000000..192b80f
--- /dev/null
+++ b/itests/hive-blobstore/src/test/results/clientpositive/ptf_parquetfile.q.out
@@ -0,0 +1,104 @@
+PREHOOK: query: DROP TABLE part_parquet
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE part_parquet
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE part_parquet(
+ p_partkey int,
+ p_name string,
+ p_mfgr string,
+ p_brand string,
+ p_type string,
+ p_size int,
+ p_container string,
+ p_retailprice double,
+ p_comment string
+)
+STORED AS PARQUET
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/ptf_parquetfile/part_parquet
+PREHOOK: Output: database:default
+PREHOOK: Output: default@part_parquet
+POSTHOOK: query: CREATE TABLE part_parquet(
+ p_partkey int,
+ p_name string,
+ p_mfgr string,
+ p_brand string,
+ p_type string,
+ p_size int,
+ p_container string,
+ p_retailprice double,
+ p_comment string
+)
+STORED AS PARQUET
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/ptf_parquetfile/part_parquet
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@part_parquet
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/part.parquet' OVERWRITE INTO TABLE part_parquet
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@part_parquet
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/part.parquet' OVERWRITE INTO TABLE part_parquet
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@part_parquet
+PREHOOK: query: SELECT
+ p_mfgr, p_name, p_size,
+ RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS r,
+ DENSE_RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS dr,
+ SUM(p_retailprice) OVER (
+ PARTITION BY p_mfgr
+ ORDER BY p_name ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+ ) AS s1
+FROM NOOP(
+ ON part_parquet
+ PARTITION BY p_mfgr
+ ORDER BY p_name
+)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part_parquet
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT
+ p_mfgr, p_name, p_size,
+ RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS r,
+ DENSE_RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS dr,
+ SUM(p_retailprice) OVER (
+ PARTITION BY p_mfgr
+ ORDER BY p_name ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+ ) AS s1
+FROM NOOP(
+ ON part_parquet
+ PARTITION BY p_mfgr
+ ORDER BY p_name
+)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part_parquet
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3
+Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001
+Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001
+Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68
+Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38
+Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001
+Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62
+Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68
+Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95
+Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34
+Manufacturer#3 almond antique misty red olive 1 4 4 6195.32
+Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61
+Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67
+Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09
+Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35
+Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27
+Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001
+Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69
+Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004
+Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18
+Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66
[14/32] hive git commit: HIVE-20060 Refactor HiveSchemaTool and
MetastoreSchemaTool (Alan Gates, reviewed by Daniel Dai)
Posted by se...@apache.org.
HIVE-20060 Refactor HiveSchemaTool and MetastoreSchemaTool (Alan Gates, reviewed by Daniel Dai)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/eae5225f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/eae5225f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/eae5225f
Branch: refs/heads/master-txnstats
Commit: eae5225f4301254cd8c5ad127bc242890bd441a8
Parents: 02a038e
Author: Alan Gates <ga...@hortonworks.com>
Authored: Fri Jul 6 09:26:17 2018 -0700
Committer: Alan Gates <ga...@hortonworks.com>
Committed: Fri Jul 6 09:26:17 2018 -0700
----------------------------------------------------------------------
.../hive/beeline/schematool/HiveSchemaTool.java | 338 +----
.../schematool/HiveSchemaToolCommandLine.java | 286 ----
.../beeline/schematool/HiveSchemaToolTask.java | 32 -
.../HiveSchemaToolTaskAlterCatalog.java | 90 --
.../HiveSchemaToolTaskCreateCatalog.java | 132 --
.../schematool/HiveSchemaToolTaskInfo.java | 43 -
.../schematool/HiveSchemaToolTaskInit.java | 73 -
.../HiveSchemaToolTaskMoveDatabase.java | 96 --
.../schematool/HiveSchemaToolTaskMoveTable.java | 142 --
.../schematool/HiveSchemaToolTaskUpgrade.java | 116 --
.../schematool/HiveSchemaToolTaskValidate.java | 631 ---------
.../beeline/schematool/TestHiveSchemaTool.java | 6 +-
.../tools/TestSchemaToolCatalogOps.java | 479 +++++++
.../hive/beeline/schematool/TestSchemaTool.java | 550 +-------
.../schematool/TestSchemaToolCatalogOps.java | 478 -------
.../metastore/tools/MetastoreSchemaTool.java | 1282 +++---------------
.../metastore/tools/SchemaToolCommandLine.java | 308 +++++
.../hive/metastore/tools/SchemaToolTask.java | 32 +
.../tools/SchemaToolTaskAlterCatalog.java | 90 ++
.../tools/SchemaToolTaskCreateCatalog.java | 132 ++
.../tools/SchemaToolTaskCreateUser.java | 115 ++
.../metastore/tools/SchemaToolTaskInfo.java | 43 +
.../metastore/tools/SchemaToolTaskInit.java | 73 +
.../tools/SchemaToolTaskMoveDatabase.java | 96 ++
.../tools/SchemaToolTaskMoveTable.java | 142 ++
.../metastore/tools/SchemaToolTaskUpgrade.java | 116 ++
.../metastore/tools/SchemaToolTaskValidate.java | 630 +++++++++
.../hive/metastore/dbinstall/DbInstallBase.java | 25 +-
.../tools/TestSchemaToolForMetastore.java | 277 ++--
29 files changed, 2708 insertions(+), 4145 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaTool.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaTool.java b/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaTool.java
index 9c3f30b..69514e5 100644
--- a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaTool.java
+++ b/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaTool.java
@@ -17,207 +17,50 @@
*/
package org.apache.hive.beeline.schematool;
-import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.OptionGroup;
import org.apache.commons.io.output.NullOutputStream;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaException;
-import org.apache.hadoop.hive.metastore.IMetaStoreSchemaInfo;
-import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory;
-import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper;
import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo;
import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.NestedScriptParser;
-import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.hive.metastore.tools.MetastoreSchemaTool;
import org.apache.hive.beeline.BeeLine;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.annotations.VisibleForTesting;
-
-import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintStream;
-import java.net.URI;
-import java.sql.Connection;
-import java.sql.SQLException;
-public class HiveSchemaTool {
+public class HiveSchemaTool extends MetastoreSchemaTool {
private static final Logger LOG = LoggerFactory.getLogger(HiveSchemaTool.class.getName());
- private final HiveConf hiveConf;
- private final String dbType;
- private final String metaDbType;
- private final IMetaStoreSchemaInfo metaStoreSchemaInfo;
- private final boolean needsQuotedIdentifier;
- private String quoteCharacter;
-
- private String url = null;
- private String driver = null;
- private String userName = null;
- private String passWord = null;
- private boolean dryRun = false;
- private boolean verbose = false;
- private String dbOpts = null;
- private URI[] validationServers = null; // The list of servers the database/partition/table can locate on
-
- private HiveSchemaTool(String dbType, String metaDbType) throws HiveMetaException {
- this(System.getenv("HIVE_HOME"), new HiveConf(HiveSchemaTool.class), dbType, metaDbType);
- }
-
- @VisibleForTesting
- public HiveSchemaTool(String hiveHome, HiveConf hiveConf, String dbType, String metaDbType)
- throws HiveMetaException {
- if (hiveHome == null || hiveHome.isEmpty()) {
- throw new HiveMetaException("No Hive home directory provided");
- }
- this.hiveConf = hiveConf;
- this.dbType = dbType;
- this.metaDbType = metaDbType;
- NestedScriptParser parser = getDbCommandParser(dbType, metaDbType);
- this.needsQuotedIdentifier = parser.needsQuotedIdentifier();
- this.quoteCharacter = parser.getQuoteCharacter();
- this.metaStoreSchemaInfo = MetaStoreSchemaInfoFactory.get(hiveConf, hiveHome, dbType);
- // If the dbType is "hive", this is setting up the information schema in Hive.
- // We will set the default jdbc url and driver.
- // It is overriden by command line options if passed (-url and -driver
- if (dbType.equalsIgnoreCase(HiveSchemaHelper.DB_HIVE)) {
- url = HiveSchemaHelper.EMBEDDED_HS2_URL;
- driver = HiveSchemaHelper.HIVE_JDBC_DRIVER;
- }
- }
-
- HiveConf getHiveConf() {
- return hiveConf;
- }
-
- String getDbType() {
- return dbType;
- }
-
- IMetaStoreSchemaInfo getMetaStoreSchemaInfo() {
- return metaStoreSchemaInfo;
- }
-
- private void setUrl(String url) {
- this.url = url;
- }
-
- private void setDriver(String driver) {
- this.driver = driver;
- }
-
- @VisibleForTesting
- public void setUserName(String userName) {
- this.userName = userName;
- }
-
- @VisibleForTesting
- public void setPassWord(String passWord) {
- this.passWord = passWord;
- }
-
- @VisibleForTesting
- public void setDryRun(boolean dryRun) {
- this.dryRun = dryRun;
- }
-
- boolean isDryRun() {
- return dryRun;
- }
-
- @VisibleForTesting
- public void setVerbose(boolean verbose) {
- this.verbose = verbose;
- }
-
- boolean isVerbose() {
- return verbose;
- }
-
- private void setDbOpts(String dbOpts) {
- this.dbOpts = dbOpts;
- }
-
- private void setValidationServers(String servers) {
- if(StringUtils.isNotEmpty(servers)) {
- String[] strServers = servers.split(",");
- this.validationServers = new URI[strServers.length];
- for (int i = 0; i < validationServers.length; i++) {
- validationServers[i] = new Path(strServers[i]).toUri();
- }
- }
- }
-
- URI[] getValidationServers() {
- return validationServers;
- }
-
- Connection getConnectionToMetastore(boolean printInfo) throws HiveMetaException {
- return HiveSchemaHelper.getConnectionToMetastore(userName, passWord, url, driver, printInfo, hiveConf,
- null);
- }
- private NestedScriptParser getDbCommandParser(String dbType, String metaDbType) {
- return HiveSchemaHelper.getDbCommandParser(dbType, dbOpts, userName, passWord, hiveConf,
+ @Override
+ protected NestedScriptParser getDbCommandParser(String dbType, String metaDbType) {
+ return HiveSchemaHelper.getDbCommandParser(dbType, dbOpts, userName, passWord, conf,
metaDbType, false);
}
- // test the connection metastore using the config property
- void testConnectionToMetastore() throws HiveMetaException {
- Connection conn = getConnectionToMetastore(true);
- try {
- conn.close();
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to close metastore connection", e);
- }
- }
-
- /**
- * check if the current schema version in metastore matches the Hive version
- * @throws MetaException
- */
- void verifySchemaVersion() throws HiveMetaException {
- // don't check version if its a dry run
- if (dryRun) {
- return;
- }
- String newSchemaVersion = metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(false));
- // verify that the new version is added to schema
- assertCompatibleVersion(metaStoreSchemaInfo.getHiveSchemaVersion(), newSchemaVersion);
- }
-
- void assertCompatibleVersion(String hiveSchemaVersion, String dbSchemaVersion)
- throws HiveMetaException {
- if (!metaStoreSchemaInfo.isVersionCompatible(hiveSchemaVersion, dbSchemaVersion)) {
- throw new HiveMetaException("Metastore schema version is not compatible. Hive Version: "
- + hiveSchemaVersion + ", Database Schema Version: " + dbSchemaVersion);
- }
- }
-
- MetaStoreConnectionInfo getConnectionInfo(boolean printInfo) {
- return new MetaStoreConnectionInfo(userName, passWord, url, driver, printInfo, hiveConf,
+ @Override
+ protected MetaStoreConnectionInfo getConnectionInfo(boolean printInfo) {
+ return new MetaStoreConnectionInfo(userName, passWord, url, driver, printInfo, conf,
dbType, metaDbType);
}
- // Quote if the database requires it
- String quote(String stmt) {
- stmt = stmt.replace("<q>", needsQuotedIdentifier ? quoteCharacter : "");
- stmt = stmt.replace("<qa>", quoteCharacter);
- return stmt;
- }
-
/***
* Run beeline with the given metastore script. Flatten the nested scripts
* into single file.
*/
- void runBeeLine(String scriptDir, String scriptFile)
+ @Override
+ protected void execSql(String scriptDir, String scriptFile)
throws IOException, HiveMetaException {
NestedScriptParser dbCommandParser = getDbCommandParser(dbType, metaDbType);
@@ -237,12 +80,12 @@ public class HiveSchemaTool {
out.write(sqlCommands);
out.write("!closeall" + System.getProperty("line.separator"));
out.close();
- runBeeLine(tmpFile.getPath());
+ execSql(tmpFile.getPath());
}
- // Generate the beeline args per hive conf and execute the given script
- void runBeeLine(String sqlScriptFile) throws IOException {
- CommandBuilder builder = new CommandBuilder(hiveConf, url, driver,
+ @Override
+ protected void execSql(String sqlScriptFile) throws IOException {
+ CommandBuilder builder = new HiveSchemaToolCommandBuilder(conf, url, driver,
userName, passWord, sqlScriptFile);
// run the script using Beeline
@@ -264,34 +107,15 @@ public class HiveSchemaTool {
}
}
- static class CommandBuilder {
- private final String userName;
- private final String password;
- private final String sqlScriptFile;
- private final String driver;
- private final String url;
-
- CommandBuilder(HiveConf hiveConf, String url, String driver, String userName, String password,
- String sqlScriptFile) throws IOException {
- this.userName = userName;
- this.password = password;
- this.url = url == null ?
- HiveSchemaHelper.getValidConfVar(MetastoreConf.ConfVars.CONNECT_URL_KEY, hiveConf) : url;
- this.driver = driver == null ?
- HiveSchemaHelper.getValidConfVar(MetastoreConf.ConfVars.CONNECTION_DRIVER, hiveConf) : driver;
- this.sqlScriptFile = sqlScriptFile;
- }
-
- String[] buildToRun() {
- return argsWith(password);
- }
+ static class HiveSchemaToolCommandBuilder extends MetastoreSchemaTool.CommandBuilder {
- String buildToLog() throws IOException {
- logScript();
- return StringUtils.join(argsWith(BeeLine.PASSWD_MASK), " ");
+ HiveSchemaToolCommandBuilder(Configuration conf, String url, String driver, String userName,
+ String password, String sqlScriptFile) throws IOException {
+ super(conf, url, driver, userName, password, sqlScriptFile);
}
- private String[] argsWith(String password) {
+ @Override
+ protected String[] argsWith(String password) {
return new String[]
{
"-u", url,
@@ -301,115 +125,17 @@ public class HiveSchemaTool {
"-f", sqlScriptFile
};
}
-
- private void logScript() throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Going to invoke file that contains:");
- try (BufferedReader reader = new BufferedReader(new FileReader(sqlScriptFile))) {
- String line;
- while ((line = reader.readLine()) != null) {
- LOG.debug("script: " + line);
- }
- }
- }
- }
}
public static void main(String[] args) {
- HiveSchemaToolCommandLine line = null;
- try {
- line = new HiveSchemaToolCommandLine(args);
- } catch (ParseException e) {
- System.exit(1);
- }
-
+ MetastoreSchemaTool tool = new HiveSchemaTool();
+ OptionGroup additionalGroup = new OptionGroup();
+ Option metaDbTypeOpt = OptionBuilder.withArgName("metaDatabaseType")
+ .hasArgs().withDescription("Used only if upgrading the system catalog for hive")
+ .create("metaDbType");
+ additionalGroup.addOption(metaDbTypeOpt);
System.setProperty(MetastoreConf.ConfVars.SCHEMA_VERIFICATION.getVarname(), "true");
- try {
- HiveSchemaTool schemaTool = createSchemaTool(line);
-
- HiveSchemaToolTask task = null;
- if (line.hasOption("info")) {
- task = new HiveSchemaToolTaskInfo();
- } else if (line.hasOption("upgradeSchema") || line.hasOption("upgradeSchemaFrom")) {
- task = new HiveSchemaToolTaskUpgrade();
- } else if (line.hasOption("initSchema") || line.hasOption("initSchemaTo")) {
- task = new HiveSchemaToolTaskInit();
- } else if (line.hasOption("validate")) {
- task = new HiveSchemaToolTaskValidate();
- } else if (line.hasOption("createCatalog")) {
- task = new HiveSchemaToolTaskCreateCatalog();
- } else if (line.hasOption("alterCatalog")) {
- task = new HiveSchemaToolTaskAlterCatalog();
- } else if (line.hasOption("moveDatabase")) {
- task = new HiveSchemaToolTaskMoveDatabase();
- } else if (line.hasOption("moveTable")) {
- task = new HiveSchemaToolTaskMoveTable();
- } else {
- throw new HiveMetaException("No task defined!");
- }
-
- task.setHiveSchemaTool(schemaTool);
- task.setCommandLineArguments(line);
- task.execute();
-
- } catch (HiveMetaException e) {
- System.err.println(e);
- if (e.getCause() != null) {
- Throwable t = e.getCause();
- System.err.println("Underlying cause: " + t.getClass().getName() + " : " + t.getMessage());
- if (e.getCause() instanceof SQLException) {
- System.err.println("SQL Error code: " + ((SQLException)t).getErrorCode());
- }
- }
- if (line.hasOption("verbose")) {
- e.printStackTrace();
- } else {
- System.err.println("Use --verbose for detailed stacktrace.");
- }
- System.err.println("*** schemaTool failed ***");
- System.exit(1);
- }
- System.out.println("schemaTool completed");
- System.exit(0);
- }
-
- private static HiveSchemaTool createSchemaTool(HiveSchemaToolCommandLine line) throws HiveMetaException {
- HiveSchemaTool schemaTool = new HiveSchemaTool(line.getDbType(), line.getMetaDbType());
-
- if (line.hasOption("userName")) {
- schemaTool.setUserName(line.getOptionValue("userName"));
- } else {
- schemaTool.setUserName(
- schemaTool.getHiveConf().get(MetastoreConf.ConfVars.CONNECTION_USER_NAME.getVarname()));
- }
- if (line.hasOption("passWord")) {
- schemaTool.setPassWord(line.getOptionValue("passWord"));
- } else {
- try {
- schemaTool.setPassWord(ShimLoader.getHadoopShims().getPassword(schemaTool.getHiveConf(),
- MetastoreConf.ConfVars.PWD.getVarname()));
- } catch (IOException err) {
- throw new HiveMetaException("Error getting metastore password", err);
- }
- }
- if (line.hasOption("url")) {
- schemaTool.setUrl(line.getOptionValue("url"));
- }
- if (line.hasOption("driver")) {
- schemaTool.setDriver(line.getOptionValue("driver"));
- }
- if (line.hasOption("dryRun")) {
- schemaTool.setDryRun(true);
- }
- if (line.hasOption("verbose")) {
- schemaTool.setVerbose(true);
- }
- if (line.hasOption("dbOpts")) {
- schemaTool.setDbOpts(line.getOptionValue("dbOpts"));
- }
- if (line.hasOption("validate") && line.hasOption("servers")) {
- schemaTool.setValidationServers(line.getOptionValue("servers"));
- }
- return schemaTool;
+ System.exit(tool.run(System.getenv("HIVE_HOME"), args, additionalGroup,
+ new HiveConf(HiveSchemaTool.class)));
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolCommandLine.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolCommandLine.java b/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolCommandLine.java
deleted file mode 100644
index 8ca8343..0000000
--- a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolCommandLine.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.beeline.schematool;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.OptionGroup;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper;
-
-import com.google.common.collect.ImmutableSet;
-
-import java.util.Set;
-
-class HiveSchemaToolCommandLine {
- private final Options cmdLineOptions = createOptions();
-
- @SuppressWarnings("static-access")
- private Options createOptions() {
- Option help = new Option("help", "print this message");
- Option infoOpt = new Option("info", "Show config and schema details");
- Option upgradeOpt = new Option("upgradeSchema", "Schema upgrade");
- Option upgradeFromOpt = OptionBuilder.withArgName("upgradeFrom").hasArg()
- .withDescription("Schema upgrade from a version")
- .create("upgradeSchemaFrom");
- Option initOpt = new Option("initSchema", "Schema initialization");
- Option initToOpt = OptionBuilder.withArgName("initTo").hasArg()
- .withDescription("Schema initialization to a version")
- .create("initSchemaTo");
- Option validateOpt = new Option("validate", "Validate the database");
- Option createCatalog = OptionBuilder
- .hasArg()
- .withDescription("Create a catalog, requires --catalogLocation parameter as well")
- .create("createCatalog");
- Option alterCatalog = OptionBuilder
- .hasArg()
- .withDescription("Alter a catalog, requires --catalogLocation and/or --catalogDescription parameter as well")
- .create("alterCatalog");
- Option moveDatabase = OptionBuilder
- .hasArg()
- .withDescription("Move a database between catalogs. Argument is the database name. " +
- "Requires --fromCatalog and --toCatalog parameters as well")
- .create("moveDatabase");
- Option moveTable = OptionBuilder
- .hasArg()
- .withDescription("Move a table to a different database. Argument is the table name. " +
- "Requires --fromCatalog, --toCatalog, --fromDatabase, and --toDatabase " +
- " parameters as well.")
- .create("moveTable");
-
- OptionGroup optGroup = new OptionGroup();
- optGroup
- .addOption(help)
- .addOption(infoOpt)
- .addOption(upgradeOpt)
- .addOption(upgradeFromOpt)
- .addOption(initOpt)
- .addOption(initToOpt)
- .addOption(validateOpt)
- .addOption(createCatalog)
- .addOption(alterCatalog)
- .addOption(moveDatabase)
- .addOption(moveTable);
- optGroup.setRequired(true);
-
- Option userNameOpt = OptionBuilder.withArgName("user")
- .hasArgs()
- .withDescription("Override config file user name")
- .create("userName");
- Option passwdOpt = OptionBuilder.withArgName("password")
- .hasArgs()
- .withDescription("Override config file password")
- .create("passWord");
- Option dbTypeOpt = OptionBuilder.withArgName("databaseType")
- .hasArgs().withDescription("Metastore database type").isRequired()
- .create("dbType");
- Option metaDbTypeOpt = OptionBuilder.withArgName("metaDatabaseType")
- .hasArgs().withDescription("Used only if upgrading the system catalog for hive")
- .create("metaDbType");
- Option urlOpt = OptionBuilder.withArgName("url")
- .hasArgs().withDescription("connection url to the database")
- .create("url");
- Option driverOpt = OptionBuilder.withArgName("driver")
- .hasArgs().withDescription("driver name for connection")
- .create("driver");
- Option dbOpts = OptionBuilder.withArgName("databaseOpts")
- .hasArgs().withDescription("Backend DB specific options")
- .create("dbOpts");
- Option dryRunOpt = new Option("dryRun", "list SQL scripts (no execute)");
- Option verboseOpt = new Option("verbose", "only print SQL statements");
- Option serversOpt = OptionBuilder.withArgName("serverList")
- .hasArgs().withDescription("a comma-separated list of servers used in location validation in the format of " +
- "scheme://authority (e.g. hdfs://localhost:8000)")
- .create("servers");
- Option catalogLocation = OptionBuilder
- .hasArg()
- .withDescription("Location of new catalog, required when adding a catalog")
- .create("catalogLocation");
- Option catalogDescription = OptionBuilder
- .hasArg()
- .withDescription("Description of new catalog")
- .create("catalogDescription");
- Option ifNotExists = OptionBuilder
- .withDescription("If passed then it is not an error to create an existing catalog")
- .create("ifNotExists");
- Option fromCatalog = OptionBuilder
- .hasArg()
- .withDescription("Catalog a moving database or table is coming from. This is " +
- "required if you are moving a database or table.")
- .create("fromCatalog");
- Option toCatalog = OptionBuilder
- .hasArg()
- .withDescription("Catalog a moving database or table is going to. This is " +
- "required if you are moving a database or table.")
- .create("toCatalog");
- Option fromDatabase = OptionBuilder
- .hasArg()
- .withDescription("Database a moving table is coming from. This is " +
- "required if you are moving a table.")
- .create("fromDatabase");
- Option toDatabase = OptionBuilder
- .hasArg()
- .withDescription("Database a moving table is going to. This is " +
- "required if you are moving a table.")
- .create("toDatabase");
-
- Options options = new Options();
- options.addOption(help);
- options.addOptionGroup(optGroup);
- options.addOption(dbTypeOpt);
- options.addOption(metaDbTypeOpt);
- options.addOption(userNameOpt);
- options.addOption(passwdOpt);
- options.addOption(urlOpt);
- options.addOption(driverOpt);
- options.addOption(dbOpts);
- options.addOption(dryRunOpt);
- options.addOption(verboseOpt);
- options.addOption(serversOpt);
- options.addOption(catalogLocation);
- options.addOption(catalogDescription);
- options.addOption(ifNotExists);
- options.addOption(fromCatalog);
- options.addOption(toCatalog);
- options.addOption(fromDatabase);
- options.addOption(toDatabase);
-
- return options;
- }
-
- private final CommandLine cl;
- private final String dbType;
- private final String metaDbType;
-
- HiveSchemaToolCommandLine(String[] args) throws ParseException {
- cl = getCommandLine(args);
- if (cl.hasOption("help")) {
- printAndExit(null);
- }
-
- dbType = cl.getOptionValue("dbType");
- metaDbType = cl.getOptionValue("metaDbType");
-
- validate();
- }
-
- private CommandLine getCommandLine(String[] args) throws ParseException {
- try {
- CommandLineParser parser = new GnuParser();
- return parser.parse(cmdLineOptions, args);
- } catch (ParseException e) {
- printAndExit("HiveSchemaTool:Parsing failed. Reason: " + e.getLocalizedMessage());
- return null;
- }
- }
-
- private static final Set<String> VALID_DB_TYPES = ImmutableSet.of(HiveSchemaHelper.DB_DERBY,
- HiveSchemaHelper.DB_HIVE, HiveSchemaHelper.DB_MSSQL, HiveSchemaHelper.DB_MYSQL,
- HiveSchemaHelper.DB_POSTGRACE, HiveSchemaHelper.DB_ORACLE);
-
- private static final Set<String> VALID_META_DB_TYPES = ImmutableSet.of(HiveSchemaHelper.DB_DERBY,
- HiveSchemaHelper.DB_MSSQL, HiveSchemaHelper.DB_MYSQL, HiveSchemaHelper.DB_POSTGRACE,
- HiveSchemaHelper.DB_ORACLE);
-
- private void validate() throws ParseException {
- if (!VALID_DB_TYPES.contains(dbType)) {
- printAndExit("Unsupported dbType " + dbType);
- }
-
- if (metaDbType != null) {
- if (!dbType.equals(HiveSchemaHelper.DB_HIVE)) {
- printAndExit("metaDbType may only be set if dbType is hive");
- }
- if (!VALID_META_DB_TYPES.contains(metaDbType)) {
- printAndExit("Unsupported metaDbType " + metaDbType);
- }
- } else if (dbType.equalsIgnoreCase(HiveSchemaHelper.DB_HIVE)) {
- System.err.println();
- printAndExit("metaDbType must be set if dbType is hive");
- }
-
- if ((cl.hasOption("createCatalog")) && !cl.hasOption("catalogLocation")) {
- System.err.println();
- printAndExit("catalogLocation must be set for createCatalog");
- }
-
- if (!cl.hasOption("createCatalog") && !cl.hasOption("alterCatalog") &&
- (cl.hasOption("catalogLocation") || cl.hasOption("catalogDescription"))) {
- printAndExit("catalogLocation and catalogDescription may be set only for createCatalog and alterCatalog");
- }
-
- if (!cl.hasOption("createCatalog") && cl.hasOption("ifNotExists")) {
- printAndExit("ifNotExists may be set only for createCatalog");
- }
-
- if (cl.hasOption("moveDatabase") &&
- (!cl.hasOption("fromCatalog") || !cl.hasOption("toCatalog"))) {
- printAndExit("fromCatalog and toCatalog must be set for moveDatabase");
- }
-
- if (cl.hasOption("moveTable") &&
- (!cl.hasOption("fromCatalog") || !cl.hasOption("toCatalog") ||
- !cl.hasOption("fromDatabase") || !cl.hasOption("toDatabase"))) {
- printAndExit("fromCatalog, toCatalog, fromDatabase and toDatabase must be set for moveTable");
- }
-
- if ((!cl.hasOption("moveDatabase") && !cl.hasOption("moveTable")) &&
- (cl.hasOption("fromCatalog") || cl.hasOption("toCatalog"))) {
- printAndExit("fromCatalog and toCatalog may be set only for moveDatabase and moveTable");
- }
-
- if (!cl.hasOption("moveTable") &&
- (cl.hasOption("fromDatabase") || cl.hasOption("toDatabase"))) {
- printAndExit("fromDatabase and toDatabase may be set only for moveTable");
- }
- }
-
- private void printAndExit(String reason) throws ParseException {
- if (reason != null) {
- System.err.println(reason);
- }
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("schemaTool", cmdLineOptions);
- if (reason != null) {
- throw new ParseException(reason);
- } else {
- System.exit(0);
- }
- }
-
- String getDbType() {
- return dbType;
- }
-
- String getMetaDbType() {
- return metaDbType;
- }
-
- boolean hasOption(String opt) {
- return cl.hasOption(opt);
- }
-
- String getOptionValue(String opt) {
- return cl.getOptionValue(opt);
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTask.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTask.java b/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTask.java
deleted file mode 100644
index 3172c2f..0000000
--- a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTask.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.beeline.schematool;
-
-import org.apache.hadoop.hive.metastore.HiveMetaException;
-
-abstract class HiveSchemaToolTask {
- protected HiveSchemaTool schemaTool;
-
- void setHiveSchemaTool(HiveSchemaTool schemaTool) {
- this.schemaTool = schemaTool;
- }
-
- abstract void setCommandLineArguments(HiveSchemaToolCommandLine cl);
-
- abstract void execute() throws HiveMetaException;
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskAlterCatalog.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskAlterCatalog.java b/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskAlterCatalog.java
deleted file mode 100644
index 802fe3a..0000000
--- a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskAlterCatalog.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.beeline.schematool;
-
-import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
-
-import java.sql.Connection;
-import java.sql.SQLException;
-import java.sql.Statement;
-
-import org.apache.hadoop.hive.metastore.HiveMetaException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Print Hive version and schema version.
- */
-class HiveSchemaToolTaskAlterCatalog extends HiveSchemaToolTask {
- private static final Logger LOG = LoggerFactory.getLogger(HiveSchemaToolTaskAlterCatalog.class.getName());
-
- private String catName;
- private String location;
- private String description;
-
- @Override
- void setCommandLineArguments(HiveSchemaToolCommandLine cl) {
- catName = normalizeIdentifier(cl.getOptionValue("alterCatalog"));
- location = cl.getOptionValue("catalogLocation");
- description = cl.getOptionValue("catalogDescription");
- }
-
- private static final String UPDATE_CATALOG_STMT =
- "update <q>CTLGS<q> " +
- " set <q>LOCATION_URI<q> = %s, " +
- " <qa>DESC<qa> = %s " +
- " where <q>NAME<q> = '%s'";
-
- @Override
- void execute() throws HiveMetaException {
- if (location == null && description == null) {
- throw new HiveMetaException("Asked to update catalog " + catName + " but not given any changes to update");
- }
- System.out.println("Updating catalog " + catName);
-
- Connection conn = schemaTool.getConnectionToMetastore(true);
- boolean success = false;
- try {
- conn.setAutoCommit(false);
- try (Statement stmt = conn.createStatement()) {
- Object updateLocation = location == null ? schemaTool.quote("<q>LOCATION_URI<q>") : "'" + location + "'";
- Object updateDescription = description == null ? schemaTool.quote("<qa>DESC<qa>") : "'" + description + "'";
- String update = String.format(schemaTool.quote(UPDATE_CATALOG_STMT), updateLocation, updateDescription,
- catName);
- LOG.debug("Going to run " + update);
- int count = stmt.executeUpdate(update);
- if (count != 1) {
- throw new HiveMetaException("Failed to find catalog " + catName + " to update");
- }
- conn.commit();
- success = true;
- }
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to update catalog", e);
- } finally {
- try {
- if (!success) {
- conn.rollback();
- }
- } catch (SQLException e) {
- // Not really much we can do here.
- LOG.error("Failed to rollback, everything will probably go bad from here.", e);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskCreateCatalog.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskCreateCatalog.java b/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskCreateCatalog.java
deleted file mode 100644
index 810dfea..0000000
--- a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskCreateCatalog.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.beeline.schematool;
-
-import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
-
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-
-import org.apache.hadoop.hive.metastore.HiveMetaException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Print Hive version and schema version.
- */
-class HiveSchemaToolTaskCreateCatalog extends HiveSchemaToolTask {
- private static final Logger LOG = LoggerFactory.getLogger(HiveSchemaToolTaskCreateCatalog.class.getName());
-
- private String catName;
- private String location;
- private String description;
- private boolean ifNotExists;
-
- @Override
- void setCommandLineArguments(HiveSchemaToolCommandLine cl) {
- catName = normalizeIdentifier(cl.getOptionValue("createCatalog"));
- location = cl.getOptionValue("catalogLocation");
- description = cl.getOptionValue("catalogDescription");
- ifNotExists = cl.hasOption("ifNotExists");
- }
-
- @Override
- void execute() throws HiveMetaException {
- System.out.println("Create catalog " + catName + " at location " + location);
-
- Connection conn = schemaTool.getConnectionToMetastore(true);
- boolean success = false;
- try {
- conn.setAutoCommit(false);
- try (Statement stmt = conn.createStatement()) {
- // If they set ifNotExists check for existence first, and bail if it exists. This is
- // more reliable then attempting to parse the error message from the SQLException.
- if (ifNotExists && catalogExists(stmt)) {
- return;
- }
-
- int catNum = getNextCatalogId(stmt);
- addCatalog(conn, stmt, catNum);
- success = true;
- }
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to add catalog", e);
- } finally {
- try {
- if (!success) {
- conn.rollback();
- }
- } catch (SQLException e) {
- // Not really much we can do here.
- LOG.error("Failed to rollback, everything will probably go bad from here.", e);
- }
- }
- }
-
- private static final String CATALOG_EXISTS_QUERY =
- "select <q>NAME<q> " +
- " from <q>CTLGS<q> " +
- " where <q>NAME<q> = '%s'";
-
- private boolean catalogExists(Statement stmt) throws SQLException {
- String query = String.format(schemaTool.quote(CATALOG_EXISTS_QUERY), catName);
- LOG.debug("Going to run " + query);
- try (ResultSet rs = stmt.executeQuery(query)) {
- if (rs.next()) {
- System.out.println("Catalog " + catName + " already exists");
- return true;
- }
- }
-
- return false;
- }
-
- private static final String NEXT_CATALOG_ID_QUERY =
- "select max(<q>CTLG_ID<q>) " +
- " from <q>CTLGS<q>";
-
- private int getNextCatalogId(Statement stmt) throws SQLException, HiveMetaException {
- String query = schemaTool.quote(NEXT_CATALOG_ID_QUERY);
- LOG.debug("Going to run " + query);
- try (ResultSet rs = stmt.executeQuery(query)) {
- if (!rs.next()) {
- throw new HiveMetaException("No catalogs found, have you upgraded the database?");
- }
- int nextId = rs.getInt(1) + 1;
- // We need to stay out of the way of any sequences used by the underlying database.
- // Otherwise the next time the client tries to add a catalog we'll get an error.
- // There should never be billions of catalogs, so we'll shift our sequence number up
- // there to avoid clashes.
- int floor = 1 << 30;
- return Math.max(nextId, floor);
- }
- }
-
- private static final String ADD_CATALOG_STMT =
- "insert into <q>CTLGS<q> (<q>CTLG_ID<q>, <q>NAME<q>, <qa>DESC<qa>, <q>LOCATION_URI<q>) " +
- " values (%d, '%s', '%s', '%s')";
-
- private void addCatalog(Connection conn, Statement stmt, int catNum) throws SQLException {
- String update = String.format(schemaTool.quote(ADD_CATALOG_STMT), catNum, catName, description, location);
- LOG.debug("Going to run " + update);
- stmt.execute(update);
- conn.commit();
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskInfo.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskInfo.java b/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskInfo.java
deleted file mode 100644
index b70ea87..0000000
--- a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskInfo.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.beeline.schematool;
-
-import org.apache.hadoop.hive.metastore.HiveMetaException;
-import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo;
-
-/**
- * Print Hive version and schema version.
- */
-class HiveSchemaToolTaskInfo extends HiveSchemaToolTask {
- @Override
- void setCommandLineArguments(HiveSchemaToolCommandLine cl) {
- // do nothing
- }
-
- @Override
- void execute() throws HiveMetaException {
- String hiveVersion = schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion();
- MetaStoreConnectionInfo connectionInfo = schemaTool.getConnectionInfo(true);
- String dbVersion = schemaTool.getMetaStoreSchemaInfo().getMetaStoreSchemaVersion(connectionInfo);
-
- System.out.println("Hive distribution version:\t " + hiveVersion);
- System.out.println("Metastore schema version:\t " + dbVersion);
-
- schemaTool.assertCompatibleVersion(hiveVersion, dbVersion);
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskInit.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskInit.java b/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskInit.java
deleted file mode 100644
index 40fd1e7..0000000
--- a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskInit.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.beeline.schematool;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hive.metastore.HiveMetaException;
-
-/**
- * Initialize the metastore schema.
- */
-class HiveSchemaToolTaskInit extends HiveSchemaToolTask {
- private boolean validate = true;
- private String toVersion;
-
- @Override
- void setCommandLineArguments(HiveSchemaToolCommandLine cl) {
- if (cl.hasOption("initSchemaTo")) {
- this.toVersion = cl.getOptionValue("initSchemaTo");
- this.validate = false;
- }
- }
-
- private void ensureToVersion() throws HiveMetaException {
- if (toVersion != null) {
- return;
- }
-
- // If null then current hive version is used
- toVersion = schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion();
- System.out.println("Initializing the schema to: " + toVersion);
- }
-
- @Override
- void execute() throws HiveMetaException {
- ensureToVersion();
-
- schemaTool.testConnectionToMetastore();
- System.out.println("Starting metastore schema initialization to " + toVersion);
-
- String initScriptDir = schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir();
- String initScriptFile = schemaTool.getMetaStoreSchemaInfo().generateInitFileName(toVersion);
-
- try {
- System.out.println("Initialization script " + initScriptFile);
- if (!schemaTool.isDryRun()) {
- schemaTool.runBeeLine(initScriptDir, initScriptFile);
- System.out.println("Initialization script completed");
- }
- } catch (IOException e) {
- throw new HiveMetaException("Schema initialization FAILED! Metastore state would be inconsistent!", e);
- }
-
- if (validate) {
- schemaTool.verifySchemaVersion();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskMoveDatabase.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskMoveDatabase.java b/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskMoveDatabase.java
deleted file mode 100644
index ee7c0f8..0000000
--- a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskMoveDatabase.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.beeline.schematool;
-
-import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
-
-import java.sql.Connection;
-import java.sql.SQLException;
-import java.sql.Statement;
-
-import org.apache.hadoop.hive.metastore.HiveMetaException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Print Hive version and schema version.
- */
-class HiveSchemaToolTaskMoveDatabase extends HiveSchemaToolTask {
- private static final Logger LOG = LoggerFactory.getLogger(HiveSchemaToolTaskMoveDatabase.class.getName());
-
- private String fromCatName;
- private String toCatName;
- private String dbName;
-
- @Override
- void setCommandLineArguments(HiveSchemaToolCommandLine cl) {
- fromCatName = normalizeIdentifier(cl.getOptionValue("fromCatalog"));
- toCatName = normalizeIdentifier(cl.getOptionValue("toCatalog"));
- dbName = normalizeIdentifier(cl.getOptionValue("moveDatabase"));
- }
-
- @Override
- void execute() throws HiveMetaException {
- System.out.println(String.format("Moving database %s from catalog %s to catalog %s",
- dbName, fromCatName, toCatName));
- Connection conn = schemaTool.getConnectionToMetastore(true);
- boolean success = false;
- try {
- conn.setAutoCommit(false);
- try (Statement stmt = conn.createStatement()) {
- updateCatalogNameInTable(stmt, "DBS", "CTLG_NAME", "NAME", fromCatName, toCatName, dbName, false);
- updateCatalogNameInTable(stmt, "TAB_COL_STATS", "CAT_NAME", "DB_NAME", fromCatName, toCatName, dbName, true);
- updateCatalogNameInTable(stmt, "PART_COL_STATS", "CAT_NAME", "DB_NAME", fromCatName, toCatName, dbName, true);
- updateCatalogNameInTable(stmt, "PARTITION_EVENTS", "CAT_NAME", "DB_NAME", fromCatName, toCatName, dbName, true);
- updateCatalogNameInTable(stmt, "NOTIFICATION_LOG", "CAT_NAME", "DB_NAME", fromCatName, toCatName, dbName, true);
- conn.commit();
- success = true;
- }
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to move database", e);
- } finally {
- try {
- if (!success) {
- conn.rollback();
- }
- } catch (SQLException e) {
- // Not really much we can do here.
- LOG.error("Failed to rollback, everything will probably go bad from here.");
- }
- }
- }
-
- private static final String UPDATE_CATALOG_NAME_STMT =
- "update <q>%s<q> " +
- " set <q>%s<q> = '%s' " +
- " where <q>%s<q> = '%s' " +
- " and <q>%s<q> = '%s'";
-
- private void updateCatalogNameInTable(Statement stmt, String tableName, String catColName, String dbColName,
- String fromCatName, String toCatName, String dbName, boolean zeroUpdatesOk)
- throws HiveMetaException, SQLException {
- String update = String.format(schemaTool.quote(UPDATE_CATALOG_NAME_STMT), tableName, catColName, toCatName,
- catColName, fromCatName, dbColName, dbName);
- LOG.debug("Going to run " + update);
- int numUpdated = stmt.executeUpdate(update);
- if (numUpdated != 1 && !(zeroUpdatesOk && numUpdated == 0)) {
- throw new HiveMetaException("Failed to properly update the " + tableName +
- " table. Expected to update 1 row but instead updated " + numUpdated);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskMoveTable.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskMoveTable.java b/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskMoveTable.java
deleted file mode 100644
index fcefef8..0000000
--- a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskMoveTable.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.beeline.schematool;
-
-import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
-
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-
-import org.apache.hadoop.hive.metastore.HiveMetaException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Print Hive version and schema version.
- */
-class HiveSchemaToolTaskMoveTable extends HiveSchemaToolTask {
- private static final Logger LOG = LoggerFactory.getLogger(HiveSchemaToolTaskMoveTable.class.getName());
-
- private String fromCat;
- private String toCat;
- private String fromDb;
- private String toDb;
- private String tableName;
-
- @Override
- void setCommandLineArguments(HiveSchemaToolCommandLine cl) {
- fromCat = normalizeIdentifier(cl.getOptionValue("fromCatalog"));
- toCat = normalizeIdentifier(cl.getOptionValue("toCatalog"));
- fromDb = normalizeIdentifier(cl.getOptionValue("fromDatabase"));
- toDb = normalizeIdentifier(cl.getOptionValue("toDatabase"));
- tableName = normalizeIdentifier(cl.getOptionValue("moveTable"));
- }
-
- @Override
- void execute() throws HiveMetaException {
- Connection conn = schemaTool.getConnectionToMetastore(true);
- boolean success = false;
- try {
- conn.setAutoCommit(false);
- try (Statement stmt = conn.createStatement()) {
- updateTableId(stmt);
- updateDbNameForTable(stmt, "TAB_COL_STATS", "TABLE_NAME", fromCat, toCat, fromDb, toDb, tableName);
- updateDbNameForTable(stmt, "PART_COL_STATS", "TABLE_NAME", fromCat, toCat, fromDb, toDb, tableName);
- updateDbNameForTable(stmt, "PARTITION_EVENTS", "TBL_NAME", fromCat, toCat, fromDb, toDb, tableName);
- updateDbNameForTable(stmt, "NOTIFICATION_LOG", "TBL_NAME", fromCat, toCat, fromDb, toDb, tableName);
- conn.commit();
- success = true;
- }
- } catch (SQLException se) {
- throw new HiveMetaException("Failed to move table", se);
- } finally {
- try {
- if (!success) {
- conn.rollback();
- }
- } catch (SQLException e) {
- // Not really much we can do here.
- LOG.error("Failed to rollback, everything will probably go bad from here.");
- }
-
- }
- }
-
- private static final String UPDATE_TABLE_ID_STMT =
- "update <q>TBLS<q> " +
- " set <q>DB_ID<q> = %d " +
- " where <q>DB_ID<q> = %d " +
- " and <q>TBL_NAME<q> = '%s'";
-
- private void updateTableId(Statement stmt) throws SQLException, HiveMetaException {
- // Find the old database id
- long oldDbId = getDbId(stmt, fromDb, fromCat);
-
- // Find the new database id
- long newDbId = getDbId(stmt, toDb, toCat);
-
- String update = String.format(schemaTool.quote(UPDATE_TABLE_ID_STMT), newDbId, oldDbId, tableName);
- LOG.debug("Going to run " + update);
- int numUpdated = stmt.executeUpdate(update);
- if (numUpdated != 1) {
- throw new HiveMetaException(
- "Failed to properly update TBLS table. Expected to update " +
- "1 row but instead updated " + numUpdated);
- }
- }
-
- private static final String DB_ID_QUERY =
- "select <q>DB_ID<q> " +
- " from <q>DBS<q> " +
- " where <q>NAME<q> = '%s' " +
- " and <q>CTLG_NAME<q> = '%s'";
-
- private long getDbId(Statement stmt, String db, String catalog) throws SQLException, HiveMetaException {
- String query = String.format(schemaTool.quote(DB_ID_QUERY), db, catalog);
- LOG.debug("Going to run " + query);
- try (ResultSet rs = stmt.executeQuery(query)) {
- if (!rs.next()) {
- throw new HiveMetaException("Unable to find database " + fromDb);
- }
- return rs.getLong(1);
- }
- }
-
- private static final String UPDATE_DB_NAME_STMT =
- "update <q>%s<q> " +
- " set <q>CAT_NAME<q> = '%s', " +
- " <q>DB_NAME<q> = '%s' " +
- " where <q>CAT_NAME<q> = '%s' " +
- " and <q>DB_NAME<q> = '%s' " +
- " and <q>%s<q> = '%s'";
-
- private void updateDbNameForTable(Statement stmt, String tableName, String tableColumnName, String fromCat,
- String toCat, String fromDb, String toDb, String hiveTblName) throws HiveMetaException, SQLException {
- String update = String.format(schemaTool.quote(UPDATE_DB_NAME_STMT), tableName, toCat, toDb, fromCat, fromDb,
- tableColumnName, hiveTblName);
-
- LOG.debug("Going to run " + update);
- int numUpdated = stmt.executeUpdate(update);
- if (numUpdated > 1 || numUpdated < 0) {
- throw new HiveMetaException("Failed to properly update the " + tableName +
- " table. Expected to update 1 row but instead updated " + numUpdated);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskUpgrade.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskUpgrade.java b/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskUpgrade.java
deleted file mode 100644
index fa4742f..0000000
--- a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskUpgrade.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.beeline.schematool;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.hive.metastore.HiveMetaException;
-import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo;
-
-/**
- * Perform metastore schema upgrade.
- */
-class HiveSchemaToolTaskUpgrade extends HiveSchemaToolTask {
- private String fromVersion;
-
- @Override
- void setCommandLineArguments(HiveSchemaToolCommandLine cl) {
- if (cl.hasOption("upgradeSchemaFrom")) {
- this.fromVersion = cl.getOptionValue("upgradeSchemaFrom");
- }
- }
-
- private void ensureFromVersion() throws HiveMetaException {
- if (fromVersion != null) {
- return;
- }
-
- // If null, then read from the metastore
- MetaStoreConnectionInfo connectionInfo = schemaTool.getConnectionInfo(false);
- fromVersion = schemaTool.getMetaStoreSchemaInfo().getMetaStoreSchemaVersion(connectionInfo);
- if (fromVersion == null || fromVersion.isEmpty()) {
- throw new HiveMetaException("Schema version not stored in the metastore. " +
- "Metastore schema is too old or corrupt. Try specifying the version manually");
- }
- System.out.println("Upgrading from the version " + fromVersion);
- }
-
- @Override
- void execute() throws HiveMetaException {
- ensureFromVersion();
-
- if (schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion().equals(fromVersion)) {
- System.out.println("No schema upgrade required from version " + fromVersion);
- return;
- }
-
- // Find the list of scripts to execute for this upgrade
- List<String> upgradeScripts = schemaTool.getMetaStoreSchemaInfo().getUpgradeScripts(fromVersion);
- schemaTool.testConnectionToMetastore();
- System.out.println("Starting upgrade metastore schema from version " + fromVersion + " to " +
- schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion());
- String scriptDir = schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir();
- try {
- for (String scriptFile : upgradeScripts) {
- System.out.println("Upgrade script " + scriptFile);
- if (!schemaTool.isDryRun()) {
- runPreUpgrade(scriptDir, scriptFile);
- schemaTool.runBeeLine(scriptDir, scriptFile);
- System.out.println("Completed " + scriptFile);
- }
- }
- } catch (IOException e) {
- throw new HiveMetaException("Upgrade FAILED! Metastore state would be inconsistent !!", e);
- }
-
- // Revalidated the new version after upgrade
- schemaTool.verifySchemaVersion();
- }
-
- /**
- * Run pre-upgrade scripts corresponding to a given upgrade script,
- * if any exist. The errors from pre-upgrade are ignored.
- * Pre-upgrade scripts typically contain setup statements which
- * may fail on some database versions and failure is ignorable.
- *
- * @param scriptDir upgrade script directory name
- * @param scriptFile upgrade script file name
- */
- private void runPreUpgrade(String scriptDir, String scriptFile) {
- for (int i = 0;; i++) {
- String preUpgradeScript = schemaTool.getMetaStoreSchemaInfo().getPreUpgradeScriptName(i, scriptFile);
- File preUpgradeScriptFile = new File(scriptDir, preUpgradeScript);
- if (!preUpgradeScriptFile.isFile()) {
- break;
- }
-
- try {
- schemaTool.runBeeLine(scriptDir, preUpgradeScript);
- System.out.println("Completed " + preUpgradeScript);
- } catch (Exception e) {
- // Ignore the pre-upgrade script errors
- System.err.println("Warning in pre-upgrade script " + preUpgradeScript + ": " + e.getMessage());
- if (schemaTool.isVerbose()) {
- e.printStackTrace();
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskValidate.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskValidate.java b/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskValidate.java
deleted file mode 100644
index c4f6d19..0000000
--- a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaToolTaskValidate.java
+++ /dev/null
@@ -1,631 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.beeline.schematool;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.net.URI;
-import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.SQLFeatureNotSupportedException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.metastore.HiveMetaException;
-import org.apache.hadoop.hive.metastore.TableType;
-import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper;
-import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo;
-import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.NestedScriptParser;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableMap;
-
-/**
- * Print Hive version and schema version.
- */
-class HiveSchemaToolTaskValidate extends HiveSchemaToolTask {
- private static final Logger LOG = LoggerFactory.getLogger(HiveSchemaToolTaskValidate.class.getName());
-
- @Override
- void setCommandLineArguments(HiveSchemaToolCommandLine cl) {
- // do nothing
- }
-
- @Override
- void execute() throws HiveMetaException {
- System.out.println("Starting metastore validation\n");
- Connection conn = schemaTool.getConnectionToMetastore(false);
- boolean success = true;
- try {
- success &= validateSchemaVersions();
- success &= validateSequences(conn);
- success &= validateSchemaTables(conn);
- success &= validateLocations(conn, schemaTool.getValidationServers());
- success &= validateColumnNullValues(conn);
- } finally {
- if (conn != null) {
- try {
- conn.close();
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to close metastore connection", e);
- }
- }
- }
-
- System.out.print("Done with metastore validation: ");
- if (!success) {
- System.out.println("[FAIL]");
- throw new HiveMetaException("Validation failed");
- } else {
- System.out.println("[SUCCESS]");
- }
- }
-
- boolean validateSchemaVersions() throws HiveMetaException {
- System.out.println("Validating schema version");
- try {
- String hiveSchemaVersion = schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion();
- MetaStoreConnectionInfo connectionInfo = schemaTool.getConnectionInfo(false);
- String newSchemaVersion = schemaTool.getMetaStoreSchemaInfo().getMetaStoreSchemaVersion(connectionInfo);
- schemaTool.assertCompatibleVersion(hiveSchemaVersion, newSchemaVersion);
- } catch (HiveMetaException hme) {
- if (hme.getMessage().contains("Metastore schema version is not compatible") ||
- hme.getMessage().contains("Multiple versions were found in metastore") ||
- hme.getMessage().contains("Could not find version info in metastore VERSION table")) {
- System.err.println(hme.getMessage());
- System.out.println("[FAIL]\n");
- return false;
- } else {
- throw hme;
- }
- }
- System.out.println("[SUCCESS]\n");
- return true;
- }
-
- private static final String QUERY_SEQ =
- " select t.<q>NEXT_VAL<q>" +
- " from <q>SEQUENCE_TABLE<q> t " +
- " where t.<q>SEQUENCE_NAME<q> = ? " +
- "order by t.<q>SEQUENCE_NAME<q>";
-
- private static final String QUERY_MAX_ID =
- "select max(<q>%s<q>)" +
- " from <q>%s<q>";
-
- @VisibleForTesting
- boolean validateSequences(Connection conn) throws HiveMetaException {
- Map<String, Pair<String, String>> seqNameToTable =
- new ImmutableMap.Builder<String, Pair<String, String>>()
- .put("MDatabase", Pair.of("DBS", "DB_ID"))
- .put("MRole", Pair.of("ROLES", "ROLE_ID"))
- .put("MGlobalPrivilege", Pair.of("GLOBAL_PRIVS", "USER_GRANT_ID"))
- .put("MTable", Pair.of("TBLS","TBL_ID"))
- .put("MStorageDescriptor", Pair.of("SDS", "SD_ID"))
- .put("MSerDeInfo", Pair.of("SERDES", "SERDE_ID"))
- .put("MColumnDescriptor", Pair.of("CDS", "CD_ID"))
- .put("MTablePrivilege", Pair.of("TBL_PRIVS", "TBL_GRANT_ID"))
- .put("MTableColumnStatistics", Pair.of("TAB_COL_STATS", "CS_ID"))
- .put("MPartition", Pair.of("PARTITIONS", "PART_ID"))
- .put("MPartitionColumnStatistics", Pair.of("PART_COL_STATS", "CS_ID"))
- .put("MFunction", Pair.of("FUNCS", "FUNC_ID"))
- .put("MIndex", Pair.of("IDXS", "INDEX_ID"))
- .put("MStringList", Pair.of("SKEWED_STRING_LIST", "STRING_LIST_ID"))
- .build();
-
- System.out.println("Validating sequence number for SEQUENCE_TABLE");
-
- boolean isValid = true;
- try {
- Statement stmt = conn.createStatement();
- for (Map.Entry<String, Pair<String, String>> e : seqNameToTable.entrySet()) {
- String tableName = e.getValue().getLeft();
- String tableKey = e.getValue().getRight();
- String fullSequenceName = "org.apache.hadoop.hive.metastore.model." + e.getKey();
- String seqQuery = schemaTool.quote(QUERY_SEQ);
- String maxIdQuery = String.format(schemaTool.quote(QUERY_MAX_ID), tableKey, tableName);
-
- ResultSet res = stmt.executeQuery(maxIdQuery);
- if (res.next()) {
- long maxId = res.getLong(1);
- if (maxId > 0) {
- PreparedStatement stmtSeq = conn.prepareStatement(seqQuery);
- stmtSeq.setString(1, fullSequenceName);
- ResultSet resSeq = stmtSeq.executeQuery();
- if (!resSeq.next()) {
- isValid = false;
- System.err.println("Missing SEQUENCE_NAME " + e.getKey() + " from SEQUENCE_TABLE");
- } else if (resSeq.getLong(1) < maxId) {
- isValid = false;
- System.err.println("NEXT_VAL for " + e.getKey() + " in SEQUENCE_TABLE < max(" + tableKey +
- ") in " + tableName);
- }
- }
- }
- }
-
- System.out.println(isValid ? "[SUCCESS]\n" :"[FAIL]\n");
- return isValid;
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to validate sequence number for SEQUENCE_TABLE", e);
- }
- }
-
- @VisibleForTesting
- boolean validateSchemaTables(Connection conn) throws HiveMetaException {
- System.out.println("Validating metastore schema tables");
- String version = null;
- try {
- MetaStoreConnectionInfo connectionInfo = schemaTool.getConnectionInfo(false);
- version = schemaTool.getMetaStoreSchemaInfo().getMetaStoreSchemaVersion(connectionInfo);
- } catch (HiveMetaException he) {
- System.err.println("Failed to determine schema version from Hive Metastore DB. " + he.getMessage());
- System.out.println("Failed in schema table validation.");
- LOG.debug("Failed to determine schema version from Hive Metastore DB," + he.getMessage(), he);
- return false;
- }
-
- Connection hmsConn = schemaTool.getConnectionToMetastore(false);
-
- LOG.debug("Validating tables in the schema for version " + version);
- List<String> dbTables = new ArrayList<String>();
- ResultSet rs = null;
- try {
- String schema = null;
- try {
- schema = hmsConn.getSchema();
- } catch (SQLFeatureNotSupportedException e) {
- LOG.debug("schema is not supported");
- }
-
- DatabaseMetaData metadata = conn.getMetaData();
- rs = metadata.getTables(null, schema, "%", new String[] {"TABLE"});
-
- while (rs.next()) {
- String table = rs.getString("TABLE_NAME");
- dbTables.add(table.toLowerCase());
- LOG.debug("Found table " + table + " in HMS dbstore");
- }
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to retrieve schema tables from Hive Metastore DB," +
- e.getMessage(), e);
- } finally {
- if (rs != null) {
- try {
- rs.close();
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to close resultset", e);
- }
- }
- }
-
- // parse the schema file to determine the tables that are expected to exist
- // we are using oracle schema because it is simpler to parse, no quotes or backticks etc
- List<String> schemaTables = new ArrayList<String>();
- List<String> subScripts = new ArrayList<String>();
-
- String baseDir = new File(schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir()).getParent();
- String schemaFile = new File(schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir(),
- schemaTool.getMetaStoreSchemaInfo().generateInitFileName(version)).getPath();
- try {
- LOG.debug("Parsing schema script " + schemaFile);
- subScripts.addAll(findCreateTable(schemaFile, schemaTables));
- while (subScripts.size() > 0) {
- schemaFile = baseDir + "/" + schemaTool.getDbType() + "/" + subScripts.remove(0);
- LOG.debug("Parsing subscript " + schemaFile);
- subScripts.addAll(findCreateTable(schemaFile, schemaTables));
- }
- } catch (Exception e) {
- System.err.println("Exception in parsing schema file. Cause:" + e.getMessage());
- System.out.println("Failed in schema table validation.");
- return false;
- }
-
- LOG.debug("Schema tables:[ " + Arrays.toString(schemaTables.toArray()) + " ]");
- LOG.debug("DB tables:[ " + Arrays.toString(dbTables.toArray()) + " ]");
-
- // now diff the lists
- schemaTables.removeAll(dbTables);
- if (schemaTables.size() > 0) {
- Collections.sort(schemaTables);
- System.err.println("Table(s) [ " + Arrays.toString(schemaTables.toArray()) + " ] " +
- "are missing from the metastore database schema.");
- System.out.println("[FAIL]\n");
- return false;
- } else {
- System.out.println("[SUCCESS]\n");
- return true;
- }
- }
-
- @VisibleForTesting
- List<String> findCreateTable(String path, List<String> tableList) throws Exception {
- if (!(new File(path)).exists()) {
- throw new Exception(path + " does not exist. Potentially incorrect version in the metastore VERSION table");
- }
-
- List<String> subs = new ArrayList<String>();
- NestedScriptParser sp = HiveSchemaHelper.getDbCommandParser(schemaTool.getDbType(), false);
- Pattern regexp = Pattern.compile("CREATE TABLE(\\s+IF NOT EXISTS)?\\s+(\\S+).*");
-
- try (BufferedReader reader = new BufferedReader(new FileReader(path));) {
- String line = null;
- while ((line = reader.readLine()) != null) {
- if (sp.isNestedScript(line)) {
- String subScript = sp.getScriptName(line);
- LOG.debug("Schema subscript " + subScript + " found");
- subs.add(subScript);
- continue;
- }
- line = line.replaceAll("( )+", " "); //suppress multi-spaces
- line = line.replaceAll("\\(", " ");
- line = line.replaceAll("IF NOT EXISTS ", "");
- line = line.replaceAll("`", "");
- line = line.replaceAll("'", "");
- line = line.replaceAll("\"", "");
- Matcher matcher = regexp.matcher(line);
-
- if (matcher.find()) {
- String table = matcher.group(2);
- if (schemaTool.getDbType().equals("derby")) {
- table = table.replaceAll("APP\\.", "");
- }
- tableList.add(table.toLowerCase());
- LOG.debug("Found table " + table + " in the schema");
- }
- }
- } catch (IOException ex){
- throw new Exception(ex.getMessage());
- }
-
- return subs;
- }
-
- @VisibleForTesting
- boolean validateLocations(Connection conn, URI[] defaultServers) throws HiveMetaException {
- System.out.println("Validating DFS locations");
- boolean rtn = true;
- rtn &= checkMetaStoreDBLocation(conn, defaultServers);
- rtn &= checkMetaStoreTableLocation(conn, defaultServers);
- rtn &= checkMetaStorePartitionLocation(conn, defaultServers);
- rtn &= checkMetaStoreSkewedColumnsLocation(conn, defaultServers);
- System.out.println(rtn ? "[SUCCESS]\n" : "[FAIL]\n");
- return rtn;
- }
-
- private static final String QUERY_DB_LOCATION =
- " select dbt.<q>DB_ID<q>, " +
- " dbt.<q>NAME<q>, " +
- " dbt.<q>DB_LOCATION_URI<q> " +
- " from <q>DBS<q> dbt " +
- "order by dbt.<q>DB_ID<q> ";
-
- private boolean checkMetaStoreDBLocation(Connection conn, URI[] defaultServers) throws HiveMetaException {
- String dbLocQuery = schemaTool.quote(QUERY_DB_LOCATION);
-
- int numOfInvalid = 0;
- try (Statement stmt = conn.createStatement();
- ResultSet res = stmt.executeQuery(dbLocQuery)) {
- while (res.next()) {
- String locValue = res.getString(3);
- String dbName = getNameOrID(res, 2, 1);
- if (!checkLocation("Database " + dbName, locValue, defaultServers)) {
- numOfInvalid++;
- }
- }
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to get DB Location Info.", e);
- }
- return numOfInvalid == 0;
- }
-
- private static final String TAB_ID_RANGE_QUERY =
- "select max(<q>TBL_ID<q>), " +
- " min(<q>TBL_ID<q>) " +
- " from <q>TBLS<q> ";
-
- private static final String TAB_LOC_QUERY =
- " select tbl.<q>TBL_ID<q>, " +
- " tbl.<q>TBL_NAME<q>, " +
- " sd.<q>LOCATION<q>, " +
- " dbt.<q>DB_ID<q>, " +
- " dbt.<q>NAME<q> " +
- " from <q>TBLS<q> tbl " +
- "inner join <q>SDS<q> sd on sd.<q>SD_ID<q> = tbl.<q>SD_ID<q> " +
- "inner join <q>DBS<q> dbt on tbl.<q>DB_ID<q> = dbt.<q>DB_ID<q> " +
- " where tbl.<q>TBL_TYPE<q> != '%s' " +
- " and tbl.<q>TBL_ID<q> >= ? " +
- " and tbl.<q>TBL_ID<q> <= ? " +
- " order by tbl.<q>TBL_ID<q> ";
-
- private static final int TAB_LOC_CHECK_SIZE = 2000;
-
- private boolean checkMetaStoreTableLocation(Connection conn, URI[] defaultServers)
- throws HiveMetaException {
- String tabIDRangeQuery = schemaTool.quote(TAB_ID_RANGE_QUERY);
- String tabLocQuery = String.format(schemaTool.quote(TAB_LOC_QUERY), TableType.VIRTUAL_VIEW);
-
- try {
- long maxID = 0, minID = 0;
- try (Statement stmt = conn.createStatement();
- ResultSet res = stmt.executeQuery(tabIDRangeQuery)) {
- if (res.next()) {
- maxID = res.getLong(1);
- minID = res.getLong(2);
- }
- }
-
- int numOfInvalid = 0;
- try (PreparedStatement pStmt = conn.prepareStatement(tabLocQuery)) {
- while (minID <= maxID) {
- pStmt.setLong(1, minID);
- pStmt.setLong(2, minID + TAB_LOC_CHECK_SIZE);
- try (ResultSet res = pStmt.executeQuery()) {
- while (res.next()) {
- String locValue = res.getString(3);
- String entity = "Database " + getNameOrID(res, 5, 4) + ", Table " + getNameOrID(res, 2, 1);
- if (!checkLocation(entity, locValue, defaultServers)) {
- numOfInvalid++;
- }
- }
- }
- minID += TAB_LOC_CHECK_SIZE + 1;
- }
- }
-
- return numOfInvalid == 0;
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to get Table Location Info.", e);
- }
- }
-
- private static final String QUERY_PART_ID_RANGE =
- "select max(<q>PART_ID<q>)," +
- " min(<q>PART_ID<q>)" +
- " from <q>PARTITIONS<q> ";
-
- private static final String QUERY_PART_LOC =
- " select pt.<q>PART_ID<q>, " +
- " pt.<q>PART_NAME<q>, " +
- " sd.<q>LOCATION<q>, " +
- " tbl.<q>TBL_ID<q>, " +
- " tbl.<q>TBL_NAME<q>, " +
- " dbt.<q>DB_ID<q>, " +
- " dbt.<q>NAME<q> " +
- " from <q>PARTITIONS<q> pt " +
- "inner join <q>SDS<q> sd on sd.<q>SD_ID<q> = pt.<q>SD_ID<q> " +
- "inner join <q>TBLS<q> tbl on tbl.<q>TBL_ID<q> = pt.<q>TBL_ID<q> " +
- "inner join <q>DBS<q> dbt on dbt.<q>DB_ID<q> = tbl.<q>DB_ID<q> " +
- " where pt.<q>PART_ID<q> >= ? " +
- " and pt.<q>PART_ID<q> <= ? " +
- " order by tbl.<q>TBL_ID<q> ";
-
- private static final int PART_LOC_CHECK_SIZE = 2000;
-
- private boolean checkMetaStorePartitionLocation(Connection conn, URI[] defaultServers)
- throws HiveMetaException {
- String queryPartIDRange = schemaTool.quote(QUERY_PART_ID_RANGE);
- String queryPartLoc = schemaTool.quote(QUERY_PART_LOC);
-
- try {
- long maxID = 0, minID = 0;
- try (Statement stmt = conn.createStatement();
- ResultSet res = stmt.executeQuery(queryPartIDRange)) {
- if (res.next()) {
- maxID = res.getLong(1);
- minID = res.getLong(2);
- }
- }
-
- int numOfInvalid = 0;
- try (PreparedStatement pStmt = conn.prepareStatement(queryPartLoc)) {
- while (minID <= maxID) {
- pStmt.setLong(1, minID);
- pStmt.setLong(2, minID + PART_LOC_CHECK_SIZE);
- try (ResultSet res = pStmt.executeQuery()) {
- while (res.next()) {
- String locValue = res.getString(3);
- String entity = "Database " + getNameOrID(res, 7, 6) + ", Table " + getNameOrID(res, 5, 4) +
- ", Partition " + getNameOrID(res, 2, 1);
- if (!checkLocation(entity, locValue, defaultServers)) {
- numOfInvalid++;
- }
- }
- }
- minID += PART_LOC_CHECK_SIZE + 1;
- }
- }
-
- return numOfInvalid == 0;
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to get Partition Location Info.", e);
- }
- }
-
- private static final String QUERY_SKEWED_COL_ID_RANGE =
- "select max(<q>STRING_LIST_ID_KID<q>), " +
- " min(<q>STRING_LIST_ID_KID<q>) " +
- " from <q>SKEWED_COL_VALUE_LOC_MAP<q> ";
-
- private static final String QUERY_SKEWED_COL_LOC =
- " select t.<q>TBL_NAME<q>, " +
- " t.<q>TBL_ID<q>, " +
- " sk.<q>STRING_LIST_ID_KID<q>, " +
- " sk.<q>LOCATION<q>, " +
- " db.<q>NAME<q>, " +
- " db.<q>DB_ID<q> " +
- " from <q>TBLS<q> t " +
- " join <q>SDS<q> s on s.<q>SD_ID<q> = t.<q>SD_ID<q> " +
- " join <q>DBS<q> db on db.<q>DB_ID<q> = t.<q>DB_ID<q> " +
- " join <q>SKEWED_COL_VALUE_LOC_MAP<q> sk on sk.<q>SD_ID<q> = s.<q>SD_ID<q> " +
- " where sk.<q>STRING_LIST_ID_KID<q> >= ? " +
- " and sk.<q>STRING_LIST_ID_KID<q> <= ? " +
- "order by t.<q>TBL_ID<q> ";
-
- private static final int SKEWED_COL_LOC_CHECK_SIZE = 2000;
-
- private boolean checkMetaStoreSkewedColumnsLocation(Connection conn, URI[] defaultServers)
- throws HiveMetaException {
- String querySkewedColIDRange = schemaTool.quote(QUERY_SKEWED_COL_ID_RANGE);
- String querySkewedColLoc = schemaTool.quote(QUERY_SKEWED_COL_LOC);
-
- try {
- long maxID = 0, minID = 0;
- try (Statement stmt = conn.createStatement();
- ResultSet res = stmt.executeQuery(querySkewedColIDRange)) {
- if (res.next()) {
- maxID = res.getLong(1);
- minID = res.getLong(2);
- }
- }
-
- int numOfInvalid = 0;
- try (PreparedStatement pStmt = conn.prepareStatement(querySkewedColLoc)) {
- while (minID <= maxID) {
- pStmt.setLong(1, minID);
- pStmt.setLong(2, minID + SKEWED_COL_LOC_CHECK_SIZE);
- try (ResultSet res = pStmt.executeQuery()) {
- while (res.next()) {
- String locValue = res.getString(4);
- String entity = "Database " + getNameOrID(res, 5, 6) + ", Table " + getNameOrID(res, 1, 2) +
- ", String list " + res.getString(3);
- if (!checkLocation(entity, locValue, defaultServers)) {
- numOfInvalid++;
- }
- }
- }
- minID += SKEWED_COL_LOC_CHECK_SIZE + 1;
- }
- }
-
- return numOfInvalid == 0;
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to get skewed columns location info.", e);
- }
- }
-
- /**
- * Check if the location is valid for the given entity.
- * @param entity the entity to represent a database, partition or table
- * @param entityLocation the location
- * @param defaultServers a list of the servers that the location needs to match.
- * The location host needs to match one of the given servers.
- * If empty, then no check against such list.
- * @return true if the location is valid
- */
- private boolean checkLocation(String entity, String entityLocation, URI[] defaultServers) {
- boolean isValid = true;
-
- if (entityLocation == null) {
- System.err.println(entity + ", Error: empty location");
- isValid = false;
- } else {
- try {
- URI currentUri = new Path(entityLocation).toUri();
- String scheme = currentUri.getScheme();
- String path = currentUri.getPath();
- if (StringUtils.isEmpty(scheme)) {
- System.err.println(entity + ", Location: "+ entityLocation + ", Error: missing location scheme.");
- isValid = false;
- } else if (StringUtils.isEmpty(path)) {
- System.err.println(entity + ", Location: "+ entityLocation + ", Error: missing location path.");
- isValid = false;
- } else if (ArrayUtils.isNotEmpty(defaultServers) && currentUri.getAuthority() != null) {
- String authority = currentUri.getAuthority();
- boolean matchServer = false;
- for(URI server : defaultServers) {
- if (StringUtils.equalsIgnoreCase(server.getScheme(), scheme) &&
- StringUtils.equalsIgnoreCase(server.getAuthority(), authority)) {
- matchServer = true;
- break;
- }
- }
- if (!matchServer) {
- System.err.println(entity + ", Location: " + entityLocation + ", Error: mismatched server.");
- isValid = false;
- }
- }
-
- // if there is no path element other than "/", report it but not fail
- if (isValid && StringUtils.containsOnly(path, "/")) {
- System.err.println(entity + ", Location: "+ entityLocation + ", Warn: location set to root, " +
- "not a recommended config.");
- }
- } catch (Exception pe) {
- System.err.println(entity + ", Error: invalid location - " + pe.getMessage());
- isValid =false;
- }
- }
-
- return isValid;
- }
-
- private String getNameOrID(ResultSet res, int nameInx, int idInx) throws SQLException {
- String itemName = res.getString(nameInx);
- return (itemName == null || itemName.isEmpty()) ? "ID: " + res.getString(idInx) : "Name: " + itemName;
- }
-
- private static final String QUERY_COLUMN_NULL_VALUES =
- " select t.*" +
- " from <q>TBLS<q> t" +
- " where t.<q>SD_ID<q> IS NULL" +
- " and (t.<q>TBL_TYPE<q> = '" + TableType.EXTERNAL_TABLE + "' or" +
- " t.<q>TBL_TYPE<q> = '" + TableType.MANAGED_TABLE + "') " +
- "order by t.<q>TBL_ID<q> ";
-
- @VisibleForTesting
- boolean validateColumnNullValues(Connection conn) throws HiveMetaException {
- System.out.println("Validating columns for incorrect NULL values.");
-
- boolean isValid = true;
- String queryColumnNullValues = schemaTool.quote(QUERY_COLUMN_NULL_VALUES);
-
- try (Statement stmt = conn.createStatement();
- ResultSet res = stmt.executeQuery(queryColumnNullValues)) {
- while (res.next()) {
- long tableId = res.getLong("TBL_ID");
- String tableName = res.getString("TBL_NAME");
- String tableType = res.getString("TBL_TYPE");
- isValid = false;
- System.err.println("SD_ID in TBLS should not be NULL for Table Name=" + tableName + ", Table ID=" + tableId + ", Table Type=" + tableType);
- }
-
- System.out.println(isValid ? "[SUCCESS]\n" : "[FAIL]\n");
- return isValid;
- } catch(SQLException e) {
- throw new HiveMetaException("Failed to validate columns for incorrect NULL values", e);
- }
- }
-}
[20/32] hive git commit: HIVE-20043: HiveServer2: SessionState has a
static sync block around an AtomicBoolean (Laszlo Bodor,
reviewed by Zoltan Haindrich)
Posted by se...@apache.org.
HIVE-20043: HiveServer2: SessionState has a static sync block around an AtomicBoolean (Laszlo Bodor, reviewed by Zoltan Haindrich)
Signed-off-by: Gopal V <go...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/db1a70eb
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/db1a70eb
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/db1a70eb
Branch: refs/heads/master-txnstats
Commit: db1a70eb361e56f57bd10fcc0cbf21b99866fcdf
Parents: eb91fa4
Author: Laszlo Bodor <lb...@hortonworks.com>
Authored: Mon Jul 9 09:31:43 2018 -0700
Committer: Gopal V <go...@apache.org>
Committed: Mon Jul 9 09:32:18 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/ql/session/SessionState.java | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/db1a70eb/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 27f0216..262bbb9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -607,10 +607,8 @@ public class SessionState {
private static void start(SessionState startSs, boolean isAsync, LogHelper console) {
setCurrentSessionState(startSs);
- synchronized(SessionState.class) {
- if (!startSs.isStarted.compareAndSet(false, true)) {
- return;
- }
+ if (!startSs.isStarted.compareAndSet(false, true)) {
+ return;
}
if (startSs.hiveHist == null){
[21/32] hive git commit: Update RELEASE_NOTES for Hive 3.1.0
Posted by se...@apache.org.
Update RELEASE_NOTES for Hive 3.1.0
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/601dd05b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/601dd05b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/601dd05b
Branch: refs/heads/master-txnstats
Commit: 601dd05b5100b449fc380fd0c273b743d8ce8564
Parents: db1a70e
Author: Vineet Garg <vg...@apache.org>
Authored: Mon Jul 9 11:35:33 2018 -0700
Committer: Vineet Garg <vg...@apache.org>
Committed: Mon Jul 9 11:35:33 2018 -0700
----------------------------------------------------------------------
RELEASE_NOTES.txt | 303 +++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 303 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/601dd05b/RELEASE_NOTES.txt
----------------------------------------------------------------------
diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt
index f2f2b8c..9536427 100644
--- a/RELEASE_NOTES.txt
+++ b/RELEASE_NOTES.txt
@@ -1,3 +1,306 @@
+Release Notes - Hive - Version 3.1.0
+
+** Sub-task
+ * [HIVE-12192] - Hive should carry out timestamp computations in UTC
+ * [HIVE-17227] - Incremental replication load should create tasks in execution phase rather than semantic phase
+ * [HIVE-17657] - export/import for MM tables is broken
+ * [HIVE-18193] - Migrate existing ACID tables to use write id per table rather than global transaction id
+ * [HIVE-18748] - Rename table impacts the ACID behavior as table names are not updated in meta-tables.
+ * [HIVE-18840] - CachedStore: Prioritize loading of recently accessed tables during prewarm
+ * [HIVE-18946] - Fix columnstats merge NPE
+ * [HIVE-18988] - Support bootstrap replication of ACID tables
+ * [HIVE-19009] - Retain and use runtime statistics during hs2 lifetime
+ * [HIVE-19096] - query result cache interferes with explain analyze
+ * [HIVE-19126] - CachedStore: Use memory estimation to limit cache size during prewarm
+ * [HIVE-19128] - Update golden files for spark perf tests
+ * [HIVE-19135] - Need tool to allow admins to create catalogs and move existing dbs to catalog during upgrade
+ * [HIVE-19141] - TestNegativeCliDriver insert_into_notnull_constraint, insert_into_acid_notnull failing
+ * [HIVE-19159] - TestMTQueries.testMTQueries1 failure
+ * [HIVE-19164] - TestMetastoreVersion failures
+ * [HIVE-19171] - Persist runtime statistics in metastore
+ * [HIVE-19193] - TestActivePassiveHA fails
+ * [HIVE-19194] - TestDruidStorageHandler fails
+ * [HIVE-19195] - Fix flaky tests and cleanup testconfiguration to run llap specific tests in llap only.
+ * [HIVE-19196] - TestTriggersMoveWorkloadManager is flaky
+ * [HIVE-19206] - Automatic memory management for open streaming writers
+ * [HIVE-19209] - Streaming ingest record writers should accept input stream
+ * [HIVE-19210] - Create separate module for streaming ingest
+ * [HIVE-19211] - New streaming ingest API and support for dynamic partitioning
+ * [HIVE-19214] - High throughput ingest ORC format
+ * [HIVE-19222] - TestNegativeCliDriver tests are failing due to "java.lang.OutOfMemoryError: GC overhead limit exceeded"
+ * [HIVE-19243] - Upgrade hadoop.version to 3.1.0
+ * [HIVE-19274] - Add an OpTreeSignature persistence checker hook
+ * [HIVE-19332] - Disable compute.query.using.stats for external table
+ * [HIVE-19336] - Disable SMB/Bucketmap join for external tables
+ * [HIVE-19340] - Disable timeout of transactions opened by replication task at target cluster
+ * [HIVE-19347] - TestTriggersWorkloadManager tests are failing consistently
+ * [HIVE-19348] - org.apache.hadoop.hive.ql.plan.mapping.TestOperatorCmp are failing
+ * [HIVE-19366] - Vectorization causing TestStreaming.testStreamBucketingMatchesRegularBucketing to fail
+ * [HIVE-19374] - Parse and process ALTER TABLE SET OWNER command syntax
+ * [HIVE-19409] - Disable incremental rewriting with outdated materialized views
+ * [HIVE-19472] - HiveStreamingConnection swallows exception on partition creation
+ * [HIVE-19494] - Accept shade prefix during reflective instantiation of output format
+ * [HIVE-19495] - Arrow SerDe itest failure
+ * [HIVE-19499] - Bootstrap REPL LOAD shall add tasks to create checkpoints for db/tables/partitions.
+ * [HIVE-19500] - Prevent multiple selectivity estimations for the same variable in conjuctions
+ * [HIVE-19562] - Flaky test: TestMiniSparkOnYarn FileNotFoundException in spark-submit
+ * [HIVE-19598] - Add Acid V1 to V2 upgrade module
+ * [HIVE-19637] - Add slow test report script to testutils
+ * [HIVE-19688] - Make catalogs updatable
+ * [HIVE-19727] - Fix Signature matching of table aliases
+ * [HIVE-19739] - Bootstrap REPL LOAD to use checkpoints to validate and skip the loaded data/metadata.
+ * [HIVE-19758] - Set hadoop.version=3.1.0 in standalone-metastore
+ * [HIVE-19768] - Utility to convert tables to conform to Hive strict managed tables mode
+ * [HIVE-19799] - remove jasper dependency
+ * [HIVE-19815] - Repl dump should not propagate the checkpoint and repl source properties
+ * [HIVE-19851] - upgrade jQuery version
+ * [HIVE-19852] - update jackson to latest
+ * [HIVE-19868] - Add support for float aggregator
+ * [HIVE-19892] - Disable query results cache for for HiveServer2 doAs=true
+ * [HIVE-19923] - Follow up of HIVE-19615, use UnaryFunction instead of prefix
+
+
+** Bug
+ * [HIVE-15190] - Field names are not preserved in ORC files written with ACID
+ * [HIVE-18434] - Type is not determined correctly for comparison between decimal column and string constant
+ * [HIVE-18816] - CREATE TABLE (ACID) doesn't work with TIMESTAMPLOCALTZ column type
+ * [HIVE-19016] - Vectorization and Parquet: Disable vectorization for nested complex types
+ * [HIVE-19054] - Function replication shall use "hive.repl.replica.functions.root.dir" as root
+ * [HIVE-19108] - Vectorization and Parquet: Turning on vectorization in parquet_ppd_decimal.q causes Wrong Query Results
+ * [HIVE-19109] - Vectorization: Enabling vectorization causes TestCliDriver delete_orig_table.q to produce Wrong Results
+ * [HIVE-19110] - Vectorization: Enabling vectorization causes TestContribCliDriver udf_example_arraymapstruct.q to produce Wrong Results
+ * [HIVE-19118] - Vectorization: Turning on vectorization in escape_crlf produces wrong results
+ * [HIVE-19120] - catalog not properly set for some tables in SQL upgrade scripts
+ * [HIVE-19131] - DecimalColumnStatsMergerTest comparison review
+ * [HIVE-19155] - Day time saving cause Druid inserts to fail with org.apache.hive.druid.io.druid.java.util.common.UOE: Cannot add overlapping segments
+ * [HIVE-19157] - Assert that Insert into Druid Table fails if the publishing of metadata by HS2 fails
+ * [HIVE-19167] - Map data type doesn't keep the order of the key/values pairs as read (Part 2, The Sequel or SQL)
+ * [HIVE-19168] - Ranger changes for llap commands
+ * [HIVE-19186] - Multi Table INSERT statements query has a flaw for partitioned table when INSERT INTO and INSERT OVERWRITE are used
+ * [HIVE-19200] - Vectorization: Disable vectorization for LLAP I/O when a non-VECTORIZED_INPUT_FILE_FORMAT mode is needed (i.e. rows) and data type conversion is needed
+ * [HIVE-19219] - Incremental REPL DUMP should throw error if requested events are cleaned-up.
+ * [HIVE-19230] - Schema column width inconsistency in Oracle
+ * [HIVE-19231] - Beeline generates garbled output when using UnsupportedTerminal
+ * [HIVE-19237] - Only use an operatorId once in a plan
+ * [HIVE-19247] - StatsOptimizer: Missing stats fast-path for Date
+ * [HIVE-19248] - REPL LOAD couldn't copy file from source CM path and also doesn't throw error if file copy fails.
+ * [HIVE-19258] - add originals support to MM tables (and make the conversion a metadata only operation)
+ * [HIVE-19264] - Vectorization: Reenable vectorization in vector_adaptor_usage_mode.q
+ * [HIVE-19269] - Vectorization: Turn On by Default
+ * [HIVE-19275] - Vectorization: Defer Wrong Results / Execution Failures when Vectorization turned on
+ * [HIVE-19277] - Active/Passive HA web endpoints does not allow cross origin requests
+ * [HIVE-19312] - MM tables don't work with BucketizedHIF
+ * [HIVE-19317] - Handle schema evolution from int like types to decimal
+ * [HIVE-19327] - qroupby_rollup_empty.q fails for insert-only transactional tables
+ * [HIVE-19331] - Repl load config in "with" clause not pass to Context.getStagingDir
+ * [HIVE-19350] - Vectorization: Turn off vectorization for explainuser_1.q / spark_explainuser_1
+ * [HIVE-19352] - Vectorization: Disable vectorization for org.apache.hive.jdbc.TestJdbcDriver2.testResultSetMetaData
+ * [HIVE-19358] - CBO decorrelation logic should generate Hive operators
+ * [HIVE-19365] - Index on COMPLETED_TXN_COMPONENTS in Metastore RDBMS has different names in different scripts
+ * [HIVE-19370] - Issue: ADD Months function on timestamp datatype fields in hive
+ * [HIVE-19381] - Function replication in cloud fail when download resource from AWS
+ * [HIVE-19382] - Acquire locks before generating valid transaction list for some operations
+ * [HIVE-19384] - Vectorization: IfExprTimestamp* do not handle NULLs correctly
+ * [HIVE-19410] - don't create serde reader in LLAP if there's no cache
+ * [HIVE-19418] - add background stats updater similar to compactor
+ * [HIVE-19423] - REPL LOAD creates staging directory in source dump directory instead of table data location
+ * [HIVE-19433] - HiveJoinPushTransitivePredicatesRule hangs
+ * [HIVE-19435] - Incremental replication cause data loss if a table is dropped followed by create and insert-into with different partition type.
+ * [HIVE-19454] - Test failure : org.apache.hadoop.hive.ql.TestTxnCommands2.testNonAcidToAcidConversion1 fails with java.lang.AssertionError
+ * [HIVE-19460] - Improve stats estimations for NOT IN operator
+ * [HIVE-19463] - TezTask - getting groups may fail (PartialGroupNameException in some tests)
+ * [HIVE-19467] - Make storage format configurable for temp tables created using LLAP external client
+ * [HIVE-19474] - Decimal type should be casted as part of the CTAS or INSERT Clause.
+ * [HIVE-19479] - encoded stream seek is incorrect for 0-length RGs in LLAP IO
+ * [HIVE-19481] - Tablesample uses incorrect logic to pick files corresponding to buckets.
+ * [HIVE-19485] - dump directory for non native tables should not be created
+ * [HIVE-19493] - VectorUDFDateDiffColCol copySelected does not handle nulls correctly
+ * [HIVE-19496] - Check untar folder
+ * [HIVE-19498] - Vectorization: CAST expressions produce wrong results
+ * [HIVE-19504] - Change default value for hive.auto.convert.join.shuffle.max.size property
+ * [HIVE-19516] - TestNegative merge_negative_5 and mm_concatenate are causing timeouts
+ * [HIVE-19529] - Vectorization: Date/Timestamp NULL issues
+ * [HIVE-19557] - stats: filters for dates are not taking advantage of min/max values
+ * [HIVE-19565] - Vectorization: Fix NULL / Wrong Results issues in STRING Functions
+ * [HIVE-19567] - Fix flakiness in TestTriggers
+ * [HIVE-19569] - alter table db1.t1 rename db2.t2 generates MetaStoreEventListener.onDropTable()
+ * [HIVE-19575] - TestAutoPurgeTables seems flaky
+ * [HIVE-19577] - CREATE TEMPORARY TABLE LIKE and INSERT generate output format mismatch errors
+ * [HIVE-19578] - HLL merges tempList on every add
+ * [HIVE-19588] - Several invocation of file listing when creating VectorizedOrcAcidRowBatchReader
+ * [HIVE-19589] - Disable TestAutoPurge tests and annotate TestTriggersWorkloadManager with retry
+ * [HIVE-19590] - mask stats in llap_smb
+ * [HIVE-19592] - TestWorkloadManager - add retry for now
+ * [HIVE-19594] - Add custom tmp folders to tests to avoid collisions
+ * [HIVE-19595] - Regenerate webui port in MiniHS2
+ * [HIVE-19604] - Incorrect Handling of Boolean in DruidSerde
+ * [HIVE-19605] - TAB_COL_STATS table has no index on db/table name
+ * [HIVE-19608] - disable flaky tests 2
+ * [HIVE-19613] - GenericUDTFGetSplits should handle fetch task with temp table rewrite
+ * [HIVE-19614] - GenericUDTFGetSplits does not honor ORDER BY
+ * [HIVE-19615] - Proper handling of is null and not is null predicate when pushed to Druid
+ * [HIVE-19619] - Allow comparisons between doubles and bigints
+ * [HIVE-19629] - Enable Decimal64 reader after orc version upgrade
+ * [HIVE-19631] - reduce epic locking in AbstractService
+ * [HIVE-19632] - Remove webapps directory from standalone jar
+ * [HIVE-19639] - a transactional Hive table cannot be imported as an external table
+ * [HIVE-19643] - MM table conversion doesn't need full ACID structure checks
+ * [HIVE-19644] - change WM syntax to avoid conflicts with identifiers starting with a number
+ * [HIVE-19646] - Filesystem closed error in HiveProtoLoggingHook
+ * [HIVE-19660] - update branch-3 to be version 3.1 and fix storage-api mismatch
+ * [HIVE-19675] - Cast to timestamps on Druid time column leads to an exception
+ * [HIVE-19677] - Disable sample6.q
+ * [HIVE-19680] - Push down limit is not applied for Druid storage handler.
+ * [HIVE-19684] - Hive stats optimizer wrongly uses stats against non native tables
+ * [HIVE-19687] - Export table on acid partitioned table is failing
+ * [HIVE-19690] - multi-insert query with multiple GBY, and distinct in only some branches can produce incorrect results
+ * [HIVE-19691] - Start SessionState in materialized views registry
+ * [HIVE-19695] - Year Month Day extraction functions need to add an implicit cast for column that are String types
+ * [HIVE-19697] - TestReOptimization#testStatCachingMetaStore is flaky
+ * [HIVE-19698] - TestAMReporter#testMultipleAM is flaky
+ * [HIVE-19700] - Workaround for JLine issue with UnsupportedTerminal
+ * [HIVE-19713] - itests/hive-jmh should not reference a concreate storage-api version
+ * [HIVE-19723] - Arrow serde: "Unsupported data type: Timestamp(NANOSECOND, null)"
+ * [HIVE-19726] - ORC date PPD is broken
+ * [HIVE-19728] - beeline with USE_BEELINE_FOR_HIVE_CLI fails when trying to set hive.aux.jars.path
+ * [HIVE-19734] - Beeline: When beeline-site.xml is present, beeline does not honor -n (username) and -p (password) arguments
+ * [HIVE-19744] - In Beeline if -u is specified the default connection should not be tried at all
+ * [HIVE-19750] - Initialize NEXT_WRITE_ID. NWI_NEXT on converting an existing table to full acid
+ * [HIVE-19753] - Strict managed tables mode in Hive
+ * [HIVE-19754] - vector_decimal_2 failing on branch-3
+ * [HIVE-19755] - insertsel_fail.q.out needs to be updated on branch-3
+ * [HIVE-19762] - Druid Queries containing Joins gives wrong results.
+ * [HIVE-19771] - allowNullColumnForMissingStats should not be false when column stats are estimated
+ * [HIVE-19772] - Streaming ingest V2 API can generate invalid orc file if interrupted
+ * [HIVE-19773] - CBO exception while running queries with tables that are not present in materialized views
+ * [HIVE-19777] - NPE in TezSessionState
+ * [HIVE-19789] - reenable orc_llap test
+ * [HIVE-19793] - disable LLAP IO batch-to-row wrapper for ACID deletes/updates
+ * [HIVE-19794] - Disable removing order by from subquery in GenericUDTFGetSplits
+ * [HIVE-19796] - Push Down TRUNC Fn to Druid Storage Handler
+ * [HIVE-19801] - JDBC: Add some missing classes to jdbc standalone jar and remove hbase classes
+ * [HIVE-19808] - GenericUDTFGetSplits should support ACID reads in the temp. table read path
+ * [HIVE-19810] - StorageHandler fail to ship jars in Tez intermittently
+ * [HIVE-19813] - SessionState.start don't have to be synchronized
+ * [HIVE-19817] - Hive streaming API + dynamic partitioning + json/regex writer does not work
+ * [HIVE-19826] - OrcRawRecordMerger doesn't work for more than one file in non vectorized case
+ * [HIVE-19827] - hiveserver2 startup should provide a way to override TEZ_CONF_DIR
+ * [HIVE-19833] - reduce LLAP IO min allocation to match ORC variable CB size
+ * [HIVE-19837] - Setting to have different default location for external tables
+ * [HIVE-19838] - simplify & fix ColumnizedDeleteEventRegistry load loop
+ * [HIVE-19853] - Arrow serializer needs to create a TimeStampMicroTZVector instead of TimeStampMicroVector
+ * [HIVE-19857] - Set 3.1.0 for sys db version
+ * [HIVE-19859] - Inspect lock components for DBHiveLock while verifying whether transaction list is valid
+ * [HIVE-19861] - Fix temp table path generation for acid table export
+ * [HIVE-19862] - Postgres init script has a glitch around UNIQUE_DATABASE
+ * [HIVE-19864] - Address TestTriggersWorkloadManager flakiness
+ * [HIVE-19866] - improve LLAP cache purge
+ * [HIVE-19869] - Remove double formatting bug followup of HIVE-19382
+ * [HIVE-19872] - hive-schema-3.1.0.hive.sql is missing on master and branch-3
+ * [HIVE-19873] - Cleanup operation log on query cancellation after some delay
+ * [HIVE-19875] - increase LLAP IO queue size for perf
+ * [HIVE-19876] - Multiple fixes for Driver.isValidTxnListState
+ * [HIVE-19877] - Remove setting hive.execution.engine as mr in HiveStreamingConnection
+ * [HIVE-19879] - Remove unused calcite sql operator.
+ * [HIVE-19884] - Invalidation cache may throw NPE when there is no data in table used by materialized view
+ * [HIVE-19889] - Wrong results due to PPD of non deterministic functions with CBO
+ * [HIVE-19890] - ACID: Inherit bucket-id from original ROW_ID for delete deltas
+ * [HIVE-19898] - Disable TransactionalValidationListener when the table is not in the Hive catalog
+ * [HIVE-19903] - Disable temporary insert-only transactional table
+ * [HIVE-19904] - Load data rewrite into Tez job fails for ACID
+ * [HIVE-19908] - Block Insert Overwrite with Union All on full CRUD ACID tables using HIVE_UNION_SUBDIR_
+ * [HIVE-19912] - Schema evolution checks prints a log line in INFO mode for each vectorized rowbatch, impacts performance
+ * [HIVE-19917] - Export of full CRUD transactional table fails if table is not in default database
+ * [HIVE-19920] - Schematool fails in embedded mode when auth is on
+ * [HIVE-19921] - Fix perf duration and queue name in HiveProtoLoggingHook
+ * [HIVE-19938] - Upgrade scripts for information schema
+ * [HIVE-19941] - Row based Filters added via Hive Ranger policies are not pushed to druid
+ * [HIVE-19946] - VectorizedRowBatchCtx.recordIdColumnVector cannot be shared between different JVMs
+ * [HIVE-19951] - Vectorization: Need to disable encoded LLAP I/O for ORC when there is data type conversion (Schema Evolution)
+ * [HIVE-19956] - Include yarn registry classes to jdbc standalone jar
+ * [HIVE-19964] - Apply resource plan fails if trigger expression has quotes
+ * [HIVE-19965] - Make HiveEndPoint use IMetaStoreClient.add_partition
+ * [HIVE-19972] - Followup to HIVE-19928 : Fix the check for managed table
+ * [HIVE-19973] - Enable materialized view rewriting by default
+ * [HIVE-19980] - GenericUDTFGetSplits fails when order by query returns 0 rows
+ * [HIVE-19997] - Batches for TestMiniDruidCliDriver
+ * [HIVE-20002] - Shipping jdbd-storage-handler dependency jars in LLAP
+ * [HIVE-20010] - Fix create view over literals
+
+
+** New Feature
+ * [HIVE-18739] - Add support for Import/Export from Acid table
+ * [HIVE-19307] - Support ArrowOutputStream in LlapOutputFormatService
+
+
+** Improvement
+ * [HIVE-17824] - msck repair table should drop the missing partitions from metastore
+ * [HIVE-18079] - Statistics: Allow HyperLogLog to be merged to the lowest-common-denominator bit-size
+ * [HIVE-18394] - Materialized view: "Create Materialized View" should default to rewritable ones
+ * [HIVE-18410] - [Performance][Avro] Reading flat Avro tables is very expensive in Hive
+ * [HIVE-18743] - CREATE TABLE on S3 data can be extremely slow. DO_NOT_UPDATE_STATS workaround is buggy.
+ * [HIVE-18792] - Allow standard compliant syntax for insert on partitioned tables
+ * [HIVE-18866] - Semijoin and analyze: Implement a Long -> Hash64 vector fast-path
+ * [HIVE-19027] - Make materializations invalidation cache work with multiple active remote metastores
+ * [HIVE-19161] - Add authorizations to information schema
+ * [HIVE-19228] - Remove commons-httpclient 3.x usage
+ * [HIVE-19259] - Create view on tables having union all fail with "Table not found"
+ * [HIVE-19344] - Change default value of msck.repair.batch.size
+ * [HIVE-19390] - Useless error messages logged for dummy table stats
+ * [HIVE-19415] - Support CORS for all HS2 web endpoints
+ * [HIVE-19421] - Upgrade version of Jetty to 9.3.20.v20170531
+ * [HIVE-19440] - Make StorageBasedAuthorizer work with information schema
+ * [HIVE-19462] - Fix mapping for char_length function to enable pushdown to Druid.
+ * [HIVE-19464] - Upgrade Parquet to 1.10.0
+ * [HIVE-19465] - Upgrade ORC to 1.5.0
+ * [HIVE-19466] - Update constraint violation error message
+ * [HIVE-19490] - Locking on Insert into for non native and managed tables.
+ * [HIVE-19534] - Allow implementations to access member variables of AbstractRecordWriter
+ * [HIVE-19560] - Retry test runner and retry rule for flaky tests
+ * [HIVE-19572] - Add option to mask stats and data size in q files
+ * [HIVE-19586] - Optimize Count(distinct X) pushdown based on the storage capabilities
+ * [HIVE-19669] - Upgrade ORC to 1.5.1
+ * [HIVE-19682] - Provide option for GenericUDTFGetSplits to return only schema metadata
+ * [HIVE-19775] - Schematool should use HS2 embedded mode in privileged auth mode
+ * [HIVE-19776] - HiveServer2.startHiveServer2 retries of start has concurrency issues
+ * [HIVE-19824] - Improve online datasize estimations for MapJoins
+ * [HIVE-19885] - Druid Kafka Ingestion - Allow user to set kafka consumer properties via table properties
+
+** Test
+ * [HIVE-19271] - TestMiniLlapLocalCliDriver default_constraint and check_constraint failing
+ * [HIVE-19515] - TestRpc.testServerPort is consistently failing
+ * [HIVE-19555] - Enable TestMiniLlapLocalCliDriver#tez_dynpart_hashjoin_1.q and TestMiniLlapLocalCliDriver#tez_vector_dynpart_hashjoin_1.q
+ * [HIVE-19573] - Fix flaky TestMiniLlapLocalCliDriver#explainuser_4.q
+ * [HIVE-19612] - Add option to mask lineage in q files
+ * [HIVE-19617] - Rename test tables to avoid collisions during execution in batches
+ * [HIVE-19620] - Change tmp directory used by PigServer in HCat tests
+ * [HIVE-19626] - Change tmp staging mapred directory for CliDriver
+ * [HIVE-19654] - Change tmp staging mapred directory for TestBlobstoreCliDriver
+ * [HIVE-19655] - Mask stats for TestMiniLlapLocalCliDriver#smb_mapjoin_15
+ * [HIVE-19699] - Re-enable TestReOptimization
+ * [HIVE-19706] - Disable TestJdbcWithMiniHS2#testHttpRetryOnServerIdleTimeout
+ * [HIVE-19731] - Change staging tmp directory used by TestHCatLoaderComplexSchema
+
+
+** Task
+ * [HIVE-18875] - Enable SMB Join by default in Tez
+ * [HIVE-19134] - Update copyright NOTICE and fix rat check failures
+ * [HIVE-19140] - Update metastore upgrade scripts to prepare for 3.1.0 development
+ * [HIVE-19257] - HIVE-19157 commit references wrong jira
+ * [HIVE-19306] - Arrow batch serializer
+ * [HIVE-19308] - Provide an Arrow stream reader for external LLAP clients
+ * [HIVE-19323] - Create metastore SQL install and upgrade scripts for 3.1
+ * [HIVE-19488] - Enable CM root based on db parameter, identifying a db as source of replication.
+ * [HIVE-19509] - Disable tests that are failing continuously
+ * [HIVE-19512] - If parallel execution is enabled, metastore is throwing out of sequence error.
+ * [HIVE-19708] - Repl copy retrying with cm path even if the failure is due to network issue
+ * [HIVE-19725] - Add ability to dump non-native tables in replication metadata dump
+ * [HIVE-19880] - Repl Load to return recoverable vs non-recoverable error codes
+ * [HIVE-19881] - Allow metadata-only dump for database which are not source of replication
+ * [HIVE-19928] - Load Data for managed tables should set the owner of loaded files to a configurable user
+ * [HIVE-19978] - Backport HIVE-18037 to branch-3
+
+
Release Notes - Hive - Version 3.0.0
** Sub-task
[03/32] hive git commit: HIVE-19765: Add Parquet specific tests to
BlobstoreCliDriver (Sahil Takiar, reviewed by Marta Kuczora)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/cbcfab9e/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out
new file mode 100644
index 0000000..15ae3d9
--- /dev/null
+++ b/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out
@@ -0,0 +1,537 @@
+PREHOOK: query: DROP TABLE src_events
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE src_events
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE src_events
+(
+ log_id BIGINT,
+ `time` BIGINT,
+ uid BIGINT,
+ user_id BIGINT,
+ type INT,
+ event_data STRING,
+ session_id STRING,
+ full_uid BIGINT,
+ run_date STRING,
+ game_id INT,
+ event_name STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/src_events
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_events
+POSTHOOK: query: CREATE TABLE src_events
+(
+ log_id BIGINT,
+ `time` BIGINT,
+ uid BIGINT,
+ user_id BIGINT,
+ type INT,
+ event_data STRING,
+ session_id STRING,
+ full_uid BIGINT,
+ run_date STRING,
+ game_id INT,
+ event_name STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/src_events
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_events
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@src_events
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@src_events
+PREHOOK: query: DROP TABLE parquet_events
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE parquet_events
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE parquet_events
+(
+ log_id BIGINT,
+ `time` BIGINT,
+ uid BIGINT,
+ user_id BIGINT,
+ type INT,
+ event_data STRING,
+ session_id STRING,
+ full_uid BIGINT
+)
+PARTITIONED BY (run_date STRING, game_id INT, event_name STRING)
+STORED AS PARQUET
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_events
+PREHOOK: Output: database:default
+PREHOOK: Output: default@parquet_events
+POSTHOOK: query: CREATE TABLE parquet_events
+(
+ log_id BIGINT,
+ `time` BIGINT,
+ uid BIGINT,
+ user_id BIGINT,
+ type INT,
+ event_data STRING,
+ session_id STRING,
+ full_uid BIGINT
+)
+PARTITIONED BY (run_date STRING, game_id INT, event_name STRING)
+STORED AS PARQUET
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_events
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@parquet_events
+PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
+SELECT * FROM src_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events
+POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
+SELECT * FROM src_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change
+POSTHOOK: Output: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change
+POSTHOOK: Output: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: SHOW PARTITIONS parquet_events
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@parquet_events
+POSTHOOK: query: SHOW PARTITIONS parquet_events
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@parquet_events
+run_date=20120921/game_id=39/event_name=hq_change
+run_date=20121021/game_id=39/event_name=hq_change
+run_date=20121121/game_id=39/event_name=hq_change
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+200
+PREHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change')
+#### A masked pattern was here ####
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-1
+PREHOOK: Output: default@parquet_events
+POSTHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change')
+#### A masked pattern was here ####
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-1
+POSTHOOK: Output: default@parquet_events
+POSTHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
+PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201211'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
+POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201211'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: SHOW PARTITIONS parquet_events
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@parquet_events
+POSTHOOK: query: SHOW PARTITIONS parquet_events
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@parquet_events
+run_date=20120921/game_id=39/event_name=hq_change
+run_date=20121021/game_id=39/event_name=hq_change
+run_date=201211/game_id=39/event_name=hq_change
+run_date=20121121/game_id=39/event_name=hq_change
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+300
+PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201211'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
+POSTHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201211'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: SHOW PARTITIONS parquet_events
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@parquet_events
+POSTHOOK: query: SHOW PARTITIONS parquet_events
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@parquet_events
+run_date=20120921/game_id=39/event_name=hq_change
+run_date=20121021/game_id=39/event_name=hq_change
+run_date=201211/game_id=39/event_name=hq_change
+run_date=20121121/game_id=39/event_name=hq_change
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+400
+PREHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+#### A masked pattern was here ####
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-2
+PREHOOK: Output: default@parquet_events
+POSTHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+#### A masked pattern was here ####
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-2
+POSTHOOK: Output: default@parquet_events
+POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+POSTHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: SHOW PARTITIONS parquet_events
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@parquet_events
+POSTHOOK: query: SHOW PARTITIONS parquet_events
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@parquet_events
+run_date=201209/game_id=39/event_name=hq_change
+run_date=20120921/game_id=39/event_name=hq_change
+run_date=20121021/game_id=39/event_name=hq_change
+run_date=201211/game_id=39/event_name=hq_change
+run_date=20121121/game_id=39/event_name=hq_change
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+500
+PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
+SELECT * FROM src_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events
+POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
+SELECT * FROM src_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change
+POSTHOOK: Output: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change
+POSTHOOK: Output: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: SHOW PARTITIONS parquet_events
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@parquet_events
+POSTHOOK: query: SHOW PARTITIONS parquet_events
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@parquet_events
+run_date=201209/game_id=39/event_name=hq_change
+run_date=20120921/game_id=39/event_name=hq_change
+run_date=20121021/game_id=39/event_name=hq_change
+run_date=201211/game_id=39/event_name=hq_change
+run_date=20121121/game_id=39/event_name=hq_change
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+500
+PREHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201207, game_id=39, event_name='hq_change')
+#### A masked pattern was here ####
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-3
+PREHOOK: Output: default@parquet_events
+POSTHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201207, game_id=39, event_name='hq_change')
+#### A masked pattern was here ####
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-3
+POSTHOOK: Output: default@parquet_events
+POSTHOOK: Output: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
+PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201207, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
+POSTHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201207, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: SHOW PARTITIONS parquet_events
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@parquet_events
+POSTHOOK: query: SHOW PARTITIONS parquet_events
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@parquet_events
+run_date=201207/game_id=39/event_name=hq_change
+run_date=201209/game_id=39/event_name=hq_change
+run_date=20120921/game_id=39/event_name=hq_change
+run_date=20121021/game_id=39/event_name=hq_change
+run_date=201211/game_id=39/event_name=hq_change
+run_date=20121121/game_id=39/event_name=hq_change
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+PREHOOK: Input: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
+PREHOOK: Input: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+PREHOOK: Input: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change
+PREHOOK: Input: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change
+PREHOOK: Input: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
+PREHOOK: Input: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+POSTHOOK: Input: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
+POSTHOOK: Input: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+POSTHOOK: Input: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change
+POSTHOOK: Input: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change
+POSTHOOK: Input: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
+POSTHOOK: Input: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change
+#### A masked pattern was here ####
+550
+PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
+SELECT * FROM src_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events
+POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
+SELECT * FROM src_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change
+POSTHOOK: Output: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change
+POSTHOOK: Output: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: SHOW PARTITIONS parquet_events
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@parquet_events
+POSTHOOK: query: SHOW PARTITIONS parquet_events
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@parquet_events
+run_date=201207/game_id=39/event_name=hq_change
+run_date=201209/game_id=39/event_name=hq_change
+run_date=20120921/game_id=39/event_name=hq_change
+run_date=20121021/game_id=39/event_name=hq_change
+run_date=201211/game_id=39/event_name=hq_change
+run_date=20121121/game_id=39/event_name=hq_change
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+PREHOOK: Input: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
+PREHOOK: Input: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+PREHOOK: Input: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change
+PREHOOK: Input: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change
+PREHOOK: Input: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
+PREHOOK: Input: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+POSTHOOK: Input: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
+POSTHOOK: Input: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+POSTHOOK: Input: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change
+POSTHOOK: Input: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change
+POSTHOOK: Input: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
+POSTHOOK: Input: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change
+#### A masked pattern was here ####
+550
+PREHOOK: query: ALTER TABLE parquet_events DROP PARTITION (run_date=201211, game_id=39, event_name='hq_change')
+PREHOOK: type: ALTERTABLE_DROPPARTS
+PREHOOK: Input: default@parquet_events
+PREHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
+POSTHOOK: query: ALTER TABLE parquet_events DROP PARTITION (run_date=201211, game_id=39, event_name='hq_change')
+POSTHOOK: type: ALTERTABLE_DROPPARTS
+POSTHOOK: Input: default@parquet_events
+POSTHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
+PREHOOK: query: ALTER TABLE parquet_events DROP PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+PREHOOK: type: ALTERTABLE_DROPPARTS
+PREHOOK: Input: default@parquet_events
+PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+POSTHOOK: query: ALTER TABLE parquet_events DROP PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+POSTHOOK: type: ALTERTABLE_DROPPARTS
+POSTHOOK: Input: default@parquet_events
+POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+PREHOOK: query: ALTER TABLE parquet_events DROP PARTITION (run_date=201207, game_id=39, event_name='hq_change')
+PREHOOK: type: ALTERTABLE_DROPPARTS
+PREHOOK: Input: default@parquet_events
+PREHOOK: Output: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
+POSTHOOK: query: ALTER TABLE parquet_events DROP PARTITION (run_date=201207, game_id=39, event_name='hq_change')
+POSTHOOK: type: ALTERTABLE_DROPPARTS
+POSTHOOK: Input: default@parquet_events
+POSTHOOK: Output: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
+PREHOOK: query: SHOW PARTITIONS parquet_events
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@parquet_events
+POSTHOOK: query: SHOW PARTITIONS parquet_events
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@parquet_events
+run_date=20120921/game_id=39/event_name=hq_change
+run_date=20121021/game_id=39/event_name=hq_change
+run_date=20121121/game_id=39/event_name=hq_change
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+200
http://git-wip-us.apache.org/repos/asf/hive/blob/cbcfab9e/itests/hive-blobstore/src/test/results/clientpositive/ptf_parquetfile.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/ptf_parquetfile.q.out b/itests/hive-blobstore/src/test/results/clientpositive/ptf_parquetfile.q.out
new file mode 100644
index 0000000..192b80f
--- /dev/null
+++ b/itests/hive-blobstore/src/test/results/clientpositive/ptf_parquetfile.q.out
@@ -0,0 +1,104 @@
+PREHOOK: query: DROP TABLE part_parquet
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE part_parquet
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE part_parquet(
+ p_partkey int,
+ p_name string,
+ p_mfgr string,
+ p_brand string,
+ p_type string,
+ p_size int,
+ p_container string,
+ p_retailprice double,
+ p_comment string
+)
+STORED AS PARQUET
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/ptf_parquetfile/part_parquet
+PREHOOK: Output: database:default
+PREHOOK: Output: default@part_parquet
+POSTHOOK: query: CREATE TABLE part_parquet(
+ p_partkey int,
+ p_name string,
+ p_mfgr string,
+ p_brand string,
+ p_type string,
+ p_size int,
+ p_container string,
+ p_retailprice double,
+ p_comment string
+)
+STORED AS PARQUET
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/ptf_parquetfile/part_parquet
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@part_parquet
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/part.parquet' OVERWRITE INTO TABLE part_parquet
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@part_parquet
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/part.parquet' OVERWRITE INTO TABLE part_parquet
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@part_parquet
+PREHOOK: query: SELECT
+ p_mfgr, p_name, p_size,
+ RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS r,
+ DENSE_RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS dr,
+ SUM(p_retailprice) OVER (
+ PARTITION BY p_mfgr
+ ORDER BY p_name ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+ ) AS s1
+FROM NOOP(
+ ON part_parquet
+ PARTITION BY p_mfgr
+ ORDER BY p_name
+)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part_parquet
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT
+ p_mfgr, p_name, p_size,
+ RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS r,
+ DENSE_RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS dr,
+ SUM(p_retailprice) OVER (
+ PARTITION BY p_mfgr
+ ORDER BY p_name ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+ ) AS s1
+FROM NOOP(
+ ON part_parquet
+ PARTITION BY p_mfgr
+ ORDER BY p_name
+)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part_parquet
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3
+Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001
+Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001
+Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68
+Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38
+Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001
+Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62
+Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68
+Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95
+Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34
+Manufacturer#3 almond antique misty red olive 1 4 4 6195.32
+Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61
+Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67
+Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09
+Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35
+Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27
+Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001
+Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69
+Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004
+Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18
+Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66
[04/32] hive git commit: HIVE-19765: Add Parquet specific tests to
BlobstoreCliDriver (Sahil Takiar, reviewed by Marta Kuczora)
Posted by se...@apache.org.
HIVE-19765: Add Parquet specific tests to BlobstoreCliDriver (Sahil Takiar, reviewed by Marta Kuczora)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cbcfab9e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cbcfab9e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cbcfab9e
Branch: refs/heads/master-txnstats
Commit: cbcfab9eb1c58e0c389b093af188d6483440beb0
Parents: c2940a0
Author: Sahil Takiar <ta...@gmail.com>
Authored: Fri Jun 1 12:25:01 2018 -0500
Committer: Sahil Takiar <st...@cloudera.com>
Committed: Thu Jul 5 17:47:42 2018 -0500
----------------------------------------------------------------------
.../queries/clientpositive/parquet_buckets.q | 31 ++
.../clientpositive/parquet_format_nonpart.q | 30 ++
.../clientpositive/parquet_format_part.q | 67 +++
.../parquet_nonstd_partitions_loc.q | 100 ++++
.../queries/clientpositive/ptf_parquetfile.q | 32 ++
.../clientpositive/parquet_buckets.q.out | 183 +++++++
.../clientpositive/parquet_format_nonpart.q.out | 201 +++++++
.../clientpositive/parquet_format_part.q.out | 290 ++++++++++
.../parquet_nonstd_partitions_loc.q.out | 537 +++++++++++++++++++
.../clientpositive/ptf_parquetfile.q.out | 104 ++++
10 files changed, 1575 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/cbcfab9e/itests/hive-blobstore/src/test/queries/clientpositive/parquet_buckets.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/parquet_buckets.q b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_buckets.q
new file mode 100644
index 0000000..c5a2cd3
--- /dev/null
+++ b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_buckets.q
@@ -0,0 +1,31 @@
+-- Test simple interaction with partitioned bucketed table with paquet format in blobstore
+
+SET hive.exec.dynamic.partition=true;
+SET hive.exec.reducers.max=10;
+SET hive.exec.dynamic.partition.mode=nonstrict;
+
+DROP TABLE blobstore_source;
+CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ' '
+COLLECTION ITEMS TERMINATED BY '\t'
+LINES TERMINATED BY '\n'
+LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_buckets/blobstore_source/';
+LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source;
+
+DROP TABLE parquet_buckets;
+CREATE TABLE parquet_buckets (a STRING, value DOUBLE)
+PARTITIONED BY (b STRING)
+CLUSTERED BY (a) INTO 10 BUCKETS
+STORED AS PARQUET
+LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_buckets/parquet_buckets';
+
+INSERT OVERWRITE TABLE parquet_buckets
+PARTITION (b)
+SELECT a, c, b FROM blobstore_source;
+SELECT * FROM parquet_buckets;
+
+INSERT INTO TABLE parquet_buckets
+PARTITION (b)
+SELECT a, c, b FROM blobstore_source;
+SELECT * FROM parquet_buckets;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/cbcfab9e/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_nonpart.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_nonpart.q b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_nonpart.q
new file mode 100644
index 0000000..a7827e9
--- /dev/null
+++ b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_nonpart.q
@@ -0,0 +1,30 @@
+-- Test INSERT OVERWRITE and INSERT INTO on parquet table in blobstore
+
+DROP TABLE blobstore_source;
+CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ' '
+COLLECTION ITEMS TERMINATED BY '\t'
+LINES TERMINATED BY '\n'
+LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_format_nonpart/blobstore_source/';
+LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source;
+
+DROP TABLE parquet_table;
+CREATE EXTERNAL TABLE parquet_table (a INT, b STRING, value DOUBLE) STORED AS PARQUET
+LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_format_nonpart/parquet_table';
+
+INSERT OVERWRITE TABLE parquet_table
+SELECT * FROM blobstore_source;
+
+SELECT * FROM parquet_table;
+SELECT a FROM parquet_table GROUP BY a;
+SELECT b FROM parquet_table GROUP BY b;
+SELECT value FROM parquet_table GROUP BY value;
+
+INSERT INTO TABLE parquet_table
+SELECT * FROM blobstore_source;
+
+SELECT * FROM parquet_table;
+SELECT a FROM parquet_table GROUP BY a;
+SELECT b FROM parquet_table GROUP BY b;
+SELECT value FROM parquet_table GROUP BY value;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/cbcfab9e/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_part.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_part.q b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_part.q
new file mode 100644
index 0000000..e9924fe
--- /dev/null
+++ b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_part.q
@@ -0,0 +1,67 @@
+-- Test INSERT INTO and INSERT OVERWRITE on partitioned parquet table in blobstore
+
+DROP TABLE src_events;
+CREATE TABLE src_events
+(
+ log_id BIGINT,
+ `time` BIGINT,
+ uid BIGINT,
+ user_id BIGINT,
+ type INT,
+ event_data STRING,
+ session_id STRING,
+ full_uid BIGINT,
+ run_date STRING,
+ game_id INT,
+ event_name STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_format_part/src_events/';
+LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events;
+
+DROP TABLE parquet_events;
+CREATE TABLE parquet_events
+(
+ log_id BIGINT,
+ `time` BIGINT,
+ uid BIGINT,
+ user_id BIGINT,
+ type INT,
+ event_data STRING,
+ session_id STRING,
+ full_uid BIGINT
+)
+PARTITIONED BY (run_date STRING, game_id INT, event_name STRING)
+STORED AS PARQUET
+LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_format_part/parquet_events';
+
+SET hive.exec.dynamic.partition=true;
+SET hive.exec.dynamic.partition.mode=nonstrict;
+
+INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
+SELECT * FROM src_events;
+SHOW PARTITIONS parquet_events;
+SELECT COUNT(*) FROM parquet_events;
+SELECT COUNT(*) FROM parquet_events WHERE run_date=20120921;
+SELECT COUNT(*) FROM parquet_events WHERE run_date=20121121;
+
+INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id, event_name)
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events
+WHERE SUBSTR(run_date,1,6)='201211';
+SHOW PARTITIONS parquet_events;
+SELECT COUNT(*) FROM parquet_events;
+
+INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name)
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39;
+SELECT COUNT(*) FROM parquet_events;
+
+INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change';
+SELECT COUNT(*) FROM parquet_events;
+
+INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change';
+SELECT COUNT(*) FROM parquet_events;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/cbcfab9e/itests/hive-blobstore/src/test/queries/clientpositive/parquet_nonstd_partitions_loc.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/parquet_nonstd_partitions_loc.q b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_nonstd_partitions_loc.q
new file mode 100644
index 0000000..2d20b99
--- /dev/null
+++ b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_nonstd_partitions_loc.q
@@ -0,0 +1,100 @@
+-- Test table in parquet format with non-standard partition locations in blobstore
+
+DROP TABLE src_events;
+CREATE TABLE src_events
+(
+ log_id BIGINT,
+ `time` BIGINT,
+ uid BIGINT,
+ user_id BIGINT,
+ type INT,
+ event_data STRING,
+ session_id STRING,
+ full_uid BIGINT,
+ run_date STRING,
+ game_id INT,
+ event_name STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_nonstd_partitions_loc/src_events/';
+LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events;
+
+DROP TABLE parquet_events;
+CREATE TABLE parquet_events
+(
+ log_id BIGINT,
+ `time` BIGINT,
+ uid BIGINT,
+ user_id BIGINT,
+ type INT,
+ event_data STRING,
+ session_id STRING,
+ full_uid BIGINT
+)
+PARTITIONED BY (run_date STRING, game_id INT, event_name STRING)
+STORED AS PARQUET
+LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_nonstd_partitions_loc/parquet_events/';
+
+SET hive.exec.dynamic.partition=true;
+SET hive.exec.dynamic.partition.mode=nonstrict;
+
+INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
+SELECT * FROM src_events;
+SHOW PARTITIONS parquet_events;
+SELECT COUNT(*) FROM parquet_events;
+
+-- verify INSERT OVERWRITE and INSERT INTO nonstandard partition location
+ALTER TABLE parquet_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change')
+LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-1/';
+INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201211';
+SHOW PARTITIONS parquet_events;
+SELECT COUNT(*) FROM parquet_events;
+INSERT INTO TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201211';
+SHOW PARTITIONS parquet_events;
+SELECT COUNT(*) FROM parquet_events;
+
+SET hive.merge.mapfiles=false;
+
+-- verify INSERT OVERWRITE and INSERT INTO nonstandard partition location with hive.merge.mapfiles false
+ALTER TABLE parquet_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-2/';
+INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209';
+INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209';
+SHOW PARTITIONS parquet_events;
+SELECT COUNT(*) FROM parquet_events;
+
+-- verify dynamic INSERT OVERWRITE over all partitions (standard and nonstandard locations) with hive.merge.mapfiles false
+INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
+SELECT * FROM src_events;
+SHOW PARTITIONS parquet_events;
+SELECT COUNT(*) FROM parquet_events;
+
+SET hive.merge.mapfiles=true;
+
+ALTER TABLE parquet_events ADD PARTITION (run_date=201207, game_id=39, event_name='hq_change')
+LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-3/';
+INSERT INTO TABLE parquet_events PARTITION (run_date=201207, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209';
+SHOW PARTITIONS parquet_events;
+SELECT COUNT(*) FROM parquet_events;
+
+-- verify dynamic INSERT OVERWRITE over all partitions (standard and nonstandard locations) with hive.merge.mapfiles true
+INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
+SELECT * FROM src_events;
+SHOW PARTITIONS parquet_events;
+SELECT COUNT(*) FROM parquet_events;
+
+ALTER TABLE parquet_events DROP PARTITION (run_date=201211, game_id=39, event_name='hq_change');
+ALTER TABLE parquet_events DROP PARTITION (run_date=201209, game_id=39, event_name='hq_change');
+ALTER TABLE parquet_events DROP PARTITION (run_date=201207, game_id=39, event_name='hq_change');
+SHOW PARTITIONS parquet_events;
+SELECT COUNT(*) FROM parquet_events;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/cbcfab9e/itests/hive-blobstore/src/test/queries/clientpositive/ptf_parquetfile.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/ptf_parquetfile.q b/itests/hive-blobstore/src/test/queries/clientpositive/ptf_parquetfile.q
new file mode 100644
index 0000000..fbf5ff3
--- /dev/null
+++ b/itests/hive-blobstore/src/test/queries/clientpositive/ptf_parquetfile.q
@@ -0,0 +1,32 @@
+-- Test several window functions on a table stored using PARQUET
+DROP TABLE part_parquet;
+CREATE TABLE part_parquet(
+ p_partkey int,
+ p_name string,
+ p_mfgr string,
+ p_brand string,
+ p_type string,
+ p_size int,
+ p_container string,
+ p_retailprice double,
+ p_comment string
+)
+STORED AS PARQUET
+LOCATION '${hiveconf:test.blobstore.path.unique}/ptf_parquetfile/part_parquet';
+
+LOAD DATA LOCAL INPATH '../../data/files/part.parquet' OVERWRITE INTO TABLE part_parquet;
+
+-- Test windowing PTFs with several partitions, using PARQUET storage
+SELECT
+ p_mfgr, p_name, p_size,
+ RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS r,
+ DENSE_RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS dr,
+ SUM(p_retailprice) OVER (
+ PARTITION BY p_mfgr
+ ORDER BY p_name ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+ ) AS s1
+FROM NOOP(
+ ON part_parquet
+ PARTITION BY p_mfgr
+ ORDER BY p_name
+);
http://git-wip-us.apache.org/repos/asf/hive/blob/cbcfab9e/itests/hive-blobstore/src/test/results/clientpositive/parquet_buckets.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_buckets.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_buckets.q.out
new file mode 100644
index 0000000..355958d
--- /dev/null
+++ b/itests/hive-blobstore/src/test/results/clientpositive/parquet_buckets.q.out
@@ -0,0 +1,183 @@
+PREHOOK: query: DROP TABLE blobstore_source
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE blobstore_source
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ' '
+COLLECTION ITEMS TERMINATED BY '\t'
+LINES TERMINATED BY '\n'
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/parquet_buckets/blobstore_source
+PREHOOK: Output: database:default
+PREHOOK: Output: default@blobstore_source
+POSTHOOK: query: CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ' '
+COLLECTION ITEMS TERMINATED BY '\t'
+LINES TERMINATED BY '\n'
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/parquet_buckets/blobstore_source
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@blobstore_source
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@blobstore_source
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@blobstore_source
+PREHOOK: query: DROP TABLE parquet_buckets
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE parquet_buckets
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE parquet_buckets (a STRING, value DOUBLE)
+PARTITIONED BY (b STRING)
+CLUSTERED BY (a) INTO 10 BUCKETS
+STORED AS PARQUET
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/parquet_buckets/parquet_buckets
+PREHOOK: Output: database:default
+PREHOOK: Output: default@parquet_buckets
+POSTHOOK: query: CREATE TABLE parquet_buckets (a STRING, value DOUBLE)
+PARTITIONED BY (b STRING)
+CLUSTERED BY (a) INTO 10 BUCKETS
+STORED AS PARQUET
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/parquet_buckets/parquet_buckets
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@parquet_buckets
+PREHOOK: query: INSERT OVERWRITE TABLE parquet_buckets
+PARTITION (b)
+SELECT a, c, b FROM blobstore_source
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_source
+PREHOOK: Output: default@parquet_buckets
+POSTHOOK: query: INSERT OVERWRITE TABLE parquet_buckets
+PARTITION (b)
+SELECT a, c, b FROM blobstore_source
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_source
+POSTHOOK: Output: default@parquet_buckets@b=abc
+POSTHOOK: Output: default@parquet_buckets@b=ajss
+POSTHOOK: Output: default@parquet_buckets@b=data
+POSTHOOK: Output: default@parquet_buckets@b=def
+POSTHOOK: Output: default@parquet_buckets@b=djns
+POSTHOOK: Output: default@parquet_buckets@b=ne
+POSTHOOK: Output: default@parquet_buckets@b=random
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=abc).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=abc).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=ajss).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=ajss).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=data).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=data).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=def).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=def).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=djns).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=djns).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=ne).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=ne).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=random).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=random).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+PREHOOK: query: SELECT * FROM parquet_buckets
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_buckets
+PREHOOK: Input: default@parquet_buckets@b=abc
+PREHOOK: Input: default@parquet_buckets@b=ajss
+PREHOOK: Input: default@parquet_buckets@b=data
+PREHOOK: Input: default@parquet_buckets@b=def
+PREHOOK: Input: default@parquet_buckets@b=djns
+PREHOOK: Input: default@parquet_buckets@b=ne
+PREHOOK: Input: default@parquet_buckets@b=random
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_buckets
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_buckets
+POSTHOOK: Input: default@parquet_buckets@b=abc
+POSTHOOK: Input: default@parquet_buckets@b=ajss
+POSTHOOK: Input: default@parquet_buckets@b=data
+POSTHOOK: Input: default@parquet_buckets@b=def
+POSTHOOK: Input: default@parquet_buckets@b=djns
+POSTHOOK: Input: default@parquet_buckets@b=ne
+POSTHOOK: Input: default@parquet_buckets@b=random
+#### A masked pattern was here ####
+1 10.5 abc
+3 90.23232 ajss
+6 3.002 data
+2 11.5 def
+4 89.02002 djns
+7 71.9084 ne
+5 2.99 random
+PREHOOK: query: INSERT INTO TABLE parquet_buckets
+PARTITION (b)
+SELECT a, c, b FROM blobstore_source
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_source
+PREHOOK: Output: default@parquet_buckets
+POSTHOOK: query: INSERT INTO TABLE parquet_buckets
+PARTITION (b)
+SELECT a, c, b FROM blobstore_source
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_source
+POSTHOOK: Output: default@parquet_buckets@b=abc
+POSTHOOK: Output: default@parquet_buckets@b=ajss
+POSTHOOK: Output: default@parquet_buckets@b=data
+POSTHOOK: Output: default@parquet_buckets@b=def
+POSTHOOK: Output: default@parquet_buckets@b=djns
+POSTHOOK: Output: default@parquet_buckets@b=ne
+POSTHOOK: Output: default@parquet_buckets@b=random
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=abc).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=abc).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=ajss).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=ajss).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=data).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=data).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=def).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=def).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=djns).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=djns).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=ne).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=ne).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=random).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=random).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+PREHOOK: query: SELECT * FROM parquet_buckets
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_buckets
+PREHOOK: Input: default@parquet_buckets@b=abc
+PREHOOK: Input: default@parquet_buckets@b=ajss
+PREHOOK: Input: default@parquet_buckets@b=data
+PREHOOK: Input: default@parquet_buckets@b=def
+PREHOOK: Input: default@parquet_buckets@b=djns
+PREHOOK: Input: default@parquet_buckets@b=ne
+PREHOOK: Input: default@parquet_buckets@b=random
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_buckets
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_buckets
+POSTHOOK: Input: default@parquet_buckets@b=abc
+POSTHOOK: Input: default@parquet_buckets@b=ajss
+POSTHOOK: Input: default@parquet_buckets@b=data
+POSTHOOK: Input: default@parquet_buckets@b=def
+POSTHOOK: Input: default@parquet_buckets@b=djns
+POSTHOOK: Input: default@parquet_buckets@b=ne
+POSTHOOK: Input: default@parquet_buckets@b=random
+#### A masked pattern was here ####
+1 10.5 abc
+1 10.5 abc
+3 90.23232 ajss
+3 90.23232 ajss
+6 3.002 data
+6 3.002 data
+2 11.5 def
+2 11.5 def
+4 89.02002 djns
+4 89.02002 djns
+7 71.9084 ne
+7 71.9084 ne
+5 2.99 random
+5 2.99 random
http://git-wip-us.apache.org/repos/asf/hive/blob/cbcfab9e/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_nonpart.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_nonpart.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_nonpart.q.out
new file mode 100644
index 0000000..cba6b1a
--- /dev/null
+++ b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_nonpart.q.out
@@ -0,0 +1,201 @@
+PREHOOK: query: DROP TABLE blobstore_source
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE blobstore_source
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ' '
+COLLECTION ITEMS TERMINATED BY '\t'
+LINES TERMINATED BY '\n'
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/parquet_format_nonpart/blobstore_source
+PREHOOK: Output: database:default
+PREHOOK: Output: default@blobstore_source
+POSTHOOK: query: CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ' '
+COLLECTION ITEMS TERMINATED BY '\t'
+LINES TERMINATED BY '\n'
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/parquet_format_nonpart/blobstore_source
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@blobstore_source
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@blobstore_source
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@blobstore_source
+PREHOOK: query: DROP TABLE parquet_table
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE parquet_table
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE EXTERNAL TABLE parquet_table (a INT, b STRING, value DOUBLE) STORED AS PARQUET
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/parquet_format_nonpart/parquet_table
+PREHOOK: Output: database:default
+PREHOOK: Output: default@parquet_table
+POSTHOOK: query: CREATE EXTERNAL TABLE parquet_table (a INT, b STRING, value DOUBLE) STORED AS PARQUET
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/parquet_format_nonpart/parquet_table
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@parquet_table
+PREHOOK: query: INSERT OVERWRITE TABLE parquet_table
+SELECT * FROM blobstore_source
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_source
+PREHOOK: Output: default@parquet_table
+POSTHOOK: query: INSERT OVERWRITE TABLE parquet_table
+SELECT * FROM blobstore_source
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_source
+POSTHOOK: Output: default@parquet_table
+POSTHOOK: Lineage: parquet_table.a EXPRESSION [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_table.b SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:b, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_table.value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+PREHOOK: query: SELECT * FROM parquet_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+1 abc 10.5
+2 def 11.5
+3 ajss 90.23232
+4 djns 89.02002
+5 random 2.99
+6 data 3.002
+7 ne 71.9084
+PREHOOK: query: SELECT a FROM parquet_table GROUP BY a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT a FROM parquet_table GROUP BY a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+1
+2
+3
+4
+5
+6
+7
+PREHOOK: query: SELECT b FROM parquet_table GROUP BY b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT b FROM parquet_table GROUP BY b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+abc
+ajss
+data
+def
+djns
+ne
+random
+PREHOOK: query: SELECT value FROM parquet_table GROUP BY value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT value FROM parquet_table GROUP BY value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+2.99
+3.002
+10.5
+11.5
+71.9084
+89.02002
+90.23232
+PREHOOK: query: INSERT INTO TABLE parquet_table
+SELECT * FROM blobstore_source
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_source
+PREHOOK: Output: default@parquet_table
+POSTHOOK: query: INSERT INTO TABLE parquet_table
+SELECT * FROM blobstore_source
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_source
+POSTHOOK: Output: default@parquet_table
+POSTHOOK: Lineage: parquet_table.a EXPRESSION [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_table.b SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:b, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_table.value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+PREHOOK: query: SELECT * FROM parquet_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+1 abc 10.5
+2 def 11.5
+3 ajss 90.23232
+4 djns 89.02002
+5 random 2.99
+6 data 3.002
+7 ne 71.9084
+1 abc 10.5
+2 def 11.5
+3 ajss 90.23232
+4 djns 89.02002
+5 random 2.99
+6 data 3.002
+7 ne 71.9084
+PREHOOK: query: SELECT a FROM parquet_table GROUP BY a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT a FROM parquet_table GROUP BY a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+1
+2
+3
+4
+5
+6
+7
+PREHOOK: query: SELECT b FROM parquet_table GROUP BY b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT b FROM parquet_table GROUP BY b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+abc
+ajss
+data
+def
+djns
+ne
+random
+PREHOOK: query: SELECT value FROM parquet_table GROUP BY value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT value FROM parquet_table GROUP BY value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+2.99
+3.002
+10.5
+11.5
+71.9084
+89.02002
+90.23232
http://git-wip-us.apache.org/repos/asf/hive/blob/cbcfab9e/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out
new file mode 100644
index 0000000..0931e3d
--- /dev/null
+++ b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out
@@ -0,0 +1,290 @@
+PREHOOK: query: DROP TABLE src_events
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE src_events
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE src_events
+(
+ log_id BIGINT,
+ `time` BIGINT,
+ uid BIGINT,
+ user_id BIGINT,
+ type INT,
+ event_data STRING,
+ session_id STRING,
+ full_uid BIGINT,
+ run_date STRING,
+ game_id INT,
+ event_name STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/parquet_format_part/src_events
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_events
+POSTHOOK: query: CREATE TABLE src_events
+(
+ log_id BIGINT,
+ `time` BIGINT,
+ uid BIGINT,
+ user_id BIGINT,
+ type INT,
+ event_data STRING,
+ session_id STRING,
+ full_uid BIGINT,
+ run_date STRING,
+ game_id INT,
+ event_name STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/parquet_format_part/src_events
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_events
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@src_events
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@src_events
+PREHOOK: query: DROP TABLE parquet_events
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE parquet_events
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE parquet_events
+(
+ log_id BIGINT,
+ `time` BIGINT,
+ uid BIGINT,
+ user_id BIGINT,
+ type INT,
+ event_data STRING,
+ session_id STRING,
+ full_uid BIGINT
+)
+PARTITIONED BY (run_date STRING, game_id INT, event_name STRING)
+STORED AS PARQUET
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/parquet_format_part/parquet_events
+PREHOOK: Output: database:default
+PREHOOK: Output: default@parquet_events
+POSTHOOK: query: CREATE TABLE parquet_events
+(
+ log_id BIGINT,
+ `time` BIGINT,
+ uid BIGINT,
+ user_id BIGINT,
+ type INT,
+ event_data STRING,
+ session_id STRING,
+ full_uid BIGINT
+)
+PARTITIONED BY (run_date STRING, game_id INT, event_name STRING)
+STORED AS PARQUET
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/parquet_format_part/parquet_events
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@parquet_events
+PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
+SELECT * FROM src_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events
+POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
+SELECT * FROM src_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change
+POSTHOOK: Output: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change
+POSTHOOK: Output: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: SHOW PARTITIONS parquet_events
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@parquet_events
+POSTHOOK: query: SHOW PARTITIONS parquet_events
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@parquet_events
+run_date=20120921/game_id=39/event_name=hq_change
+run_date=20121021/game_id=39/event_name=hq_change
+run_date=20121121/game_id=39/event_name=hq_change
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+200
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20120921
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20120921
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+50
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20121121
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20121121
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+100
+PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id, event_name)
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events
+WHERE SUBSTR(run_date,1,6)='201211'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events@run_date=201211
+POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id, event_name)
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events
+WHERE SUBSTR(run_date,1,6)='201211'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: SHOW PARTITIONS parquet_events
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@parquet_events
+POSTHOOK: query: SHOW PARTITIONS parquet_events
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@parquet_events
+run_date=20120921/game_id=39/event_name=hq_change
+run_date=20121021/game_id=39/event_name=hq_change
+run_date=201211/game_id=39/event_name=hq_change
+run_date=20121121/game_id=39/event_name=hq_change
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+300
+PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name)
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39
+POSTHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name)
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+350
+PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+POSTHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+400
+PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+350
[32/32] hive git commit: HIVE-19416 : merge master into branch
(Sergey Shelukhin) 0710
Posted by se...@apache.org.
HIVE-19416 : merge master into branch (Sergey Shelukhin) 0710
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/04ea1455
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/04ea1455
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/04ea1455
Branch: refs/heads/master-txnstats
Commit: 04ea1455febfbddeab505a71232465f0f24fa164
Parents: f384d4c b574bcd
Author: sergey <se...@apache.org>
Authored: Tue Jul 10 18:59:34 2018 -0700
Committer: sergey <se...@apache.org>
Committed: Tue Jul 10 18:59:34 2018 -0700
----------------------------------------------------------------------
RELEASE_NOTES.txt | 303 +
.../hive/accumulo/AccumuloStorageHandler.java | 19 +-
.../accumulo/TestAccumuloStorageHandler.java | 44 +-
.../test/queries/positive/accumulo_custom_key.q | 5 +-
.../queries/positive/accumulo_custom_key2.q | 5 +-
.../src/test/queries/positive/accumulo_index.q | 5 +-
.../src/test/queries/positive/accumulo_joins.q | 25 +-
.../positive/accumulo_predicate_pushdown.q | 5 +-
.../test/queries/positive/accumulo_queries.q | 35 +-
.../accumulo_single_sourced_multi_insert.q | 5 +-
.../results/positive/accumulo_custom_key.q.out | 6 +-
.../results/positive/accumulo_custom_key2.q.out | 6 +-
.../test/results/positive/accumulo_index.q.out | 6 +-
.../test/results/positive/accumulo_joins.q.out | 30 +-
.../positive/accumulo_predicate_pushdown.q.out | 6 +-
.../results/positive/accumulo_queries.q.out | 46 +-
.../accumulo_single_sourced_multi_insert.q.out | 6 +-
.../hive/beeline/schematool/HiveSchemaTool.java | 338 +-
.../schematool/HiveSchemaToolCommandLine.java | 286 -
.../beeline/schematool/HiveSchemaToolTask.java | 32 -
.../HiveSchemaToolTaskAlterCatalog.java | 90 -
.../HiveSchemaToolTaskCreateCatalog.java | 132 -
.../schematool/HiveSchemaToolTaskInfo.java | 43 -
.../schematool/HiveSchemaToolTaskInit.java | 73 -
.../HiveSchemaToolTaskMoveDatabase.java | 96 -
.../schematool/HiveSchemaToolTaskMoveTable.java | 142 -
.../schematool/HiveSchemaToolTaskUpgrade.java | 116 -
.../schematool/HiveSchemaToolTaskValidate.java | 631 --
.../beeline/schematool/TestHiveSchemaTool.java | 6 +-
.../org/apache/hadoop/hive/conf/HiveConf.java | 2 +
.../hive/common/util/HiveStringUtils.java | 21 +-
data/files/part.parquet | Bin 0 -> 4758 bytes
data/files/studenttab10k | 10000 +++++++++++++++++
.../hadoop/hive/druid/DruidStorageHandler.java | 71 +-
.../hadoop/hive/druid/serde/DruidSerDe.java | 3 +-
.../hive/druid/TestDruidStorageHandler.java | 4 +-
hcatalog/bin/hcat | 41 +-
.../apache/hive/hcatalog/data/JsonSerDe.java | 612 +-
.../hive/hcatalog/data/TestJsonSerDe.java | 12 +-
.../insert_into_dynamic_partitions.q | 3 +-
.../clientpositive/insert_overwrite_directory.q | 3 +-
.../insert_overwrite_dynamic_partitions.q | 3 +-
.../queries/clientpositive/parquet_buckets.q | 31 +
.../clientpositive/parquet_format_nonpart.q | 30 +
.../clientpositive/parquet_format_part.q | 67 +
.../parquet_nonstd_partitions_loc.q | 100 +
.../queries/clientpositive/ptf_parquetfile.q | 32 +
.../insert_into_dynamic_partitions.q.out | 8 +-
.../insert_overwrite_dynamic_partitions.q.out | 12 +-
.../clientpositive/parquet_buckets.q.out | 183 +
.../clientpositive/parquet_format_nonpart.q.out | 201 +
.../clientpositive/parquet_format_part.q.out | 290 +
.../parquet_nonstd_partitions_loc.q.out | 537 +
.../clientpositive/ptf_parquetfile.q.out | 104 +
.../benchmark/udf/json_read/JsonReadBench.java | 83 +
.../hive/benchmark/udf/json_read/val1.json | 86 +
.../hive/benchmark/udf/json_read/val1.type | 1 +
.../hive/metastore/TestMetaStoreMetrics.java | 5 +-
.../tools/TestSchemaToolCatalogOps.java | 479 +
.../TestReplicationScenariosAcidTables.java | 2 +
.../hive/beeline/schematool/TestSchemaTool.java | 550 +-
.../schematool/TestSchemaToolCatalogOps.java | 478 -
.../apache/hive/jdbc/TestJdbcWithMiniHS2.java | 2 +-
.../jdbc/TestJdbcWithMiniHS2ErasureCoding.java | 215 +
.../test/resources/testconfiguration.properties | 16 +-
.../org/apache/hadoop/hive/ql/QTestUtil.java | 4 +-
.../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 23 +-
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 44 +-
.../hadoop/hive/ql/exec/FunctionRegistry.java | 1 +
.../ql/exec/tez/monitoring/TezJobMonitor.java | 19 +-
.../hadoop/hive/ql/io/orc/OrcInputFormat.java | 6 +-
.../ql/io/orc/VectorizedOrcInputFormat.java | 2 +-
.../ql/metadata/SessionHiveMetaStoreClient.java | 2 +-
.../formatting/MetaDataFormatUtils.java | 11 +-
.../formatting/TextMetaDataFormatter.java | 10 +
.../annotation/OpTraitsRulesProcFactory.java | 59 +-
.../optimizer/spark/SparkMapJoinOptimizer.java | 2 +-
.../stats/annotation/StatsRulesProcFactory.java | 2 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 2 +-
.../hadoop/hive/ql/plan/PartitionDesc.java | 4 +-
.../apache/hadoop/hive/ql/plan/PlanUtils.java | 31 +
.../apache/hadoop/hive/ql/plan/Statistics.java | 12 +-
.../apache/hadoop/hive/ql/plan/TableDesc.java | 3 +-
.../hadoop/hive/ql/session/SessionState.java | 6 +-
.../hive/ql/stats/BasicStatsNoJobTask.java | 7 +-
.../hadoop/hive/ql/stats/BasicStatsTask.java | 4 +-
.../apache/hadoop/hive/ql/stats/StatsUtils.java | 20 +-
.../hive/ql/txn/compactor/CompactorMR.java | 4 +-
.../hive/ql/udf/generic/GenericUDFJsonRead.java | 92 +
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 4 +-
.../ql/udf/generic/TestGenericUDFJsonRead.java | 204 +
ql/src/test/queries/clientnegative/ctas.q | 2 +-
.../queries/clientnegative/druid_datasource2.q | 1 +
.../queries/clientpositive/druid_timestamptz.q | 5 +-
.../queries/clientpositive/druid_timestamptz2.q | 14 +-
.../clientpositive/druidkafkamini_basic.q | 2 +-
.../druidmini_dynamic_partition.q | 10 +-
.../clientpositive/druidmini_expressions.q | 3 +-
.../clientpositive/druidmini_extractTime.q | 6 +-
.../clientpositive/druidmini_floorTime.q | 4 +-
.../queries/clientpositive/druidmini_joins.q | 5 +-
.../queries/clientpositive/druidmini_masking.q | 4 +-
.../queries/clientpositive/druidmini_test1.q | 5 +-
.../clientpositive/druidmini_test_alter.q | 5 +-
.../clientpositive/druidmini_test_insert.q | 8 +-
ql/src/test/queries/clientpositive/dst.q | 12 +
.../queries/clientpositive/erasure_explain.q | 24 +
.../queries/clientpositive/erasure_simple.q | 13 +
.../test/queries/clientpositive/explainuser_4.q | 1 +
.../test/queries/clientpositive/json_serde2.q | 37 +
ql/src/test/queries/clientpositive/mm_all.q | 1 +
.../clientpositive/orc_schema_evol_quoted.q | 42 +
.../clientpositive/orc_schema_evolution_float.q | 1 +
.../clientpositive/parquet_ppd_multifiles.q | 1 +
.../test/queries/clientpositive/selectindate.q | 1 +
.../test/queries/clientpositive/udf_json_read.q | 44 +
ql/src/test/queries/clientpositive/union38.q | 1 +
.../test/queries/clientpositive/vector_bround.q | 1 +
.../queries/clientpositive/vector_case_when_1.q | 3 +-
.../queries/clientpositive/vector_coalesce_2.q | 1 +
.../queries/clientpositive/vector_coalesce_3.q | 1 +
.../queries/clientpositive/vector_interval_1.q | 1 +
.../clientpositive/vectorized_parquet_types.q | 1 +
.../clientnegative/druid_datasource2.q.out | 2 +-
.../clientnegative/unset_table_property.q.out | 1 +
.../materialized_view_create_rewrite.q.out | 2 +
.../druid/druid_timestamptz.q.out | 4 +-
.../druid/druid_timestamptz2.q.out | 20 +-
.../druid/druidkafkamini_basic.q.out | 6 +-
.../druid/druidmini_dynamic_partition.q.out | 18 +-
.../druid/druidmini_expressions.q.out | 6 +-
.../druid/druidmini_extractTime.q.out | 8 +-
.../druid/druidmini_floorTime.q.out | 4 +-
.../clientpositive/druid/druidmini_joins.q.out | 4 +-
.../druid/druidmini_masking.q.out | 4 +-
.../clientpositive/druid/druidmini_mv.q.out | 3 +
.../clientpositive/druid/druidmini_test1.q.out | 4 +-
.../druid/druidmini_test_alter.q.out | 4 +-
.../druid/druidmini_test_insert.q.out | 12 +-
.../erasurecoding/erasure_explain.q.out | 409 +
.../erasurecoding/erasure_simple.q.out | 20 +-
.../results/clientpositive/json_serde2.q.out | 113 +
.../test/results/clientpositive/llap/dst.q.out | 108 +
.../clientpositive/llap/explainuser_4.q.out | 28 +-
.../llap/materialized_view_create.q.out | 1 +
.../llap/materialized_view_create_rewrite.q.out | 2 +
...materialized_view_create_rewrite_dummy.q.out | 2 +
...erialized_view_create_rewrite_multi_db.q.out | 2 +
.../llap/materialized_view_describe.q.out | 1 +
.../llap/orc_schema_evol_quoted.q.out | 258 +
.../clientpositive/llap/subquery_notin.q.out | 118 +-
.../results/clientpositive/llap/tez_join.q.out | 51 +-
.../clientpositive/llap/vector_bround.q.out | 4 +-
.../clientpositive/llap/vector_coalesce_2.q.out | 8 +-
.../clientpositive/llap/vector_interval_1.q.out | 16 +-
.../llap/vectorized_parquet_types.q.out | 8 +-
.../clientpositive/orc_schema_evol_quoted.q.out | 258 +
.../orc_schema_evolution_float.q.out | 12 +-
.../results/clientpositive/show_functions.q.out | 1 +
.../clientpositive/show_tblproperties.q.out | 5 +
.../results/clientpositive/udf_json_read.q.out | 107 +
.../test/results/clientpositive/union38.q.out | 14 +-
.../unset_table_view_property.q.out | 9 +
.../results/clientpositive/vector_bround.q.out | 4 +-
.../clientpositive/vector_coalesce_2.q.out | 4 +-
.../clientpositive/vector_interval_1.q.out | 16 +-
.../vectorized_parquet_types.q.out | 8 +-
.../apache/hadoop/hive/serde2/JsonSerDe.java | 645 +-
.../hive/serde2/json/HiveJsonStructReader.java | 402 +
.../java/org/apache/hive/http/LlapServlet.java | 2 +-
.../hadoop/hive/common/StatsSetupConst.java | 21 +-
.../hadoop/hive/metastore/HiveMetaStore.java | 5 +-
.../hive/metastore/StatObjectConverter.java | 10 +-
.../hive/metastore/events/InsertEvent.java | 9 +-
.../metastore/messaging/PartitionFiles.java | 2 +-
.../messaging/json/JSONAbortTxnMessage.java | 3 +-
.../messaging/json/JSONAcidWriteMessage.java | 2 +-
.../json/JSONAddForeignKeyMessage.java | 3 +-
.../json/JSONAddNotNullConstraintMessage.java | 3 +-
.../messaging/json/JSONAddPartitionMessage.java | 26 +-
.../json/JSONAddPrimaryKeyMessage.java | 3 +-
.../json/JSONAddUniqueConstraintMessage.java | 3 +-
.../messaging/json/JSONAllocWriteIdMessage.java | 8 +-
.../json/JSONAlterDatabaseMessage.java | 3 +-
.../json/JSONAlterPartitionMessage.java | 11 +-
.../messaging/json/JSONAlterTableMessage.java | 9 +-
.../messaging/json/JSONCommitTxnMessage.java | 3 +-
.../json/JSONCreateCatalogMessage.java | 3 +-
.../json/JSONCreateDatabaseMessage.java | 4 +-
.../json/JSONCreateFunctionMessage.java | 3 +-
.../messaging/json/JSONCreateTableMessage.java | 8 +-
.../messaging/json/JSONDropCatalogMessage.java | 3 +-
.../json/JSONDropConstraintMessage.java | 3 +-
.../messaging/json/JSONDropDatabaseMessage.java | 3 +-
.../messaging/json/JSONDropFunctionMessage.java | 3 +-
.../json/JSONDropPartitionMessage.java | 13 +-
.../messaging/json/JSONDropTableMessage.java | 9 +-
.../messaging/json/JSONInsertMessage.java | 14 +-
.../messaging/json/JSONMessageDeserializer.java | 22 +-
.../messaging/json/JSONMessageFactory.java | 21 +-
.../messaging/json/JSONOpenTxnMessage.java | 6 +-
.../metastore/tools/MetastoreSchemaTool.java | 1282 +--
.../metastore/tools/SchemaToolCommandLine.java | 308 +
.../hive/metastore/tools/SchemaToolTask.java | 32 +
.../tools/SchemaToolTaskAlterCatalog.java | 90 +
.../tools/SchemaToolTaskCreateCatalog.java | 132 +
.../tools/SchemaToolTaskCreateUser.java | 115 +
.../metastore/tools/SchemaToolTaskInfo.java | 43 +
.../metastore/tools/SchemaToolTaskInit.java | 73 +
.../tools/SchemaToolTaskMoveDatabase.java | 96 +
.../tools/SchemaToolTaskMoveTable.java | 142 +
.../metastore/tools/SchemaToolTaskUpgrade.java | 116 +
.../metastore/tools/SchemaToolTaskValidate.java | 630 ++
.../hive/metastore/utils/MetaStoreUtils.java | 30 +-
.../hive/metastore/dbinstall/DbInstallBase.java | 25 +-
.../json/TestJSONMessageDeserializer.java | 28 +-
.../tools/TestSchemaToolForMetastore.java | 277 +-
.../metastore/utils/TestMetaStoreUtils.java | 37 +-
.../apache/hive/streaming/StrictJsonWriter.java | 2 +-
219 files changed, 18301 insertions(+), 5801 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/04ea1455/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/04ea1455/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/04ea1455/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/04ea1455/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
index cb6913e,95a4440..ae51b4d
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
@@@ -263,11 -261,10 +263,13 @@@ public class StatsUtils
long nr = basicStats.getNumRows();
List<ColStatistics> colStats = Lists.newArrayList();
+ long numErasureCodedFiles = getErasureCodedFiles(table);
+
if (fetchColStats) {
- colStats = getTableColumnStats(table, schema, neededColumns, colStatsCache);
+ // Note: this is currently called from two notable places (w/false for checkTxn)
+ // 1) StatsRulesProcFactory.TableScanStatsRule via collectStatistics
+ // 2) RelOptHiveTable via getColStats and updateColStats.
+ colStats = getTableColumnStats(table, schema, neededColumns, colStatsCache, false);
if(colStats == null) {
colStats = Lists.newArrayList();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/04ea1455/standalone-metastore/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/04ea1455/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
[11/32] hive git commit: HIVE-20060 Refactor HiveSchemaTool and
MetastoreSchemaTool (Alan Gates, reviewed by Daniel Dai)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInfo.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInfo.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInfo.java
new file mode 100644
index 0000000..cd1d57b
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInfo.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools;
+
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo;
+
+/**
+ * Print Hive version and schema version.
+ */
+class SchemaToolTaskInfo extends SchemaToolTask {
+ @Override
+ void setCommandLineArguments(SchemaToolCommandLine cl) {
+ // do nothing
+ }
+
+ @Override
+ void execute() throws HiveMetaException {
+ String hiveVersion = schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion();
+ MetaStoreConnectionInfo connectionInfo = schemaTool.getConnectionInfo(true);
+ String dbVersion = schemaTool.getMetaStoreSchemaInfo().getMetaStoreSchemaVersion(connectionInfo);
+
+ System.out.println("Hive distribution version:\t " + hiveVersion);
+ System.out.println("Metastore schema version:\t " + dbVersion);
+
+ schemaTool.assertCompatibleVersion(hiveVersion, dbVersion);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInit.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInit.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInit.java
new file mode 100644
index 0000000..e3fa495
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInit.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+
+/**
+ * Initialize the metastore schema.
+ */
+class SchemaToolTaskInit extends SchemaToolTask {
+ private boolean validate = true;
+ private String toVersion;
+
+ @Override
+ void setCommandLineArguments(SchemaToolCommandLine cl) {
+ if (cl.hasOption("initSchemaTo")) {
+ this.toVersion = cl.getOptionValue("initSchemaTo");
+ this.validate = false;
+ }
+ }
+
+ private void ensureToVersion() throws HiveMetaException {
+ if (toVersion != null) {
+ return;
+ }
+
+ // If null then current hive version is used
+ toVersion = schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion();
+ System.out.println("Initializing the schema to: " + toVersion);
+ }
+
+ @Override
+ void execute() throws HiveMetaException {
+ ensureToVersion();
+
+ schemaTool.testConnectionToMetastore();
+ System.out.println("Starting metastore schema initialization to " + toVersion);
+
+ String initScriptDir = schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir();
+ String initScriptFile = schemaTool.getMetaStoreSchemaInfo().generateInitFileName(toVersion);
+
+ try {
+ System.out.println("Initialization script " + initScriptFile);
+ if (!schemaTool.isDryRun()) {
+ schemaTool.execSql(initScriptDir, initScriptFile);
+ System.out.println("Initialization script completed");
+ }
+ } catch (IOException e) {
+ throw new HiveMetaException("Schema initialization FAILED! Metastore state would be inconsistent!", e);
+ }
+
+ if (validate) {
+ schemaTool.verifySchemaVersion();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskMoveDatabase.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskMoveDatabase.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskMoveDatabase.java
new file mode 100644
index 0000000..8a9b9d1
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskMoveDatabase.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools;
+
+import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Print Hive version and schema version.
+ */
+class SchemaToolTaskMoveDatabase extends SchemaToolTask {
+ private static final Logger LOG = LoggerFactory.getLogger(SchemaToolTaskMoveDatabase.class.getName());
+
+ private String fromCatName;
+ private String toCatName;
+ private String dbName;
+
+ @Override
+ void setCommandLineArguments(SchemaToolCommandLine cl) {
+ fromCatName = normalizeIdentifier(cl.getOptionValue("fromCatalog"));
+ toCatName = normalizeIdentifier(cl.getOptionValue("toCatalog"));
+ dbName = normalizeIdentifier(cl.getOptionValue("moveDatabase"));
+ }
+
+ @Override
+ void execute() throws HiveMetaException {
+ System.out.println(String.format("Moving database %s from catalog %s to catalog %s",
+ dbName, fromCatName, toCatName));
+ Connection conn = schemaTool.getConnectionToMetastore(true);
+ boolean success = false;
+ try {
+ conn.setAutoCommit(false);
+ try (Statement stmt = conn.createStatement()) {
+ updateCatalogNameInTable(stmt, "DBS", "CTLG_NAME", "NAME", fromCatName, toCatName, dbName, false);
+ updateCatalogNameInTable(stmt, "TAB_COL_STATS", "CAT_NAME", "DB_NAME", fromCatName, toCatName, dbName, true);
+ updateCatalogNameInTable(stmt, "PART_COL_STATS", "CAT_NAME", "DB_NAME", fromCatName, toCatName, dbName, true);
+ updateCatalogNameInTable(stmt, "PARTITION_EVENTS", "CAT_NAME", "DB_NAME", fromCatName, toCatName, dbName, true);
+ updateCatalogNameInTable(stmt, "NOTIFICATION_LOG", "CAT_NAME", "DB_NAME", fromCatName, toCatName, dbName, true);
+ conn.commit();
+ success = true;
+ }
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to move database", e);
+ } finally {
+ try {
+ if (!success) {
+ conn.rollback();
+ }
+ } catch (SQLException e) {
+ // Not really much we can do here.
+ LOG.error("Failed to rollback, everything will probably go bad from here.");
+ }
+ }
+ }
+
+ private static final String UPDATE_CATALOG_NAME_STMT =
+ "update <q>%s<q> " +
+ " set <q>%s<q> = '%s' " +
+ " where <q>%s<q> = '%s' " +
+ " and <q>%s<q> = '%s'";
+
+ private void updateCatalogNameInTable(Statement stmt, String tableName, String catColName, String dbColName,
+ String fromCatName, String toCatName, String dbName, boolean zeroUpdatesOk)
+ throws HiveMetaException, SQLException {
+ String update = String.format(schemaTool.quote(UPDATE_CATALOG_NAME_STMT), tableName, catColName, toCatName,
+ catColName, fromCatName, dbColName, dbName);
+ LOG.debug("Going to run " + update);
+ int numUpdated = stmt.executeUpdate(update);
+ if (numUpdated != 1 && !(zeroUpdatesOk && numUpdated == 0)) {
+ throw new HiveMetaException("Failed to properly update the " + tableName +
+ " table. Expected to update 1 row but instead updated " + numUpdated);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskMoveTable.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskMoveTable.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskMoveTable.java
new file mode 100644
index 0000000..a8f9228
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskMoveTable.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools;
+
+import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Print Hive version and schema version.
+ */
+class SchemaToolTaskMoveTable extends SchemaToolTask {
+ private static final Logger LOG = LoggerFactory.getLogger(SchemaToolTaskMoveTable.class.getName());
+
+ private String fromCat;
+ private String toCat;
+ private String fromDb;
+ private String toDb;
+ private String tableName;
+
+ @Override
+ void setCommandLineArguments(SchemaToolCommandLine cl) {
+ fromCat = normalizeIdentifier(cl.getOptionValue("fromCatalog"));
+ toCat = normalizeIdentifier(cl.getOptionValue("toCatalog"));
+ fromDb = normalizeIdentifier(cl.getOptionValue("fromDatabase"));
+ toDb = normalizeIdentifier(cl.getOptionValue("toDatabase"));
+ tableName = normalizeIdentifier(cl.getOptionValue("moveTable"));
+ }
+
+ @Override
+ void execute() throws HiveMetaException {
+ Connection conn = schemaTool.getConnectionToMetastore(true);
+ boolean success = false;
+ try {
+ conn.setAutoCommit(false);
+ try (Statement stmt = conn.createStatement()) {
+ updateTableId(stmt);
+ updateDbNameForTable(stmt, "TAB_COL_STATS", "TABLE_NAME", fromCat, toCat, fromDb, toDb, tableName);
+ updateDbNameForTable(stmt, "PART_COL_STATS", "TABLE_NAME", fromCat, toCat, fromDb, toDb, tableName);
+ updateDbNameForTable(stmt, "PARTITION_EVENTS", "TBL_NAME", fromCat, toCat, fromDb, toDb, tableName);
+ updateDbNameForTable(stmt, "NOTIFICATION_LOG", "TBL_NAME", fromCat, toCat, fromDb, toDb, tableName);
+ conn.commit();
+ success = true;
+ }
+ } catch (SQLException se) {
+ throw new HiveMetaException("Failed to move table", se);
+ } finally {
+ try {
+ if (!success) {
+ conn.rollback();
+ }
+ } catch (SQLException e) {
+ // Not really much we can do here.
+ LOG.error("Failed to rollback, everything will probably go bad from here.");
+ }
+
+ }
+ }
+
+ private static final String UPDATE_TABLE_ID_STMT =
+ "update <q>TBLS<q> " +
+ " set <q>DB_ID<q> = %d " +
+ " where <q>DB_ID<q> = %d " +
+ " and <q>TBL_NAME<q> = '%s'";
+
+ private void updateTableId(Statement stmt) throws SQLException, HiveMetaException {
+ // Find the old database id
+ long oldDbId = getDbId(stmt, fromDb, fromCat);
+
+ // Find the new database id
+ long newDbId = getDbId(stmt, toDb, toCat);
+
+ String update = String.format(schemaTool.quote(UPDATE_TABLE_ID_STMT), newDbId, oldDbId, tableName);
+ LOG.debug("Going to run " + update);
+ int numUpdated = stmt.executeUpdate(update);
+ if (numUpdated != 1) {
+ throw new HiveMetaException(
+ "Failed to properly update TBLS table. Expected to update " +
+ "1 row but instead updated " + numUpdated);
+ }
+ }
+
+ private static final String DB_ID_QUERY =
+ "select <q>DB_ID<q> " +
+ " from <q>DBS<q> " +
+ " where <q>NAME<q> = '%s' " +
+ " and <q>CTLG_NAME<q> = '%s'";
+
+ private long getDbId(Statement stmt, String db, String catalog) throws SQLException, HiveMetaException {
+ String query = String.format(schemaTool.quote(DB_ID_QUERY), db, catalog);
+ LOG.debug("Going to run " + query);
+ try (ResultSet rs = stmt.executeQuery(query)) {
+ if (!rs.next()) {
+ throw new HiveMetaException("Unable to find database " + fromDb);
+ }
+ return rs.getLong(1);
+ }
+ }
+
+ private static final String UPDATE_DB_NAME_STMT =
+ "update <q>%s<q> " +
+ " set <q>CAT_NAME<q> = '%s', " +
+ " <q>DB_NAME<q> = '%s' " +
+ " where <q>CAT_NAME<q> = '%s' " +
+ " and <q>DB_NAME<q> = '%s' " +
+ " and <q>%s<q> = '%s'";
+
+ private void updateDbNameForTable(Statement stmt, String tableName, String tableColumnName, String fromCat,
+ String toCat, String fromDb, String toDb, String hiveTblName) throws HiveMetaException, SQLException {
+ String update = String.format(schemaTool.quote(UPDATE_DB_NAME_STMT), tableName, toCat, toDb, fromCat, fromDb,
+ tableColumnName, hiveTblName);
+
+ LOG.debug("Going to run " + update);
+ int numUpdated = stmt.executeUpdate(update);
+ if (numUpdated > 1 || numUpdated < 0) {
+ throw new HiveMetaException("Failed to properly update the " + tableName +
+ " table. Expected to update 1 row but instead updated " + numUpdated);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskUpgrade.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskUpgrade.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskUpgrade.java
new file mode 100644
index 0000000..5e71609
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskUpgrade.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo;
+
+/**
+ * Perform metastore schema upgrade.
+ */
+class SchemaToolTaskUpgrade extends SchemaToolTask {
+ private String fromVersion;
+
+ @Override
+ void setCommandLineArguments(SchemaToolCommandLine cl) {
+ if (cl.hasOption("upgradeSchemaFrom")) {
+ this.fromVersion = cl.getOptionValue("upgradeSchemaFrom");
+ }
+ }
+
+ private void ensureFromVersion() throws HiveMetaException {
+ if (fromVersion != null) {
+ return;
+ }
+
+ // If null, then read from the metastore
+ MetaStoreConnectionInfo connectionInfo = schemaTool.getConnectionInfo(false);
+ fromVersion = schemaTool.getMetaStoreSchemaInfo().getMetaStoreSchemaVersion(connectionInfo);
+ if (fromVersion == null || fromVersion.isEmpty()) {
+ throw new HiveMetaException("Schema version not stored in the metastore. " +
+ "Metastore schema is too old or corrupt. Try specifying the version manually");
+ }
+ System.out.println("Upgrading from the version " + fromVersion);
+ }
+
+ @Override
+ void execute() throws HiveMetaException {
+ ensureFromVersion();
+
+ if (schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion().equals(fromVersion)) {
+ System.out.println("No schema upgrade required from version " + fromVersion);
+ return;
+ }
+
+ // Find the list of scripts to execute for this upgrade
+ List<String> upgradeScripts = schemaTool.getMetaStoreSchemaInfo().getUpgradeScripts(fromVersion);
+ schemaTool.testConnectionToMetastore();
+ System.out.println("Starting upgrade metastore schema from version " + fromVersion + " to " +
+ schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion());
+ String scriptDir = schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir();
+ try {
+ for (String scriptFile : upgradeScripts) {
+ System.out.println("Upgrade script " + scriptFile);
+ if (!schemaTool.isDryRun()) {
+ runPreUpgrade(scriptDir, scriptFile);
+ schemaTool.execSql(scriptDir, scriptFile);
+ System.out.println("Completed " + scriptFile);
+ }
+ }
+ } catch (IOException e) {
+ throw new HiveMetaException("Upgrade FAILED! Metastore state would be inconsistent !!", e);
+ }
+
+ // Revalidated the new version after upgrade
+ schemaTool.verifySchemaVersion();
+ }
+
+ /**
+ * Run pre-upgrade scripts corresponding to a given upgrade script,
+ * if any exist. The errors from pre-upgrade are ignored.
+ * Pre-upgrade scripts typically contain setup statements which
+ * may fail on some database versions and failure is ignorable.
+ *
+ * @param scriptDir upgrade script directory name
+ * @param scriptFile upgrade script file name
+ */
+ private void runPreUpgrade(String scriptDir, String scriptFile) {
+ for (int i = 0;; i++) {
+ String preUpgradeScript = schemaTool.getMetaStoreSchemaInfo().getPreUpgradeScriptName(i, scriptFile);
+ File preUpgradeScriptFile = new File(scriptDir, preUpgradeScript);
+ if (!preUpgradeScriptFile.isFile()) {
+ break;
+ }
+
+ try {
+ schemaTool.execSql(scriptDir, preUpgradeScript);
+ System.out.println("Completed " + preUpgradeScript);
+ } catch (Exception e) {
+ // Ignore the pre-upgrade script errors
+ System.err.println("Warning in pre-upgrade script " + preUpgradeScript + ": " + e.getMessage());
+ if (schemaTool.isVerbose()) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskValidate.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskValidate.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskValidate.java
new file mode 100644
index 0000000..d86b457
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskValidate.java
@@ -0,0 +1,630 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.net.URI;
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.SQLFeatureNotSupportedException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.NestedScriptParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
+
+/**
+ * Print Hive version and schema version.
+ */
+class SchemaToolTaskValidate extends SchemaToolTask {
+ private static final Logger LOG = LoggerFactory.getLogger(SchemaToolTaskValidate.class.getName());
+
+ @Override
+ void setCommandLineArguments(SchemaToolCommandLine cl) {
+ // do nothing
+ }
+
+ @Override
+ void execute() throws HiveMetaException {
+ System.out.println("Starting metastore validation\n");
+ Connection conn = schemaTool.getConnectionToMetastore(false);
+ boolean success = true;
+ try {
+ success &= validateSchemaVersions();
+ success &= validateSequences(conn);
+ success &= validateSchemaTables(conn);
+ success &= validateLocations(conn, schemaTool.getValidationServers());
+ success &= validateColumnNullValues(conn);
+ } finally {
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to close metastore connection", e);
+ }
+ }
+ }
+
+ System.out.print("Done with metastore validation: ");
+ if (!success) {
+ System.out.println("[FAIL]");
+ throw new HiveMetaException("Validation failed");
+ } else {
+ System.out.println("[SUCCESS]");
+ }
+ }
+
+ boolean validateSchemaVersions() throws HiveMetaException {
+ System.out.println("Validating schema version");
+ try {
+ String hiveSchemaVersion = schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion();
+ MetaStoreConnectionInfo connectionInfo = schemaTool.getConnectionInfo(false);
+ String newSchemaVersion = schemaTool.getMetaStoreSchemaInfo().getMetaStoreSchemaVersion(connectionInfo);
+ schemaTool.assertCompatibleVersion(hiveSchemaVersion, newSchemaVersion);
+ } catch (HiveMetaException hme) {
+ if (hme.getMessage().contains("Metastore schema version is not compatible") ||
+ hme.getMessage().contains("Multiple versions were found in metastore") ||
+ hme.getMessage().contains("Could not find version info in metastore VERSION table")) {
+ System.err.println(hme.getMessage());
+ System.out.println("[FAIL]\n");
+ return false;
+ } else {
+ throw hme;
+ }
+ }
+ System.out.println("[SUCCESS]\n");
+ return true;
+ }
+
+ private static final String QUERY_SEQ =
+ " select t.<q>NEXT_VAL<q>" +
+ " from <q>SEQUENCE_TABLE<q> t " +
+ " where t.<q>SEQUENCE_NAME<q> = ? " +
+ "order by t.<q>SEQUENCE_NAME<q>";
+
+ private static final String QUERY_MAX_ID =
+ "select max(<q>%s<q>)" +
+ " from <q>%s<q>";
+
+ @VisibleForTesting
+ boolean validateSequences(Connection conn) throws HiveMetaException {
+ Map<String, Pair<String, String>> seqNameToTable =
+ new ImmutableMap.Builder<String, Pair<String, String>>()
+ .put("MDatabase", Pair.of("DBS", "DB_ID"))
+ .put("MRole", Pair.of("ROLES", "ROLE_ID"))
+ .put("MGlobalPrivilege", Pair.of("GLOBAL_PRIVS", "USER_GRANT_ID"))
+ .put("MTable", Pair.of("TBLS","TBL_ID"))
+ .put("MStorageDescriptor", Pair.of("SDS", "SD_ID"))
+ .put("MSerDeInfo", Pair.of("SERDES", "SERDE_ID"))
+ .put("MColumnDescriptor", Pair.of("CDS", "CD_ID"))
+ .put("MTablePrivilege", Pair.of("TBL_PRIVS", "TBL_GRANT_ID"))
+ .put("MTableColumnStatistics", Pair.of("TAB_COL_STATS", "CS_ID"))
+ .put("MPartition", Pair.of("PARTITIONS", "PART_ID"))
+ .put("MPartitionColumnStatistics", Pair.of("PART_COL_STATS", "CS_ID"))
+ .put("MFunction", Pair.of("FUNCS", "FUNC_ID"))
+ .put("MIndex", Pair.of("IDXS", "INDEX_ID"))
+ .put("MStringList", Pair.of("SKEWED_STRING_LIST", "STRING_LIST_ID"))
+ .build();
+
+ System.out.println("Validating sequence number for SEQUENCE_TABLE");
+
+ boolean isValid = true;
+ try {
+ Statement stmt = conn.createStatement();
+ for (Map.Entry<String, Pair<String, String>> e : seqNameToTable.entrySet()) {
+ String tableName = e.getValue().getLeft();
+ String tableKey = e.getValue().getRight();
+ String fullSequenceName = "org.apache.hadoop.hive.metastore.model." + e.getKey();
+ String seqQuery = schemaTool.quote(QUERY_SEQ);
+ String maxIdQuery = String.format(schemaTool.quote(QUERY_MAX_ID), tableKey, tableName);
+
+ ResultSet res = stmt.executeQuery(maxIdQuery);
+ if (res.next()) {
+ long maxId = res.getLong(1);
+ if (maxId > 0) {
+ PreparedStatement stmtSeq = conn.prepareStatement(seqQuery);
+ stmtSeq.setString(1, fullSequenceName);
+ ResultSet resSeq = stmtSeq.executeQuery();
+ if (!resSeq.next()) {
+ isValid = false;
+ System.err.println("Missing SEQUENCE_NAME " + e.getKey() + " from SEQUENCE_TABLE");
+ } else if (resSeq.getLong(1) < maxId) {
+ isValid = false;
+ System.err.println("NEXT_VAL for " + e.getKey() + " in SEQUENCE_TABLE < max(" + tableKey +
+ ") in " + tableName);
+ }
+ }
+ }
+ }
+
+ System.out.println(isValid ? "[SUCCESS]\n" :"[FAIL]\n");
+ return isValid;
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to validate sequence number for SEQUENCE_TABLE", e);
+ }
+ }
+
+ @VisibleForTesting
+ boolean validateSchemaTables(Connection conn) throws HiveMetaException {
+ System.out.println("Validating metastore schema tables");
+ String version = null;
+ try {
+ MetaStoreConnectionInfo connectionInfo = schemaTool.getConnectionInfo(false);
+ version = schemaTool.getMetaStoreSchemaInfo().getMetaStoreSchemaVersion(connectionInfo);
+ } catch (HiveMetaException he) {
+ System.err.println("Failed to determine schema version from Hive Metastore DB. " + he.getMessage());
+ System.out.println("Failed in schema table validation.");
+ LOG.debug("Failed to determine schema version from Hive Metastore DB," + he.getMessage(), he);
+ return false;
+ }
+
+ Connection hmsConn = schemaTool.getConnectionToMetastore(false);
+
+ LOG.debug("Validating tables in the schema for version " + version);
+ List<String> dbTables = new ArrayList<>();
+ ResultSet rs = null;
+ try {
+ String schema = null;
+ try {
+ schema = hmsConn.getSchema();
+ } catch (SQLFeatureNotSupportedException e) {
+ LOG.debug("schema is not supported");
+ }
+
+ DatabaseMetaData metadata = conn.getMetaData();
+ rs = metadata.getTables(null, schema, "%", new String[] {"TABLE"});
+
+ while (rs.next()) {
+ String table = rs.getString("TABLE_NAME");
+ dbTables.add(table.toLowerCase());
+ LOG.debug("Found table " + table + " in HMS dbstore");
+ }
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to retrieve schema tables from Hive Metastore DB," +
+ e.getMessage(), e);
+ } finally {
+ if (rs != null) {
+ try {
+ rs.close();
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to close resultset", e);
+ }
+ }
+ }
+
+ // parse the schema file to determine the tables that are expected to exist
+ // we are using oracle schema because it is simpler to parse, no quotes or backticks etc
+ List<String> schemaTables = new ArrayList<>();
+ List<String> subScripts = new ArrayList<>();
+
+ String baseDir = new File(schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir()).getParent();
+ String schemaFile = new File(schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir(),
+ schemaTool.getMetaStoreSchemaInfo().generateInitFileName(version)).getPath();
+ try {
+ LOG.debug("Parsing schema script " + schemaFile);
+ subScripts.addAll(findCreateTable(schemaFile, schemaTables));
+ while (subScripts.size() > 0) {
+ schemaFile = baseDir + "/" + schemaTool.getDbType() + "/" + subScripts.remove(0);
+ LOG.debug("Parsing subscript " + schemaFile);
+ subScripts.addAll(findCreateTable(schemaFile, schemaTables));
+ }
+ } catch (Exception e) {
+ System.err.println("Exception in parsing schema file. Cause:" + e.getMessage());
+ System.out.println("Failed in schema table validation.");
+ return false;
+ }
+
+ LOG.debug("Schema tables:[ " + Arrays.toString(schemaTables.toArray()) + " ]");
+ LOG.debug("DB tables:[ " + Arrays.toString(dbTables.toArray()) + " ]");
+
+ // now diff the lists
+ schemaTables.removeAll(dbTables);
+ if (schemaTables.size() > 0) {
+ Collections.sort(schemaTables);
+ System.err.println("Table(s) [ " + Arrays.toString(schemaTables.toArray()) + " ] " +
+ "are missing from the metastore database schema.");
+ System.out.println("[FAIL]\n");
+ return false;
+ } else {
+ System.out.println("[SUCCESS]\n");
+ return true;
+ }
+ }
+
+ @VisibleForTesting
+ List<String> findCreateTable(String path, List<String> tableList) throws Exception {
+ if (!(new File(path)).exists()) {
+ throw new Exception(path + " does not exist. Potentially incorrect version in the metastore VERSION table");
+ }
+
+ List<String> subs = new ArrayList<>();
+ NestedScriptParser sp = HiveSchemaHelper.getDbCommandParser(schemaTool.getDbType(), false);
+ Pattern regexp = Pattern.compile("CREATE TABLE(\\s+IF NOT EXISTS)?\\s+(\\S+).*");
+
+ try (BufferedReader reader = new BufferedReader(new FileReader(path))) {
+ String line = null;
+ while ((line = reader.readLine()) != null) {
+ if (sp.isNestedScript(line)) {
+ String subScript = sp.getScriptName(line);
+ LOG.debug("Schema subscript " + subScript + " found");
+ subs.add(subScript);
+ continue;
+ }
+ line = line.replaceAll("( )+", " "); //suppress multi-spaces
+ line = line.replaceAll("\\(", " ");
+ line = line.replaceAll("IF NOT EXISTS ", "");
+ line = line.replaceAll("`", "");
+ line = line.replaceAll("'", "");
+ line = line.replaceAll("\"", "");
+ Matcher matcher = regexp.matcher(line);
+
+ if (matcher.find()) {
+ String table = matcher.group(2);
+ if (schemaTool.getDbType().equals("derby")) {
+ table = table.replaceAll("APP\\.", "");
+ }
+ tableList.add(table.toLowerCase());
+ LOG.debug("Found table " + table + " in the schema");
+ }
+ }
+ } catch (IOException ex){
+ throw new Exception(ex.getMessage());
+ }
+
+ return subs;
+ }
+
+ @VisibleForTesting
+ boolean validateLocations(Connection conn, URI[] defaultServers) throws HiveMetaException {
+ System.out.println("Validating DFS locations");
+ boolean rtn = true;
+ rtn &= checkMetaStoreDBLocation(conn, defaultServers);
+ rtn &= checkMetaStoreTableLocation(conn, defaultServers);
+ rtn &= checkMetaStorePartitionLocation(conn, defaultServers);
+ rtn &= checkMetaStoreSkewedColumnsLocation(conn, defaultServers);
+ System.out.println(rtn ? "[SUCCESS]\n" : "[FAIL]\n");
+ return rtn;
+ }
+
+ private static final String QUERY_DB_LOCATION =
+ " select dbt.<q>DB_ID<q>, " +
+ " dbt.<q>NAME<q>, " +
+ " dbt.<q>DB_LOCATION_URI<q> " +
+ " from <q>DBS<q> dbt " +
+ "order by dbt.<q>DB_ID<q> ";
+
+ private boolean checkMetaStoreDBLocation(Connection conn, URI[] defaultServers) throws HiveMetaException {
+ String dbLocQuery = schemaTool.quote(QUERY_DB_LOCATION);
+
+ int numOfInvalid = 0;
+ try (Statement stmt = conn.createStatement();
+ ResultSet res = stmt.executeQuery(dbLocQuery)) {
+ while (res.next()) {
+ String locValue = res.getString(3);
+ String dbName = getNameOrID(res, 2, 1);
+ if (!checkLocation("Database " + dbName, locValue, defaultServers)) {
+ numOfInvalid++;
+ }
+ }
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to get DB Location Info.", e);
+ }
+ return numOfInvalid == 0;
+ }
+
+ private static final String TAB_ID_RANGE_QUERY =
+ "select max(<q>TBL_ID<q>), " +
+ " min(<q>TBL_ID<q>) " +
+ " from <q>TBLS<q> ";
+
+ private static final String TAB_LOC_QUERY =
+ " select tbl.<q>TBL_ID<q>, " +
+ " tbl.<q>TBL_NAME<q>, " +
+ " sd.<q>LOCATION<q>, " +
+ " dbt.<q>DB_ID<q>, " +
+ " dbt.<q>NAME<q> " +
+ " from <q>TBLS<q> tbl " +
+ "inner join <q>SDS<q> sd on sd.<q>SD_ID<q> = tbl.<q>SD_ID<q> " +
+ "inner join <q>DBS<q> dbt on tbl.<q>DB_ID<q> = dbt.<q>DB_ID<q> " +
+ " where tbl.<q>TBL_TYPE<q> != '%s' " +
+ " and tbl.<q>TBL_ID<q> >= ? " +
+ " and tbl.<q>TBL_ID<q> <= ? " +
+ " order by tbl.<q>TBL_ID<q> ";
+
+ private static final int TAB_LOC_CHECK_SIZE = 2000;
+
+ private boolean checkMetaStoreTableLocation(Connection conn, URI[] defaultServers)
+ throws HiveMetaException {
+ String tabIDRangeQuery = schemaTool.quote(TAB_ID_RANGE_QUERY);
+ String tabLocQuery = String.format(schemaTool.quote(TAB_LOC_QUERY), TableType.VIRTUAL_VIEW);
+
+ try {
+ long maxID = 0, minID = 0;
+ try (Statement stmt = conn.createStatement();
+ ResultSet res = stmt.executeQuery(tabIDRangeQuery)) {
+ if (res.next()) {
+ maxID = res.getLong(1);
+ minID = res.getLong(2);
+ }
+ }
+
+ int numOfInvalid = 0;
+ try (PreparedStatement pStmt = conn.prepareStatement(tabLocQuery)) {
+ while (minID <= maxID) {
+ pStmt.setLong(1, minID);
+ pStmt.setLong(2, minID + TAB_LOC_CHECK_SIZE);
+ try (ResultSet res = pStmt.executeQuery()) {
+ while (res.next()) {
+ String locValue = res.getString(3);
+ String entity = "Database " + getNameOrID(res, 5, 4) + ", Table " + getNameOrID(res, 2, 1);
+ if (!checkLocation(entity, locValue, defaultServers)) {
+ numOfInvalid++;
+ }
+ }
+ }
+ minID += TAB_LOC_CHECK_SIZE + 1;
+ }
+ }
+
+ return numOfInvalid == 0;
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to get Table Location Info.", e);
+ }
+ }
+
+ private static final String QUERY_PART_ID_RANGE =
+ "select max(<q>PART_ID<q>)," +
+ " min(<q>PART_ID<q>)" +
+ " from <q>PARTITIONS<q> ";
+
+ private static final String QUERY_PART_LOC =
+ " select pt.<q>PART_ID<q>, " +
+ " pt.<q>PART_NAME<q>, " +
+ " sd.<q>LOCATION<q>, " +
+ " tbl.<q>TBL_ID<q>, " +
+ " tbl.<q>TBL_NAME<q>, " +
+ " dbt.<q>DB_ID<q>, " +
+ " dbt.<q>NAME<q> " +
+ " from <q>PARTITIONS<q> pt " +
+ "inner join <q>SDS<q> sd on sd.<q>SD_ID<q> = pt.<q>SD_ID<q> " +
+ "inner join <q>TBLS<q> tbl on tbl.<q>TBL_ID<q> = pt.<q>TBL_ID<q> " +
+ "inner join <q>DBS<q> dbt on dbt.<q>DB_ID<q> = tbl.<q>DB_ID<q> " +
+ " where pt.<q>PART_ID<q> >= ? " +
+ " and pt.<q>PART_ID<q> <= ? " +
+ " order by tbl.<q>TBL_ID<q> ";
+
+ private static final int PART_LOC_CHECK_SIZE = 2000;
+
+ private boolean checkMetaStorePartitionLocation(Connection conn, URI[] defaultServers)
+ throws HiveMetaException {
+ String queryPartIDRange = schemaTool.quote(QUERY_PART_ID_RANGE);
+ String queryPartLoc = schemaTool.quote(QUERY_PART_LOC);
+
+ try {
+ long maxID = 0, minID = 0;
+ try (Statement stmt = conn.createStatement();
+ ResultSet res = stmt.executeQuery(queryPartIDRange)) {
+ if (res.next()) {
+ maxID = res.getLong(1);
+ minID = res.getLong(2);
+ }
+ }
+
+ int numOfInvalid = 0;
+ try (PreparedStatement pStmt = conn.prepareStatement(queryPartLoc)) {
+ while (minID <= maxID) {
+ pStmt.setLong(1, minID);
+ pStmt.setLong(2, minID + PART_LOC_CHECK_SIZE);
+ try (ResultSet res = pStmt.executeQuery()) {
+ while (res.next()) {
+ String locValue = res.getString(3);
+ String entity = "Database " + getNameOrID(res, 7, 6) + ", Table " + getNameOrID(res, 5, 4) +
+ ", Partition " + getNameOrID(res, 2, 1);
+ if (!checkLocation(entity, locValue, defaultServers)) {
+ numOfInvalid++;
+ }
+ }
+ }
+ minID += PART_LOC_CHECK_SIZE + 1;
+ }
+ }
+
+ return numOfInvalid == 0;
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to get Partition Location Info.", e);
+ }
+ }
+
+ private static final String QUERY_SKEWED_COL_ID_RANGE =
+ "select max(<q>STRING_LIST_ID_KID<q>), " +
+ " min(<q>STRING_LIST_ID_KID<q>) " +
+ " from <q>SKEWED_COL_VALUE_LOC_MAP<q> ";
+
+ private static final String QUERY_SKEWED_COL_LOC =
+ " select t.<q>TBL_NAME<q>, " +
+ " t.<q>TBL_ID<q>, " +
+ " sk.<q>STRING_LIST_ID_KID<q>, " +
+ " sk.<q>LOCATION<q>, " +
+ " db.<q>NAME<q>, " +
+ " db.<q>DB_ID<q> " +
+ " from <q>TBLS<q> t " +
+ " join <q>SDS<q> s on s.<q>SD_ID<q> = t.<q>SD_ID<q> " +
+ " join <q>DBS<q> db on db.<q>DB_ID<q> = t.<q>DB_ID<q> " +
+ " join <q>SKEWED_COL_VALUE_LOC_MAP<q> sk on sk.<q>SD_ID<q> = s.<q>SD_ID<q> " +
+ " where sk.<q>STRING_LIST_ID_KID<q> >= ? " +
+ " and sk.<q>STRING_LIST_ID_KID<q> <= ? " +
+ "order by t.<q>TBL_ID<q> ";
+
+ private static final int SKEWED_COL_LOC_CHECK_SIZE = 2000;
+
+ private boolean checkMetaStoreSkewedColumnsLocation(Connection conn, URI[] defaultServers)
+ throws HiveMetaException {
+ String querySkewedColIDRange = schemaTool.quote(QUERY_SKEWED_COL_ID_RANGE);
+ String querySkewedColLoc = schemaTool.quote(QUERY_SKEWED_COL_LOC);
+
+ try {
+ long maxID = 0, minID = 0;
+ try (Statement stmt = conn.createStatement();
+ ResultSet res = stmt.executeQuery(querySkewedColIDRange)) {
+ if (res.next()) {
+ maxID = res.getLong(1);
+ minID = res.getLong(2);
+ }
+ }
+
+ int numOfInvalid = 0;
+ try (PreparedStatement pStmt = conn.prepareStatement(querySkewedColLoc)) {
+ while (minID <= maxID) {
+ pStmt.setLong(1, minID);
+ pStmt.setLong(2, minID + SKEWED_COL_LOC_CHECK_SIZE);
+ try (ResultSet res = pStmt.executeQuery()) {
+ while (res.next()) {
+ String locValue = res.getString(4);
+ String entity = "Database " + getNameOrID(res, 5, 6) + ", Table " + getNameOrID(res, 1, 2) +
+ ", String list " + res.getString(3);
+ if (!checkLocation(entity, locValue, defaultServers)) {
+ numOfInvalid++;
+ }
+ }
+ }
+ minID += SKEWED_COL_LOC_CHECK_SIZE + 1;
+ }
+ }
+
+ return numOfInvalid == 0;
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to get skewed columns location info.", e);
+ }
+ }
+
+ /**
+ * Check if the location is valid for the given entity.
+ * @param entity the entity to represent a database, partition or table
+ * @param entityLocation the location
+ * @param defaultServers a list of the servers that the location needs to match.
+ * The location host needs to match one of the given servers.
+ * If empty, then no check against such list.
+ * @return true if the location is valid
+ */
+ private boolean checkLocation(String entity, String entityLocation, URI[] defaultServers) {
+ boolean isValid = true;
+
+ if (entityLocation == null) {
+ System.err.println(entity + ", Error: empty location");
+ isValid = false;
+ } else {
+ try {
+ URI currentUri = new Path(entityLocation).toUri();
+ String scheme = currentUri.getScheme();
+ String path = currentUri.getPath();
+ if (StringUtils.isEmpty(scheme)) {
+ System.err.println(entity + ", Location: "+ entityLocation + ", Error: missing location scheme.");
+ isValid = false;
+ } else if (StringUtils.isEmpty(path)) {
+ System.err.println(entity + ", Location: "+ entityLocation + ", Error: missing location path.");
+ isValid = false;
+ } else if (ArrayUtils.isNotEmpty(defaultServers) && currentUri.getAuthority() != null) {
+ String authority = currentUri.getAuthority();
+ boolean matchServer = false;
+ for(URI server : defaultServers) {
+ if (StringUtils.equalsIgnoreCase(server.getScheme(), scheme) &&
+ StringUtils.equalsIgnoreCase(server.getAuthority(), authority)) {
+ matchServer = true;
+ break;
+ }
+ }
+ if (!matchServer) {
+ System.err.println(entity + ", Location: " + entityLocation + ", Error: mismatched server.");
+ isValid = false;
+ }
+ }
+
+ // if there is no path element other than "/", report it but not fail
+ if (isValid && StringUtils.containsOnly(path, "/")) {
+ System.err.println(entity + ", Location: "+ entityLocation + ", Warn: location set to root, " +
+ "not a recommended config.");
+ }
+ } catch (Exception pe) {
+ System.err.println(entity + ", Error: invalid location - " + pe.getMessage());
+ isValid =false;
+ }
+ }
+
+ return isValid;
+ }
+
+ private String getNameOrID(ResultSet res, int nameInx, int idInx) throws SQLException {
+ String itemName = res.getString(nameInx);
+ return (itemName == null || itemName.isEmpty()) ? "ID: " + res.getString(idInx) : "Name: " + itemName;
+ }
+
+ private static final String QUERY_COLUMN_NULL_VALUES =
+ " select t.*" +
+ " from <q>TBLS<q> t" +
+ " where t.<q>SD_ID<q> IS NULL" +
+ " and (t.<q>TBL_TYPE<q> = '" + TableType.EXTERNAL_TABLE + "' or" +
+ " t.<q>TBL_TYPE<q> = '" + TableType.MANAGED_TABLE + "') " +
+ "order by t.<q>TBL_ID<q> ";
+
+ @VisibleForTesting
+ boolean validateColumnNullValues(Connection conn) throws HiveMetaException {
+ System.out.println("Validating columns for incorrect NULL values.");
+
+ boolean isValid = true;
+ String queryColumnNullValues = schemaTool.quote(QUERY_COLUMN_NULL_VALUES);
+
+ try (Statement stmt = conn.createStatement();
+ ResultSet res = stmt.executeQuery(queryColumnNullValues)) {
+ while (res.next()) {
+ long tableId = res.getLong("TBL_ID");
+ String tableName = res.getString("TBL_NAME");
+ String tableType = res.getString("TBL_TYPE");
+ isValid = false;
+ System.err.println("SD_ID in TBLS should not be NULL for Table Name=" + tableName + ", Table ID=" + tableId + ", Table Type=" + tableType);
+ }
+
+ System.out.println(isValid ? "[SUCCESS]\n" : "[FAIL]\n");
+ return isValid;
+ } catch(SQLException e) {
+ throw new HiveMetaException("Failed to validate columns for incorrect NULL values", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/DbInstallBase.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/DbInstallBase.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/DbInstallBase.java
index 4722a56..2915720 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/DbInstallBase.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/DbInstallBase.java
@@ -19,8 +19,6 @@ package org.apache.hadoop.hive.metastore.dbinstall;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.metastore.HiveMetaException;
-import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.tools.MetastoreSchemaTool;
import org.junit.After;
import org.junit.Assert;
@@ -46,8 +44,6 @@ public abstract class DbInstallBase {
private static final String FIRST_VERSION = "1.2.0";
private static final int MAX_STARTUP_WAIT = 5 * 60 * 1000;
- private String metastoreHome;
-
protected abstract String getDockerContainerName();
protected abstract String getDockerImageName();
protected abstract String[] getDockerAdditionalArgs();
@@ -86,7 +82,7 @@ public abstract class DbInstallBase {
throw new RuntimeException("Container failed to be ready in " + MAX_STARTUP_WAIT/1000 +
" seconds");
}
- MetastoreSchemaTool.homeDir = metastoreHome = System.getProperty("test.tmp.dir", "target/tmp");
+ MetastoreSchemaTool.setHomeDirForTesting();
}
@After
@@ -145,7 +141,7 @@ public abstract class DbInstallBase {
}
private int createUser() {
- return MetastoreSchemaTool.run(buildArray(
+ return new MetastoreSchemaTool().run(buildArray(
"-createUser",
"-dbType",
getDbType(),
@@ -167,7 +163,7 @@ public abstract class DbInstallBase {
}
private int installLatest() {
- return MetastoreSchemaTool.run(buildArray(
+ return new MetastoreSchemaTool().run(buildArray(
"-initSchema",
"-dbType",
getDbType(),
@@ -183,7 +179,7 @@ public abstract class DbInstallBase {
}
private int installAVersion(String version) {
- return MetastoreSchemaTool.run(buildArray(
+ return new MetastoreSchemaTool().run(buildArray(
"-initSchemaTo",
version,
"-dbType",
@@ -200,7 +196,7 @@ public abstract class DbInstallBase {
}
private int upgradeToLatest() {
- return MetastoreSchemaTool.run(buildArray(
+ return new MetastoreSchemaTool().run(buildArray(
"-upgradeSchema",
"-dbType",
getDbType(),
@@ -219,17 +215,6 @@ public abstract class DbInstallBase {
return strs;
}
- private String getCurrentVersionMinusOne() throws HiveMetaException {
- List<String> scripts = MetaStoreSchemaInfoFactory.get(
- MetastoreConf.newMetastoreConf(), metastoreHome, getDbType()
- ).getUpgradeScripts(FIRST_VERSION);
- Assert.assertTrue(scripts.size() > 0);
- String lastUpgradePath = scripts.get(scripts.size() - 1);
- String version = lastUpgradePath.split("-")[1];
- LOG.info("Current version minus 1 is " + version);
- return version;
- }
-
@Test
public void install() {
Assert.assertEquals(0, createUser());
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java
index 9e425cf..0657ae5 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java
@@ -27,10 +27,13 @@ import java.io.OutputStream;
import java.io.PrintStream;
import java.net.URI;
import java.sql.Connection;
+import java.sql.DatabaseMetaData;
import java.sql.SQLException;
import java.util.Random;
+import org.apache.commons.dbcp.DelegatingConnection;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.text.StrTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.HiveMetaException;
import org.apache.hadoop.hive.metastore.IMetaStoreSchemaInfo;
@@ -56,6 +59,8 @@ public class TestSchemaToolForMetastore {
private String testMetastoreDB;
private PrintStream errStream;
private PrintStream outStream;
+ private String argsBase;
+ private SchemaToolTaskValidate validator;
@Before
public void setUp() throws HiveMetaException, IOException {
@@ -64,14 +69,21 @@ public class TestSchemaToolForMetastore {
System.setProperty(ConfVars.CONNECT_URL_KEY.toString(),
"jdbc:derby:" + testMetastoreDB + ";create=true");
conf = MetastoreConf.newMetastoreConf();
- schemaTool = new MetastoreSchemaTool(
- System.getProperty("test.tmp.dir", "target/tmp"), conf, "derby");
- schemaTool.setUserName(MetastoreConf.getVar(schemaTool.getConf(), ConfVars.CONNECTION_USER_NAME));
- schemaTool.setPassWord(MetastoreConf.getPassword(schemaTool.getConf(), ConfVars.PWD));
+ schemaTool = new MetastoreSchemaTool();
+ schemaTool.init(System.getProperty("test.tmp.dir", "target/tmp"),
+ new String[]{"-dbType", "derby", "--info"}, null, conf);
+ String userName = MetastoreConf.getVar(schemaTool.getConf(), ConfVars.CONNECTION_USER_NAME);
+ String passWord = MetastoreConf.getPassword(schemaTool.getConf(), ConfVars.PWD);
+ schemaTool.setUserName(userName);
+ schemaTool.setPassWord(passWord);
+ argsBase = "-dbType derby -userName " + userName + " -passWord " + passWord + " ";
System.setProperty("beeLine.system.exit", "true");
errStream = System.err;
outStream = System.out;
conn = schemaTool.getConnectionToMetastore(false);
+
+ validator = new SchemaToolTaskValidate();
+ validator.setHiveSchemaTool(schemaTool);
}
@After
@@ -87,24 +99,26 @@ public class TestSchemaToolForMetastore {
}
}
- // Test the sequence validation functionality
+ /*
+ * Test the sequence validation functionality
+ */
@Test
public void testValidateSequences() throws Exception {
- schemaTool.doInit();
+ execute(new SchemaToolTaskInit(), "-initSchema");
// Test empty database
- boolean isValid = schemaTool.validateSequences(conn);
+ boolean isValid = validator.validateSequences(conn);
Assert.assertTrue(isValid);
// Test valid case
String[] scripts = new String[] {
+ "insert into CTLGS values(99, 'test_cat_1', 'description', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb');",
"insert into SEQUENCE_TABLE values('org.apache.hadoop.hive.metastore.model.MDatabase', 100);",
- "insert into CTLGS values(37, 'mycat', 'my description', 'hdfs://tmp');",
- "insert into DBS values(99, 'test db1', 'hdfs:///tmp', 'db1', 'test', 'test', 'mycat');"
+ "insert into DBS values(99, 'test db1', 'hdfs:///tmp', 'db1', 'test', 'test', 'test_cat_1');"
};
File scriptFile = generateTestScript(scripts);
- schemaTool.runSqlLine(scriptFile.getPath());
- isValid = schemaTool.validateSequences(conn);
+ schemaTool.execSql(scriptFile.getPath());
+ isValid = validator.validateSequences(conn);
Assert.assertTrue(isValid);
// Test invalid case
@@ -112,25 +126,27 @@ public class TestSchemaToolForMetastore {
"delete from SEQUENCE_TABLE;",
"delete from DBS;",
"insert into SEQUENCE_TABLE values('org.apache.hadoop.hive.metastore.model.MDatabase', 100);",
- "insert into DBS values(102, 'test db1', 'hdfs:///tmp', 'db1', 'test', 'test', 'mycat');"
+ "insert into DBS values(102, 'test db1', 'hdfs:///tmp', 'db1', 'test', 'test', 'test_cat_1');"
};
scriptFile = generateTestScript(scripts);
- schemaTool.runSqlLine(scriptFile.getPath());
- isValid = schemaTool.validateSequences(conn);
+ schemaTool.execSql(scriptFile.getPath());
+ isValid = validator.validateSequences(conn);
Assert.assertFalse(isValid);
}
- // Test to validate that all tables exist in the HMS metastore.
+ /*
+ * Test to validate that all tables exist in the HMS metastore.
+ */
@Test
public void testValidateSchemaTables() throws Exception {
- schemaTool.doInit("1.2.0");
+ execute(new SchemaToolTaskInit(), "-initSchemaTo 1.2.0");
- boolean isValid = schemaTool.validateSchemaTables(conn);
+ boolean isValid = validator.validateSchemaTables(conn);
Assert.assertTrue(isValid);
- // upgrade from 2.0.0 schema and re-validate
- schemaTool.doUpgrade("1.2.0");
- isValid = schemaTool.validateSchemaTables(conn);
+ // upgrade from 1.2.0 schema and re-validate
+ execute(new SchemaToolTaskUpgrade(), "-upgradeSchemaFrom 1.2.0");
+ isValid = validator.validateSchemaTables(conn);
Assert.assertTrue(isValid);
// Simulate a missing table scenario by renaming a couple of tables
@@ -140,8 +156,8 @@ public class TestSchemaToolForMetastore {
};
File scriptFile = generateTestScript(scripts);
- schemaTool.runSqlLine(scriptFile.getPath());
- isValid = schemaTool.validateSchemaTables(conn);
+ schemaTool.execSql(scriptFile.getPath());
+ isValid = validator.validateSchemaTables(conn);
Assert.assertFalse(isValid);
// Restored the renamed tables
@@ -151,31 +167,49 @@ public class TestSchemaToolForMetastore {
};
scriptFile = generateTestScript(scripts);
- schemaTool.runSqlLine(scriptFile.getPath());
- isValid = schemaTool.validateSchemaTables(conn);
+ schemaTool.execSql(scriptFile.getPath());
+ isValid = validator.validateSchemaTables(conn);
Assert.assertTrue(isValid);
- }
+
+ // Check that an exception from getMetaData() is reported correctly
+ try {
+ // Make a Connection object that will throw an exception
+ BadMetaDataConnection bad = new BadMetaDataConnection(conn);
+ validator.validateSchemaTables(bad);
+ Assert.fail("did not get expected exception");
+ } catch (HiveMetaException hme) {
+ String message = hme.getMessage();
+ Assert.assertTrue("Bad HiveMetaException message :" + message,
+ message.contains("Failed to retrieve schema tables from Hive Metastore DB"));
+ Throwable cause = hme.getCause();
+ Assert.assertNotNull("HiveMetaException did not contain a cause", cause);
+ String causeMessage = cause.getMessage();
+ Assert.assertTrue("Bad SQLException message: " + causeMessage, causeMessage.contains(
+ BadMetaDataConnection.FAILURE_TEXT));
+ }
+ }
+
// Test the validation of incorrect NULL values in the tables
@Test
public void testValidateNullValues() throws Exception {
- schemaTool.doInit();
+ execute(new SchemaToolTaskInit(), "-initSchema");
// Test empty database
- boolean isValid = schemaTool.validateColumnNullValues(conn);
+ boolean isValid = validator.validateColumnNullValues(conn);
Assert.assertTrue(isValid);
// Test valid case
createTestHiveTableSchemas();
- isValid = schemaTool.validateColumnNullValues(conn);
+ isValid = validator.validateColumnNullValues(conn);
// Test invalid case
String[] scripts = new String[] {
"update TBLS set SD_ID=null"
};
File scriptFile = generateTestScript(scripts);
- schemaTool.runSqlLine(scriptFile.getPath());
- isValid = schemaTool.validateColumnNullValues(conn);
+ schemaTool.execSql(scriptFile.getPath());
+ isValid = validator.validateColumnNullValues(conn);
Assert.assertFalse(isValid);
}
@@ -183,7 +217,7 @@ public class TestSchemaToolForMetastore {
@Test
public void testSchemaInitDryRun() throws Exception {
schemaTool.setDryRun(true);
- schemaTool.doInit("3.0.0");
+ execute(new SchemaToolTaskInit(), "-initSchemaTo 1.2.0");
schemaTool.setDryRun(false);
try {
schemaTool.verifySchemaVersion();
@@ -197,10 +231,10 @@ public class TestSchemaToolForMetastore {
// Test dryrun of schema upgrade
@Test
public void testSchemaUpgradeDryRun() throws Exception {
- schemaTool.doInit("1.2.0");
+ execute(new SchemaToolTaskInit(), "-initSchemaTo 1.2.0");
schemaTool.setDryRun(true);
- schemaTool.doUpgrade("1.2.0");
+ execute(new SchemaToolTaskUpgrade(), "-upgradeSchemaFrom 1.2.0");
schemaTool.setDryRun(false);
try {
schemaTool.verifySchemaVersion();
@@ -218,8 +252,7 @@ public class TestSchemaToolForMetastore {
public void testSchemaInit() throws Exception {
IMetaStoreSchemaInfo metastoreSchemaInfo = MetaStoreSchemaInfoFactory.get(conf,
System.getProperty("test.tmp.dir", "target/tmp"), "derby");
- LOG.info("Starting testSchemaInit");
- schemaTool.doInit(metastoreSchemaInfo.getHiveSchemaVersion());
+ execute(new SchemaToolTaskInit(), "-initSchemaTo " + metastoreSchemaInfo.getHiveSchemaVersion());
schemaTool.verifySchemaVersion();
}
@@ -227,35 +260,35 @@ public class TestSchemaToolForMetastore {
* Test validation for schema versions
*/
@Test
- public void testValidateSchemaVersions() throws Exception {
- schemaTool.doInit();
- boolean isValid = schemaTool.validateSchemaVersions();
- // Test an invalid case with multiple versions
- String[] scripts = new String[] {
- "insert into VERSION values(100, '2.2.0', 'Hive release version 2.2.0')"
- };
- File scriptFile = generateTestScript(scripts);
- schemaTool.runSqlLine(scriptFile.getPath());
- isValid = schemaTool.validateSchemaVersions();
- Assert.assertFalse(isValid);
-
- scripts = new String[] {
- "delete from VERSION where VER_ID = 100"
- };
- scriptFile = generateTestScript(scripts);
- schemaTool.runSqlLine(scriptFile.getPath());
- isValid = schemaTool.validateSchemaVersions();
- Assert.assertTrue(isValid);
-
- // Test an invalid case without version
- scripts = new String[] {
- "delete from VERSION"
- };
- scriptFile = generateTestScript(scripts);
- schemaTool.runSqlLine(scriptFile.getPath());
- isValid = schemaTool.validateSchemaVersions();
- Assert.assertFalse(isValid);
- }
+ public void testValidateSchemaVersions() throws Exception {
+ execute(new SchemaToolTaskInit(), "-initSchema");
+ boolean isValid = validator.validateSchemaVersions();
+ // Test an invalid case with multiple versions
+ String[] scripts = new String[] {
+ "insert into VERSION values(100, '2.2.0', 'Hive release version 2.2.0')"
+ };
+ File scriptFile = generateTestScript(scripts);
+ schemaTool.execSql(scriptFile.getPath());
+ isValid = validator.validateSchemaVersions();
+ Assert.assertFalse(isValid);
+
+ scripts = new String[] {
+ "delete from VERSION where VER_ID = 100"
+ };
+ scriptFile = generateTestScript(scripts);
+ schemaTool.execSql(scriptFile.getPath());
+ isValid = validator.validateSchemaVersions();
+ Assert.assertTrue(isValid);
+
+ // Test an invalid case without version
+ scripts = new String[] {
+ "delete from VERSION"
+ };
+ scriptFile = generateTestScript(scripts);
+ schemaTool.execSql(scriptFile.getPath());
+ isValid = validator.validateSchemaVersions();
+ Assert.assertFalse(isValid);
+ }
/**
* Test schema upgrade
@@ -264,7 +297,7 @@ public class TestSchemaToolForMetastore {
public void testSchemaUpgrade() throws Exception {
boolean foundException = false;
// Initialize 1.2.0 schema
- schemaTool.doInit("1.2.0");
+ execute(new SchemaToolTaskInit(), "-initSchemaTo 1.2.0");
// verify that driver fails due to older version schema
try {
schemaTool.verifySchemaVersion();
@@ -298,17 +331,7 @@ public class TestSchemaToolForMetastore {
System.setOut(outPrintStream);
// Upgrade schema from 0.7.0 to latest
- Exception caught = null;
- try {
- schemaTool.doUpgrade("1.2.0");
- } catch (Exception e) {
- caught = e;
- }
-
- LOG.info("stdout is " + stdout.toString());
- LOG.info("stderr is " + stderr.toString());
-
- if (caught != null) Assert.fail(caught.getMessage());
+ execute(new SchemaToolTaskUpgrade(), "-upgradeSchemaFrom 1.2.0");
// Verify that the schemaTool ran pre-upgrade scripts and ignored errors
Assert.assertTrue(stderr.toString().contains(invalidPreUpgradeScript));
@@ -327,38 +350,38 @@ public class TestSchemaToolForMetastore {
*/
@Test
public void testValidateLocations() throws Exception {
- schemaTool.doInit();
+ execute(new SchemaToolTaskInit(), "-initSchema");
URI defaultRoot = new URI("hdfs://myhost.com:8020");
URI defaultRoot2 = new URI("s3://myhost2.com:8888");
//check empty DB
- boolean isValid = schemaTool.validateLocations(conn, null);
+ boolean isValid = validator.validateLocations(conn, null);
Assert.assertTrue(isValid);
- isValid = schemaTool.validateLocations(conn, new URI[] {defaultRoot,defaultRoot2});
+ isValid = validator.validateLocations(conn, new URI[] {defaultRoot, defaultRoot2});
Assert.assertTrue(isValid);
- // Test valid case
+ // Test valid case
String[] scripts = new String[] {
- "insert into CTLGS values (1, 'mycat', 'mydescription', 'hdfs://myhost.com:8020/user/hive/warehouse');",
- "insert into DBS values(2, 'my db', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'mycat');",
- "insert into DBS values(7, 'db with bad port', 'hdfs://myhost.com:8020/', 'haDB', 'public', 'role', 'mycat');",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3,null,'org.apache.hadoop.mapred.TextInputFormat','N','N',null,-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n');",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (3 ,1435255431,2,0 ,'hive',0,3,'myView','VIRTUAL_VIEW','select a.col1,a.col2 from foo','select * from foo','n');",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4012 ,1435255431,7,0 ,'hive',0,4000,'mytal4012','MANAGED_TABLE',NULL,NULL,'n');",
- "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(1, 1441402388,0, 'd1=1/d2=1',2,2);",
- "insert into SKEWED_STRING_LIST values(1);",
- "insert into SKEWED_STRING_LIST values(2);",
- "insert into SKEWED_COL_VALUE_LOC_MAP values(1,1,'hdfs://myhost.com:8020/user/hive/warehouse/mytal/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/');",
- "insert into SKEWED_COL_VALUE_LOC_MAP values(2,2,'s3://myhost.com:8020/user/hive/warehouse/mytal/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/');"
- };
+ "insert into CTLGS values(3, 'test_cat_2', 'description', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb');",
+ "insert into DBS values(2, 'my db', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'test_cat_2');",
+ "insert into DBS values(7, 'db with bad port', 'hdfs://myhost.com:8020/', 'haDB', 'public', 'role', 'test_cat_2');",
+ "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);",
+ "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);",
+ "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3,null,'org.apache.hadoop.mapred.TextInputFormat','N','N',null,-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);",
+ "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);",
+ "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n');",
+ "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (3 ,1435255431,2,0 ,'hive',0,3,'myView','VIRTUAL_VIEW','select a.col1,a.col2 from foo','select * from foo','n');",
+ "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4012 ,1435255431,7,0 ,'hive',0,4000,'mytal4012','MANAGED_TABLE',NULL,NULL,'n');",
+ "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(1, 1441402388,0, 'd1=1/d2=1',2,2);",
+ "insert into SKEWED_STRING_LIST values(1);",
+ "insert into SKEWED_STRING_LIST values(2);",
+ "insert into SKEWED_COL_VALUE_LOC_MAP values(1,1,'hdfs://myhost.com:8020/user/hive/warehouse/mytal/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/');",
+ "insert into SKEWED_COL_VALUE_LOC_MAP values(2,2,'s3://myhost.com:8020/user/hive/warehouse/mytal/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/');"
+ };
File scriptFile = generateTestScript(scripts);
- schemaTool.runSqlLine(scriptFile.getPath());
- isValid = schemaTool.validateLocations(conn, null);
+ schemaTool.execSql(scriptFile.getPath());
+ isValid = validator.validateLocations(conn, null);
Assert.assertTrue(isValid);
- isValid = schemaTool.validateLocations(conn, new URI[] {defaultRoot, defaultRoot2});
+ isValid = validator.validateLocations(conn, new URI[] {defaultRoot, defaultRoot2});
Assert.assertTrue(isValid);
scripts = new String[] {
"delete from SKEWED_COL_VALUE_LOC_MAP;",
@@ -367,10 +390,10 @@ public class TestSchemaToolForMetastore {
"delete from TBLS;",
"delete from SDS;",
"delete from DBS;",
- "insert into DBS values(2, 'my db', '/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'mycat');",
- "insert into DBS values(4, 'my db2', 'hdfs://myhost.com:8020', '', 'public', 'role', 'mycat');",
- "insert into DBS values(6, 'db with bad port', 'hdfs://myhost.com:8020:', 'zDB', 'public', 'role', 'mycat');",
- "insert into DBS values(7, 'db with bad port', 'hdfs://mynameservice.com/', 'haDB', 'public', 'role', 'mycat');",
+ "insert into DBS values(2, 'my db', '/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'test_cat_2');",
+ "insert into DBS values(4, 'my db2', 'hdfs://myhost.com:8020', '', 'public', 'role', 'test_cat_2');",
+ "insert into DBS values(6, 'db with bad port', 'hdfs://myhost.com:8020:', 'zDB', 'public', 'role', 'test_cat_2');",
+ "insert into DBS values(7, 'db with bad port', 'hdfs://mynameservice.com/', 'haDB', 'public', 'role', 'test_cat_2');",
"insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://yourhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);",
"insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','file:///user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);",
"insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n');",
@@ -398,23 +421,23 @@ public class TestSchemaToolForMetastore {
"insert into SKEWED_COL_VALUE_LOC_MAP values(2,2,'file:///user/admin/warehouse/mytal/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/');"
};
scriptFile = generateTestScript(scripts);
- schemaTool.runSqlLine(scriptFile.getPath());
- isValid = schemaTool.validateLocations(conn, null);
+ schemaTool.execSql(scriptFile.getPath());
+ isValid = validator.validateLocations(conn, null);
Assert.assertFalse(isValid);
- isValid = schemaTool.validateLocations(conn, new URI[] {defaultRoot, defaultRoot2});
+ isValid = validator.validateLocations(conn, new URI[] {defaultRoot, defaultRoot2});
Assert.assertFalse(isValid);
}
@Test
public void testHiveMetastoreDbPropertiesTable() throws HiveMetaException, IOException {
- schemaTool.doInit("3.0.0");
+ execute(new SchemaToolTaskInit(), "-initSchemaTo 3.0.0");
validateMetastoreDbPropertiesTable();
}
@Test
public void testMetastoreDbPropertiesAfterUpgrade() throws HiveMetaException, IOException {
- schemaTool.doInit("1.2.0");
- schemaTool.doUpgrade();
+ execute(new SchemaToolTaskInit(), "-initSchemaTo 1.2.0");
+ execute(new SchemaToolTaskUpgrade(), "-upgradeSchema");
validateMetastoreDbPropertiesTable();
}
@@ -432,7 +455,7 @@ public class TestSchemaToolForMetastore {
}
private void validateMetastoreDbPropertiesTable() throws HiveMetaException, IOException {
- boolean isValid = schemaTool.validateSchemaTables(conn);
+ boolean isValid = (boolean) validator.validateSchemaTables(conn);
Assert.assertTrue(isValid);
// adding same property key twice should throw unique key constraint violation exception
String[] scripts = new String[] {
@@ -441,12 +464,13 @@ public class TestSchemaToolForMetastore {
File scriptFile = generateTestScript(scripts);
Exception ex = null;
try {
- schemaTool.runSqlLine(scriptFile.getPath());
+ schemaTool.execSql(scriptFile.getPath());
} catch (Exception iox) {
ex = iox;
}
Assert.assertTrue(ex != null && ex instanceof IOException);
}
+
/**
* Write out a dummy pre-upgrade script with given SQL statement.
*/
@@ -476,6 +500,35 @@ public class TestSchemaToolForMetastore {
"insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(1, 1441402388,0, 'd1=1/d2=1',2,2);"
};
File scriptFile = generateTestScript(scripts);
- schemaTool.runSqlLine(scriptFile.getPath());
+ schemaTool.execSql(scriptFile.getPath());
+ }
+
+ /**
+ * A mock Connection class that throws an exception out of getMetaData().
+ */
+ class BadMetaDataConnection extends DelegatingConnection {
+ static final String FAILURE_TEXT = "fault injected";
+
+ BadMetaDataConnection(Connection connection) {
+ super(connection);
+ }
+
+ @Override
+ public DatabaseMetaData getMetaData() throws SQLException {
+ throw new SQLException(FAILURE_TEXT);
+ }
+ }
+
+ private void execute(SchemaToolTask task, String taskArgs) throws HiveMetaException {
+ try {
+ StrTokenizer tokenizer = new StrTokenizer(argsBase + taskArgs, ' ', '\"');
+ SchemaToolCommandLine cl = new SchemaToolCommandLine(tokenizer.getTokenArray(), null);
+ task.setCommandLineArguments(cl);
+ } catch (Exception e) {
+ throw new IllegalStateException("Could not parse comman line \n" + argsBase + taskArgs, e);
+ }
+
+ task.setHiveSchemaTool(schemaTool);
+ task.execute();
}
}
[02/32] hive git commit: HIVE-18118: Explain Extended should indicate
if a file being read is an EC file (Andrew Sherman, reviewed by Sahil Takiar)
Posted by se...@apache.org.
HIVE-18118: Explain Extended should indicate if a file being read is an EC file (Andrew Sherman, reviewed by Sahil Takiar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c2940a07
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c2940a07
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c2940a07
Branch: refs/heads/master-txnstats
Commit: c2940a07cf0891e922672782b73ec22551a7eedd
Parents: e865b44
Author: Andrew Sherman <as...@cloudera.com>
Authored: Thu Jul 5 16:08:16 2018 -0500
Committer: Sahil Takiar <st...@cloudera.com>
Committed: Thu Jul 5 16:08:16 2018 -0500
----------------------------------------------------------------------
.../hive/common/util/HiveStringUtils.java | 21 +-
.../apache/hive/jdbc/TestJdbcWithMiniHS2.java | 2 +-
.../jdbc/TestJdbcWithMiniHS2ErasureCoding.java | 215 ++++++++++
.../test/resources/testconfiguration.properties | 3 +-
.../org/apache/hadoop/hive/ql/QTestUtil.java | 4 +-
.../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 23 +-
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 44 +-
.../ql/metadata/SessionHiveMetaStoreClient.java | 2 +-
.../formatting/MetaDataFormatUtils.java | 11 +-
.../formatting/TextMetaDataFormatter.java | 10 +
.../optimizer/spark/SparkMapJoinOptimizer.java | 2 +-
.../stats/annotation/StatsRulesProcFactory.java | 2 +-
.../hadoop/hive/ql/plan/PartitionDesc.java | 4 +-
.../apache/hadoop/hive/ql/plan/PlanUtils.java | 31 ++
.../apache/hadoop/hive/ql/plan/Statistics.java | 12 +-
.../apache/hadoop/hive/ql/plan/TableDesc.java | 3 +-
.../hive/ql/stats/BasicStatsNoJobTask.java | 7 +-
.../hadoop/hive/ql/stats/BasicStatsTask.java | 4 +-
.../apache/hadoop/hive/ql/stats/StatsUtils.java | 20 +-
.../hive/ql/txn/compactor/CompactorMR.java | 4 +-
.../queries/clientpositive/erasure_explain.q | 24 ++
.../queries/clientpositive/erasure_simple.q | 13 +
.../clientnegative/unset_table_property.q.out | 1 +
.../materialized_view_create_rewrite.q.out | 2 +
.../clientpositive/druid/druidmini_mv.q.out | 3 +
.../erasurecoding/erasure_explain.q.out | 409 +++++++++++++++++++
.../erasurecoding/erasure_simple.q.out | 20 +-
.../llap/materialized_view_create.q.out | 1 +
.../llap/materialized_view_create_rewrite.q.out | 2 +
...materialized_view_create_rewrite_dummy.q.out | 2 +
...erialized_view_create_rewrite_multi_db.q.out | 2 +
.../llap/materialized_view_describe.q.out | 1 +
.../clientpositive/show_tblproperties.q.out | 5 +
.../unset_table_view_property.q.out | 9 +
.../hadoop/hive/common/StatsSetupConst.java | 21 +-
.../hadoop/hive/metastore/HiveMetaStore.java | 3 +-
.../hive/metastore/utils/MetaStoreUtils.java | 11 +-
.../metastore/utils/TestMetaStoreUtils.java | 37 +-
38 files changed, 877 insertions(+), 113 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/common/src/java/org/apache/hive/common/util/HiveStringUtils.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hive/common/util/HiveStringUtils.java b/common/src/java/org/apache/hive/common/util/HiveStringUtils.java
index cfe9b22..6b14ad9 100644
--- a/common/src/java/org/apache/hive/common/util/HiveStringUtils.java
+++ b/common/src/java/org/apache/hive/common/util/HiveStringUtils.java
@@ -31,19 +31,15 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
-import java.util.HashMap;
import java.util.Locale;
-import java.util.Properties;
+import java.util.Map;
import java.util.StringTokenizer;
import java.util.regex.Pattern;
import com.google.common.base.Splitter;
-import com.google.common.collect.Interner;
-import com.google.common.collect.Interners;
-
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.text.translate.CharSequenceTranslator;
import org.apache.commons.lang3.text.translate.EntityArrays;
@@ -1062,19 +1058,6 @@ public class HiveStringUtils {
return identifier.trim().toLowerCase();
}
- public static Map getPropertiesExplain(Properties properties) {
- if (properties != null) {
- String value = properties.getProperty("columns.comments");
- if (value != null) {
- // should copy properties first
- Map clone = new HashMap(properties);
- clone.put("columns.comments", quoteComments(value));
- return clone;
- }
- }
- return properties;
- }
-
public static String quoteComments(String value) {
char[] chars = value.toCharArray();
if (!commentProvided(chars)) {
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
index d7d7097..7ef2ced 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
@@ -1658,7 +1658,7 @@ public class TestJdbcWithMiniHS2 {
/**
* Get Detailed Table Information via jdbc
*/
- private String getDetailedTableDescription(Statement stmt, String table) throws SQLException {
+ static String getDetailedTableDescription(Statement stmt, String table) throws SQLException {
String extendedDescription = null;
try (ResultSet rs = stmt.executeQuery("describe extended " + table)) {
while (rs.next()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2ErasureCoding.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2ErasureCoding.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2ErasureCoding.java
new file mode 100644
index 0000000..b0a0145
--- /dev/null
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2ErasureCoding.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.jdbc;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Collections;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.shims.HadoopShims;
+import org.apache.hadoop.hive.shims.HadoopShims.HdfsErasureCodingShim;
+import org.apache.hadoop.hive.shims.HadoopShims.MiniDFSShim;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hive.jdbc.miniHS2.MiniHS2;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.apache.hadoop.hive.ql.QTestUtil.DEFAULT_TEST_EC_POLICY;
+import static org.apache.hive.jdbc.TestJdbcWithMiniHS2.getDetailedTableDescription;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Run erasure Coding tests with jdbc.
+ */
+public class TestJdbcWithMiniHS2ErasureCoding {
+ private static final String DB_NAME = "ecTestDb";
+ private static MiniHS2 miniHS2 = null;
+ private static HiveConf conf;
+ private Connection hs2Conn = null;
+
+ private static HiveConf createHiveOnSparkConf() {
+ HiveConf hiveConf = new HiveConf();
+ // Tell dfs not to consider load when choosing a datanode as this can cause failure as
+ // in a test we do not have spare datanode capacity.
+ hiveConf.setBoolean("dfs.namenode.redundancy.considerLoad", false);
+ hiveConf.set("hive.execution.engine", "spark");
+ hiveConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
+ hiveConf.set("spark.master", "local-cluster[2,2,1024]");
+ hiveConf.set("hive.spark.client.connect.timeout", "30000ms");
+ hiveConf.set("spark.local.dir",
+ Paths.get(System.getProperty("test.tmp.dir"), "TestJdbcWithMiniHS2ErasureCoding-local-dir")
+ .toString());
+ hiveConf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false); // avoid ZK errors
+ return hiveConf;
+ }
+
+ /**
+ * Setup a mini HS2 with miniMR.
+ */
+ @BeforeClass
+ public static void beforeTest() throws Exception {
+ Class.forName(MiniHS2.getJdbcDriverName());
+ conf = createHiveOnSparkConf();
+ DriverManager.setLoginTimeout(0);
+ miniHS2 = new MiniHS2.Builder()
+ .withConf(conf)
+ .withMiniMR()
+ .withDataNodes(5) // sufficient for RS-3-2-1024k
+ .build();
+ miniHS2.start(Collections.emptyMap());
+ createDb();
+ MiniDFSShim dfs = miniHS2.getDfs();
+ addErasurePolicy(dfs, "hdfs:///", DEFAULT_TEST_EC_POLICY);
+ }
+
+ /**
+ * Shutdown the mini HS2.
+ */
+ @AfterClass
+ public static void afterTest() {
+ if (miniHS2 != null && miniHS2.isStarted()) {
+ miniHS2.stop();
+ }
+ }
+
+ /**
+ * Setup a connection to the test database before each test.
+ */
+ @Before
+ public void setUp() throws Exception {
+ hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(DB_NAME),
+ System.getProperty("user.name"), "bar");
+ }
+
+ /**
+ * Close connection after each test.
+ */
+ @After
+ public void tearDown() throws Exception {
+ if (hs2Conn != null) {
+ hs2Conn.close();
+ }
+ }
+
+ /**
+ * Create a database.
+ */
+ private static void createDb() throws Exception {
+ try (Connection conn = DriverManager.getConnection(miniHS2.getJdbcURL(),
+ System.getProperty("user.name"), "bar");
+ Statement stmt2 = conn.createStatement()) {
+ stmt2.execute("DROP DATABASE IF EXISTS " + DB_NAME + " CASCADE");
+ stmt2.execute("CREATE DATABASE " + DB_NAME);
+ }
+ }
+
+ /**
+ * Test EXPLAIN on fs with Erasure Coding.
+ */
+ @Test
+ public void testExplainErasureCoding() throws Exception {
+ try (Statement stmt = hs2Conn.createStatement()) {
+ String tableName = "pTableEc";
+ stmt.execute(
+ " CREATE TABLE " + tableName + " (userid VARCHAR(64), link STRING, source STRING) "
+ + "PARTITIONED BY (datestamp STRING, i int) "
+ + "CLUSTERED BY (userid) INTO 4 BUCKETS STORED AS PARQUET");
+ // insert data to create 2 partitions
+ stmt.execute("INSERT INTO TABLE " + tableName
+ + " PARTITION (datestamp = '2014-09-23', i = 1)(userid,link) VALUES ('jsmith', 'mail.com')");
+ stmt.execute("INSERT INTO TABLE " + tableName
+ + " PARTITION (datestamp = '2014-09-24', i = 2)(userid,link) VALUES ('mac', 'superchunk.com')");
+ String explain = getExtendedExplain(stmt, "select userid from " + tableName);
+ assertMatchAndCount(explain, " numFiles 4", 2);
+ assertMatchAndCount(explain, " numFilesErasureCoded 4", 2);
+ }
+ }
+
+ /**
+ * Test DESCRIBE on fs with Erasure Coding.
+ */
+ @Test
+ public void testDescribeErasureCoding() throws Exception {
+ try (Statement stmt = hs2Conn.createStatement()) {
+ String table = "pageviews";
+ stmt.execute(" CREATE TABLE " + table + " (userid VARCHAR(64), link STRING, source STRING) "
+ + "PARTITIONED BY (datestamp STRING, i int) CLUSTERED BY (userid) INTO 4 BUCKETS STORED AS PARQUET");
+ stmt.execute("INSERT INTO TABLE " + table + " PARTITION (datestamp = '2014-09-23', i = 1)"
+ + "(userid,link) VALUES ('jsmith', 'mail.com')");
+ stmt.execute("INSERT INTO TABLE " + table + " PARTITION (datestamp = '2014-09-24', i = 1)"
+ + "(userid,link) VALUES ('dpatel', 'gmail.com')");
+ String description = getDetailedTableDescription(stmt, table);
+ assertMatchAndCount(description, "numFiles=8", 1);
+ assertMatchAndCount(description, "numFilesErasureCoded=8", 1);
+ assertMatchAndCount(description, "numPartitions=2", 1);
+ }
+ }
+
+ /**
+ * Add a Erasure Coding Policy to a Path.
+ */
+ private static void addErasurePolicy(MiniDFSShim dfs, String pathString, String policyName) throws IOException {
+ HadoopShims hadoopShims = ShimLoader.getHadoopShims();
+ HdfsErasureCodingShim erasureCodingShim = hadoopShims.createHdfsErasureCodingShim(dfs.getFileSystem(), conf);
+ erasureCodingShim.enableErasureCodingPolicy(policyName);
+ Path fsRoot = new Path(pathString);
+ erasureCodingShim.setErasureCodingPolicy(fsRoot, policyName);
+ HadoopShims.HdfsFileErasureCodingPolicy erasureCodingPolicy =
+ erasureCodingShim.getErasureCodingPolicy(fsRoot);
+ assertEquals(policyName, erasureCodingPolicy.getName());
+ }
+
+ /**
+ * Get Extended Explain output via jdbc.
+ */
+ private static String getExtendedExplain(Statement stmt, String query) throws SQLException {
+ StringBuilder sb = new StringBuilder(2048);
+ try (ResultSet rs = stmt.executeQuery("explain extended " + query)) {
+ while (rs.next()) {
+ sb.append(rs.getString(1)).append('\n');
+ }
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Check that the expected string occurs correctly in the output string.
+ * @param output string to probe
+ * @param expectedString string to find in output
+ * @param expectedCount the expected number of occurrences of the expected string
+ */
+ private void assertMatchAndCount(String output, String expectedString, int expectedCount) {
+ assertTrue("Did not find expected '" + expectedString + "' in text " +
+ output, output.contains(expectedString));
+ assertEquals("wrong count of matches of '" + expectedString + "' in text " +
+ output, expectedCount, StringUtils.countMatches(output, expectedString));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 978a806..fedb77b 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -1714,4 +1714,5 @@ druid.kafka.query.files=druidkafkamini_basic.q
erasurecoding.shared.query.files=erasure_commands.q
# tests to be run only by TestErasureCodingHDFSCliDriver
-erasurecoding.only.query.files=erasure_simple.q
+erasurecoding.only.query.files=erasure_simple.q,\
+ erasure_explain.q
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 2dfd2aa..98aae5c 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -150,9 +150,9 @@ public class QTestUtil {
public static final String TEST_HIVE_USER_PROPERTY = "test.hive.user";
/**
- * The Erasure Coding Policy to use in TestErasureCodingHDFSCliDriver.
+ * The default Erasure Coding Policy to use in Erasure Coding tests.
*/
- private static final String DEFAULT_TEST_EC_POLICY = "RS-3-2-1024k";
+ public static final String DEFAULT_TEST_EC_POLICY = "RS-3-2-1024k";
private String testWarehouse;
@Deprecated
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
index 1700c08..a78dd73 100644
--- a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
+++ b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
@@ -66,6 +66,7 @@ public class MiniHS2 extends AbstractHiveService {
private static final FsPermission FULL_PERM = new FsPermission((short)00777);
private static final FsPermission WRITE_ALL_PERM = new FsPermission((short)00733);
private static final String tmpDir = System.getProperty("test.tmp.dir");
+ private static final int DEFAULT_DATANODE_COUNT = 4;
private HiveServer2 hiveServer2 = null;
private final File baseDir;
private final Path baseFsDir;
@@ -104,6 +105,7 @@ public class MiniHS2 extends AbstractHiveService {
private boolean isMetastoreSecure;
private String metastoreServerPrincipal;
private String metastoreServerKeyTab;
+ private int dataNodes = DEFAULT_DATANODE_COUNT; // default number of datanodes for miniHS2
public Builder() {
}
@@ -162,6 +164,16 @@ public class MiniHS2 extends AbstractHiveService {
return this;
}
+ /**
+ * Set the number of datanodes to be used by HS2.
+ * @param count the number of datanodes
+ * @return this Builder
+ */
+ public Builder withDataNodes(int count) {
+ this.dataNodes = count;
+ return this;
+ }
+
public MiniHS2 build() throws Exception {
if (miniClusterType == MiniClusterType.MR && useMiniKdc) {
throw new IOException("Can't create secure miniMr ... yet");
@@ -173,7 +185,7 @@ public class MiniHS2 extends AbstractHiveService {
}
return new MiniHS2(hiveConf, miniClusterType, useMiniKdc, serverPrincipal, serverKeytab,
isMetastoreRemote, usePortsFromConf, authType, isHA, cleanupLocalDirOnStartup,
- isMetastoreSecure, metastoreServerPrincipal, metastoreServerKeyTab);
+ isMetastoreSecure, metastoreServerPrincipal, metastoreServerKeyTab, dataNodes);
}
}
@@ -212,9 +224,8 @@ public class MiniHS2 extends AbstractHiveService {
private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useMiniKdc,
String serverPrincipal, String serverKeytab, boolean isMetastoreRemote,
boolean usePortsFromConf, String authType, boolean isHA, boolean cleanupLocalDirOnStartup,
- boolean isMetastoreSecure,
- String metastoreServerPrincipal,
- String metastoreKeyTab) throws Exception {
+ boolean isMetastoreSecure, String metastoreServerPrincipal, String metastoreKeyTab,
+ int dataNodes) throws Exception {
// Always use localhost for hostname as some tests like SSL CN validation ones
// are tied to localhost being present in the certificate name
super(
@@ -242,7 +253,7 @@ public class MiniHS2 extends AbstractHiveService {
if (miniClusterType != MiniClusterType.LOCALFS_ONLY) {
// Initialize dfs
- dfs = ShimLoader.getHadoopShims().getMiniDfs(hiveConf, 4, true, null, isHA);
+ dfs = ShimLoader.getHadoopShims().getMiniDfs(hiveConf, dataNodes, true, null, isHA);
fs = dfs.getFileSystem();
String uriString = fs.getUri().toString();
@@ -334,7 +345,7 @@ public class MiniHS2 extends AbstractHiveService {
throws Exception {
this(hiveConf, clusterType, false, null, null,
false, usePortsFromConf, "KERBEROS", false, true,
- false, null, null);
+ false, null, null, DEFAULT_DATANODE_COUNT);
}
public void start(Map<String, String> confOverlay) throws Exception {
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index b7babd6..ba0070d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -60,16 +60,12 @@ import com.google.common.util.concurrent.ListenableFuture;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
-import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.common.StatsSetupConst;
-import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.common.ValidTxnWriteIdList;
-import org.apache.hadoop.hive.common.ValidWriteIdList;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.conf.Constants;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -293,39 +289,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.stringtemplate.v4.ST;
-import java.io.BufferedWriter;
-import java.io.DataOutputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.io.Serializable;
-import java.io.Writer;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.charset.StandardCharsets;
-import java.sql.SQLException;
-import java.util.AbstractList;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.concurrent.ExecutionException;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
import static org.apache.commons.lang.StringUtils.join;
-import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE;
/**
* DDLTask implementation.
@@ -2713,7 +2677,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
String tbl_location = " '" + HiveStringUtils.escapeHiveCommand(sd.getLocation()) + "'";
// Table properties
- duplicateProps.addAll(Arrays.asList(StatsSetupConst.TABLE_PARAMS_STATS_KEYS));
+ duplicateProps.addAll(StatsSetupConst.TABLE_PARAMS_STATS_KEYS);
String tbl_properties = propertiesToString(tbl.getParameters(), duplicateProps);
createTab_stmt.add(TEMPORARY, tbl_temp);
@@ -3679,7 +3643,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
Map<String,String> tblProps = tbl.getParameters() == null ? new HashMap<String,String>() : tbl.getParameters();
Map<String, Long> valueMap = new HashMap<>();
Map<String, Boolean> stateMap = new HashMap<>();
- for (String stat : StatsSetupConst.supportedStats) {
+ for (String stat : StatsSetupConst.SUPPORTED_STATS) {
valueMap.put(stat, 0L);
stateMap.put(stat, true);
}
@@ -3688,7 +3652,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
for (Partition partition : parts) {
Map<String, String> props = partition.getParameters();
Boolean state = StatsSetupConst.areBasicStatsUptoDate(props);
- for (String stat : StatsSetupConst.supportedStats) {
+ for (String stat : StatsSetupConst.SUPPORTED_STATS) {
stateMap.put(stat, stateMap.get(stat) && state);
if (props != null && props.get(stat) != null) {
valueMap.put(stat, valueMap.get(stat) + Long.parseLong(props.get(stat)));
@@ -3696,7 +3660,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
}
numParts++;
}
- for (String stat : StatsSetupConst.supportedStats) {
+ for (String stat : StatsSetupConst.SUPPORTED_STATS) {
StatsSetupConst.setBasicStatsState(tblProps, Boolean.toString(stateMap.get(stat)));
tblProps.put(stat, valueMap.get(stat).toString());
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
index 58c8960..5d382ae 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
@@ -593,7 +593,7 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I
return false;
}
boolean statsPresent = false;
- for (String stat : StatsSetupConst.supportedStats) {
+ for (String stat : StatsSetupConst.SUPPORTED_STATS) {
String statVal = props.get(stat);
if (statVal != null && Long.parseLong(statVal) > 0) {
statsPresent = true;
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java
index 7af6dab..36cd46a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql.metadata.formatting;
import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
@@ -474,10 +475,16 @@ public final class MetaDataFormatUtils {
List<String> keys = new ArrayList<String>(params.keySet());
Collections.sort(keys);
for (String key : keys) {
+ String value = params.get(key);
+ if (key.equals(StatsSetupConst.NUM_ERASURE_CODED_FILES)) {
+ if ("0".equals(value)) {
+ continue;
+ }
+ }
tableInfo.append(FIELD_DELIM); // Ensures all params are indented.
formatOutput(key,
- escapeUnicode ? StringEscapeUtils.escapeJava(params.get(key))
- : HiveStringUtils.escapeJava(params.get(key)),
+ escapeUnicode ? StringEscapeUtils.escapeJava(value)
+ : HiveStringUtils.escapeJava(value),
tableInfo, isOutputPadded);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java
index 326cbed..705365b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java
@@ -366,6 +366,7 @@ class TextMetaDataFormatter implements MetaDataFormatter {
public long lastAccessTime = 0;
public long lastUpdateTime = 0;
public int numOfFiles = 0;
+ int numOfErasureCodedFiles = 0;
}
// TODO: why is this in text formatter?!!
@@ -416,6 +417,12 @@ class TextMetaDataFormatter implements MetaDataFormatter {
outStream.write((unknown ? unknownString : "" + fd.numOfFiles).getBytes("UTF-8"));
outStream.write(terminator);
+ if (fd.numOfErasureCodedFiles > 0) {
+ outStream.write("totalNumberErasureCodedFiles:".getBytes("UTF-8"));
+ outStream.write((unknown ? unknownString : "" + fd.numOfErasureCodedFiles).getBytes("UTF-8"));
+ outStream.write(terminator);
+ }
+
for (int k = 0; k < indent; k++) {
outStream.write(Utilities.INDENT.getBytes("UTF-8"));
}
@@ -473,6 +480,9 @@ class TextMetaDataFormatter implements MetaDataFormatter {
continue;
}
fd.numOfFiles++;
+ if (currentStatus.isErasureCoded()) {
+ fd.numOfErasureCodedFiles++;
+ }
long fileLen = currentStatus.getLen();
fd.totalFileSize += fileLen;
if (fileLen > fd.maxFileSize) {
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java
index 8e75db9..689c888 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java
@@ -216,7 +216,7 @@ public class SparkMapJoinOptimizer implements NodeProcessor {
LOG.debug("Found a big table branch with parent operator {} and position {}", parentOp, pos);
bigTablePosition = pos;
bigTableFound = true;
- bigInputStat = new Statistics(0, Long.MAX_VALUE);
+ bigInputStat = new Statistics(0, Long.MAX_VALUE, 0);
} else {
// Either we've found multiple big table branches, or the current branch cannot
// be a big table branch. Disable mapjoin for these cases.
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
index d0be33b..3c2b085 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
@@ -1818,7 +1818,7 @@ public class StatsRulesProcFactory {
}
}
- Statistics wcStats = new Statistics(newNumRows, newDataSize);
+ Statistics wcStats = new Statistics(newNumRows, newDataSize, 0);
wcStats.setBasicStatsState(statsState);
// evaluate filter expression and update statistics
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
index 61458b4..821e428 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
@@ -45,10 +45,8 @@ import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.OutputFormat;
import org.apache.hive.common.util.ReflectionUtil;
-import org.apache.hive.common.util.HiveStringUtils;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
-
/**
* PartitionDesc.
*
@@ -221,7 +219,7 @@ public class PartitionDesc implements Serializable, Cloneable {
@Explain(displayName = "properties", explainLevels = { Level.EXTENDED })
public Map getPropertiesExplain() {
- return HiveStringUtils.getPropertiesExplain(getProperties());
+ return PlanUtils.getPropertiesExplain(getProperties());
}
public void setProperties(final Properties properties) {
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
index 2c5b655..250a085 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -31,6 +32,7 @@ import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.JavaUtils;
+import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.llap.LlapOutputFormat;
@@ -78,6 +80,7 @@ import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.hadoop.mapred.TextInputFormat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.hive.common.util.HiveStringUtils.quoteComments;
/**
* PlanUtils.
@@ -1203,4 +1206,32 @@ public final class PlanUtils {
public static Class<? extends AbstractSerDe> getDefaultSerDe() {
return LazySimpleSerDe.class;
}
+
+ /**
+ * Get a Map of table or partition properties to be used in explain extended output.
+ * Do some filtering to make output readable and/or concise.
+ */
+ static Map getPropertiesExplain(Properties properties) {
+ if (properties != null) {
+ Map<Object, Object> clone = null;
+ String value = properties.getProperty("columns.comments");
+ if (value != null) {
+ // should copy properties first
+ clone = new HashMap<>(properties);
+ clone.put("columns.comments", quoteComments(value));
+ }
+ value = properties.getProperty(StatsSetupConst.NUM_ERASURE_CODED_FILES);
+ if ("0".equals(value)) {
+ // should copy properties first
+ if (clone == null) {
+ clone = new HashMap<>(properties);
+ }
+ clone.remove(StatsSetupConst.NUM_ERASURE_CODED_FILES);
+ }
+ if (clone != null) {
+ return clone;
+ }
+ }
+ return properties;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java
index 6babe49..bc5f9d9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java
@@ -53,18 +53,20 @@ public class Statistics implements Serializable {
private long numRows;
private long runTimeNumRows;
private long dataSize;
+ private long numErasureCodedFiles;
private State basicStatsState;
private Map<String, ColStatistics> columnStats;
private State columnStatsState;
private boolean runtimeStats;
public Statistics() {
- this(0, 0);
+ this(0, 0, 0);
}
- public Statistics(long nr, long ds) {
+ public Statistics(long nr, long ds, long numEcFiles) {
numRows = nr;
dataSize = ds;
+ numErasureCodedFiles = numEcFiles;
runTimeNumRows = -1;
columnStats = null;
columnStatsState = State.NONE;
@@ -137,6 +139,10 @@ public class Statistics implements Serializable {
}
sb.append(" Data size: ");
sb.append(dataSize);
+ if (numErasureCodedFiles > 0) {
+ sb.append(" Erasure files: ");
+ sb.append(numErasureCodedFiles);
+ }
sb.append(" Basic stats: ");
sb.append(basicStatsState);
sb.append(" Column stats: ");
@@ -185,7 +191,7 @@ public class Statistics implements Serializable {
@Override
public Statistics clone() {
- Statistics clone = new Statistics(numRows, dataSize);
+ Statistics clone = new Statistics(numRows, dataSize, numErasureCodedFiles);
clone.setRunTimeNumRows(runTimeNumRows);
clone.setBasicStatsState(basicStatsState);
clone.setColumnStatsState(columnStatsState);
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java
index 4068e56..bbce940 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.OutputFormat;
-import org.apache.hive.common.util.HiveStringUtils;
import org.apache.hive.common.util.ReflectionUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -127,7 +126,7 @@ public class TableDesc implements Serializable, Cloneable {
@Explain(displayName = "properties", explainLevels = { Level.EXTENDED })
public Map getPropertiesExplain() {
- return HiveStringUtils.getPropertiesExplain(getProperties());
+ return PlanUtils.getPropertiesExplain(getProperties());
}
public void setProperties(final Properties properties) {
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java
index d4d46a3..3128ee8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java
@@ -163,6 +163,7 @@ public class BasicStatsNoJobTask implements IStatsProcessor {
long rawDataSize = 0;
long fileSize = 0;
long numFiles = 0;
+ long numErasureCodedFiles = 0;
// Note: this code would be invalid for transactional tables of any kind.
Utilities.FILE_OP_LOGGER.debug("Aggregating stats for {}", dir);
List<FileStatus> fileList = null;
@@ -190,6 +191,9 @@ public class BasicStatsNoJobTask implements IStatsProcessor {
numRows += statsRR.getStats().getRowCount();
fileSize += file.getLen();
numFiles += 1;
+ if (file.isErasureCoded()) {
+ numErasureCodedFiles++;
+ }
} else {
throw new HiveException(String.format("Unexpected file found during reading footers for: %s ", file));
}
@@ -206,6 +210,7 @@ public class BasicStatsNoJobTask implements IStatsProcessor {
parameters.put(StatsSetupConst.RAW_DATA_SIZE, String.valueOf(rawDataSize));
parameters.put(StatsSetupConst.TOTAL_SIZE, String.valueOf(fileSize));
parameters.put(StatsSetupConst.NUM_FILES, String.valueOf(numFiles));
+ parameters.put(StatsSetupConst.NUM_ERASURE_CODED_FILES, String.valueOf(numErasureCodedFiles));
if (partish.getPartition() != null) {
result = new Partition(partish.getTable(), partish.getPartition().getTPartition());
@@ -224,7 +229,7 @@ public class BasicStatsNoJobTask implements IStatsProcessor {
private String toString(Map<String, String> parameters) {
StringBuilder builder = new StringBuilder();
- for (String statType : StatsSetupConst.supportedStats) {
+ for (String statType : StatsSetupConst.SUPPORTED_STATS) {
String value = parameters.get(statType);
if (value != null) {
if (builder.length() > 0) {
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java
index f31c170..0db90b0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java
@@ -207,7 +207,7 @@ public class BasicStatsTask implements Serializable, IStatsProcessor {
private void updateStats(StatsAggregator statsAggregator, Map<String, String> parameters,
String aggKey) throws HiveException {
- for (String statType : StatsSetupConst.statsRequireCompute) {
+ for (String statType : StatsSetupConst.STATS_REQUIRE_COMPUTE) {
String value = statsAggregator.aggregateStats(aggKey, statType);
if (value != null && !value.isEmpty()) {
long longValue = Long.parseLong(value);
@@ -411,7 +411,7 @@ public class BasicStatsTask implements Serializable, IStatsProcessor {
private String toString(Map<String, String> parameters) {
StringBuilder builder = new StringBuilder();
- for (String statType : StatsSetupConst.supportedStats) {
+ for (String statType : StatsSetupConst.SUPPORTED_STATS) {
String value = parameters.get(statType);
if (value != null) {
if (builder.length() > 0) {
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
index 494939a..95a4440 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
@@ -261,6 +261,8 @@ public class StatsUtils {
long nr = basicStats.getNumRows();
List<ColStatistics> colStats = Lists.newArrayList();
+ long numErasureCodedFiles = getErasureCodedFiles(table);
+
if (fetchColStats) {
colStats = getTableColumnStats(table, schema, neededColumns, colStatsCache);
if(colStats == null) {
@@ -273,7 +275,7 @@ public class StatsUtils {
long betterDS = getDataSizeFromColumnStats(nr, colStats);
ds = (betterDS < 1 || colStats.isEmpty()) ? ds : betterDS;
}
- stats = new Statistics(nr, ds);
+ stats = new Statistics(nr, ds, numErasureCodedFiles);
// infer if any column can be primary key based on column statistics
inferAndSetPrimaryKey(stats.getNumRows(), colStats);
@@ -308,10 +310,14 @@ public class StatsUtils {
long nr = bbs.getNumRows();
long ds = bbs.getDataSize();
+ List<Long> erasureCodedFiles = getBasicStatForPartitions(table, partList.getNotDeniedPartns(),
+ StatsSetupConst.NUM_ERASURE_CODED_FILES);
+ long numErasureCodedFiles = getSumIgnoreNegatives(erasureCodedFiles);
+
if (nr == 0) {
nr=1;
}
- stats = new Statistics(nr, ds);
+ stats = new Statistics(nr, ds, numErasureCodedFiles);
stats.setBasicStatsState(bbs.getState());
if (nr > 0) {
// FIXME: this promotion process should be removed later
@@ -1656,6 +1662,14 @@ public class StatsUtils {
}
/**
+ * Get number of Erasure Coded files for a table
+ * @return count of EC files
+ */
+ public static long getErasureCodedFiles(Table table) {
+ return getBasicStatForTable(table, StatsSetupConst.NUM_ERASURE_CODED_FILES);
+ }
+
+ /**
* Get basic stats of table
* @param table
* - table
@@ -1782,7 +1796,7 @@ public class StatsUtils {
}
/**
- * Get qualified column name from output key column names
+ * Get qualified column name from output key column names.
* @param keyExprs
* - output key names
* @return list of qualified names
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
index 6044719..611f85a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
@@ -21,8 +21,6 @@ import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.HashSet;
-import com.google.common.collect.Lists;
-
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
@@ -469,7 +467,7 @@ public class CompactorMR {
HiveStringUtils.escapeHiveCommand(location)).append("' TBLPROPERTIES (");
// Exclude all standard table properties.
Set<String> excludes = getHiveMetastoreConstants();
- excludes.addAll(Lists.newArrayList(StatsSetupConst.TABLE_PARAMS_STATS_KEYS));
+ excludes.addAll(StatsSetupConst.TABLE_PARAMS_STATS_KEYS);
isFirst = true;
for (Map.Entry<String, String> e : t.getParameters().entrySet()) {
if (e.getValue() == null) continue;
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/test/queries/clientpositive/erasure_explain.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/erasure_explain.q b/ql/src/test/queries/clientpositive/erasure_explain.q
new file mode 100644
index 0000000..e2954d4
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/erasure_explain.q
@@ -0,0 +1,24 @@
+--! qt:dataset:src
+--! qt:dataset:srcpart
+-- Test explain diagnostics with Erasure Coding
+
+ERASURE echo listPolicies originally was;
+ERASURE listPolicies;
+
+show table extended like srcpart;
+
+desc formatted srcpart;
+
+explain select key, value from srcpart;
+
+explain extended select key, value from srcpart;
+
+show table extended like src;
+
+desc formatted src;
+
+explain select key, value from src;
+
+explain extended select key, value from src;
+
+
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/test/queries/clientpositive/erasure_simple.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/erasure_simple.q b/ql/src/test/queries/clientpositive/erasure_simple.q
index c08409c..cc886c2 100644
--- a/ql/src/test/queries/clientpositive/erasure_simple.q
+++ b/ql/src/test/queries/clientpositive/erasure_simple.q
@@ -5,6 +5,7 @@
ERASURE echo listPolicies originally was;
ERASURE listPolicies;
ERASURE enablePolicy --policy RS-10-4-1024k;
+ERASURE enablePolicy --policy XOR-2-1-1024k;
ERASURE echo listPolicies after enablePolicy;
ERASURE listPolicies;
@@ -25,8 +26,20 @@ ERASURE getPolicy --path hdfs:///tmp/erasure_coding1;
create table erasure_table (a int) location 'hdfs:///tmp/erasure_coding1/location1';
+-- insert some data with the default policy (RS-3-2-1024k) from the fs root
insert into erasure_table values(4);
+
+-- set a new policy on the directory and insert some data
+ERASURE setPolicy --path hdfs:///tmp/erasure_coding1 --policy XOR-2-1-1024k;
+insert into erasure_table values(5);
+
+ERASURE echo policy on older file is;
+ERASURE getPolicy --path hdfs:///tmp/erasure_coding1/location1/000000_0;
+ERASURE echo policy on newer file is;
+ERASURE getPolicy --path hdfs:///tmp/erasure_coding1/location1/000000_0_copy_1;
+
+-- show that data is present
select * from erasure_table;
drop table if exists erasure_table2;
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/test/results/clientnegative/unset_table_property.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/unset_table_property.q.out b/ql/src/test/results/clientnegative/unset_table_property.q.out
index eb308eb..20378a1 100644
--- a/ql/src/test/results/clientnegative/unset_table_property.q.out
+++ b/ql/src/test/results/clientnegative/unset_table_property.q.out
@@ -24,6 +24,7 @@ bucketing_version 2
c 3
#### A masked pattern was here ####
numFiles 0
+numFilesErasureCoded 0
numRows 0
rawDataSize 0
totalSize 0
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out b/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out
index ff02643..4db5d70 100644
--- a/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out
+++ b/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out
@@ -67,6 +67,7 @@ POSTHOOK: type: SHOW_TBLPROPERTIES
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucketing_version 2
numFiles 1
+numFilesErasureCoded 0
numRows 2
rawDataSize 408
totalSize 457
@@ -100,6 +101,7 @@ POSTHOOK: type: SHOW_TBLPROPERTIES
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucketing_version 2
numFiles 1
+numFilesErasureCoded 0
numRows 2
rawDataSize 232
totalSize 326
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out b/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out
index 2e44e14..383f2dc 100644
--- a/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out
+++ b/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out
@@ -89,6 +89,7 @@ bucketing_version 2
druid.datasource default.cmv_mat_view_n2
druid.segment.granularity HOUR
numFiles 0
+numFilesErasureCoded 0
numRows 2
rawDataSize 0
storage_handler org.apache.hadoop.hive.druid.DruidStorageHandler
@@ -136,6 +137,7 @@ bucketing_version 2
druid.datasource default.cmv_mat_view2_n0
druid.segment.granularity HOUR
numFiles 0
+numFilesErasureCoded 0
numRows 3
rawDataSize 0
storage_handler org.apache.hadoop.hive.druid.DruidStorageHandler
@@ -515,6 +517,7 @@ druid.datasource default.cmv_mat_view2_n0
druid.segment.granularity HOUR
#### A masked pattern was here ####
numFiles 0
+numFilesErasureCoded 0
numRows 3
rawDataSize 0
storage_handler org.apache.hadoop.hive.druid.DruidStorageHandler
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/test/results/clientpositive/erasurecoding/erasure_explain.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/erasurecoding/erasure_explain.q.out b/ql/src/test/results/clientpositive/erasurecoding/erasure_explain.q.out
new file mode 100644
index 0000000..8ada9b6
--- /dev/null
+++ b/ql/src/test/results/clientpositive/erasurecoding/erasure_explain.q.out
@@ -0,0 +1,409 @@
+ECHO listPolicies originally was
+Policy: RS-10-4-1024k DISABLED
+Policy: RS-3-2-1024k ENABLED
+Policy: RS-6-3-1024k ENABLED
+Policy: RS-LEGACY-6-3-1024k DISABLED
+Policy: XOR-2-1-1024k DISABLED
+PREHOOK: query: show table extended like srcpart
+PREHOOK: type: SHOW_TABLESTATUS
+POSTHOOK: query: show table extended like srcpart
+POSTHOOK: type: SHOW_TABLESTATUS
+tableName:srcpart
+#### A masked pattern was here ####
+location:hdfs://### HDFS PATH ###
+inputformat:org.apache.hadoop.mapred.TextInputFormat
+outputformat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+columns:struct columns { string key, string value}
+partitioned:true
+partitionColumns:struct partition_columns { string ds, string hr}
+totalNumberFiles:4
+totalNumberErasureCodedFiles:4
+totalFileSize:23248
+maxFileSize:5812
+minFileSize:5812
+#### A masked pattern was here ####
+
+PREHOOK: query: desc formatted srcpart
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@srcpart
+POSTHOOK: query: desc formatted srcpart
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@srcpart
+# col_name data_type comment
+key string default
+value string default
+
+# Partition Information
+# col_name data_type comment
+ds string
+hr string
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ bucketing_version 2
+ numFiles 4
+ numFilesErasureCoded 4
+ numPartitions 4
+ numRows 2000
+ rawDataSize 21248
+ totalSize 23248
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: explain select key, value from srcpart
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select key, value from srcpart
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ TableScan
+ alias: srcpart
+ Statistics: Num rows: 2000 Data size: 21248 Erasure files: 4 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2000 Data size: 21248 Erasure files: 4 Basic stats: COMPLETE Column stats: NONE
+ ListSink
+
+PREHOOK: query: explain extended select key, value from srcpart
+PREHOOK: type: QUERY
+POSTHOOK: query: explain extended select key, value from srcpart
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Partition Description:
+ Partition
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 2008-04-08
+ hr 11
+ properties:
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
+ bucket_count -1
+ column.name.delimiter ,
+ columns key,value
+ columns.comments 'default','default'
+ columns.types string:string
+#### A masked pattern was here ####
+ location hdfs://### HDFS PATH ###
+ name default.srcpart
+ numFiles 1
+ numFilesErasureCoded 1
+ numRows 500
+ partition_columns ds/hr
+ partition_columns.types string:string
+ rawDataSize 5312
+ serialization.ddl struct srcpart { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ bucketing_version 2
+ column.name.delimiter ,
+ columns key,value
+ columns.comments 'default','default'
+ columns.types string:string
+#### A masked pattern was here ####
+ location hdfs://### HDFS PATH ###
+ name default.srcpart
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.ddl struct srcpart { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.srcpart
+ name: default.srcpart
+ Partition
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 2008-04-08
+ hr 12
+ properties:
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
+ bucket_count -1
+ column.name.delimiter ,
+ columns key,value
+ columns.comments 'default','default'
+ columns.types string:string
+#### A masked pattern was here ####
+ location hdfs://### HDFS PATH ###
+ name default.srcpart
+ numFiles 1
+ numFilesErasureCoded 1
+ numRows 500
+ partition_columns ds/hr
+ partition_columns.types string:string
+ rawDataSize 5312
+ serialization.ddl struct srcpart { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ bucketing_version 2
+ column.name.delimiter ,
+ columns key,value
+ columns.comments 'default','default'
+ columns.types string:string
+#### A masked pattern was here ####
+ location hdfs://### HDFS PATH ###
+ name default.srcpart
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.ddl struct srcpart { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.srcpart
+ name: default.srcpart
+ Partition
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 2008-04-09
+ hr 11
+ properties:
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
+ bucket_count -1
+ column.name.delimiter ,
+ columns key,value
+ columns.comments 'default','default'
+ columns.types string:string
+#### A masked pattern was here ####
+ location hdfs://### HDFS PATH ###
+ name default.srcpart
+ numFiles 1
+ numFilesErasureCoded 1
+ numRows 500
+ partition_columns ds/hr
+ partition_columns.types string:string
+ rawDataSize 5312
+ serialization.ddl struct srcpart { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ bucketing_version 2
+ column.name.delimiter ,
+ columns key,value
+ columns.comments 'default','default'
+ columns.types string:string
+#### A masked pattern was here ####
+ location hdfs://### HDFS PATH ###
+ name default.srcpart
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.ddl struct srcpart { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.srcpart
+ name: default.srcpart
+ Partition
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 2008-04-09
+ hr 12
+ properties:
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
+ bucket_count -1
+ column.name.delimiter ,
+ columns key,value
+ columns.comments 'default','default'
+ columns.types string:string
+#### A masked pattern was here ####
+ location hdfs://### HDFS PATH ###
+ name default.srcpart
+ numFiles 1
+ numFilesErasureCoded 1
+ numRows 500
+ partition_columns ds/hr
+ partition_columns.types string:string
+ rawDataSize 5312
+ serialization.ddl struct srcpart { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ bucketing_version 2
+ column.name.delimiter ,
+ columns key,value
+ columns.comments 'default','default'
+ columns.types string:string
+#### A masked pattern was here ####
+ location hdfs://### HDFS PATH ###
+ name default.srcpart
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.ddl struct srcpart { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.srcpart
+ name: default.srcpart
+ Processor Tree:
+ TableScan
+ alias: srcpart
+ Statistics: Num rows: 2000 Data size: 21248 Erasure files: 4 Basic stats: COMPLETE Column stats: NONE
+ GatherStats: false
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2000 Data size: 21248 Erasure files: 4 Basic stats: COMPLETE Column stats: NONE
+ ListSink
+
+PREHOOK: query: show table extended like src
+PREHOOK: type: SHOW_TABLESTATUS
+POSTHOOK: query: show table extended like src
+POSTHOOK: type: SHOW_TABLESTATUS
+tableName:src
+#### A masked pattern was here ####
+location:hdfs://### HDFS PATH ###
+inputformat:org.apache.hadoop.mapred.TextInputFormat
+outputformat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+columns:struct columns { string key, string value}
+partitioned:false
+partitionColumns:
+totalNumberFiles:1
+totalNumberErasureCodedFiles:1
+totalFileSize:5812
+maxFileSize:5812
+minFileSize:5812
+#### A masked pattern was here ####
+
+PREHOOK: query: desc formatted src
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@src
+POSTHOOK: query: desc formatted src
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@src
+# col_name data_type comment
+key string default
+value string default
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
+ bucketing_version 2
+ numFiles 1
+ numFilesErasureCoded 1
+ numRows 500
+ rawDataSize 5312
+ totalSize 5812
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: explain select key, value from src
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select key, value from src
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Erasure files: 1 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Erasure files: 1 Basic stats: COMPLETE Column stats: NONE
+ ListSink
+
+PREHOOK: query: explain extended select key, value from src
+PREHOOK: type: QUERY
+POSTHOOK: query: explain extended select key, value from src
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Erasure files: 1 Basic stats: COMPLETE Column stats: NONE
+ GatherStats: false
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Erasure files: 1 Basic stats: COMPLETE Column stats: NONE
+ ListSink
+
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/test/results/clientpositive/erasurecoding/erasure_simple.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/erasurecoding/erasure_simple.q.out b/ql/src/test/results/clientpositive/erasurecoding/erasure_simple.q.out
index 01f6015..b44cb7d 100644
--- a/ql/src/test/results/clientpositive/erasurecoding/erasure_simple.q.out
+++ b/ql/src/test/results/clientpositive/erasurecoding/erasure_simple.q.out
@@ -5,12 +5,13 @@ Policy: RS-6-3-1024k ENABLED
Policy: RS-LEGACY-6-3-1024k DISABLED
Policy: XOR-2-1-1024k DISABLED
Enabled EC policy 'RS-10-4-1024k'
+Enabled EC policy 'XOR-2-1-1024k'
ECHO listPolicies after enablePolicy
Policy: RS-10-4-1024k ENABLED
Policy: RS-3-2-1024k ENABLED
Policy: RS-6-3-1024k ENABLED
Policy: RS-LEGACY-6-3-1024k DISABLED
-Policy: XOR-2-1-1024k DISABLED
+Policy: XOR-2-1-1024k ENABLED
ECHO original policy on erasure_coding1
EC policy is 'RS-3-2-1024k'
ECHO set the default policy on erasure_coding1
@@ -39,6 +40,20 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@erasure_table
POSTHOOK: Lineage: erasure_table.a SCRIPT []
+Set EC policy' XOR-2-1-1024k
+PREHOOK: query: insert into erasure_table values(5)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@erasure_table
+POSTHOOK: query: insert into erasure_table values(5)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@erasure_table
+POSTHOOK: Lineage: erasure_table.a SCRIPT []
+ECHO policy on older file is
+EC policy is 'RS-3-2-1024k'
+ECHO policy on newer file is
+EC policy is 'XOR-2-1-1024k'
PREHOOK: query: select * from erasure_table
PREHOOK: type: QUERY
PREHOOK: Input: default@erasure_table
@@ -48,6 +63,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@erasure_table
POSTHOOK: Output: hdfs://### HDFS PATH ###
4
+5
PREHOOK: query: drop table if exists erasure_table2
PREHOOK: type: DROPTABLE
POSTHOOK: query: drop table if exists erasure_table2
@@ -88,6 +104,7 @@ columns:struct columns { string key, string value}
partitioned:false
partitionColumns:
totalNumberFiles:1
+totalNumberErasureCodedFiles:1
totalFileSize:5812
maxFileSize:5812
minFileSize:5812
@@ -100,6 +117,7 @@ POSTHOOK: query: SHOW TBLPROPERTIES erasure_table2
POSTHOOK: type: SHOW_TBLPROPERTIES
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
numFiles 1
+numFilesErasureCoded 1
numRows 500
rawDataSize 5312
totalSize 5812
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out
index 9a70096..95f8966 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out
@@ -245,6 +245,7 @@ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucketing_version 2
key value
numFiles 1
+numFilesErasureCoded 0
numRows 5
rawDataSize 1605
totalSize 703
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out
index c3cd893..71adebb 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out
@@ -67,6 +67,7 @@ POSTHOOK: type: SHOW_TBLPROPERTIES
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucketing_version 2
numFiles 1
+numFilesErasureCoded 0
numRows 2
rawDataSize 408
totalSize 457
@@ -100,6 +101,7 @@ POSTHOOK: type: SHOW_TBLPROPERTIES
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucketing_version 2
numFiles 1
+numFilesErasureCoded 0
numRows 2
rawDataSize 232
totalSize 326
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out
index e2972f3..ce1c281 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out
@@ -67,6 +67,7 @@ POSTHOOK: type: SHOW_TBLPROPERTIES
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucketing_version 2
numFiles 1
+numFilesErasureCoded 0
numRows 2
rawDataSize 408
totalSize 457
@@ -100,6 +101,7 @@ POSTHOOK: type: SHOW_TBLPROPERTIES
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucketing_version 2
numFiles 1
+numFilesErasureCoded 0
numRows 2
rawDataSize 232
totalSize 326
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_multi_db.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_multi_db.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_multi_db.q.out
index e5fb23d..98f7437 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_multi_db.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_multi_db.q.out
@@ -91,6 +91,7 @@ POSTHOOK: type: SHOW_TBLPROPERTIES
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucketing_version 2
numFiles 1
+numFilesErasureCoded 0
numRows 2
rawDataSize 408
totalSize 457
@@ -124,6 +125,7 @@ POSTHOOK: type: SHOW_TBLPROPERTIES
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucketing_version 2
numFiles 1
+numFilesErasureCoded 0
numRows 2
rawDataSize 232
totalSize 326
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/test/results/clientpositive/llap/materialized_view_describe.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_describe.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_describe.q.out
index 85092a0..c68c127 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_describe.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_describe.q.out
@@ -98,6 +98,7 @@ bucketing_version 2
comment this is the first view
key foo
numFiles 1
+numFilesErasureCoded 0
numRows 5
rawDataSize 580
totalSize 345
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/test/results/clientpositive/show_tblproperties.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/show_tblproperties.q.out b/ql/src/test/results/clientpositive/show_tblproperties.q.out
index e4bda1d..83e1ebd 100644
--- a/ql/src/test/results/clientpositive/show_tblproperties.q.out
+++ b/ql/src/test/results/clientpositive/show_tblproperties.q.out
@@ -41,6 +41,7 @@ bar bar value
bucketing_version 2
#### A masked pattern was here ####
numFiles 0
+numFilesErasureCoded 0
numRows 0
rawDataSize 0
tmp true
@@ -60,6 +61,7 @@ bar bar value
bucketing_version 2
#### A masked pattern was here ####
numFiles 0
+numFilesErasureCoded 0
numRows 0
rawDataSize 0
tmp true
@@ -115,6 +117,7 @@ bar bar value
bucketing_version 2
#### A masked pattern was here ####
numFiles 0
+numFilesErasureCoded 0
numRows 0
rawDataSize 0
tmp true
@@ -134,6 +137,7 @@ bar bar value1
bucketing_version 2
#### A masked pattern was here ####
numFiles 0
+numFilesErasureCoded 0
numRows 0
rawDataSize 0
tmp true1
@@ -159,6 +163,7 @@ bar bar value1
bucketing_version 2
#### A masked pattern was here ####
numFiles 0
+numFilesErasureCoded 0
numRows 0
rawDataSize 0
tmp true1
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/ql/src/test/results/clientpositive/unset_table_view_property.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/unset_table_view_property.q.out b/ql/src/test/results/clientpositive/unset_table_view_property.q.out
index 5887971..5d140d6 100644
--- a/ql/src/test/results/clientpositive/unset_table_view_property.q.out
+++ b/ql/src/test/results/clientpositive/unset_table_view_property.q.out
@@ -19,6 +19,7 @@ POSTHOOK: type: SHOW_TBLPROPERTIES
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"col1":"true","col2":"true"}}
bucketing_version 2
numFiles 0
+numFilesErasureCoded 0
numRows 0
rawDataSize 0
totalSize 0
@@ -41,6 +42,7 @@ bucketing_version 2
c 3
#### A masked pattern was here ####
numFiles 0
+numFilesErasureCoded 0
numRows 0
rawDataSize 0
totalSize 0
@@ -61,6 +63,7 @@ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"col1":"true","col2"
bucketing_version 2
#### A masked pattern was here ####
numFiles 0
+numFilesErasureCoded 0
numRows 0
rawDataSize 0
totalSize 0
@@ -84,6 +87,7 @@ c 3
d 4
#### A masked pattern was here ####
numFiles 0
+numFilesErasureCoded 0
numRows 0
rawDataSize 0
totalSize 0
@@ -105,6 +109,7 @@ bucketing_version 2
c 3
#### A masked pattern was here ####
numFiles 0
+numFilesErasureCoded 0
numRows 0
rawDataSize 0
totalSize 0
@@ -125,6 +130,7 @@ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"col1":"true","col2"
bucketing_version 2
#### A masked pattern was here ####
numFiles 0
+numFilesErasureCoded 0
numRows 0
rawDataSize 0
totalSize 0
@@ -149,6 +155,7 @@ c 3
d 4
#### A masked pattern was here ####
numFiles 0
+numFilesErasureCoded 0
numRows 0
rawDataSize 0
totalSize 0
@@ -171,6 +178,7 @@ bucketing_version 2
c 3
#### A masked pattern was here ####
numFiles 0
+numFilesErasureCoded 0
numRows 0
rawDataSize 0
totalSize 0
@@ -192,6 +200,7 @@ a 1
bucketing_version 2
#### A masked pattern was here ####
numFiles 0
+numFilesErasureCoded 0
numRows 0
rawDataSize 0
totalSize 0
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/standalone-metastore/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java
index 78ea01d..a7ca05a 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java
@@ -22,6 +22,7 @@ import java.util.List;
import java.util.Map;
import java.util.TreeMap;
+import com.google.common.collect.ImmutableList;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
@@ -105,6 +106,11 @@ public class StatsSetupConst {
public static final String RAW_DATA_SIZE = "rawDataSize";
/**
+ * The name of the statistic for Number of Erasure Coded Files - to be published or gathered.
+ */
+ public static final String NUM_ERASURE_CODED_FILES = "numFilesErasureCoded";
+
+ /**
* Temp dir for writing stats from tasks.
*/
public static final String STATS_TMP_LOC = "hive.stats.tmp.loc";
@@ -113,18 +119,20 @@ public class StatsSetupConst {
/**
* List of all supported statistics
*/
- public static final String[] supportedStats = {NUM_FILES,ROW_COUNT,TOTAL_SIZE,RAW_DATA_SIZE};
+ public static final List<String> SUPPORTED_STATS = ImmutableList.of(
+ NUM_FILES, ROW_COUNT, TOTAL_SIZE, RAW_DATA_SIZE, NUM_ERASURE_CODED_FILES);
/**
* List of all statistics that need to be collected during query execution. These are
* statistics that inherently require a scan of the data.
*/
- public static final String[] statsRequireCompute = new String[] {ROW_COUNT,RAW_DATA_SIZE};
+ public static final List<String> STATS_REQUIRE_COMPUTE = ImmutableList.of(ROW_COUNT, RAW_DATA_SIZE);
/**
* List of statistics that can be collected quickly without requiring a scan of the data.
*/
- public static final String[] fastStats = new String[] {NUM_FILES,TOTAL_SIZE};
+ public static final List<String> FAST_STATS = ImmutableList.of(
+ NUM_FILES, TOTAL_SIZE, NUM_ERASURE_CODED_FILES);
// This string constant is used to indicate to AlterHandler that
// alterPartition/alterTable is happening via statsTask or via user.
@@ -154,8 +162,9 @@ public class StatsSetupConst {
public static final String FALSE = "false";
// The parameter keys for the table statistics. Those keys are excluded from 'show create table' command output.
- public static final String[] TABLE_PARAMS_STATS_KEYS = new String[] {
- COLUMN_STATS_ACCURATE, NUM_FILES, TOTAL_SIZE,ROW_COUNT, RAW_DATA_SIZE, NUM_PARTITIONS};
+ public static final List<String> TABLE_PARAMS_STATS_KEYS = ImmutableList.of(
+ COLUMN_STATS_ACCURATE, NUM_FILES, TOTAL_SIZE, ROW_COUNT, RAW_DATA_SIZE, NUM_PARTITIONS,
+ NUM_ERASURE_CODED_FILES);
private static class ColumnStatsAccurate {
private static ObjectReader objectReader;
@@ -299,7 +308,7 @@ public class StatsSetupConst {
public static void setStatsStateForCreateTable(Map<String, String> params,
List<String> cols, String setting) {
if (TRUE.equals(setting)) {
- for (String stat : StatsSetupConst.supportedStats) {
+ for (String stat : StatsSetupConst.SUPPORTED_STATS) {
params.put(stat, "0");
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index c6c04b7..31bf615 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.metastore;
import static org.apache.commons.lang.StringUtils.join;
-import static org.apache.hadoop.hive.metastore.ReplChangeManager.SOURCE_OF_REPLICATION;
import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_COMMENT;
import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME;
import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME;
@@ -2674,7 +2673,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
if (null == props) {
return;
}
- for (String stat : StatsSetupConst.supportedStats) {
+ for (String stat : StatsSetupConst.SUPPORTED_STATS) {
String statVal = props.get(stat);
if (statVal != null) {
//In the case of truncate table, we set the stats to be 0.
http://git-wip-us.apache.org/repos/asf/hive/blob/c2940a07/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
index cbe89b6..73924ee 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
@@ -95,7 +95,6 @@ import java.util.Properties;
import java.util.Map.Entry;
import java.util.SortedMap;
import java.util.SortedSet;
-import java.util.StringJoiner;
import java.util.TimeZone;
import java.util.TreeMap;
import java.util.TreeSet;
@@ -628,7 +627,7 @@ public class MetaStoreUtils {
* @return True if the passed Parameters Map contains values for all "Fast Stats".
*/
private static boolean containsAllFastStats(Map<String, String> partParams) {
- for (String stat : StatsSetupConst.fastStats) {
+ for (String stat : StatsSetupConst.FAST_STATS) {
if (!partParams.containsKey(stat)) {
return false;
}
@@ -639,7 +638,7 @@ public class MetaStoreUtils {
public static boolean isFastStatsSame(Partition oldPart, Partition newPart) {
// requires to calculate stats if new and old have different fast stats
if ((oldPart != null) && (oldPart.getParameters() != null)) {
- for (String stat : StatsSetupConst.fastStats) {
+ for (String stat : StatsSetupConst.FAST_STATS) {
if (oldPart.getParameters().containsKey(stat)) {
Long oldStat = Long.parseLong(oldPart.getParameters().get(stat));
Long newStat = Long.parseLong(newPart.getParameters().get(stat));
@@ -720,20 +719,26 @@ public class MetaStoreUtils {
LOG.trace("Populating quick stats based on {} files", fileStatus.size());
int numFiles = 0;
long tableSize = 0L;
+ int numErasureCodedFiles = 0;
for (FileStatus status : fileStatus) {
// don't take directories into account for quick stats TODO: wtf?
if (!status.isDir()) {
tableSize += status.getLen();
numFiles += 1;
+ if (status.isErasureCoded()) {
+ numErasureCodedFiles++;
+ }
}
}
params.put(StatsSetupConst.NUM_FILES, Integer.toString(numFiles));
params.put(StatsSetupConst.TOTAL_SIZE, Long.toString(tableSize));
+ params.put(StatsSetupConst.NUM_ERASURE_CODED_FILES, Integer.toString(numErasureCodedFiles));
}
public static void clearQuickStats(Map<String, String> params) {
params.remove(StatsSetupConst.NUM_FILES);
params.remove(StatsSetupConst.TOTAL_SIZE);
+ params.remove(StatsSetupConst.NUM_ERASURE_CODED_FILES);
}
[07/32] hive git commit: Revert "HIVE-19765: Add Parquet specific
tests to BlobstoreCliDriver (Sahil Takiar, reviewed by Marta Kuczora)"
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9bd040ad/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out
deleted file mode 100644
index 15ae3d9..0000000
--- a/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out
+++ /dev/null
@@ -1,537 +0,0 @@
-PREHOOK: query: DROP TABLE src_events
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: DROP TABLE src_events
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: CREATE TABLE src_events
-(
- log_id BIGINT,
- `time` BIGINT,
- uid BIGINT,
- user_id BIGINT,
- type INT,
- event_data STRING,
- session_id STRING,
- full_uid BIGINT,
- run_date STRING,
- game_id INT,
- event_name STRING
-)
-ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-#### A masked pattern was here ####
-PREHOOK: type: CREATETABLE
-PREHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/src_events
-PREHOOK: Output: database:default
-PREHOOK: Output: default@src_events
-POSTHOOK: query: CREATE TABLE src_events
-(
- log_id BIGINT,
- `time` BIGINT,
- uid BIGINT,
- user_id BIGINT,
- type INT,
- event_data STRING,
- session_id STRING,
- full_uid BIGINT,
- run_date STRING,
- game_id INT,
- event_name STRING
-)
-ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-#### A masked pattern was here ####
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/src_events
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@src_events
-PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@src_events
-POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@src_events
-PREHOOK: query: DROP TABLE parquet_events
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: DROP TABLE parquet_events
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: CREATE TABLE parquet_events
-(
- log_id BIGINT,
- `time` BIGINT,
- uid BIGINT,
- user_id BIGINT,
- type INT,
- event_data STRING,
- session_id STRING,
- full_uid BIGINT
-)
-PARTITIONED BY (run_date STRING, game_id INT, event_name STRING)
-STORED AS PARQUET
-#### A masked pattern was here ####
-PREHOOK: type: CREATETABLE
-PREHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_events
-PREHOOK: Output: database:default
-PREHOOK: Output: default@parquet_events
-POSTHOOK: query: CREATE TABLE parquet_events
-(
- log_id BIGINT,
- `time` BIGINT,
- uid BIGINT,
- user_id BIGINT,
- type INT,
- event_data STRING,
- session_id STRING,
- full_uid BIGINT
-)
-PARTITIONED BY (run_date STRING, game_id INT, event_name STRING)
-STORED AS PARQUET
-#### A masked pattern was here ####
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_events
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@parquet_events
-PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
-SELECT * FROM src_events
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src_events
-PREHOOK: Output: default@parquet_events
-POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
-SELECT * FROM src_events
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src_events
-POSTHOOK: Output: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change
-POSTHOOK: Output: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change
-POSTHOOK: Output: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
-PREHOOK: query: SHOW PARTITIONS parquet_events
-PREHOOK: type: SHOWPARTITIONS
-PREHOOK: Input: default@parquet_events
-POSTHOOK: query: SHOW PARTITIONS parquet_events
-POSTHOOK: type: SHOWPARTITIONS
-POSTHOOK: Input: default@parquet_events
-run_date=20120921/game_id=39/event_name=hq_change
-run_date=20121021/game_id=39/event_name=hq_change
-run_date=20121121/game_id=39/event_name=hq_change
-PREHOOK: query: SELECT COUNT(*) FROM parquet_events
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-200
-PREHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change')
-#### A masked pattern was here ####
-PREHOOK: type: ALTERTABLE_ADDPARTS
-PREHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-1
-PREHOOK: Output: default@parquet_events
-POSTHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change')
-#### A masked pattern was here ####
-POSTHOOK: type: ALTERTABLE_ADDPARTS
-POSTHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-1
-POSTHOOK: Output: default@parquet_events
-POSTHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
-PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change')
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
-WHERE SUBSTR(run_date,1,6)='201211'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src_events
-PREHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
-POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change')
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
-WHERE SUBSTR(run_date,1,6)='201211'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src_events
-POSTHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
-PREHOOK: query: SHOW PARTITIONS parquet_events
-PREHOOK: type: SHOWPARTITIONS
-PREHOOK: Input: default@parquet_events
-POSTHOOK: query: SHOW PARTITIONS parquet_events
-POSTHOOK: type: SHOWPARTITIONS
-POSTHOOK: Input: default@parquet_events
-run_date=20120921/game_id=39/event_name=hq_change
-run_date=20121021/game_id=39/event_name=hq_change
-run_date=201211/game_id=39/event_name=hq_change
-run_date=20121121/game_id=39/event_name=hq_change
-PREHOOK: query: SELECT COUNT(*) FROM parquet_events
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-300
-PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change')
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
-WHERE SUBSTR(run_date,1,6)='201211'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src_events
-PREHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
-POSTHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change')
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
-WHERE SUBSTR(run_date,1,6)='201211'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src_events
-POSTHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
-PREHOOK: query: SHOW PARTITIONS parquet_events
-PREHOOK: type: SHOWPARTITIONS
-PREHOOK: Input: default@parquet_events
-POSTHOOK: query: SHOW PARTITIONS parquet_events
-POSTHOOK: type: SHOWPARTITIONS
-POSTHOOK: Input: default@parquet_events
-run_date=20120921/game_id=39/event_name=hq_change
-run_date=20121021/game_id=39/event_name=hq_change
-run_date=201211/game_id=39/event_name=hq_change
-run_date=20121121/game_id=39/event_name=hq_change
-PREHOOK: query: SELECT COUNT(*) FROM parquet_events
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-400
-PREHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change')
-#### A masked pattern was here ####
-PREHOOK: type: ALTERTABLE_ADDPARTS
-PREHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-2
-PREHOOK: Output: default@parquet_events
-POSTHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change')
-#### A masked pattern was here ####
-POSTHOOK: type: ALTERTABLE_ADDPARTS
-POSTHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-2
-POSTHOOK: Output: default@parquet_events
-POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
-PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
-WHERE SUBSTR(run_date,1,6)='201209'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src_events
-PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
-POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
-WHERE SUBSTR(run_date,1,6)='201209'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src_events
-POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
-PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
-WHERE SUBSTR(run_date,1,6)='201209'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src_events
-PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
-POSTHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
-WHERE SUBSTR(run_date,1,6)='201209'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src_events
-POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
-PREHOOK: query: SHOW PARTITIONS parquet_events
-PREHOOK: type: SHOWPARTITIONS
-PREHOOK: Input: default@parquet_events
-POSTHOOK: query: SHOW PARTITIONS parquet_events
-POSTHOOK: type: SHOWPARTITIONS
-POSTHOOK: Input: default@parquet_events
-run_date=201209/game_id=39/event_name=hq_change
-run_date=20120921/game_id=39/event_name=hq_change
-run_date=20121021/game_id=39/event_name=hq_change
-run_date=201211/game_id=39/event_name=hq_change
-run_date=20121121/game_id=39/event_name=hq_change
-PREHOOK: query: SELECT COUNT(*) FROM parquet_events
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-500
-PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
-SELECT * FROM src_events
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src_events
-PREHOOK: Output: default@parquet_events
-POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
-SELECT * FROM src_events
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src_events
-POSTHOOK: Output: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change
-POSTHOOK: Output: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change
-POSTHOOK: Output: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
-PREHOOK: query: SHOW PARTITIONS parquet_events
-PREHOOK: type: SHOWPARTITIONS
-PREHOOK: Input: default@parquet_events
-POSTHOOK: query: SHOW PARTITIONS parquet_events
-POSTHOOK: type: SHOWPARTITIONS
-POSTHOOK: Input: default@parquet_events
-run_date=201209/game_id=39/event_name=hq_change
-run_date=20120921/game_id=39/event_name=hq_change
-run_date=20121021/game_id=39/event_name=hq_change
-run_date=201211/game_id=39/event_name=hq_change
-run_date=20121121/game_id=39/event_name=hq_change
-PREHOOK: query: SELECT COUNT(*) FROM parquet_events
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-500
-PREHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201207, game_id=39, event_name='hq_change')
-#### A masked pattern was here ####
-PREHOOK: type: ALTERTABLE_ADDPARTS
-PREHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-3
-PREHOOK: Output: default@parquet_events
-POSTHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201207, game_id=39, event_name='hq_change')
-#### A masked pattern was here ####
-POSTHOOK: type: ALTERTABLE_ADDPARTS
-POSTHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-3
-POSTHOOK: Output: default@parquet_events
-POSTHOOK: Output: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
-PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201207, game_id=39, event_name='hq_change')
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
-WHERE SUBSTR(run_date,1,6)='201209'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src_events
-PREHOOK: Output: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
-POSTHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201207, game_id=39, event_name='hq_change')
-SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
-WHERE SUBSTR(run_date,1,6)='201209'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src_events
-POSTHOOK: Output: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
-PREHOOK: query: SHOW PARTITIONS parquet_events
-PREHOOK: type: SHOWPARTITIONS
-PREHOOK: Input: default@parquet_events
-POSTHOOK: query: SHOW PARTITIONS parquet_events
-POSTHOOK: type: SHOWPARTITIONS
-POSTHOOK: Input: default@parquet_events
-run_date=201207/game_id=39/event_name=hq_change
-run_date=201209/game_id=39/event_name=hq_change
-run_date=20120921/game_id=39/event_name=hq_change
-run_date=20121021/game_id=39/event_name=hq_change
-run_date=201211/game_id=39/event_name=hq_change
-run_date=20121121/game_id=39/event_name=hq_change
-PREHOOK: query: SELECT COUNT(*) FROM parquet_events
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet_events
-PREHOOK: Input: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
-PREHOOK: Input: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
-PREHOOK: Input: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change
-PREHOOK: Input: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change
-PREHOOK: Input: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
-PREHOOK: Input: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet_events
-POSTHOOK: Input: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
-POSTHOOK: Input: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
-POSTHOOK: Input: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change
-POSTHOOK: Input: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change
-POSTHOOK: Input: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
-POSTHOOK: Input: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change
-#### A masked pattern was here ####
-550
-PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
-SELECT * FROM src_events
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src_events
-PREHOOK: Output: default@parquet_events
-POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
-SELECT * FROM src_events
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src_events
-POSTHOOK: Output: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change
-POSTHOOK: Output: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change
-POSTHOOK: Output: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
-POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
-PREHOOK: query: SHOW PARTITIONS parquet_events
-PREHOOK: type: SHOWPARTITIONS
-PREHOOK: Input: default@parquet_events
-POSTHOOK: query: SHOW PARTITIONS parquet_events
-POSTHOOK: type: SHOWPARTITIONS
-POSTHOOK: Input: default@parquet_events
-run_date=201207/game_id=39/event_name=hq_change
-run_date=201209/game_id=39/event_name=hq_change
-run_date=20120921/game_id=39/event_name=hq_change
-run_date=20121021/game_id=39/event_name=hq_change
-run_date=201211/game_id=39/event_name=hq_change
-run_date=20121121/game_id=39/event_name=hq_change
-PREHOOK: query: SELECT COUNT(*) FROM parquet_events
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet_events
-PREHOOK: Input: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
-PREHOOK: Input: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
-PREHOOK: Input: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change
-PREHOOK: Input: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change
-PREHOOK: Input: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
-PREHOOK: Input: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet_events
-POSTHOOK: Input: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
-POSTHOOK: Input: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
-POSTHOOK: Input: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change
-POSTHOOK: Input: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change
-POSTHOOK: Input: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
-POSTHOOK: Input: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change
-#### A masked pattern was here ####
-550
-PREHOOK: query: ALTER TABLE parquet_events DROP PARTITION (run_date=201211, game_id=39, event_name='hq_change')
-PREHOOK: type: ALTERTABLE_DROPPARTS
-PREHOOK: Input: default@parquet_events
-PREHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
-POSTHOOK: query: ALTER TABLE parquet_events DROP PARTITION (run_date=201211, game_id=39, event_name='hq_change')
-POSTHOOK: type: ALTERTABLE_DROPPARTS
-POSTHOOK: Input: default@parquet_events
-POSTHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
-PREHOOK: query: ALTER TABLE parquet_events DROP PARTITION (run_date=201209, game_id=39, event_name='hq_change')
-PREHOOK: type: ALTERTABLE_DROPPARTS
-PREHOOK: Input: default@parquet_events
-PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
-POSTHOOK: query: ALTER TABLE parquet_events DROP PARTITION (run_date=201209, game_id=39, event_name='hq_change')
-POSTHOOK: type: ALTERTABLE_DROPPARTS
-POSTHOOK: Input: default@parquet_events
-POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
-PREHOOK: query: ALTER TABLE parquet_events DROP PARTITION (run_date=201207, game_id=39, event_name='hq_change')
-PREHOOK: type: ALTERTABLE_DROPPARTS
-PREHOOK: Input: default@parquet_events
-PREHOOK: Output: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
-POSTHOOK: query: ALTER TABLE parquet_events DROP PARTITION (run_date=201207, game_id=39, event_name='hq_change')
-POSTHOOK: type: ALTERTABLE_DROPPARTS
-POSTHOOK: Input: default@parquet_events
-POSTHOOK: Output: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change
-PREHOOK: query: SHOW PARTITIONS parquet_events
-PREHOOK: type: SHOWPARTITIONS
-PREHOOK: Input: default@parquet_events
-POSTHOOK: query: SHOW PARTITIONS parquet_events
-POSTHOOK: type: SHOWPARTITIONS
-POSTHOOK: Input: default@parquet_events
-run_date=20120921/game_id=39/event_name=hq_change
-run_date=20121021/game_id=39/event_name=hq_change
-run_date=20121121/game_id=39/event_name=hq_change
-PREHOOK: query: SELECT COUNT(*) FROM parquet_events
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet_events
-#### A masked pattern was here ####
-200
http://git-wip-us.apache.org/repos/asf/hive/blob/9bd040ad/itests/hive-blobstore/src/test/results/clientpositive/ptf_parquetfile.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/ptf_parquetfile.q.out b/itests/hive-blobstore/src/test/results/clientpositive/ptf_parquetfile.q.out
deleted file mode 100644
index 192b80f..0000000
--- a/itests/hive-blobstore/src/test/results/clientpositive/ptf_parquetfile.q.out
+++ /dev/null
@@ -1,104 +0,0 @@
-PREHOOK: query: DROP TABLE part_parquet
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: DROP TABLE part_parquet
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: CREATE TABLE part_parquet(
- p_partkey int,
- p_name string,
- p_mfgr string,
- p_brand string,
- p_type string,
- p_size int,
- p_container string,
- p_retailprice double,
- p_comment string
-)
-STORED AS PARQUET
-#### A masked pattern was here ####
-PREHOOK: type: CREATETABLE
-PREHOOK: Input: ### test.blobstore.path ###/ptf_parquetfile/part_parquet
-PREHOOK: Output: database:default
-PREHOOK: Output: default@part_parquet
-POSTHOOK: query: CREATE TABLE part_parquet(
- p_partkey int,
- p_name string,
- p_mfgr string,
- p_brand string,
- p_type string,
- p_size int,
- p_container string,
- p_retailprice double,
- p_comment string
-)
-STORED AS PARQUET
-#### A masked pattern was here ####
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Input: ### test.blobstore.path ###/ptf_parquetfile/part_parquet
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@part_parquet
-PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/part.parquet' OVERWRITE INTO TABLE part_parquet
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@part_parquet
-POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/part.parquet' OVERWRITE INTO TABLE part_parquet
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@part_parquet
-PREHOOK: query: SELECT
- p_mfgr, p_name, p_size,
- RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS r,
- DENSE_RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS dr,
- SUM(p_retailprice) OVER (
- PARTITION BY p_mfgr
- ORDER BY p_name ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
- ) AS s1
-FROM NOOP(
- ON part_parquet
- PARTITION BY p_mfgr
- ORDER BY p_name
-)
-PREHOOK: type: QUERY
-PREHOOK: Input: default@part_parquet
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT
- p_mfgr, p_name, p_size,
- RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS r,
- DENSE_RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS dr,
- SUM(p_retailprice) OVER (
- PARTITION BY p_mfgr
- ORDER BY p_name ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
- ) AS s1
-FROM NOOP(
- ON part_parquet
- PARTITION BY p_mfgr
- ORDER BY p_name
-)
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@part_parquet
-#### A masked pattern was here ####
-Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15
-Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3
-Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06
-Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001
-Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001
-Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001
-Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68
-Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38
-Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001
-Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02
-Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62
-Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68
-Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95
-Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34
-Manufacturer#3 almond antique misty red olive 1 4 4 6195.32
-Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61
-Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67
-Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09
-Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35
-Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27
-Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001
-Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69
-Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004
-Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08
-Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18
-Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66
[31/32] hive git commit: HIVE-19765: Add Parquet specific tests to
BlobstoreCliDriver (Sahil Takiar, reviewed by Marta Kuczora)
Posted by se...@apache.org.
HIVE-19765: Add Parquet specific tests to BlobstoreCliDriver (Sahil Takiar, reviewed by Marta Kuczora)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b574bcdd
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b574bcdd
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b574bcdd
Branch: refs/heads/master-txnstats
Commit: b574bcdd01c154c46c7ae95b9b5d3dd57f5e32f5
Parents: c4c55d2
Author: Sahil Takiar <st...@cloudera.com>
Authored: Fri Jul 6 10:25:37 2018 -0500
Committer: Sahil Takiar <st...@cloudera.com>
Committed: Tue Jul 10 18:46:08 2018 -0500
----------------------------------------------------------------------
data/files/part.parquet | Bin 0 -> 4758 bytes
.../queries/clientpositive/parquet_buckets.q | 31 ++
.../clientpositive/parquet_format_nonpart.q | 30 ++
.../clientpositive/parquet_format_part.q | 67 +++
.../parquet_nonstd_partitions_loc.q | 100 ++++
.../queries/clientpositive/ptf_parquetfile.q | 32 ++
.../clientpositive/parquet_buckets.q.out | 183 +++++++
.../clientpositive/parquet_format_nonpart.q.out | 201 +++++++
.../clientpositive/parquet_format_part.q.out | 290 ++++++++++
.../parquet_nonstd_partitions_loc.q.out | 537 +++++++++++++++++++
.../clientpositive/ptf_parquetfile.q.out | 104 ++++
11 files changed, 1575 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b574bcdd/data/files/part.parquet
----------------------------------------------------------------------
diff --git a/data/files/part.parquet b/data/files/part.parquet
new file mode 100644
index 0000000..5b48321
Binary files /dev/null and b/data/files/part.parquet differ
http://git-wip-us.apache.org/repos/asf/hive/blob/b574bcdd/itests/hive-blobstore/src/test/queries/clientpositive/parquet_buckets.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/parquet_buckets.q b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_buckets.q
new file mode 100644
index 0000000..c5a2cd3
--- /dev/null
+++ b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_buckets.q
@@ -0,0 +1,31 @@
+-- Test simple interaction with partitioned bucketed table with paquet format in blobstore
+
+SET hive.exec.dynamic.partition=true;
+SET hive.exec.reducers.max=10;
+SET hive.exec.dynamic.partition.mode=nonstrict;
+
+DROP TABLE blobstore_source;
+CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ' '
+COLLECTION ITEMS TERMINATED BY '\t'
+LINES TERMINATED BY '\n'
+LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_buckets/blobstore_source/';
+LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source;
+
+DROP TABLE parquet_buckets;
+CREATE TABLE parquet_buckets (a STRING, value DOUBLE)
+PARTITIONED BY (b STRING)
+CLUSTERED BY (a) INTO 10 BUCKETS
+STORED AS PARQUET
+LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_buckets/parquet_buckets';
+
+INSERT OVERWRITE TABLE parquet_buckets
+PARTITION (b)
+SELECT a, c, b FROM blobstore_source;
+SELECT * FROM parquet_buckets;
+
+INSERT INTO TABLE parquet_buckets
+PARTITION (b)
+SELECT a, c, b FROM blobstore_source;
+SELECT * FROM parquet_buckets;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/b574bcdd/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_nonpart.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_nonpart.q b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_nonpart.q
new file mode 100644
index 0000000..a7827e9
--- /dev/null
+++ b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_nonpart.q
@@ -0,0 +1,30 @@
+-- Test INSERT OVERWRITE and INSERT INTO on parquet table in blobstore
+
+DROP TABLE blobstore_source;
+CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ' '
+COLLECTION ITEMS TERMINATED BY '\t'
+LINES TERMINATED BY '\n'
+LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_format_nonpart/blobstore_source/';
+LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source;
+
+DROP TABLE parquet_table;
+CREATE EXTERNAL TABLE parquet_table (a INT, b STRING, value DOUBLE) STORED AS PARQUET
+LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_format_nonpart/parquet_table';
+
+INSERT OVERWRITE TABLE parquet_table
+SELECT * FROM blobstore_source;
+
+SELECT * FROM parquet_table;
+SELECT a FROM parquet_table GROUP BY a;
+SELECT b FROM parquet_table GROUP BY b;
+SELECT value FROM parquet_table GROUP BY value;
+
+INSERT INTO TABLE parquet_table
+SELECT * FROM blobstore_source;
+
+SELECT * FROM parquet_table;
+SELECT a FROM parquet_table GROUP BY a;
+SELECT b FROM parquet_table GROUP BY b;
+SELECT value FROM parquet_table GROUP BY value;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/b574bcdd/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_part.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_part.q b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_part.q
new file mode 100644
index 0000000..e9924fe
--- /dev/null
+++ b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_part.q
@@ -0,0 +1,67 @@
+-- Test INSERT INTO and INSERT OVERWRITE on partitioned parquet table in blobstore
+
+DROP TABLE src_events;
+CREATE TABLE src_events
+(
+ log_id BIGINT,
+ `time` BIGINT,
+ uid BIGINT,
+ user_id BIGINT,
+ type INT,
+ event_data STRING,
+ session_id STRING,
+ full_uid BIGINT,
+ run_date STRING,
+ game_id INT,
+ event_name STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_format_part/src_events/';
+LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events;
+
+DROP TABLE parquet_events;
+CREATE TABLE parquet_events
+(
+ log_id BIGINT,
+ `time` BIGINT,
+ uid BIGINT,
+ user_id BIGINT,
+ type INT,
+ event_data STRING,
+ session_id STRING,
+ full_uid BIGINT
+)
+PARTITIONED BY (run_date STRING, game_id INT, event_name STRING)
+STORED AS PARQUET
+LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_format_part/parquet_events';
+
+SET hive.exec.dynamic.partition=true;
+SET hive.exec.dynamic.partition.mode=nonstrict;
+
+INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
+SELECT * FROM src_events;
+SHOW PARTITIONS parquet_events;
+SELECT COUNT(*) FROM parquet_events;
+SELECT COUNT(*) FROM parquet_events WHERE run_date=20120921;
+SELECT COUNT(*) FROM parquet_events WHERE run_date=20121121;
+
+INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id, event_name)
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events
+WHERE SUBSTR(run_date,1,6)='201211';
+SHOW PARTITIONS parquet_events;
+SELECT COUNT(*) FROM parquet_events;
+
+INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name)
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39;
+SELECT COUNT(*) FROM parquet_events;
+
+INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change';
+SELECT COUNT(*) FROM parquet_events;
+
+INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change';
+SELECT COUNT(*) FROM parquet_events;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/b574bcdd/itests/hive-blobstore/src/test/queries/clientpositive/parquet_nonstd_partitions_loc.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/parquet_nonstd_partitions_loc.q b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_nonstd_partitions_loc.q
new file mode 100644
index 0000000..2d20b99
--- /dev/null
+++ b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_nonstd_partitions_loc.q
@@ -0,0 +1,100 @@
+-- Test table in parquet format with non-standard partition locations in blobstore
+
+DROP TABLE src_events;
+CREATE TABLE src_events
+(
+ log_id BIGINT,
+ `time` BIGINT,
+ uid BIGINT,
+ user_id BIGINT,
+ type INT,
+ event_data STRING,
+ session_id STRING,
+ full_uid BIGINT,
+ run_date STRING,
+ game_id INT,
+ event_name STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_nonstd_partitions_loc/src_events/';
+LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events;
+
+DROP TABLE parquet_events;
+CREATE TABLE parquet_events
+(
+ log_id BIGINT,
+ `time` BIGINT,
+ uid BIGINT,
+ user_id BIGINT,
+ type INT,
+ event_data STRING,
+ session_id STRING,
+ full_uid BIGINT
+)
+PARTITIONED BY (run_date STRING, game_id INT, event_name STRING)
+STORED AS PARQUET
+LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_nonstd_partitions_loc/parquet_events/';
+
+SET hive.exec.dynamic.partition=true;
+SET hive.exec.dynamic.partition.mode=nonstrict;
+
+INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
+SELECT * FROM src_events;
+SHOW PARTITIONS parquet_events;
+SELECT COUNT(*) FROM parquet_events;
+
+-- verify INSERT OVERWRITE and INSERT INTO nonstandard partition location
+ALTER TABLE parquet_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change')
+LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-1/';
+INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201211';
+SHOW PARTITIONS parquet_events;
+SELECT COUNT(*) FROM parquet_events;
+INSERT INTO TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201211';
+SHOW PARTITIONS parquet_events;
+SELECT COUNT(*) FROM parquet_events;
+
+SET hive.merge.mapfiles=false;
+
+-- verify INSERT OVERWRITE and INSERT INTO nonstandard partition location with hive.merge.mapfiles false
+ALTER TABLE parquet_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-2/';
+INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209';
+INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209';
+SHOW PARTITIONS parquet_events;
+SELECT COUNT(*) FROM parquet_events;
+
+-- verify dynamic INSERT OVERWRITE over all partitions (standard and nonstandard locations) with hive.merge.mapfiles false
+INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
+SELECT * FROM src_events;
+SHOW PARTITIONS parquet_events;
+SELECT COUNT(*) FROM parquet_events;
+
+SET hive.merge.mapfiles=true;
+
+ALTER TABLE parquet_events ADD PARTITION (run_date=201207, game_id=39, event_name='hq_change')
+LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-3/';
+INSERT INTO TABLE parquet_events PARTITION (run_date=201207, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209';
+SHOW PARTITIONS parquet_events;
+SELECT COUNT(*) FROM parquet_events;
+
+-- verify dynamic INSERT OVERWRITE over all partitions (standard and nonstandard locations) with hive.merge.mapfiles true
+INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
+SELECT * FROM src_events;
+SHOW PARTITIONS parquet_events;
+SELECT COUNT(*) FROM parquet_events;
+
+ALTER TABLE parquet_events DROP PARTITION (run_date=201211, game_id=39, event_name='hq_change');
+ALTER TABLE parquet_events DROP PARTITION (run_date=201209, game_id=39, event_name='hq_change');
+ALTER TABLE parquet_events DROP PARTITION (run_date=201207, game_id=39, event_name='hq_change');
+SHOW PARTITIONS parquet_events;
+SELECT COUNT(*) FROM parquet_events;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/b574bcdd/itests/hive-blobstore/src/test/queries/clientpositive/ptf_parquetfile.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/ptf_parquetfile.q b/itests/hive-blobstore/src/test/queries/clientpositive/ptf_parquetfile.q
new file mode 100644
index 0000000..fbf5ff3
--- /dev/null
+++ b/itests/hive-blobstore/src/test/queries/clientpositive/ptf_parquetfile.q
@@ -0,0 +1,32 @@
+-- Test several window functions on a table stored using PARQUET
+DROP TABLE part_parquet;
+CREATE TABLE part_parquet(
+ p_partkey int,
+ p_name string,
+ p_mfgr string,
+ p_brand string,
+ p_type string,
+ p_size int,
+ p_container string,
+ p_retailprice double,
+ p_comment string
+)
+STORED AS PARQUET
+LOCATION '${hiveconf:test.blobstore.path.unique}/ptf_parquetfile/part_parquet';
+
+LOAD DATA LOCAL INPATH '../../data/files/part.parquet' OVERWRITE INTO TABLE part_parquet;
+
+-- Test windowing PTFs with several partitions, using PARQUET storage
+SELECT
+ p_mfgr, p_name, p_size,
+ RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS r,
+ DENSE_RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS dr,
+ SUM(p_retailprice) OVER (
+ PARTITION BY p_mfgr
+ ORDER BY p_name ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+ ) AS s1
+FROM NOOP(
+ ON part_parquet
+ PARTITION BY p_mfgr
+ ORDER BY p_name
+);
http://git-wip-us.apache.org/repos/asf/hive/blob/b574bcdd/itests/hive-blobstore/src/test/results/clientpositive/parquet_buckets.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_buckets.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_buckets.q.out
new file mode 100644
index 0000000..355958d
--- /dev/null
+++ b/itests/hive-blobstore/src/test/results/clientpositive/parquet_buckets.q.out
@@ -0,0 +1,183 @@
+PREHOOK: query: DROP TABLE blobstore_source
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE blobstore_source
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ' '
+COLLECTION ITEMS TERMINATED BY '\t'
+LINES TERMINATED BY '\n'
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/parquet_buckets/blobstore_source
+PREHOOK: Output: database:default
+PREHOOK: Output: default@blobstore_source
+POSTHOOK: query: CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ' '
+COLLECTION ITEMS TERMINATED BY '\t'
+LINES TERMINATED BY '\n'
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/parquet_buckets/blobstore_source
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@blobstore_source
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@blobstore_source
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@blobstore_source
+PREHOOK: query: DROP TABLE parquet_buckets
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE parquet_buckets
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE parquet_buckets (a STRING, value DOUBLE)
+PARTITIONED BY (b STRING)
+CLUSTERED BY (a) INTO 10 BUCKETS
+STORED AS PARQUET
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/parquet_buckets/parquet_buckets
+PREHOOK: Output: database:default
+PREHOOK: Output: default@parquet_buckets
+POSTHOOK: query: CREATE TABLE parquet_buckets (a STRING, value DOUBLE)
+PARTITIONED BY (b STRING)
+CLUSTERED BY (a) INTO 10 BUCKETS
+STORED AS PARQUET
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/parquet_buckets/parquet_buckets
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@parquet_buckets
+PREHOOK: query: INSERT OVERWRITE TABLE parquet_buckets
+PARTITION (b)
+SELECT a, c, b FROM blobstore_source
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_source
+PREHOOK: Output: default@parquet_buckets
+POSTHOOK: query: INSERT OVERWRITE TABLE parquet_buckets
+PARTITION (b)
+SELECT a, c, b FROM blobstore_source
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_source
+POSTHOOK: Output: default@parquet_buckets@b=abc
+POSTHOOK: Output: default@parquet_buckets@b=ajss
+POSTHOOK: Output: default@parquet_buckets@b=data
+POSTHOOK: Output: default@parquet_buckets@b=def
+POSTHOOK: Output: default@parquet_buckets@b=djns
+POSTHOOK: Output: default@parquet_buckets@b=ne
+POSTHOOK: Output: default@parquet_buckets@b=random
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=abc).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=abc).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=ajss).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=ajss).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=data).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=data).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=def).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=def).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=djns).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=djns).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=ne).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=ne).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=random).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=random).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+PREHOOK: query: SELECT * FROM parquet_buckets
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_buckets
+PREHOOK: Input: default@parquet_buckets@b=abc
+PREHOOK: Input: default@parquet_buckets@b=ajss
+PREHOOK: Input: default@parquet_buckets@b=data
+PREHOOK: Input: default@parquet_buckets@b=def
+PREHOOK: Input: default@parquet_buckets@b=djns
+PREHOOK: Input: default@parquet_buckets@b=ne
+PREHOOK: Input: default@parquet_buckets@b=random
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_buckets
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_buckets
+POSTHOOK: Input: default@parquet_buckets@b=abc
+POSTHOOK: Input: default@parquet_buckets@b=ajss
+POSTHOOK: Input: default@parquet_buckets@b=data
+POSTHOOK: Input: default@parquet_buckets@b=def
+POSTHOOK: Input: default@parquet_buckets@b=djns
+POSTHOOK: Input: default@parquet_buckets@b=ne
+POSTHOOK: Input: default@parquet_buckets@b=random
+#### A masked pattern was here ####
+1 10.5 abc
+3 90.23232 ajss
+6 3.002 data
+2 11.5 def
+4 89.02002 djns
+7 71.9084 ne
+5 2.99 random
+PREHOOK: query: INSERT INTO TABLE parquet_buckets
+PARTITION (b)
+SELECT a, c, b FROM blobstore_source
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_source
+PREHOOK: Output: default@parquet_buckets
+POSTHOOK: query: INSERT INTO TABLE parquet_buckets
+PARTITION (b)
+SELECT a, c, b FROM blobstore_source
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_source
+POSTHOOK: Output: default@parquet_buckets@b=abc
+POSTHOOK: Output: default@parquet_buckets@b=ajss
+POSTHOOK: Output: default@parquet_buckets@b=data
+POSTHOOK: Output: default@parquet_buckets@b=def
+POSTHOOK: Output: default@parquet_buckets@b=djns
+POSTHOOK: Output: default@parquet_buckets@b=ne
+POSTHOOK: Output: default@parquet_buckets@b=random
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=abc).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=abc).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=ajss).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=ajss).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=data).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=data).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=def).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=def).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=djns).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=djns).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=ne).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=ne).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=random).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_buckets PARTITION(b=random).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+PREHOOK: query: SELECT * FROM parquet_buckets
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_buckets
+PREHOOK: Input: default@parquet_buckets@b=abc
+PREHOOK: Input: default@parquet_buckets@b=ajss
+PREHOOK: Input: default@parquet_buckets@b=data
+PREHOOK: Input: default@parquet_buckets@b=def
+PREHOOK: Input: default@parquet_buckets@b=djns
+PREHOOK: Input: default@parquet_buckets@b=ne
+PREHOOK: Input: default@parquet_buckets@b=random
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_buckets
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_buckets
+POSTHOOK: Input: default@parquet_buckets@b=abc
+POSTHOOK: Input: default@parquet_buckets@b=ajss
+POSTHOOK: Input: default@parquet_buckets@b=data
+POSTHOOK: Input: default@parquet_buckets@b=def
+POSTHOOK: Input: default@parquet_buckets@b=djns
+POSTHOOK: Input: default@parquet_buckets@b=ne
+POSTHOOK: Input: default@parquet_buckets@b=random
+#### A masked pattern was here ####
+1 10.5 abc
+1 10.5 abc
+3 90.23232 ajss
+3 90.23232 ajss
+6 3.002 data
+6 3.002 data
+2 11.5 def
+2 11.5 def
+4 89.02002 djns
+4 89.02002 djns
+7 71.9084 ne
+7 71.9084 ne
+5 2.99 random
+5 2.99 random
http://git-wip-us.apache.org/repos/asf/hive/blob/b574bcdd/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_nonpart.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_nonpart.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_nonpart.q.out
new file mode 100644
index 0000000..cba6b1a
--- /dev/null
+++ b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_nonpart.q.out
@@ -0,0 +1,201 @@
+PREHOOK: query: DROP TABLE blobstore_source
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE blobstore_source
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ' '
+COLLECTION ITEMS TERMINATED BY '\t'
+LINES TERMINATED BY '\n'
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/parquet_format_nonpart/blobstore_source
+PREHOOK: Output: database:default
+PREHOOK: Output: default@blobstore_source
+POSTHOOK: query: CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ' '
+COLLECTION ITEMS TERMINATED BY '\t'
+LINES TERMINATED BY '\n'
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/parquet_format_nonpart/blobstore_source
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@blobstore_source
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@blobstore_source
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@blobstore_source
+PREHOOK: query: DROP TABLE parquet_table
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE parquet_table
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE EXTERNAL TABLE parquet_table (a INT, b STRING, value DOUBLE) STORED AS PARQUET
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/parquet_format_nonpart/parquet_table
+PREHOOK: Output: database:default
+PREHOOK: Output: default@parquet_table
+POSTHOOK: query: CREATE EXTERNAL TABLE parquet_table (a INT, b STRING, value DOUBLE) STORED AS PARQUET
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/parquet_format_nonpart/parquet_table
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@parquet_table
+PREHOOK: query: INSERT OVERWRITE TABLE parquet_table
+SELECT * FROM blobstore_source
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_source
+PREHOOK: Output: default@parquet_table
+POSTHOOK: query: INSERT OVERWRITE TABLE parquet_table
+SELECT * FROM blobstore_source
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_source
+POSTHOOK: Output: default@parquet_table
+POSTHOOK: Lineage: parquet_table.a EXPRESSION [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_table.b SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:b, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_table.value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+PREHOOK: query: SELECT * FROM parquet_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+1 abc 10.5
+2 def 11.5
+3 ajss 90.23232
+4 djns 89.02002
+5 random 2.99
+6 data 3.002
+7 ne 71.9084
+PREHOOK: query: SELECT a FROM parquet_table GROUP BY a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT a FROM parquet_table GROUP BY a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+1
+2
+3
+4
+5
+6
+7
+PREHOOK: query: SELECT b FROM parquet_table GROUP BY b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT b FROM parquet_table GROUP BY b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+abc
+ajss
+data
+def
+djns
+ne
+random
+PREHOOK: query: SELECT value FROM parquet_table GROUP BY value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT value FROM parquet_table GROUP BY value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+2.99
+3.002
+10.5
+11.5
+71.9084
+89.02002
+90.23232
+PREHOOK: query: INSERT INTO TABLE parquet_table
+SELECT * FROM blobstore_source
+PREHOOK: type: QUERY
+PREHOOK: Input: default@blobstore_source
+PREHOOK: Output: default@parquet_table
+POSTHOOK: query: INSERT INTO TABLE parquet_table
+SELECT * FROM blobstore_source
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@blobstore_source
+POSTHOOK: Output: default@parquet_table
+POSTHOOK: Lineage: parquet_table.a EXPRESSION [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_table.b SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:b, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_table.value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ]
+PREHOOK: query: SELECT * FROM parquet_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+1 abc 10.5
+2 def 11.5
+3 ajss 90.23232
+4 djns 89.02002
+5 random 2.99
+6 data 3.002
+7 ne 71.9084
+1 abc 10.5
+2 def 11.5
+3 ajss 90.23232
+4 djns 89.02002
+5 random 2.99
+6 data 3.002
+7 ne 71.9084
+PREHOOK: query: SELECT a FROM parquet_table GROUP BY a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT a FROM parquet_table GROUP BY a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+1
+2
+3
+4
+5
+6
+7
+PREHOOK: query: SELECT b FROM parquet_table GROUP BY b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT b FROM parquet_table GROUP BY b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+abc
+ajss
+data
+def
+djns
+ne
+random
+PREHOOK: query: SELECT value FROM parquet_table GROUP BY value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT value FROM parquet_table GROUP BY value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_table
+#### A masked pattern was here ####
+2.99
+3.002
+10.5
+11.5
+71.9084
+89.02002
+90.23232
http://git-wip-us.apache.org/repos/asf/hive/blob/b574bcdd/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out
new file mode 100644
index 0000000..0931e3d
--- /dev/null
+++ b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out
@@ -0,0 +1,290 @@
+PREHOOK: query: DROP TABLE src_events
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE src_events
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE src_events
+(
+ log_id BIGINT,
+ `time` BIGINT,
+ uid BIGINT,
+ user_id BIGINT,
+ type INT,
+ event_data STRING,
+ session_id STRING,
+ full_uid BIGINT,
+ run_date STRING,
+ game_id INT,
+ event_name STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/parquet_format_part/src_events
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_events
+POSTHOOK: query: CREATE TABLE src_events
+(
+ log_id BIGINT,
+ `time` BIGINT,
+ uid BIGINT,
+ user_id BIGINT,
+ type INT,
+ event_data STRING,
+ session_id STRING,
+ full_uid BIGINT,
+ run_date STRING,
+ game_id INT,
+ event_name STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/parquet_format_part/src_events
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_events
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@src_events
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@src_events
+PREHOOK: query: DROP TABLE parquet_events
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE parquet_events
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE parquet_events
+(
+ log_id BIGINT,
+ `time` BIGINT,
+ uid BIGINT,
+ user_id BIGINT,
+ type INT,
+ event_data STRING,
+ session_id STRING,
+ full_uid BIGINT
+)
+PARTITIONED BY (run_date STRING, game_id INT, event_name STRING)
+STORED AS PARQUET
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: ### test.blobstore.path ###/parquet_format_part/parquet_events
+PREHOOK: Output: database:default
+PREHOOK: Output: default@parquet_events
+POSTHOOK: query: CREATE TABLE parquet_events
+(
+ log_id BIGINT,
+ `time` BIGINT,
+ uid BIGINT,
+ user_id BIGINT,
+ type INT,
+ event_data STRING,
+ session_id STRING,
+ full_uid BIGINT
+)
+PARTITIONED BY (run_date STRING, game_id INT, event_name STRING)
+STORED AS PARQUET
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: ### test.blobstore.path ###/parquet_format_part/parquet_events
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@parquet_events
+PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
+SELECT * FROM src_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events
+POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
+SELECT * FROM src_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change
+POSTHOOK: Output: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change
+POSTHOOK: Output: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: SHOW PARTITIONS parquet_events
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@parquet_events
+POSTHOOK: query: SHOW PARTITIONS parquet_events
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@parquet_events
+run_date=20120921/game_id=39/event_name=hq_change
+run_date=20121021/game_id=39/event_name=hq_change
+run_date=20121121/game_id=39/event_name=hq_change
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+200
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20120921
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20120921
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+50
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20121121
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20121121
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+100
+PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id, event_name)
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events
+WHERE SUBSTR(run_date,1,6)='201211'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events@run_date=201211
+POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id, event_name)
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events
+WHERE SUBSTR(run_date,1,6)='201211'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: SHOW PARTITIONS parquet_events
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@parquet_events
+POSTHOOK: query: SHOW PARTITIONS parquet_events
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@parquet_events
+run_date=20120921/game_id=39/event_name=hq_change
+run_date=20121021/game_id=39/event_name=hq_change
+run_date=201211/game_id=39/event_name=hq_change
+run_date=20121121/game_id=39/event_name=hq_change
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+300
+PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name)
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39
+POSTHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name)
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+350
+PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+POSTHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+400
+PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_events
+PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
+SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
+WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_events
+POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ]
+POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ]
+PREHOOK: query: SELECT COUNT(*) FROM parquet_events
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_events
+#### A masked pattern was here ####
+350
[22/32] hive git commit: HIVE-19806 Several tests do not properly
sort their output (Alan Gates, reviewed by Vineet Garg)
Posted by se...@apache.org.
HIVE-19806 Several tests do not properly sort their output (Alan Gates, reviewed by Vineet Garg)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5016d6f0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5016d6f0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5016d6f0
Branch: refs/heads/master-txnstats
Commit: 5016d6f0564eb11090d9c3ad3b7cda3672e2f79e
Parents: 601dd05
Author: Alan Gates <ga...@hortonworks.com>
Authored: Mon Jul 9 13:25:53 2018 -0700
Committer: Alan Gates <ga...@hortonworks.com>
Committed: Mon Jul 9 13:25:53 2018 -0700
----------------------------------------------------------------------
.../insert_into_dynamic_partitions.q | 3 ++-
.../clientpositive/insert_overwrite_directory.q | 3 ++-
.../insert_overwrite_dynamic_partitions.q | 3 ++-
.../insert_into_dynamic_partitions.q.out | 8 +++---
.../insert_overwrite_dynamic_partitions.q.out | 12 ++++-----
.../hive/metastore/TestMetaStoreMetrics.java | 5 ++--
.../test/resources/testconfiguration.properties | 9 -------
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 4 +--
.../test/queries/clientpositive/explainuser_4.q | 1 +
.../clientpositive/orc_schema_evolution_float.q | 1 +
.../clientpositive/parquet_ppd_multifiles.q | 1 +
.../test/queries/clientpositive/selectindate.q | 1 +
ql/src/test/queries/clientpositive/union38.q | 1 +
.../test/queries/clientpositive/vector_bround.q | 1 +
.../queries/clientpositive/vector_case_when_1.q | 3 +--
.../queries/clientpositive/vector_coalesce_2.q | 1 +
.../queries/clientpositive/vector_coalesce_3.q | 1 +
.../queries/clientpositive/vector_interval_1.q | 1 +
.../clientpositive/vectorized_parquet_types.q | 1 +
.../clientpositive/llap/explainuser_4.q.out | 28 ++++++++++----------
.../clientpositive/llap/vector_bround.q.out | 4 +--
.../clientpositive/llap/vector_coalesce_2.q.out | 8 +++---
.../clientpositive/llap/vector_interval_1.q.out | 16 +++++------
.../llap/vectorized_parquet_types.q.out | 8 +++---
.../orc_schema_evolution_float.q.out | 12 ++++-----
.../test/results/clientpositive/union38.q.out | 14 +++++-----
.../results/clientpositive/vector_bround.q.out | 4 +--
.../clientpositive/vector_coalesce_2.q.out | 4 +--
.../clientpositive/vector_interval_1.q.out | 16 +++++------
.../vectorized_parquet_types.q.out | 8 +++---
30 files changed, 93 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/itests/hive-blobstore/src/test/queries/clientpositive/insert_into_dynamic_partitions.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/insert_into_dynamic_partitions.q b/itests/hive-blobstore/src/test/queries/clientpositive/insert_into_dynamic_partitions.q
index 7269e29..dc269eb 100644
--- a/itests/hive-blobstore/src/test/queries/clientpositive/insert_into_dynamic_partitions.q
+++ b/itests/hive-blobstore/src/test/queries/clientpositive/insert_into_dynamic_partitions.q
@@ -1,3 +1,4 @@
+-- SORT_QUERY_RESULTS
SET hive.blobstore.optimizations.enabled=true;
SET hive.exec.dynamic.partition.mode=nonstrict;
@@ -21,4 +22,4 @@ INSERT INTO TABLE table2 PARTITION (country='MEX', state) VALUES ('Peter Mo', 87
SHOW PARTITIONS table2;
SELECT * FROM table2;
DROP TABLE table2;
-DROP TABLE table1;
\ No newline at end of file
+DROP TABLE table1;
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_directory.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_directory.q b/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_directory.q
index f1b5a0b..8dc44b2 100644
--- a/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_directory.q
+++ b/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_directory.q
@@ -1,5 +1,6 @@
SET hive.blobstore.optimizations.enabled=true;
SET hive.blobstore.use.blobstore.as.scratchdir=true;
+-- SORT_BEFORE_DIFF
-- Create a simple source table;
DROP TABLE table1;
@@ -24,4 +25,4 @@ EXPLAIN EXTENDED INSERT OVERWRITE DIRECTORY '${hiveconf:test.blobstore.path.uniq
EXPLAIN EXTENDED FROM table1
INSERT OVERWRITE DIRECTORY '${hiveconf:test.blobstore.path.unique}/table1.dir/' SELECT id
- INSERT OVERWRITE DIRECTORY '${hiveconf:test.blobstore.path.unique}/table2.dir/' SELECT key;
\ No newline at end of file
+ INSERT OVERWRITE DIRECTORY '${hiveconf:test.blobstore.path.unique}/table2.dir/' SELECT key;
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_dynamic_partitions.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_dynamic_partitions.q b/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_dynamic_partitions.q
index 0be4f16..a7129bd 100644
--- a/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_dynamic_partitions.q
+++ b/itests/hive-blobstore/src/test/queries/clientpositive/insert_overwrite_dynamic_partitions.q
@@ -1,5 +1,6 @@
SET hive.blobstore.optimizations.enabled=true;
SET hive.exec.dynamic.partition.mode=nonstrict;
+-- SORT_QUERY_RESULTS
-- Single partition with buckets
DROP TABLE table1;
@@ -27,4 +28,4 @@ INSERT OVERWRITE TABLE table2 PARTITION (country='MEX', state) VALUES ('Peter Mo
SHOW PARTITIONS table2;
SELECT * FROM table2;
DROP TABLE table2;
-DROP TABLE table1;
\ No newline at end of file
+DROP TABLE table1;
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out
index a0ebed7..74a9a56 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out
@@ -339,13 +339,13 @@ POSTHOOK: Input: default@table2@country=MEX/state=SON
POSTHOOK: Input: default@table2@country=USA/state=CA
POSTHOOK: Input: default@table2@country=USA/state=TX
#### A masked pattern was here ####
-Peter Mo 87 MEX SON
-Mirna Cage 37 USA CA
-Mark Cage 38 USA CA
+Bill Rose 52 USA TX
Jane Doe 22 USA CA
John Doe 23 USA CA
Maria Full 50 USA TX
-Bill Rose 52 USA TX
+Mark Cage 38 USA CA
+Mirna Cage 37 USA CA
+Peter Mo 87 MEX SON
PREHOOK: query: DROP TABLE table2
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@table2
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out
index 6dba301..ee02c36 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out
@@ -344,12 +344,12 @@ POSTHOOK: Input: default@table2
POSTHOOK: Input: default@table2@country=USA/state=CA
POSTHOOK: Input: default@table2@country=USA/state=TX
#### A masked pattern was here ####
-Mirna Cage 37 USA CA
-Mark Cage 38 USA CA
+Bill Rose 52 USA TX
Jane Doe 22 USA CA
John Doe 23 USA CA
Maria Full 50 USA TX
-Bill Rose 52 USA TX
+Mark Cage 38 USA CA
+Mirna Cage 37 USA CA
PREHOOK: query: INSERT OVERWRITE TABLE table2 PARTITION (country, state) SELECT * FROM table1 WHERE age < 30
PREHOOK: type: QUERY
PREHOOK: Input: default@table1
@@ -384,10 +384,10 @@ POSTHOOK: Input: default@table2
POSTHOOK: Input: default@table2@country=USA/state=CA
POSTHOOK: Input: default@table2@country=USA/state=TX
#### A masked pattern was here ####
+Bill Rose 52 USA TX
Jane Doe 22 USA CA
John Doe 23 USA CA
Maria Full 50 USA TX
-Bill Rose 52 USA TX
PREHOOK: query: INSERT OVERWRITE TABLE table2 PARTITION (country='MEX', state) VALUES ('Peter Mo', 87, 'SON')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
@@ -421,11 +421,11 @@ POSTHOOK: Input: default@table2@country=MEX/state=SON
POSTHOOK: Input: default@table2@country=USA/state=CA
POSTHOOK: Input: default@table2@country=USA/state=TX
#### A masked pattern was here ####
-Peter Mo 87 MEX SON
+Bill Rose 52 USA TX
Jane Doe 22 USA CA
John Doe 23 USA CA
Maria Full 50 USA TX
-Bill Rose 52 USA TX
+Peter Mo 87 MEX SON
PREHOOK: query: DROP TABLE table2
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@table2
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
index 3f7135c..03028a4 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
@@ -132,6 +132,7 @@ public class TestMetaStoreMetrics {
@Test
public void testConnections() throws Exception {
+ Thread.sleep(2000); // TODO Evil! Need to figure out a way to remove this sleep.
//initial state is one connection
int initialCount =
(Integer)Metrics.getRegistry().getGauges().get(MetricsConstants.OPEN_CONNECTIONS).getValue();
@@ -145,13 +146,13 @@ public class TestMetaStoreMetrics {
//close one connection, verify still two left
msc.close();
- Thread.sleep(500); // TODO Evil! Need to figure out a way to remove this sleep.
+ Thread.sleep(2000); // TODO Evil! Need to figure out a way to remove this sleep.
Assert.assertEquals(initialCount + 1,
Metrics.getRegistry().getGauges().get(MetricsConstants.OPEN_CONNECTIONS).getValue());
//close one connection, verify still one left
msc2.close();
- Thread.sleep(500); // TODO Evil! Need to figure out a way to remove this sleep.
+ Thread.sleep(2000); // TODO Evil! Need to figure out a way to remove this sleep.
Assert.assertEquals(initialCount,
Metrics.getRegistry().getGauges().get(MetricsConstants.OPEN_CONNECTIONS).getValue());
}
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index a27ce5e..2795e3d 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -64,7 +64,6 @@ minillap.shared.query.files=insert_into1.q,\
parquet_map_type_vectorization.q,\
parquet_struct_type_vectorization.q,\
orc_struct_type_vectorization.q,\
- union_type_chk.q,\
cte_2.q,\
cte_4.q
@@ -313,7 +312,6 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\
vector_reduce3.q,\
vector_reduce_groupby_decimal.q,\
vector_reduce_groupby_duplicate_cols.q,\
- vector_row__id.q,\
vector_string_concat.q,\
vector_struct_in.q,\
vector_udf_character_length.q,\
@@ -385,8 +383,6 @@ minillap.query.files=acid_bucket_pruning.q,\
reduce_deduplicate.q,\
reduce_deduplicate_distinct.q, \
remote_script.q,\
- tez_aggr_part_stats.q,\
- tez_union_view.q,\
file_with_header_footer.q,\
external_table_purge.q,\
external_table_with_space_in_location_path.q,\
@@ -421,7 +417,6 @@ minillaplocal.query.files=\
dec_str.q,\
dp_counter_non_mm.q,\
dp_counter_mm.q,\
- acid_export.q,\
acid_no_buckets.q, \
acid_globallimit.q,\
acid_vectorization_missing_cols.q,\
@@ -547,7 +542,6 @@ minillaplocal.query.files=\
llap_uncompressed.q,\
llap_decimal64_reader.q,\
llap_text.q,\
- load_data_acid_rename.q,\
load_data_using_job.q,\
load_dyn_part5.q,\
lvj_mapjoin.q,\
@@ -838,7 +832,6 @@ minillaplocal.query.files=\
dynpart_sort_optimization2.q,\
multi_column_in_single.q,\
join32_lessSize.q,\
- alter_table_invalidate_column_stats.q,\
bucketmapjoin1.q,\
ppr_pushdown.q,\
smb_mapjoin_14.q,\
@@ -914,7 +907,6 @@ encrypted.query.files=encryption_join_unencrypted_tbl.q,\
encryption_insert_values.q, \
encryption_drop_view.q, \
encryption_drop_partition.q, \
- encryption_with_trash.q, \
encryption_ctas.q, \
encryption_auto_purge_tables.q, \
encryption_drop_table_in_encrypted_db.q
@@ -1661,7 +1653,6 @@ miniSparkOnYarn.query.files=auto_sortmerge_join_16.q,\
schemeAuthority.q,\
schemeAuthority2.q,\
scriptfile1.q,\
- scriptfile1_win.q,\
temp_table_external.q,\
truncate_column_buckets.q,\
uber_reduce.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
index 3e98a09..c5589b9 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
@@ -511,7 +511,7 @@ public class TestTxnCommands2 {
FileStatus[] buckets = fs.listStatus(status[i].getPath(), FileUtils.HIDDEN_FILES_PATH_FILTER);
Arrays.sort(buckets);
Assert.assertEquals(2, buckets.length);
- Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_00000"));
+ Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]"));
}
}
Assert.assertTrue(sawNewBase);
@@ -545,7 +545,7 @@ public class TestTxnCommands2 {
FileStatus[] buckets = fs.listStatus(status[0].getPath(), FileUtils.HIDDEN_FILES_PATH_FILTER);
Arrays.sort(buckets);
Assert.assertEquals(2, buckets.length);
- Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_00000"));
+ Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]"));
rs = runStatementOnDriver("select a,b from " + Table.NONACIDORCTBL);
resultData = new int[][] {{3, 4}, {1, 2}};
Assert.assertEquals(stringifyValuesNoSort(resultData), rs);
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/ql/src/test/queries/clientpositive/explainuser_4.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/explainuser_4.q b/ql/src/test/queries/clientpositive/explainuser_4.q
index a2c25bc..2125d09 100644
--- a/ql/src/test/queries/clientpositive/explainuser_4.q
+++ b/ql/src/test/queries/clientpositive/explainuser_4.q
@@ -1,3 +1,4 @@
+-- SORT_QUERY_RESULTS
--! qt:dataset:alltypesorc
set hive.mapred.mode=nonstrict;
set hive.llap.memory.oversubscription.max.executors.per.query=0;
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/ql/src/test/queries/clientpositive/orc_schema_evolution_float.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/orc_schema_evolution_float.q b/ql/src/test/queries/clientpositive/orc_schema_evolution_float.q
index c2d9840..0b31902 100644
--- a/ql/src/test/queries/clientpositive/orc_schema_evolution_float.q
+++ b/ql/src/test/queries/clientpositive/orc_schema_evolution_float.q
@@ -1,6 +1,7 @@
set hive.vectorized.execution.enabled=false;
set hive.optimize.index.filter=false;
set hive.metastore.disallow.incompatible.col.type.changes=false;
+-- SORT_QUERY_RESULTS
drop table float_text;
create table float_text(f float);
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/ql/src/test/queries/clientpositive/parquet_ppd_multifiles.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/parquet_ppd_multifiles.q b/ql/src/test/queries/clientpositive/parquet_ppd_multifiles.q
index 075e854..2edaefb 100644
--- a/ql/src/test/queries/clientpositive/parquet_ppd_multifiles.q
+++ b/ql/src/test/queries/clientpositive/parquet_ppd_multifiles.q
@@ -1,4 +1,5 @@
set hive.vectorized.execution.enabled=false;
+-- SORT_QUERY_RESULTS
CREATE TABLE parquet_ppd_multifiles (
name string,
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/ql/src/test/queries/clientpositive/selectindate.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/selectindate.q b/ql/src/test/queries/clientpositive/selectindate.q
index 00f6a5a..5ca6da1 100644
--- a/ql/src/test/queries/clientpositive/selectindate.q
+++ b/ql/src/test/queries/clientpositive/selectindate.q
@@ -1,3 +1,4 @@
+-- SORT_QUERY_RESULTS
drop table if exists datetest;
create table datetest(dValue date, iValue int);
insert into datetest values('2000-03-22', 1);
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/ql/src/test/queries/clientpositive/union38.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/union38.q b/ql/src/test/queries/clientpositive/union38.q
index 2ea02bf..156e60e 100644
--- a/ql/src/test/queries/clientpositive/union38.q
+++ b/ql/src/test/queries/clientpositive/union38.q
@@ -1,3 +1,4 @@
+-- SORT_QUERY_RESULTS
--create table union38_test1
create table union38_test1( name String, id int, address String);
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/ql/src/test/queries/clientpositive/vector_bround.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_bround.q b/ql/src/test/queries/clientpositive/vector_bround.q
index 0bada68..3f84412 100644
--- a/ql/src/test/queries/clientpositive/vector_bround.q
+++ b/ql/src/test/queries/clientpositive/vector_bround.q
@@ -1,6 +1,7 @@
set hive.mapred.mode=nonstrict;
set hive.explain.user=false;
SET hive.fetch.task.conversion=none;
+-- SORT_QUERY_RESULTS
create table test_vector_bround(v0 double, v1 double) stored as orc;
insert into table test_vector_bround
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/ql/src/test/queries/clientpositive/vector_case_when_1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_case_when_1.q b/ql/src/test/queries/clientpositive/vector_case_when_1.q
index 8614087..d027524 100644
--- a/ql/src/test/queries/clientpositive/vector_case_when_1.q
+++ b/ql/src/test/queries/clientpositive/vector_case_when_1.q
@@ -1,3 +1,4 @@
+-- SORT_QUERY_RESULTS
--! qt:dataset:lineitem
set hive.cli.print.header=true;
set hive.mapred.mode=nonstrict;
@@ -5,8 +6,6 @@ set hive.explain.user=false;
set hive.fetch.task.conversion=none;
set hive.vectorized.execution.enabled=true;
--- SORT_QUERY_RESULTS
-
CREATE TABLE lineitem_test_txt (L_ORDERKEY INT,
L_PARTKEY INT,
L_SUPPKEY INT,
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/ql/src/test/queries/clientpositive/vector_coalesce_2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_coalesce_2.q b/ql/src/test/queries/clientpositive/vector_coalesce_2.q
index a264edd..43b71cf 100644
--- a/ql/src/test/queries/clientpositive/vector_coalesce_2.q
+++ b/ql/src/test/queries/clientpositive/vector_coalesce_2.q
@@ -1,3 +1,4 @@
+-- SORT_QUERY_RESULTS
set hive.mapred.mode=nonstrict;
set hive.explain.user=false;
SET hive.vectorized.execution.enabled=true;
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/ql/src/test/queries/clientpositive/vector_coalesce_3.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_coalesce_3.q b/ql/src/test/queries/clientpositive/vector_coalesce_3.q
index 9e00bdce..44e9301 100644
--- a/ql/src/test/queries/clientpositive/vector_coalesce_3.q
+++ b/ql/src/test/queries/clientpositive/vector_coalesce_3.q
@@ -1,3 +1,4 @@
+-- SORT_QUERY_RESULTS
set hive.mapred.mode=nonstrict;
set hive.explain.user=false;
SET hive.vectorized.execution.enabled=true;
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/ql/src/test/queries/clientpositive/vector_interval_1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_interval_1.q b/ql/src/test/queries/clientpositive/vector_interval_1.q
index 4320c88..4e98ce6 100644
--- a/ql/src/test/queries/clientpositive/vector_interval_1.q
+++ b/ql/src/test/queries/clientpositive/vector_interval_1.q
@@ -1,3 +1,4 @@
+-- SORT_QUERY_RESULTS
--! qt:dataset:src
set hive.mapred.mode=nonstrict;
set hive.explain.user=false;
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/ql/src/test/queries/clientpositive/vectorized_parquet_types.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vectorized_parquet_types.q b/ql/src/test/queries/clientpositive/vectorized_parquet_types.q
index f5d28c4..abcf955 100644
--- a/ql/src/test/queries/clientpositive/vectorized_parquet_types.q
+++ b/ql/src/test/queries/clientpositive/vectorized_parquet_types.q
@@ -1,3 +1,4 @@
+-- SORT_QUERY_RESULTS
set hive.mapred.mode=nonstrict;
set hive.explain.user=false;
set hive.llap.cache.allow.synthetic.fileid=true;
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/ql/src/test/results/clientpositive/llap/explainuser_4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/explainuser_4.q.out b/ql/src/test/results/clientpositive/llap/explainuser_4.q.out
index c7a7153..fe33072 100644
--- a/ql/src/test/results/clientpositive/llap/explainuser_4.q.out
+++ b/ql/src/test/results/clientpositive/llap/explainuser_4.q.out
@@ -69,16 +69,16 @@ order by a.cint
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
+-51 NULL 2089466 -240556350 -51.0 NULL cXX24dH7tblSj46j2g C31eea0wrHHqvj 1969-12-31 16:00:08.451 NULL true true -51 NULL 2089466 -240556350 -51.0 NULL cXX24dH7tblSj46j2g C31eea0wrHHqvj 1969-12-31 16:00:08.451 NULL true true
+-51 NULL 2949963 -1580871111 -51.0 NULL 0K68k3bdl7jO7 TPPAu 1969-12-31 16:00:08.451 NULL true false -51 NULL 2949963 -1580871111 -51.0 NULL 0K68k3bdl7jO7 TPPAu 1969-12-31 16:00:08.451 NULL true false
11 NULL 1000828 1531084669 11.0 NULL wM316f6NqGIkoP388j3F6 poWQQo3Upvt3Wh 1969-12-31 16:00:02.351 NULL false true 11 NULL 1000828 1531084669 11.0 NULL wM316f6NqGIkoP388j3F6 poWQQo3Upvt3Wh 1969-12-31 16:00:02.351 NULL false true
-NULL -3799 1248059 1864027286 NULL -3799.0 Uhps6mMh3IfHB3j7yH62K 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:54.622 false true NULL -3799 1248059 1864027286 NULL -3799.0 Uhps6mMh3IfHB3j7yH62K 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:54.622 false true
-NULL 10782 1286921 1864027286 NULL 10782.0 ODLrXI8882q8LS8 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:52.138 true true NULL 10782 1286921 1864027286 NULL 10782.0 ODLrXI8882q8LS8 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:52.138 true true
-NULL -13036 1288927 -1645852809 NULL -13036.0 yinBY725P7V2 xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:00.763 true false NULL -13036 1288927 -1645852809 NULL -13036.0 yinBY725P7V2 xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:00.763 true false
11 NULL 1310786 -413875656 11.0 NULL W0rvA4H1xn0xMG4uk0 8yVVjG 1969-12-31 16:00:02.351 NULL false true 11 NULL 1310786 -413875656 11.0 NULL W0rvA4H1xn0xMG4uk0 8yVVjG 1969-12-31 16:00:02.351 NULL false true
--51 NULL 2089466 -240556350 -51.0 NULL cXX24dH7tblSj46j2g C31eea0wrHHqvj 1969-12-31 16:00:08.451 NULL true true -51 NULL 2089466 -240556350 -51.0 NULL cXX24dH7tblSj46j2g C31eea0wrHHqvj 1969-12-31 16:00:08.451 NULL true true
-NULL -8915 2101183 1864027286 NULL -8915.0 x7By66525 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:05.831 false true NULL -8915 2101183 1864027286 NULL -8915.0 x7By66525 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:05.831 false true
8 NULL 2229621 -381406148 8.0 NULL q7onkS7QRPh5ghOK oKb0bi 1969-12-31 16:00:15.892 NULL true false 8 NULL 2229621 -381406148 8.0 NULL q7onkS7QRPh5ghOK oKb0bi 1969-12-31 16:00:15.892 NULL true false
8 NULL 2433892 -1611863517 8.0 NULL 674ILv3V2TxFqXP6wSbL VLprkK2XfX 1969-12-31 16:00:15.892 NULL false true 8 NULL 2433892 -1611863517 8.0 NULL 674ILv3V2TxFqXP6wSbL VLprkK2XfX 1969-12-31 16:00:15.892 NULL false true
--51 NULL 2949963 -1580871111 -51.0 NULL 0K68k3bdl7jO7 TPPAu 1969-12-31 16:00:08.451 NULL true false -51 NULL 2949963 -1580871111 -51.0 NULL 0K68k3bdl7jO7 TPPAu 1969-12-31 16:00:08.451 NULL true false
+NULL -13036 1288927 -1645852809 NULL -13036.0 yinBY725P7V2 xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:00.763 true false NULL -13036 1288927 -1645852809 NULL -13036.0 yinBY725P7V2 xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:00.763 true false
+NULL -3799 1248059 1864027286 NULL -3799.0 Uhps6mMh3IfHB3j7yH62K 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:54.622 false true NULL -3799 1248059 1864027286 NULL -3799.0 Uhps6mMh3IfHB3j7yH62K 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:54.622 false true
+NULL -8915 2101183 1864027286 NULL -8915.0 x7By66525 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:05.831 false true NULL -8915 2101183 1864027286 NULL -8915.0 x7By66525 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:05.831 false true
+NULL 10782 1286921 1864027286 NULL 10782.0 ODLrXI8882q8LS8 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:52.138 true true NULL 10782 1286921 1864027286 NULL 10782.0 ODLrXI8882q8LS8 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:52.138 true true
PREHOOK: query: explain
select
count(*)
@@ -233,8 +233,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-13036 1
--8915 1
-3799 1
+-8915 1
10782 1
NULL 6
PREHOOK: query: explain
@@ -308,16 +308,16 @@ order by a.cint
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
+-51 NULL 2089466 -240556350 -51.0 NULL cXX24dH7tblSj46j2g C31eea0wrHHqvj 1969-12-31 16:00:08.451 NULL true true -51 NULL 2089466 -240556350 -51.0 NULL cXX24dH7tblSj46j2g C31eea0wrHHqvj 1969-12-31 16:00:08.451 NULL true true
+-51 NULL 2949963 -1580871111 -51.0 NULL 0K68k3bdl7jO7 TPPAu 1969-12-31 16:00:08.451 NULL true false -51 NULL 2949963 -1580871111 -51.0 NULL 0K68k3bdl7jO7 TPPAu 1969-12-31 16:00:08.451 NULL true false
11 NULL 1000828 1531084669 11.0 NULL wM316f6NqGIkoP388j3F6 poWQQo3Upvt3Wh 1969-12-31 16:00:02.351 NULL false true 11 NULL 1000828 1531084669 11.0 NULL wM316f6NqGIkoP388j3F6 poWQQo3Upvt3Wh 1969-12-31 16:00:02.351 NULL false true
-NULL -3799 1248059 1864027286 NULL -3799.0 Uhps6mMh3IfHB3j7yH62K 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:54.622 false true NULL -3799 1248059 1864027286 NULL -3799.0 Uhps6mMh3IfHB3j7yH62K 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:54.622 false true
-NULL 10782 1286921 1864027286 NULL 10782.0 ODLrXI8882q8LS8 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:52.138 true true NULL 10782 1286921 1864027286 NULL 10782.0 ODLrXI8882q8LS8 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:52.138 true true
-NULL -13036 1288927 -1645852809 NULL -13036.0 yinBY725P7V2 xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:00.763 true false NULL -13036 1288927 -1645852809 NULL -13036.0 yinBY725P7V2 xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:00.763 true false
11 NULL 1310786 -413875656 11.0 NULL W0rvA4H1xn0xMG4uk0 8yVVjG 1969-12-31 16:00:02.351 NULL false true 11 NULL 1310786 -413875656 11.0 NULL W0rvA4H1xn0xMG4uk0 8yVVjG 1969-12-31 16:00:02.351 NULL false true
--51 NULL 2089466 -240556350 -51.0 NULL cXX24dH7tblSj46j2g C31eea0wrHHqvj 1969-12-31 16:00:08.451 NULL true true -51 NULL 2089466 -240556350 -51.0 NULL cXX24dH7tblSj46j2g C31eea0wrHHqvj 1969-12-31 16:00:08.451 NULL true true
-NULL -8915 2101183 1864027286 NULL -8915.0 x7By66525 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:05.831 false true NULL -8915 2101183 1864027286 NULL -8915.0 x7By66525 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:05.831 false true
8 NULL 2229621 -381406148 8.0 NULL q7onkS7QRPh5ghOK oKb0bi 1969-12-31 16:00:15.892 NULL true false 8 NULL 2229621 -381406148 8.0 NULL q7onkS7QRPh5ghOK oKb0bi 1969-12-31 16:00:15.892 NULL true false
8 NULL 2433892 -1611863517 8.0 NULL 674ILv3V2TxFqXP6wSbL VLprkK2XfX 1969-12-31 16:00:15.892 NULL false true 8 NULL 2433892 -1611863517 8.0 NULL 674ILv3V2TxFqXP6wSbL VLprkK2XfX 1969-12-31 16:00:15.892 NULL false true
--51 NULL 2949963 -1580871111 -51.0 NULL 0K68k3bdl7jO7 TPPAu 1969-12-31 16:00:08.451 NULL true false -51 NULL 2949963 -1580871111 -51.0 NULL 0K68k3bdl7jO7 TPPAu 1969-12-31 16:00:08.451 NULL true false
+NULL -13036 1288927 -1645852809 NULL -13036.0 yinBY725P7V2 xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:00.763 true false NULL -13036 1288927 -1645852809 NULL -13036.0 yinBY725P7V2 xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:00.763 true false
+NULL -3799 1248059 1864027286 NULL -3799.0 Uhps6mMh3IfHB3j7yH62K 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:54.622 false true NULL -3799 1248059 1864027286 NULL -3799.0 Uhps6mMh3IfHB3j7yH62K 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:54.622 false true
+NULL -8915 2101183 1864027286 NULL -8915.0 x7By66525 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:05.831 false true NULL -8915 2101183 1864027286 NULL -8915.0 x7By66525 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:05.831 false true
+NULL 10782 1286921 1864027286 NULL 10782.0 ODLrXI8882q8LS8 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:52.138 true true NULL 10782 1286921 1864027286 NULL 10782.0 ODLrXI8882q8LS8 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:52.138 true true
PREHOOK: query: explain
select
count(*)
@@ -472,8 +472,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-13036 1
--8915 1
-3799 1
+-8915 1
10782 1
NULL 6
Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/ql/src/test/results/clientpositive/llap/vector_bround.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_bround.q.out b/ql/src/test/results/clientpositive/llap/vector_bround.q.out
index 02770a2..3e9a370 100644
--- a/ql/src/test/results/clientpositive/llap/vector_bround.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_bround.q.out
@@ -122,12 +122,12 @@ POSTHOOK: query: select bround(v0), bround(v1, 1) from test_vector_bround
POSTHOOK: type: QUERY
POSTHOOK: Input: default@test_vector_bround
#### A masked pattern was here ####
-2.0 1.2
-4.0 1.4
-2.0 -1.2
-4.0 -1.4
2.0 1.2
+2.0 1.2
3.0 1.3
3.0 1.3
4.0 1.4
+4.0 1.4
NULL NULL
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out b/ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out
index f43c9ea..3776f3e 100644
--- a/ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out
@@ -168,8 +168,8 @@ GROUP BY str2
POSTHOOK: type: QUERY
POSTHOOK: Input: default@str_str_orc
#### A masked pattern was here ####
-X 0.02
NULL 0.0
+X 0.02
y 0.0
PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION
SELECT COALESCE(str1, 0) as result
@@ -248,9 +248,9 @@ POSTHOOK: Input: default@str_str_orc
#### A masked pattern was here ####
0
0
-1
0
0
+1
PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION
SELECT
str2, ROUND(sum(cast(COALESCE(str1, 0) as int))/60, 2) as result
@@ -393,8 +393,8 @@ GROUP BY str2
POSTHOOK: type: QUERY
POSTHOOK: Input: default@str_str_orc
#### A masked pattern was here ####
-X 0.02
NULL 0.0
+X 0.02
y 0.0
PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION
SELECT COALESCE(str1, 0) as result
@@ -473,6 +473,6 @@ POSTHOOK: Input: default@str_str_orc
#### A masked pattern was here ####
0
0
-1
0
0
+1
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/ql/src/test/results/clientpositive/llap/vector_interval_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_interval_1.q.out b/ql/src/test/results/clientpositive/llap/vector_interval_1.q.out
index 43c5761..6e0663c 100644
--- a/ql/src/test/results/clientpositive/llap/vector_interval_1.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_interval_1.q.out
@@ -170,8 +170,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@vector_interval_1
#### A masked pattern was here ####
str1 _c1 _c2 _c3 _c4
-NULL 1-2 NULL 1 02:03:04.000000000 NULL
1-2 1-2 1-2 1 02:03:04.000000000 1 02:03:04.000000000
+NULL 1-2 NULL 1 02:03:04.000000000 NULL
PREHOOK: query: explain vectorization expression
select
dt,
@@ -307,8 +307,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@vector_interval_1
#### A masked pattern was here ####
dt _c1 _c2 _c3 _c4 _c5 _c6
-NULL 2-4 NULL NULL 0-0 NULL NULL
2001-01-01 2-4 2-4 2-4 0-0 0-0 0-0
+NULL 2-4 NULL NULL 0-0 NULL NULL
PREHOOK: query: explain vectorization expression
select
dt,
@@ -444,8 +444,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@vector_interval_1
#### A masked pattern was here ####
dt _c1 _c2 _c3 _c4 _c5 _c6
-NULL 2 04:06:08.000000000 NULL NULL 0 00:00:00.000000000 NULL NULL
2001-01-01 2 04:06:08.000000000 2 04:06:08.000000000 2 04:06:08.000000000 0 00:00:00.000000000 0 00:00:00.000000000 0 00:00:00.000000000
+NULL 2 04:06:08.000000000 NULL NULL 0 00:00:00.000000000 NULL NULL
PREHOOK: query: explain vectorization expression
select
dt,
@@ -604,8 +604,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@vector_interval_1
#### A masked pattern was here ####
dt _c1 _c2 _c3 _c4 _c5 _c6 _c7 _c8 _c9 _c10 _c11 _c12
-NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
2001-01-01 2002-03-01 2002-03-01 2002-03-01 2002-03-01 1999-11-01 1999-11-01 2001-01-02 02:03:04 2001-01-02 02:03:04 2001-01-02 02:03:04 2001-01-02 02:03:04 2000-12-30 21:56:56 2000-12-30 21:56:56
+NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
PREHOOK: query: explain vectorization expression
select
ts,
@@ -764,8 +764,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@vector_interval_1
#### A masked pattern was here ####
ts _c1 _c2 _c3 _c4 _c5 _c6 _c7 _c8 _c9 _c10 _c11 _c12
-NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
2001-01-01 01:02:03 2002-03-01 01:02:03 2002-03-01 01:02:03 2002-03-01 01:02:03 2002-03-01 01:02:03 1999-11-01 01:02:03 1999-11-01 01:02:03 2001-01-02 03:05:07 2001-01-02 03:05:07 2001-01-02 03:05:07 2001-01-02 03:05:07 2000-12-30 22:58:59 2000-12-30 22:58:59
+NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
PREHOOK: query: explain vectorization expression
select
ts,
@@ -888,8 +888,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@vector_interval_1
#### A masked pattern was here ####
ts _c1 _c2 _c3
-NULL NULL NULL NULL
2001-01-01 01:02:03 0 00:00:00.000000000 0 00:00:00.000000000 0 00:00:00.000000000
+NULL NULL NULL NULL
PREHOOK: query: explain vectorization expression
select
dt,
@@ -1012,8 +1012,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@vector_interval_1
#### A masked pattern was here ####
dt _c1 _c2 _c3
-NULL NULL NULL NULL
2001-01-01 0 00:00:00.000000000 0 00:00:00.000000000 0 00:00:00.000000000
+NULL NULL NULL NULL
PREHOOK: query: explain vectorization expression
select
dt,
@@ -1148,5 +1148,5 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@vector_interval_1
#### A masked pattern was here ####
dt _c1 _c2 _c3 _c4 _c5 _c6
-NULL NULL NULL NULL NULL NULL NULL
2001-01-01 0 01:02:03.000000000 0 01:02:03.000000000 0 01:02:03.000000000 -0 01:02:03.000000000 -0 01:02:03.000000000 -0 01:02:03.000000000
+NULL NULL NULL NULL NULL NULL NULL
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/ql/src/test/results/clientpositive/llap/vectorized_parquet_types.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vectorized_parquet_types.q.out b/ql/src/test/results/clientpositive/llap/vectorized_parquet_types.q.out
index d5cf9f5..d0d13ba 100644
--- a/ql/src/test/results/clientpositive/llap/vectorized_parquet_types.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorized_parquet_types.q.out
@@ -218,13 +218,14 @@ abcd 4 abcd 4 3.89 1
abcde 5 abcde 5 56.23 1
abcde 5 abcdef 6 90.21 1
abcde 5 abcdefg 7 6.09 1
+bcdef 5 abcde 5 90.33 1
bcdef 5 abcdefgh 8 9.44 1
cdefg 5 B4F3CAFDBE 10 77.54 1
klmno 5 abcdedef 8 25.42 1
-pqrst 5 abcdede 7 60.12 1
nopqr 5 abcded 6 49.56 1
opqrs 5 abcdd 5 80.76 1
pqrst 5 abc 3 23.23 1
+pqrst 5 abcdede 7 60.12 1
qrstu 5 b 1 1.01 1
rstuv 5 abcded 6 5.98 1
stuvw 5 abcded 6 11.22 1
@@ -232,7 +233,6 @@ tuvwx 5 abcded 6 9.88 1
uvwzy 5 abcdede 7 4.76 1
vwxyz 5 abcdede 7 12.83 1
wxyza 5 abcde 5 73.04 1
-bcdef 5 abcde 5 90.33 1
PREHOOK: query: explain vectorization expression
SELECT ctinyint,
MAX(cint),
@@ -559,13 +559,14 @@ abcd 4 abcd 4 3.89 1
abcde 5 abcde 5 56.23 1
abcde 5 abcdef 6 90.21 1
abcde 5 abcdefg 7 6.09 1
+bcdef 5 abcde 5 90.33 1
bcdef 5 abcdefgh 8 9.44 1
cdefg 5 B4F3CAFDBE 10 77.54 1
klmno 5 abcdedef 8 25.42 1
-pqrst 5 abcdede 7 60.12 1
nopqr 5 abcded 6 49.56 1
opqrs 5 abcdd 5 80.76 1
pqrst 5 abc 3 23.23 1
+pqrst 5 abcdede 7 60.12 1
qrstu 5 b 1 1.01 1
rstuv 5 abcded 6 5.98 1
stuvw 5 abcded 6 11.22 1
@@ -573,7 +574,6 @@ tuvwx 5 abcded 6 9.88 1
uvwzy 5 abcdede 7 4.76 1
vwxyz 5 abcdede 7 12.83 1
wxyza 5 abcde 5 73.04 1
-bcdef 5 abcde 5 90.33 1
PREHOOK: query: explain vectorization select max(t), min(t) from parquet_type_nodict
PREHOOK: type: QUERY
POSTHOOK: query: explain vectorization select max(t), min(t) from parquet_type_nodict
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/ql/src/test/results/clientpositive/orc_schema_evolution_float.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/orc_schema_evolution_float.q.out b/ql/src/test/results/clientpositive/orc_schema_evolution_float.q.out
index 1e6e473..19b5ca1 100644
--- a/ql/src/test/results/clientpositive/orc_schema_evolution_float.q.out
+++ b/ql/src/test/results/clientpositive/orc_schema_evolution_float.q.out
@@ -36,8 +36,8 @@ POSTHOOK: query: select f from float_text
POSTHOOK: type: QUERY
POSTHOOK: Input: default@float_text
#### A masked pattern was here ####
-74.72
0.22
+74.72
PREHOOK: query: alter table float_text change column f f double
PREHOOK: type: ALTERTABLE_RENAMECOL
PREHOOK: Input: default@float_text
@@ -54,8 +54,8 @@ POSTHOOK: query: select f from float_text
POSTHOOK: type: QUERY
POSTHOOK: Input: default@float_text
#### A masked pattern was here ####
-74.72
0.22
+74.72
PREHOOK: query: select f from float_text where f=74.72
PREHOOK: type: QUERY
PREHOOK: Input: default@float_text
@@ -90,8 +90,8 @@ POSTHOOK: query: select f from float_text
POSTHOOK: type: QUERY
POSTHOOK: Input: default@float_text
#### A masked pattern was here ####
-74.72000
0.22000
+74.72000
PREHOOK: query: select f from float_text where f=74.72
PREHOOK: type: QUERY
PREHOOK: Input: default@float_text
@@ -135,8 +135,8 @@ POSTHOOK: query: select f from float_orc
POSTHOOK: type: QUERY
POSTHOOK: Input: default@float_orc
#### A masked pattern was here ####
-74.72
0.22
+74.72
PREHOOK: query: alter table float_orc change column f f double
PREHOOK: type: ALTERTABLE_RENAMECOL
PREHOOK: Input: default@float_orc
@@ -153,8 +153,8 @@ POSTHOOK: query: select f from float_orc
POSTHOOK: type: QUERY
POSTHOOK: Input: default@float_orc
#### A masked pattern was here ####
-74.72
0.22
+74.72
PREHOOK: query: select f from float_orc where f=74.72
PREHOOK: type: QUERY
PREHOOK: Input: default@float_orc
@@ -207,8 +207,8 @@ POSTHOOK: query: select f from float_orc
POSTHOOK: type: QUERY
POSTHOOK: Input: default@float_orc
#### A masked pattern was here ####
-74.72000
0.22000
+74.72000
PREHOOK: query: select f from float_orc where f=74.72
PREHOOK: type: QUERY
PREHOOK: Input: default@float_orc
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/ql/src/test/results/clientpositive/union38.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union38.q.out b/ql/src/test/results/clientpositive/union38.q.out
index 285d4a4..8fe3c7d 100644
--- a/ql/src/test/results/clientpositive/union38.q.out
+++ b/ql/src/test/results/clientpositive/union38.q.out
@@ -50,8 +50,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@union38_test1
POSTHOOK: Input: default@union38_test1_view
#### A masked pattern was here ####
-Young 1 Sydney
Jin 2 Mel
+Young 1 Sydney
PREHOOK: query: create table union38_test2( name String, id int, address String)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
@@ -186,10 +186,10 @@ POSTHOOK: Input: default@union38_test1
POSTHOOK: Input: default@union38_test2
POSTHOOK: Input: default@union38_test_view1
#### A masked pattern was here ####
-Young 1 Sydney
-Jin 2 Mel
Eun 3 Bri
+Jin 2 Mel
Kim 4 Ad
+Young 1 Sydney
PREHOOK: query: create view union38_test_view2 as with union38_tmp_1 as ( select * from union38_test1 ), union38_tmp_2 as (select * from union38_test2 ) select * from union38_tmp_1 union select * from union38_tmp_2
PREHOOK: type: CREATEVIEW
PREHOOK: Input: default@union38_test1
@@ -248,10 +248,10 @@ POSTHOOK: Input: default@union38_test1
POSTHOOK: Input: default@union38_test2
POSTHOOK: Input: default@union38_test_view3
#### A masked pattern was here ####
-Young 1 Sydney
-Jin 2 Mel
Eun 3 Bri
+Jin 2 Mel
Kim 4 Ad
+Young 1 Sydney
PREHOOK: query: create table union38_test_table1 as with union38_tmp_1 as ( select * from union38_test1 ), union38_tmp_2 as (select * from union38_test2 ) select * from union38_tmp_1 union all select * from union38_tmp_2
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@union38_test1
@@ -275,7 +275,7 @@ POSTHOOK: query: select * from union38_test_table1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@union38_test_table1
#### A masked pattern was here ####
-Young 1 Sydney
-Jin 2 Mel
Eun 3 Bri
+Jin 2 Mel
Kim 4 Ad
+Young 1 Sydney
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/ql/src/test/results/clientpositive/vector_bround.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_bround.q.out b/ql/src/test/results/clientpositive/vector_bround.q.out
index 68086c2..6dcc0f1 100644
--- a/ql/src/test/results/clientpositive/vector_bround.q.out
+++ b/ql/src/test/results/clientpositive/vector_bround.q.out
@@ -118,12 +118,12 @@ POSTHOOK: query: select bround(v0), bround(v1, 1) from test_vector_bround
POSTHOOK: type: QUERY
POSTHOOK: Input: default@test_vector_bround
#### A masked pattern was here ####
-2.0 1.2
-4.0 1.4
-2.0 -1.2
-4.0 -1.4
2.0 1.2
+2.0 1.2
3.0 1.3
3.0 1.3
4.0 1.4
+4.0 1.4
NULL NULL
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/ql/src/test/results/clientpositive/vector_coalesce_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_coalesce_2.q.out b/ql/src/test/results/clientpositive/vector_coalesce_2.q.out
index 6030750..918ac59 100644
--- a/ql/src/test/results/clientpositive/vector_coalesce_2.q.out
+++ b/ql/src/test/results/clientpositive/vector_coalesce_2.q.out
@@ -218,9 +218,9 @@ POSTHOOK: Input: default@str_str_orc
#### A masked pattern was here ####
0
0
-1
0
0
+1
PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION
SELECT
str2, ROUND(sum(cast(COALESCE(str1, 0) as int))/60, 2) as result
@@ -413,6 +413,6 @@ POSTHOOK: Input: default@str_str_orc
#### A masked pattern was here ####
0
0
-1
0
0
+1
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/ql/src/test/results/clientpositive/vector_interval_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_interval_1.q.out b/ql/src/test/results/clientpositive/vector_interval_1.q.out
index 7bbd4a7..9a2a18a 100644
--- a/ql/src/test/results/clientpositive/vector_interval_1.q.out
+++ b/ql/src/test/results/clientpositive/vector_interval_1.q.out
@@ -152,8 +152,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@vector_interval_1
#### A masked pattern was here ####
str1 _c1 _c2 _c3 _c4
-NULL 1-2 NULL 1 02:03:04.000000000 NULL
1-2 1-2 1-2 1 02:03:04.000000000 1 02:03:04.000000000
+NULL 1-2 NULL 1 02:03:04.000000000 NULL
PREHOOK: query: explain vectorization expression
select
dt,
@@ -271,8 +271,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@vector_interval_1
#### A masked pattern was here ####
dt _c1 _c2 _c3 _c4 _c5 _c6
-NULL 2-4 NULL NULL 0-0 NULL NULL
2001-01-01 2-4 2-4 2-4 0-0 0-0 0-0
+NULL 2-4 NULL NULL 0-0 NULL NULL
PREHOOK: query: explain vectorization expression
select
dt,
@@ -390,8 +390,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@vector_interval_1
#### A masked pattern was here ####
dt _c1 _c2 _c3 _c4 _c5 _c6
-NULL 2 04:06:08.000000000 NULL NULL 0 00:00:00.000000000 NULL NULL
2001-01-01 2 04:06:08.000000000 2 04:06:08.000000000 2 04:06:08.000000000 0 00:00:00.000000000 0 00:00:00.000000000 0 00:00:00.000000000
+NULL 2 04:06:08.000000000 NULL NULL 0 00:00:00.000000000 NULL NULL
PREHOOK: query: explain vectorization expression
select
dt,
@@ -533,8 +533,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@vector_interval_1
#### A masked pattern was here ####
dt _c1 _c2 _c3 _c4 _c5 _c6 _c7 _c8 _c9 _c10 _c11 _c12
-NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
2001-01-01 2002-03-01 2002-03-01 2002-03-01 2002-03-01 1999-11-01 1999-11-01 2001-01-02 02:03:04 2001-01-02 02:03:04 2001-01-02 02:03:04 2001-01-02 02:03:04 2000-12-30 21:56:56 2000-12-30 21:56:56
+NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
PREHOOK: query: explain vectorization expression
select
ts,
@@ -676,8 +676,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@vector_interval_1
#### A masked pattern was here ####
ts _c1 _c2 _c3 _c4 _c5 _c6 _c7 _c8 _c9 _c10 _c11 _c12
-NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
2001-01-01 01:02:03 2002-03-01 01:02:03 2002-03-01 01:02:03 2002-03-01 01:02:03 2002-03-01 01:02:03 1999-11-01 01:02:03 1999-11-01 01:02:03 2001-01-02 03:05:07 2001-01-02 03:05:07 2001-01-02 03:05:07 2001-01-02 03:05:07 2000-12-30 22:58:59 2000-12-30 22:58:59
+NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
PREHOOK: query: explain vectorization expression
select
ts,
@@ -783,8 +783,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@vector_interval_1
#### A masked pattern was here ####
ts _c1 _c2 _c3
-NULL NULL NULL NULL
2001-01-01 01:02:03 0 00:00:00.000000000 0 00:00:00.000000000 0 00:00:00.000000000
+NULL NULL NULL NULL
PREHOOK: query: explain vectorization expression
select
dt,
@@ -890,8 +890,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@vector_interval_1
#### A masked pattern was here ####
dt _c1 _c2 _c3
-NULL NULL NULL NULL
2001-01-01 0 00:00:00.000000000 0 00:00:00.000000000 0 00:00:00.000000000
+NULL NULL NULL NULL
PREHOOK: query: explain vectorization expression
select
dt,
@@ -1009,5 +1009,5 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@vector_interval_1
#### A masked pattern was here ####
dt _c1 _c2 _c3 _c4 _c5 _c6
-NULL NULL NULL NULL NULL NULL NULL
2001-01-01 0 01:02:03.000000000 0 01:02:03.000000000 0 01:02:03.000000000 -0 01:02:03.000000000 -0 01:02:03.000000000 -0 01:02:03.000000000
+NULL NULL NULL NULL NULL NULL NULL
http://git-wip-us.apache.org/repos/asf/hive/blob/5016d6f0/ql/src/test/results/clientpositive/vectorized_parquet_types.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorized_parquet_types.q.out b/ql/src/test/results/clientpositive/vectorized_parquet_types.q.out
index 9429b04..c1f2d54 100644
--- a/ql/src/test/results/clientpositive/vectorized_parquet_types.q.out
+++ b/ql/src/test/results/clientpositive/vectorized_parquet_types.q.out
@@ -286,13 +286,14 @@ abcd 4 abcd 4 3.89 1
abcde 5 abcde 5 56.23 1
abcde 5 abcdef 6 90.21 1
abcde 5 abcdefg 7 6.09 1
+bcdef 5 abcde 5 90.33 1
bcdef 5 abcdefgh 8 9.44 1
cdefg 5 B4F3CAFDBE 10 77.54 1
klmno 5 abcdedef 8 25.42 1
-pqrst 5 abcdede 7 60.12 1
nopqr 5 abcded 6 49.56 1
opqrs 5 abcdd 5 80.76 1
pqrst 5 abc 3 23.23 1
+pqrst 5 abcdede 7 60.12 1
qrstu 5 b 1 1.01 1
rstuv 5 abcded 6 5.98 1
stuvw 5 abcded 6 11.22 1
@@ -300,7 +301,6 @@ tuvwx 5 abcded 6 9.88 1
uvwzy 5 abcdede 7 4.76 1
vwxyz 5 abcdede 7 12.83 1
wxyza 5 abcde 5 73.04 1
-bcdef 5 abcde 5 90.33 1
PREHOOK: query: explain vectorization expression
SELECT ctinyint,
MAX(cint),
@@ -685,13 +685,14 @@ abcd 4 abcd 4 3.89 1
abcde 5 abcde 5 56.23 1
abcde 5 abcdef 6 90.21 1
abcde 5 abcdefg 7 6.09 1
+bcdef 5 abcde 5 90.33 1
bcdef 5 abcdefgh 8 9.44 1
cdefg 5 B4F3CAFDBE 10 77.54 1
klmno 5 abcdedef 8 25.42 1
-pqrst 5 abcdede 7 60.12 1
nopqr 5 abcded 6 49.56 1
opqrs 5 abcdd 5 80.76 1
pqrst 5 abc 3 23.23 1
+pqrst 5 abcdede 7 60.12 1
qrstu 5 b 1 1.01 1
rstuv 5 abcded 6 5.98 1
stuvw 5 abcded 6 11.22 1
@@ -699,7 +700,6 @@ tuvwx 5 abcded 6 9.88 1
uvwzy 5 abcdede 7 4.76 1
vwxyz 5 abcdede 7 12.83 1
wxyza 5 abcde 5 73.04 1
-bcdef 5 abcde 5 90.33 1
PREHOOK: query: explain vectorization select max(t), min(t) from parquet_type_nodict
PREHOOK: type: QUERY
POSTHOOK: query: explain vectorization select max(t), min(t) from parquet_type_nodict
[28/32] hive git commit: HIVE-18545: Add UDF to parse complex types
from json (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Posted by se...@apache.org.
HIVE-18545: Add UDF to parse complex types from json (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1105ef39
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1105ef39
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1105ef39
Branch: refs/heads/master-txnstats
Commit: 1105ef3974d8a324637d3d35881a739af3aeb382
Parents: 18fb1b3
Author: Zoltan Haindrich <ki...@rxd.hu>
Authored: Tue Jul 10 16:05:10 2018 +0200
Committer: Zoltan Haindrich <ki...@rxd.hu>
Committed: Tue Jul 10 16:05:10 2018 +0200
----------------------------------------------------------------------
.../apache/hive/hcatalog/data/JsonSerDe.java | 612 ++----------------
.../hive/hcatalog/data/TestJsonSerDe.java | 12 +-
.../benchmark/udf/json_read/JsonReadBench.java | 83 +++
.../hive/benchmark/udf/json_read/val1.json | 86 +++
.../hive/benchmark/udf/json_read/val1.type | 1 +
.../hadoop/hive/ql/exec/FunctionRegistry.java | 1 +
.../hive/ql/udf/generic/GenericUDFJsonRead.java | 92 +++
.../ql/udf/generic/TestGenericUDFJsonRead.java | 204 ++++++
.../test/queries/clientpositive/json_serde2.q | 37 ++
.../test/queries/clientpositive/udf_json_read.q | 44 ++
.../results/clientpositive/json_serde2.q.out | 113 ++++
.../results/clientpositive/show_functions.q.out | 1 +
.../results/clientpositive/udf_json_read.q.out | 107 +++
.../apache/hadoop/hive/serde2/JsonSerDe.java | 645 ++++++-------------
.../hive/serde2/json/HiveJsonStructReader.java | 402 ++++++++++++
.../apache/hive/streaming/StrictJsonWriter.java | 2 +-
16 files changed, 1428 insertions(+), 1014 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1105ef39/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java
index af80c02..87611ad 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java
@@ -18,125 +18,52 @@
*/
package org.apache.hive.hcatalog.data;
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.nio.charset.CharacterCodingException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Properties;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
+import java.util.Set;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.common.type.Date;
-import org.apache.hadoop.hive.common.type.HiveChar;
-import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hadoop.hive.common.type.HiveVarchar;
-import org.apache.hadoop.hive.common.type.Timestamp;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeSpec;
import org.apache.hadoop.hive.serde2.SerDeStats;
-import org.apache.hadoop.hive.serde2.SerDeUtils;
-import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructField;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveCharObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
-import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
-import org.apache.hive.common.util.HiveStringUtils;
-import org.apache.hive.common.util.TimestampParser;
import org.apache.hive.hcatalog.common.HCatException;
-import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
-import org.apache.hive.hcatalog.data.schema.HCatFieldSchema.Type;
import org.apache.hive.hcatalog.data.schema.HCatSchema;
import org.apache.hive.hcatalog.data.schema.HCatSchemaUtils;
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.JsonParseException;
-import org.codehaus.jackson.JsonParser;
-import org.codehaus.jackson.JsonToken;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@SerDeSpec(schemaProps = {serdeConstants.LIST_COLUMNS,
serdeConstants.LIST_COLUMN_TYPES,
serdeConstants.TIMESTAMP_FORMATS})
-
public class JsonSerDe extends AbstractSerDe {
private static final Logger LOG = LoggerFactory.getLogger(JsonSerDe.class);
- private List<String> columnNames;
private HCatSchema schema;
- private JsonFactory jsonFactory = null;
-
private HCatRecordObjectInspector cachedObjectInspector;
- private TimestampParser tsParser;
+ private org.apache.hadoop.hive.serde2.JsonSerDe jsonSerde = new org.apache.hadoop.hive.serde2.JsonSerDe();
@Override
public void initialize(Configuration conf, Properties tbl)
throws SerDeException {
- List<TypeInfo> columnTypes;
- StructTypeInfo rowTypeInfo;
-
- LOG.debug("Initializing JsonSerDe: {}", tbl.entrySet());
-
- // Get column names and types
- String columnNameProperty = tbl.getProperty(serdeConstants.LIST_COLUMNS);
- String columnTypeProperty = tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES);
- final String columnNameDelimiter = tbl.containsKey(serdeConstants.COLUMN_NAME_DELIMITER) ? tbl
- .getProperty(serdeConstants.COLUMN_NAME_DELIMITER) : String.valueOf(SerDeUtils.COMMA);
- // all table column names
- if (columnNameProperty.isEmpty()) {
- columnNames = Collections.emptyList();
- } else {
- columnNames = Arrays.asList(columnNameProperty.split(columnNameDelimiter));
- }
-
- // all column types
- if (columnTypeProperty.isEmpty()) {
- columnTypes = Collections.emptyList();
- } else {
- columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty);
- }
- LOG.debug("columns: {}, {}", columnNameProperty, columnNames);
- LOG.debug("types: {}, {} ", columnTypeProperty, columnTypes);
-
- assert (columnNames.size() == columnTypes.size());
-
- rowTypeInfo = (StructTypeInfo) TypeInfoFactory.getStructTypeInfo(columnNames, columnTypes);
+ jsonSerde.initialize(conf, tbl);
+ jsonSerde.setWriteablesUsage(false);
+ StructTypeInfo rowTypeInfo = jsonSerde.getTypeInfo();
cachedObjectInspector = HCatRecordObjectInspectorFactory.getHCatRecordObjectInspector(rowTypeInfo);
try {
schema = HCatSchemaUtils.getHCatSchema(rowTypeInfo).get(0).getStructSubSchema();
@@ -145,10 +72,6 @@ public class JsonSerDe extends AbstractSerDe {
} catch (HCatException e) {
throw new SerDeException(e);
}
-
- jsonFactory = new JsonFactory();
- tsParser = new TimestampParser(
- HiveStringUtils.splitAndUnEscape(tbl.getProperty(serdeConstants.TIMESTAMP_FORMATS)));
}
/**
@@ -160,274 +83,82 @@ public class JsonSerDe extends AbstractSerDe {
*/
@Override
public Object deserialize(Writable blob) throws SerDeException {
-
- Text t = (Text) blob;
- JsonParser p;
- List<Object> r = new ArrayList<Object>(Collections.nCopies(columnNames.size(), null));
try {
- p = jsonFactory.createJsonParser(new ByteArrayInputStream((t.getBytes())));
- if (p.nextToken() != JsonToken.START_OBJECT) {
- throw new IOException("Start token not found where expected");
- }
- JsonToken token;
- while (((token = p.nextToken()) != JsonToken.END_OBJECT) && (token != null)) {
- // iterate through each token, and create appropriate object here.
- populateRecord(r, token, p, schema);
- }
- } catch (JsonParseException e) {
- LOG.warn("Error [{}] parsing json text [{}].", e, t);
- throw new SerDeException(e);
- } catch (IOException e) {
- LOG.warn("Error [{}] parsing json text [{}].", e, t);
+ Object row = jsonSerde.deserialize(blob);
+ List fatRow = fatLand((Object[]) row);
+ return new DefaultHCatRecord(fatRow);
+ } catch (Exception e) {
throw new SerDeException(e);
}
-
- return new DefaultHCatRecord(r);
}
- private void populateRecord(List<Object> r, JsonToken token, JsonParser p, HCatSchema s) throws IOException {
- if (token != JsonToken.FIELD_NAME) {
- throw new IOException("Field name expected");
- }
- String fieldName = p.getText();
- Integer fpos = s.getPosition(fieldName);
- if (fpos == null) {
- fpos = getPositionFromHiveInternalColumnName(fieldName);
- LOG.debug("NPE finding position for field [{}] in schema [{}],"
- +" attempting to check if it is an internal column name like _col0", fieldName, s);
- if (fpos == -1) {
- skipValue(p);
- return; // unknown field, we return. We'll continue from the next field onwards.
- }
- // If we get past this, then the column name did match the hive pattern for an internal
- // column name, such as _col0, etc, so it *MUST* match the schema for the appropriate column.
- // This means people can't use arbitrary column names such as _col0, and expect us to ignore it
- // if we find it.
- if (!fieldName.equalsIgnoreCase(getHiveInternalColumnName(fpos))) {
- LOG.error("Hive internal column name {} and position "
- + "encoding {} for the column name are at odds", fieldName, fpos);
- throw new IOException("Hive internal column name ("+ fieldName
- + ") and position encoding (" + fpos
- + ") for the column name are at odds");
+ @SuppressWarnings({"rawtypes", "unchecked" })
+ private static List fatLand(Object[] arr) {
+ List ret = new ArrayList<>();
+ for (Object o : arr) {
+ if (o != null && o instanceof Map<?, ?>) {
+ ret.add(fatMap(((Map) o)));
+ } else if (o != null && o instanceof List<?>) {
+ ret.add(fatLand(((List) o).toArray()));
+ } else if (o != null && o.getClass().isArray() && o.getClass().getComponentType() != byte.class) {
+ Class<?> ct = o.getClass().getComponentType();
+ if (ct.isPrimitive()) {
+ ret.add(primitiveArrayToList(o));
+ } else {
+ ret.add(fatLand((Object[]) o));
+ }
+ } else {
+ ret.add(o);
}
- // If we reached here, then we were successful at finding an alternate internal
- // column mapping, and we're about to proceed.
}
- HCatFieldSchema hcatFieldSchema = s.getFields().get(fpos);
- Object currField = extractCurrentField(p, hcatFieldSchema, false);
- r.set(fpos, currField);
+ return ret;
}
- public String getHiveInternalColumnName(int fpos) {
- return HiveConf.getColumnInternalName(fpos);
- }
-
- public int getPositionFromHiveInternalColumnName(String internalName) {
-// return HiveConf.getPositionFromInternalName(fieldName);
- // The above line should have been all the implementation that
- // we need, but due to a bug in that impl which recognizes
- // only single-digit columns, we need another impl here.
- Pattern internalPattern = Pattern.compile("_col([0-9]+)");
- Matcher m = internalPattern.matcher(internalName);
- if (!m.matches()) {
- return -1;
- } else {
- return Integer.parseInt(m.group(1));
+ @SuppressWarnings("rawtypes")
+ private static Object fatMap(Map<Object, Object> map) {
+ Map ret = new LinkedHashMap<>();
+ Set<Entry<Object, Object>> es = map.entrySet();
+ for (Entry<Object, Object> e : es) {
+ Object oldV = e.getValue();
+ Object newV;
+ if (oldV != null && oldV.getClass().isArray()) {
+ newV = fatLand((Object[]) oldV);
+ } else {
+ newV = oldV;
+ }
+ ret.put(e.getKey(), newV);
}
+ return ret;
}
- /**
- * Utility method to extract (and forget) the next value token from the JsonParser,
- * as a whole. The reason this function gets called is to yank out the next value altogether,
- * because it corresponds to a field name that we do not recognize, and thus, do not have
- * a schema/type for. Thus, this field is to be ignored.
- * @throws IOException
- * @throws JsonParseException
- */
- private void skipValue(JsonParser p) throws JsonParseException, IOException {
- JsonToken valueToken = p.nextToken();
-
- if ((valueToken == JsonToken.START_ARRAY) || (valueToken == JsonToken.START_OBJECT)){
- // if the currently read token is a beginning of an array or object, move stream forward
- // skipping any child tokens till we're at the corresponding END_ARRAY or END_OBJECT token
- p.skipChildren();
+ private static Object primitiveArrayToList(Object arr) {
+ Class<?> ct = arr.getClass().getComponentType();
+ if (int.class.equals(ct)) {
+ return Arrays.asList(ArrayUtils.toObject((int[]) arr));
}
- // At the end of this function, the stream should be pointing to the last token that
- // corresponds to the value being skipped. This way, the next call to nextToken
- // will advance it to the next field name.
- }
-
- /**
- * Utility method to extract current expected field from given JsonParser
- *
- * isTokenCurrent is a boolean variable also passed in, which determines
- * if the JsonParser is already at the token we expect to read next, or
- * needs advancing to the next before we read.
- */
- private Object extractCurrentField(JsonParser p, HCatFieldSchema hcatFieldSchema,
- boolean isTokenCurrent) throws IOException {
- Object val = null;
- JsonToken valueToken;
- if (isTokenCurrent) {
- valueToken = p.getCurrentToken();
- } else {
- valueToken = p.nextToken();
+ if (long.class.equals(ct)) {
+ return Arrays.asList(ArrayUtils.toObject((long[]) arr));
}
- switch (hcatFieldSchema.getType()) {
- case INT:
- val = (valueToken == JsonToken.VALUE_NULL) ? null : p.getIntValue();
- break;
- case TINYINT:
- val = (valueToken == JsonToken.VALUE_NULL) ? null : p.getByteValue();
- break;
- case SMALLINT:
- val = (valueToken == JsonToken.VALUE_NULL) ? null : p.getShortValue();
- break;
- case BIGINT:
- val = (valueToken == JsonToken.VALUE_NULL) ? null : p.getLongValue();
- break;
- case BOOLEAN:
- String bval = (valueToken == JsonToken.VALUE_NULL) ? null : p.getText();
- if (bval != null) {
- val = Boolean.valueOf(bval);
- } else {
- val = null;
- }
- break;
- case FLOAT:
- val = (valueToken == JsonToken.VALUE_NULL) ? null : p.getFloatValue();
- break;
- case DOUBLE:
- val = (valueToken == JsonToken.VALUE_NULL) ? null : p.getDoubleValue();
- break;
- case STRING:
- val = (valueToken == JsonToken.VALUE_NULL) ? null : p.getText();
- break;
- case BINARY:
- String b = (valueToken == JsonToken.VALUE_NULL) ? null : p.getText();
- if (b != null) {
- try {
- String t = Text.decode(b.getBytes(), 0, b.getBytes().length);
- return t.getBytes();
- } catch (CharacterCodingException e) {
- LOG.warn("Error generating json binary type from object.", e);
- return null;
- }
- } else {
- val = null;
- }
- break;
- case DATE:
- val = (valueToken == JsonToken.VALUE_NULL) ? null : Date.valueOf(p.getText());
- break;
- case TIMESTAMP:
- val = (valueToken == JsonToken.VALUE_NULL) ? null : tsParser.parseTimestamp(p.getText());
- break;
- case DECIMAL:
- val = (valueToken == JsonToken.VALUE_NULL) ? null : HiveDecimal.create(p.getText());
- break;
- case VARCHAR:
- int vLen = ((BaseCharTypeInfo)hcatFieldSchema.getTypeInfo()).getLength();
- val = (valueToken == JsonToken.VALUE_NULL) ? null : new HiveVarchar(p.getText(), vLen);
- break;
- case CHAR:
- int cLen = ((BaseCharTypeInfo)hcatFieldSchema.getTypeInfo()).getLength();
- val = (valueToken == JsonToken.VALUE_NULL) ? null : new HiveChar(p.getText(), cLen);
- break;
- case ARRAY:
- if (valueToken == JsonToken.VALUE_NULL) {
- val = null;
- break;
- }
- if (valueToken != JsonToken.START_ARRAY) {
- throw new IOException("Start of Array expected");
- }
- List<Object> arr = new ArrayList<Object>();
- while ((valueToken = p.nextToken()) != JsonToken.END_ARRAY) {
- arr.add(extractCurrentField(p, hcatFieldSchema.getArrayElementSchema().get(0), true));
- }
- val = arr;
- break;
- case MAP:
- if (valueToken == JsonToken.VALUE_NULL) {
- val = null;
- break;
- }
- if (valueToken != JsonToken.START_OBJECT) {
- throw new IOException("Start of Object expected");
- }
- Map<Object, Object> map = new LinkedHashMap<Object, Object>();
- HCatFieldSchema valueSchema = hcatFieldSchema.getMapValueSchema().get(0);
- while ((valueToken = p.nextToken()) != JsonToken.END_OBJECT) {
- Object k = getObjectOfCorrespondingPrimitiveType(p.getCurrentName(), hcatFieldSchema.getMapKeyTypeInfo());
- Object v = extractCurrentField(p, valueSchema, false);
- map.put(k, v);
- }
- val = map;
- break;
- case STRUCT:
- if (valueToken == JsonToken.VALUE_NULL) {
- val = null;
- break;
- }
- if (valueToken != JsonToken.START_OBJECT) {
- throw new IOException("Start of Object expected");
- }
- HCatSchema subSchema = hcatFieldSchema.getStructSubSchema();
- int sz = subSchema.getFieldNames().size();
-
- List<Object> struct = new ArrayList<Object>(Collections.nCopies(sz, null));
- while ((valueToken = p.nextToken()) != JsonToken.END_OBJECT) {
- populateRecord(struct, valueToken, p, subSchema);
- }
- val = struct;
- break;
- default:
- LOG.error("Unknown type found: " + hcatFieldSchema.getType());
- return null;
+ if (char.class.equals(ct)) {
+ return Arrays.asList(ArrayUtils.toObject((char[]) arr));
+ }
+ if (byte.class.equals(ct)) {
+ return Arrays.asList(ArrayUtils.toObject((byte[]) arr));
+ }
+ if (short.class.equals(ct)) {
+ return Arrays.asList(ArrayUtils.toObject((short[]) arr));
}
- return val;
+ if (float.class.equals(ct)) {
+ return Arrays.asList(ArrayUtils.toObject((float[]) arr));
+ }
+ if (double.class.equals(ct)) {
+ return Arrays.asList(ArrayUtils.toObject((double[]) arr));
+ }
+ throw new RuntimeException("Unhandled primitiveArrayToList for type: " + ct);
}
- private Object getObjectOfCorrespondingPrimitiveType(String s, PrimitiveTypeInfo mapKeyType)
- throws IOException {
- switch (Type.getPrimitiveHType(mapKeyType)) {
- case INT:
- return Integer.valueOf(s);
- case TINYINT:
- return Byte.valueOf(s);
- case SMALLINT:
- return Short.valueOf(s);
- case BIGINT:
- return Long.valueOf(s);
- case BOOLEAN:
- return (s.equalsIgnoreCase("true"));
- case FLOAT:
- return Float.valueOf(s);
- case DOUBLE:
- return Double.valueOf(s);
- case STRING:
- return s;
- case BINARY:
- try {
- String t = Text.decode(s.getBytes(), 0, s.getBytes().length);
- return t.getBytes();
- } catch (CharacterCodingException e) {
- LOG.warn("Error generating json binary type from object.", e);
- return null;
- }
- case DATE:
- return Date.valueOf(s);
- case TIMESTAMP:
- return Timestamp.valueOf(s);
- case DECIMAL:
- return HiveDecimal.create(s);
- case VARCHAR:
- return new HiveVarchar(s, ((BaseCharTypeInfo)mapKeyType).getLength());
- case CHAR:
- return new HiveChar(s, ((BaseCharTypeInfo)mapKeyType).getLength());
- }
- throw new IOException("Could not convert from string to map type " + mapKeyType.getTypeName());
+ public String getHiveInternalColumnName(int fpos) {
+ return HiveConf.getColumnInternalName(fpos);
}
/**
@@ -437,217 +168,8 @@ public class JsonSerDe extends AbstractSerDe {
@Override
public Writable serialize(Object obj, ObjectInspector objInspector)
throws SerDeException {
- StringBuilder sb = new StringBuilder();
- try {
-
- StructObjectInspector soi = (StructObjectInspector) objInspector;
- List<? extends StructField> structFields = soi.getAllStructFieldRefs();
- assert (columnNames.size() == structFields.size());
- if (obj == null) {
- sb.append("null");
- } else {
- sb.append(SerDeUtils.LBRACE);
- for (int i = 0; i < structFields.size(); i++) {
- if (i > 0) {
- sb.append(SerDeUtils.COMMA);
- }
- appendWithQuotes(sb, columnNames.get(i));
- sb.append(SerDeUtils.COLON);
- buildJSONString(sb, soi.getStructFieldData(obj, structFields.get(i)),
- structFields.get(i).getFieldObjectInspector());
- }
- sb.append(SerDeUtils.RBRACE);
- }
-
- } catch (IOException e) {
- LOG.warn("Error generating json text from object.", e);
- throw new SerDeException(e);
- }
- return new Text(sb.toString());
- }
- private static StringBuilder appendWithQuotes(StringBuilder sb, String value) {
- return sb == null ? null : sb.append(SerDeUtils.QUOTE).append(value).append(SerDeUtils.QUOTE);
+ return jsonSerde.serialize(obj, objInspector);
}
- // TODO : code section copied over from SerDeUtils because of non-standard json production there
- // should use quotes for all field names. We should fix this there, and then remove this copy.
- // See http://jackson.codehaus.org/1.7.3/javadoc/org/codehaus/jackson/JsonParser.Feature.html#ALLOW_UNQUOTED_FIELD_NAMES
- // for details - trying to enable Jackson to ignore that doesn't seem to work(compilation failure
- // when attempting to use that feature, so having to change the production itself.
- // Also, throws IOException when Binary is detected.
- private static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi) throws IOException {
-
- switch (oi.getCategory()) {
- case PRIMITIVE: {
- PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
- if (o == null) {
- sb.append("null");
- } else {
- switch (poi.getPrimitiveCategory()) {
- case BOOLEAN: {
- boolean b = ((BooleanObjectInspector) poi).get(o);
- sb.append(b ? "true" : "false");
- break;
- }
- case BYTE: {
- sb.append(((ByteObjectInspector) poi).get(o));
- break;
- }
- case SHORT: {
- sb.append(((ShortObjectInspector) poi).get(o));
- break;
- }
- case INT: {
- sb.append(((IntObjectInspector) poi).get(o));
- break;
- }
- case LONG: {
- sb.append(((LongObjectInspector) poi).get(o));
- break;
- }
- case FLOAT: {
- sb.append(((FloatObjectInspector) poi).get(o));
- break;
- }
- case DOUBLE: {
- sb.append(((DoubleObjectInspector) poi).get(o));
- break;
- }
- case STRING: {
- String s =
- SerDeUtils.escapeString(((StringObjectInspector) poi).getPrimitiveJavaObject(o));
- appendWithQuotes(sb, s);
- break;
- }
- case BINARY:
- byte[] b = ((BinaryObjectInspector) oi).getPrimitiveJavaObject(o);
- Text txt = new Text();
- txt.set(b, 0, b.length);
- appendWithQuotes(sb, SerDeUtils.escapeString(txt.toString()));
- break;
- case DATE:
- Date d = ((DateObjectInspector)poi).getPrimitiveJavaObject(o);
- appendWithQuotes(sb, d.toString());
- break;
- case TIMESTAMP: {
- Timestamp t = ((TimestampObjectInspector) poi).getPrimitiveJavaObject(o);
- appendWithQuotes(sb, t.toString());
- break;
- }
- case DECIMAL:
- sb.append(((HiveDecimalObjectInspector)poi).getPrimitiveJavaObject(o));
- break;
- case VARCHAR: {
- String s = SerDeUtils.escapeString(
- ((HiveVarcharObjectInspector) poi).getPrimitiveJavaObject(o).toString());
- appendWithQuotes(sb, s);
- break;
- }
- case CHAR: {
- //this should use HiveChar.getPaddedValue() but it's protected; currently (v0.13)
- // HiveChar.toString() returns getPaddedValue()
- String s = SerDeUtils.escapeString(
- ((HiveCharObjectInspector) poi).getPrimitiveJavaObject(o).toString());
- appendWithQuotes(sb, s);
- break;
- }
- default:
- throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory());
- }
- }
- break;
- }
- case LIST: {
- ListObjectInspector loi = (ListObjectInspector) oi;
- ObjectInspector listElementObjectInspector = loi
- .getListElementObjectInspector();
- List<?> olist = loi.getList(o);
- if (olist == null) {
- sb.append("null");
- } else {
- sb.append(SerDeUtils.LBRACKET);
- for (int i = 0; i < olist.size(); i++) {
- if (i > 0) {
- sb.append(SerDeUtils.COMMA);
- }
- buildJSONString(sb, olist.get(i), listElementObjectInspector);
- }
- sb.append(SerDeUtils.RBRACKET);
- }
- break;
- }
- case MAP: {
- MapObjectInspector moi = (MapObjectInspector) oi;
- ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
- ObjectInspector mapValueObjectInspector = moi
- .getMapValueObjectInspector();
- Map<?, ?> omap = moi.getMap(o);
- if (omap == null) {
- sb.append("null");
- } else {
- sb.append(SerDeUtils.LBRACE);
- boolean first = true;
- for (Object entry : omap.entrySet()) {
- if (first) {
- first = false;
- } else {
- sb.append(SerDeUtils.COMMA);
- }
- Map.Entry<?, ?> e = (Map.Entry<?, ?>) entry;
- StringBuilder keyBuilder = new StringBuilder();
- buildJSONString(keyBuilder, e.getKey(), mapKeyObjectInspector);
- String keyString = keyBuilder.toString().trim();
- if((!keyString.isEmpty()) && (keyString.charAt(0) != SerDeUtils.QUOTE)) {
- appendWithQuotes(sb, keyString);
- }
- else {
- sb.append(keyString);
- }
- sb.append(SerDeUtils.COLON);
- buildJSONString(sb, e.getValue(), mapValueObjectInspector);
- }
- sb.append(SerDeUtils.RBRACE);
- }
- break;
- }
- case STRUCT: {
- StructObjectInspector soi = (StructObjectInspector) oi;
- List<? extends StructField> structFields = soi.getAllStructFieldRefs();
- if (o == null) {
- sb.append("null");
- } else {
- sb.append(SerDeUtils.LBRACE);
- for (int i = 0; i < structFields.size(); i++) {
- if (i > 0) {
- sb.append(SerDeUtils.COMMA);
- }
- appendWithQuotes(sb, structFields.get(i).getFieldName());
- sb.append(SerDeUtils.COLON);
- buildJSONString(sb, soi.getStructFieldData(o, structFields.get(i)),
- structFields.get(i).getFieldObjectInspector());
- }
- sb.append(SerDeUtils.RBRACE);
- }
- break;
- }
- case UNION: {
- UnionObjectInspector uoi = (UnionObjectInspector) oi;
- if (o == null) {
- sb.append("null");
- } else {
- sb.append(SerDeUtils.LBRACE);
- sb.append(uoi.getTag(o));
- sb.append(SerDeUtils.COLON);
- buildJSONString(sb, uoi.getField(o),
- uoi.getObjectInspectors().get(uoi.getTag(o)));
- sb.append(SerDeUtils.RBRACE);
- }
- break;
- }
- default:
- throw new RuntimeException("Unknown type in ObjectInspector!");
- }
- }
-
/**
* Returns an object inspector for the specified schema that
http://git-wip-us.apache.org/repos/asf/hive/blob/1105ef39/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java
index 6770d44..d476b43 100644
--- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java
+++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java
@@ -27,8 +27,6 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
-import junit.framework.TestCase;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.type.Date;
import org.apache.hadoop.hive.common.type.HiveChar;
@@ -43,6 +41,8 @@ import org.apache.hadoop.io.Writable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import junit.framework.TestCase;
+
public class TestJsonSerDe extends TestCase {
private static final Logger LOG = LoggerFactory.getLogger(TestJsonSerDe.class);
@@ -158,17 +158,17 @@ public class TestJsonSerDe extends TestCase {
Writable s = hrsd.serialize(r, hrsd.getObjectInspector());
LOG.info("ONE:{}", s);
- Object o1 = hrsd.deserialize(s);
+ HCatRecord o1 = (HCatRecord) hrsd.deserialize(s);
StringBuilder msg = new StringBuilder();
- boolean isEqual = HCatDataCheckUtil.recordsEqual(r, (HCatRecord) o1);
+ boolean isEqual = HCatDataCheckUtil.recordsEqual(r, o1);
assertTrue(msg.toString(), isEqual);
Writable s2 = jsde.serialize(o1, hrsd.getObjectInspector());
LOG.info("TWO:{}", s2);
- Object o2 = jsde.deserialize(s2);
+ HCatRecord o2 = (HCatRecord) jsde.deserialize(s2);
LOG.info("deserialized TWO : {} ", o2);
msg.setLength(0);
- isEqual = HCatDataCheckUtil.recordsEqual(r, (HCatRecord) o2, msg);
+ isEqual = HCatDataCheckUtil.recordsEqual(r, o2, msg);
assertTrue(msg.toString(), isEqual);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1105ef39/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/udf/json_read/JsonReadBench.java
----------------------------------------------------------------------
diff --git a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/udf/json_read/JsonReadBench.java b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/udf/json_read/JsonReadBench.java
new file mode 100644
index 0000000..aae247c
--- /dev/null
+++ b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/udf/json_read/JsonReadBench.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.benchmark.udf.json_read;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFJsonRead;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.io.Text;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+
+public class JsonReadBench {
+
+ @State(Scope.Thread)
+ public static class MyState {
+
+ public final String json;
+ public final String type;
+
+ public MyState() {
+ try {
+ json = getResource("val1.json");
+ type = getResource("val1.type").toLowerCase().trim();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private String getResource(String fname) throws IOException {
+ return IOUtils.toString(JsonReadBench.class.getResourceAsStream(fname), Charset.defaultCharset());
+ }
+ }
+
+ public void checkBenchMarkMethod() throws Exception {
+ benchmarkMethod(new MyState());
+ }
+
+ @Benchmark
+ public void benchmarkMethod(MyState state) throws Exception {
+ try (GenericUDFJsonRead udf = new GenericUDFJsonRead()) {
+ ObjectInspector[] arguments = buildArguments(state.type);
+ udf.initialize(arguments);
+
+ udf.evaluate(evalArgs(state.json));
+ }
+ }
+
+ private DeferredObject[] evalArgs(String string) {
+ return new DeferredObject[] { new GenericUDF.DeferredJavaObject(new Text(string)), null };
+ }
+
+ private ObjectInspector[] buildArguments(String typeStr) {
+ ObjectInspector valueOI = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+ ObjectInspector[] arguments = { valueOI, PrimitiveObjectInspectorFactory
+ .getPrimitiveWritableConstantObjectInspector(TypeInfoFactory.stringTypeInfo, new Text(typeStr)) };
+ return arguments;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/1105ef39/itests/hive-jmh/src/main/resources/org/apache/hive/benchmark/udf/json_read/val1.json
----------------------------------------------------------------------
diff --git a/itests/hive-jmh/src/main/resources/org/apache/hive/benchmark/udf/json_read/val1.json b/itests/hive-jmh/src/main/resources/org/apache/hive/benchmark/udf/json_read/val1.json
new file mode 100644
index 0000000..4466539
--- /dev/null
+++ b/itests/hive-jmh/src/main/resources/org/apache/hive/benchmark/udf/json_read/val1.json
@@ -0,0 +1,86 @@
+[
+{
+ "t0":"2017-08-1414:45:23.522000",
+ "business_id": "vcNAWiLM4dR7D2nwwJ7nCA",
+ "hours": {
+ "Tuesday": {
+ "close": "17:00",
+ "open": "08:00"
+ },
+ "Friday": {
+ "close": "17:00",
+ "open": "08:00"
+ }
+ },
+ "open": true,
+ "categories": [
+ "Doctors",
+ "Health & Medical"
+ ],
+ "review_count": 9,
+ "name": "Eric Goldberg, MD",
+ "neighborhoods": [],
+ "attributes": {
+ "By Appointment Only": true,
+ "Accepts Credit Cards": true,
+ "Good For Groups": 1
+ },
+ "type": "business"
+}
+,
+{
+ "business_id": "vcNAWiLM4dR7D2nwwJ7nCA",
+ "hours": {
+ "Tuesday": {
+ "close": "17:00",
+ "open": "08:00"
+ },
+ "Friday": {
+ "close": "17:00",
+ "open": "08:00"
+ }
+ },
+ "open": true,
+ "categories": [
+ "Doctors",
+ "Health & Medical"
+ ],
+ "review_count": 9,
+ "name": "Eric Goldberg, MD",
+ "neighborhoods": [],
+ "attributes": {
+ "By Appointment Only": true,
+ "Accepts Credit Cards": true,
+ "Good For Groups": 1
+ },
+ "type": "business"
+}
+,
+{
+ "business_id": "vcNAWiLM4dR7D2nwwJ7nCA",
+ "hours": {
+ "Tuesday": {
+ "close": "17:00",
+ "open": "08:00"
+ },
+ "Friday": {
+ "close": "17:00",
+ "open": "08:00"
+ }
+ },
+ "open": true,
+ "categories": [
+ "Doctors",
+ "Health & Medical"
+ ],
+ "review_count": 9,
+ "name": "Eric Goldberg, MD",
+ "neighborhoods": [],
+ "attributes": {
+ "By Appointment Only": true,
+ "Accepts Credit Cards": true,
+ "Good For Groups": 1
+ },
+ "type": "business"
+}
+]
http://git-wip-us.apache.org/repos/asf/hive/blob/1105ef39/itests/hive-jmh/src/main/resources/org/apache/hive/benchmark/udf/json_read/val1.type
----------------------------------------------------------------------
diff --git a/itests/hive-jmh/src/main/resources/org/apache/hive/benchmark/udf/json_read/val1.type b/itests/hive-jmh/src/main/resources/org/apache/hive/benchmark/udf/json_read/val1.type
new file mode 100644
index 0000000..3543223
--- /dev/null
+++ b/itests/hive-jmh/src/main/resources/org/apache/hive/benchmark/udf/json_read/val1.type
@@ -0,0 +1 @@
+array<struct<t0:timestamp,attributes:struct<accepts credit cards:boolean,by appointment only:boolean,good for groups:int>,business_id:string,categories:array<string>,hours:map<string,struct<close:string,open:string>>,name:string,neighborhoods:array<string>,open:boolean,review_count:int,type:string>>
http://git-wip-us.apache.org/repos/asf/hive/blob/1105ef39/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
index 10517ad..0800a10 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
@@ -213,6 +213,7 @@ public final class FunctionRegistry {
system.registerGenericUDF("ceiling", GenericUDFCeil.class);
system.registerUDF("rand", UDFRand.class, false);
system.registerGenericUDF("abs", GenericUDFAbs.class);
+ system.registerGenericUDF("json_read", GenericUDFJsonRead.class);
system.registerGenericUDF("sq_count_check", GenericUDFSQCountCheck.class);
system.registerGenericUDF("enforce_constraint", GenericUDFEnforceConstraint.class);
system.registerGenericUDF("pmod", GenericUDFPosMod.class);
http://git-wip-us.apache.org/repos/asf/hive/blob/1105ef39/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFJsonRead.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFJsonRead.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFJsonRead.java
new file mode 100644
index 0000000..f5814ed
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFJsonRead.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.json.HiveJsonStructReader;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.TextConverter;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+
+/**
+ * Parses a json string representation into a Hive struct.
+ */
+@Description(name = "json_read", value = "_FUNC_(json,type) - "
+ + "Parses the given json according to the given complex type specification", extended = ""
+ + "Parsed as null: if the json is null, it is the empty string or if it contains only whitespaces\n"
+ + "Example:\n" + "select _FUNC_('[]','array<struct<a:string>>' ")
+public class GenericUDFJsonRead extends GenericUDF {
+
+ private TextConverter inputConverter;
+ private HiveJsonStructReader jsonReader;
+
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
+
+ checkArgsSize(arguments, 2, 2);
+ checkArgPrimitive(arguments, 0);
+ checkArgPrimitive(arguments, 1);
+ if (!ObjectInspectorUtils.isConstantObjectInspector(arguments[1])) {
+ throw new UDFArgumentTypeException(1, getFuncName() + " argument 2 may only be a constant");
+ }
+
+ inputConverter = new TextConverter((PrimitiveObjectInspector) arguments[0]);
+ String typeStr = getConstantStringValue(arguments, 1);
+
+ try {
+ TypeInfo t = TypeInfoUtils.getTypeInfoFromTypeString(typeStr);
+ jsonReader = new HiveJsonStructReader(t);
+ jsonReader.setWritablesUsage(true);
+ } catch (Exception e) {
+ throw new UDFArgumentException(getFuncName() + ": Error parsing typestring: " + e.getMessage());
+ }
+
+ return jsonReader.getObjectInspector();
+ }
+
+ @Override
+ public Object evaluate(DeferredObject[] arguments) throws HiveException {
+ Object valObject = arguments[0].get();
+ if (valObject == null) {
+ return null;
+ }
+
+ try {
+ String text = inputConverter.convert(valObject).toString();
+ if (text.trim().length() == 0) {
+ return null;
+ }
+ return jsonReader.parseStruct(text);
+ } catch (Exception e) {
+ throw new HiveException("Error parsing json: " + e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public String getDisplayString(String[] children) {
+ return getStandardDisplayString("json_read", children);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/1105ef39/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFJsonRead.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFJsonRead.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFJsonRead.java
new file mode 100644
index 0000000..3016eaf
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFJsonRead.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class TestGenericUDFJsonRead {
+
+ @Test(expected = UDFArgumentException.class)
+ public void testArgCnt1() throws Exception {
+ try (GenericUDFJsonRead udf = new GenericUDFJsonRead()) {
+ ObjectInspector valueOI = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+ ObjectInspector[] arguments = { valueOI };
+ udf.initialize(arguments);
+ }
+ }
+
+ @Test(expected = UDFArgumentException.class)
+ public void testArgCnt3() throws Exception {
+ try (GenericUDFJsonRead udf = new GenericUDFJsonRead()) {
+ ObjectInspector valueOI = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+ ObjectInspector[] arguments = { valueOI, valueOI };
+ udf.initialize(arguments);
+ }
+ }
+
+ @Test(expected = UDFArgumentException.class)
+ public void testArgInvalidType() throws Exception {
+ try (GenericUDFJsonRead udf = new GenericUDFJsonRead()) {
+ ObjectInspector[] arguments = buildArguments("__invalid__type__");
+ udf.initialize(arguments);
+ }
+ }
+
+ @Test
+ public void testList() throws Exception {
+ try (GenericUDFJsonRead udf = new GenericUDFJsonRead()) {
+ ObjectInspector[] arguments = buildArguments("array<string>");
+ udf.initialize(arguments);
+
+ Object res = udf.evaluate(evalArgs("[\"a\",\"b\",null]"));
+ assertTrue(res instanceof List<?>);
+ List<?> l = (List<?>) res;
+ assertEquals(3, l.size());
+ assertEquals(new Text("a"), l.get(0));
+ assertEquals(new Text("b"), l.get(1));
+ assertEquals(null, l.get(2));
+ }
+ }
+
+ @Test
+ public void testListNull() throws Exception {
+ try (GenericUDFJsonRead udf = new GenericUDFJsonRead()) {
+ ObjectInspector[] arguments = buildArguments("array<string>");
+ udf.initialize(arguments);
+
+ Object res = udf.evaluate(evalArgs("null"));
+ assertNull(res);
+ }
+ }
+
+ @Test
+ public void testSimpleStruct() throws Exception {
+ try (GenericUDFJsonRead udf = new GenericUDFJsonRead()) {
+ ObjectInspector[] arguments = buildArguments("struct<a:string>");
+ udf.initialize(arguments);
+
+ Object res = udf.evaluate(evalArgs("{\"a\":\"b\"}"));
+ assertTrue(res instanceof Object[]);
+ Object o[] = (Object[]) res;
+ assertEquals(new Text("b"), o[0]);
+ }
+ }
+
+ @Test
+ public void testStructNullField() throws Exception {
+ try (GenericUDFJsonRead udf = new GenericUDFJsonRead()) {
+ ObjectInspector[] arguments = buildArguments("struct<a:string>");
+ udf.initialize(arguments);
+
+ Object res = udf.evaluate(evalArgs("{\"a\":null}"));
+ assertTrue(res instanceof Object[]);
+ Object o[] = (Object[]) res;
+ assertEquals(null, o[0]);
+ }
+ }
+
+ @Test
+ public void testStructEmptyString() throws Exception {
+ try (GenericUDFJsonRead udf = new GenericUDFJsonRead()) {
+ ObjectInspector[] arguments = buildArguments("struct<a:string>");
+ udf.initialize(arguments);
+
+ Object res = udf.evaluate(evalArgs(""));
+ assertNull(res);
+ }
+ }
+
+ @Test
+ public void testStructNull() throws Exception {
+ try (GenericUDFJsonRead udf = new GenericUDFJsonRead()) {
+ ObjectInspector[] arguments = buildArguments("struct<a:string>");
+ udf.initialize(arguments);
+
+ Object res = udf.evaluate(new DeferredObject[] { new DeferredJavaObject(null), null });
+ assertNull(res);
+ }
+ }
+
+ @Test
+ public void testStructNullComplexField() throws Exception {
+ try (GenericUDFJsonRead udf = new GenericUDFJsonRead()) {
+ ObjectInspector[] arguments = buildArguments("struct<a:struct<x:string>>");
+ udf.initialize(arguments);
+
+ Object res = udf.evaluate(evalArgs("{\"a\":null}"));
+ assertTrue(res instanceof Object[]);
+ Object o[] = (Object[]) res;
+ assertEquals(null, o[0]);
+ }
+ }
+
+ @Test(expected = HiveException.class)
+ public void testUndeclaredStructField() throws Exception {
+ try (GenericUDFJsonRead udf = new GenericUDFJsonRead()) {
+ ObjectInspector[] arguments = buildArguments("struct<a:int>");
+ udf.initialize(arguments);
+
+ Object res = udf.evaluate(evalArgs("{\"b\":null}"));
+ assertTrue(res instanceof Object[]);
+ Object o[] = (Object[]) res;
+ assertEquals(null, o[0]);
+ }
+ }
+
+ @Test(expected = HiveException.class)
+ public void testUnexpectedStruct() throws Exception {
+ try (GenericUDFJsonRead udf = new GenericUDFJsonRead()) {
+ ObjectInspector[] arguments = buildArguments("array<int>");
+ udf.initialize(arguments);
+
+ Object res = udf.evaluate(evalArgs("[1,22,2,{\"b\":null}]"));
+ assertTrue(res instanceof Object[]);
+ Object o[] = (Object[]) res;
+ assertEquals(null, o[0]);
+ }
+ }
+
+ @Test
+ public void testMap() throws Exception {
+ try (GenericUDFJsonRead udf = new GenericUDFJsonRead()) {
+ ObjectInspector[] arguments = buildArguments("map<string,string>");
+ udf.initialize(arguments);
+
+ Object res = udf.evaluate(evalArgs("{\"a\":\"v\"}"));
+ assertTrue(res instanceof Map);
+ Map o = (Map) res;
+ assertEquals(1, o.size());
+ assertEquals(new Text("v"), o.get(new Text("a")));
+ }
+ }
+
+ private DeferredObject[] evalArgs(String string) {
+ return new DeferredObject[] { new DeferredJavaObject(new Text(string)), null };
+ }
+
+ private ObjectInspector[] buildArguments(String typeStr) {
+ ObjectInspector valueOI = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+ ObjectInspector[] arguments = { valueOI, PrimitiveObjectInspectorFactory
+ .getPrimitiveWritableConstantObjectInspector(TypeInfoFactory.stringTypeInfo, new Text(typeStr)) };
+ return arguments;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/1105ef39/ql/src/test/queries/clientpositive/json_serde2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/json_serde2.q b/ql/src/test/queries/clientpositive/json_serde2.q
new file mode 100644
index 0000000..c6088b7
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/json_serde2.q
@@ -0,0 +1,37 @@
+--! qt:dataset:src
+
+add jar ${system:maven.local.repository}/org/apache/hive/hcatalog/hive-hcatalog-core/${system:hive.version}/hive-hcatalog-core-${system:hive.version}.jar;
+
+drop table if exists json_serde1_1;
+drop table if exists json_serde1_2;
+
+create table json_serde1_1 (a array<string>,b map<string,int>)
+ row format serde 'org.apache.hadoop.hive.serde2.JsonSerDe';
+
+insert into table json_serde1_1
+ select array('aaa'),map('aaa',1) from src limit 2;
+
+select * from json_serde1_1;
+
+create table json_serde1_2 (
+ a array<int>,
+ b map<int,date>,
+ c struct<c1:int, c2:string, c3:array<string>, c4:map<string, int>, c5:struct<c5_1:string, c5_2:int>>
+) row format serde 'org.apache.hadoop.hive.serde2.JsonSerDe';
+
+insert into table json_serde1_2
+ select
+ array(3, 2, 1),
+ map(1, date '2001-01-01', 2, null),
+ named_struct(
+ 'c1', 123456,
+ 'c2', 'hello',
+ 'c3', array('aa', 'bb', 'cc'),
+ 'c4', map('abc', 123, 'xyz', 456),
+ 'c5', named_struct('c5_1', 'bye', 'c5_2', 88))
+ from src limit 2;
+
+select * from json_serde1_2;
+
+drop table json_serde1_1;
+drop table json_serde1_2;
http://git-wip-us.apache.org/repos/asf/hive/blob/1105ef39/ql/src/test/queries/clientpositive/udf_json_read.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/udf_json_read.q b/ql/src/test/queries/clientpositive/udf_json_read.q
new file mode 100644
index 0000000..30c297b
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/udf_json_read.q
@@ -0,0 +1,44 @@
+DESCRIBE FUNCTION java_read;
+DESCRIBE FUNCTION EXTENDED java_read;
+
+
+select json_read('[{"name":"john","alias":"j","address":{"city":"LA"}},{"name":"kinga","alias":"binga","age":2}]',
+ 'array<struct<name:string,age:int,alias:string,address:struct<city:string,street:string>>>');
+
+create table t (info array<struct<name:string,age:int,alias:string,address:struct<city:string,street:string>>>);
+
+insert into t
+ select json_read('[{"name":"john","alias":"j","address":{"city":"LA"}},{"name":"kinga","alias":"binga","age":2}]',
+ 'array<struct<name:string,age:int,alias:string,address:struct<city:string,street:string>>>');
+
+
+
+select json_read('[
+{
+ "business_id": "vcNAWiLM4dR7D2nwwJ7nCA",
+ "hours": {
+ "Tuesday": {
+ "close": "17:00",
+ "open": "08:00"
+ },
+ "Friday": {
+ "close": "17:00",
+ "open": "08:00"
+ }
+ },
+ "open": true,
+ "categories": [
+ "Doctors",
+ "Health & Medical"
+ ],
+ "review_count": 9,
+ "name": "Eric Goldberg, MD",
+ "neighborhoods": [],
+ "attributes": {
+ "By Appointment Only": true,
+ "Accepts Credit Cards": true,
+ "Good For Groups": 1
+ },
+ "type": "business"
+}
+]','array<struct<attributes:struct<accepts credit cards:boolean,by appointment only:boolean,good for groups:int>,business_id:string,categories:array<string>,hours:map<string,struct<close:string,open:string>>,name:string,neighborhoods:array<string>,open:boolean,review_count:int,type:string>>');
http://git-wip-us.apache.org/repos/asf/hive/blob/1105ef39/ql/src/test/results/clientpositive/json_serde2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/json_serde2.q.out b/ql/src/test/results/clientpositive/json_serde2.q.out
new file mode 100644
index 0000000..cfdb051
--- /dev/null
+++ b/ql/src/test/results/clientpositive/json_serde2.q.out
@@ -0,0 +1,113 @@
+PREHOOK: query: drop table if exists json_serde1_1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists json_serde1_1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table if exists json_serde1_2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists json_serde1_2
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table json_serde1_1 (a array<string>,b map<string,int>)
+ row format serde 'org.apache.hadoop.hive.serde2.JsonSerDe'
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@json_serde1_1
+POSTHOOK: query: create table json_serde1_1 (a array<string>,b map<string,int>)
+ row format serde 'org.apache.hadoop.hive.serde2.JsonSerDe'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@json_serde1_1
+PREHOOK: query: insert into table json_serde1_1
+ select array('aaa'),map('aaa',1) from src limit 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@json_serde1_1
+POSTHOOK: query: insert into table json_serde1_1
+ select array('aaa'),map('aaa',1) from src limit 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@json_serde1_1
+POSTHOOK: Lineage: json_serde1_1.a EXPRESSION []
+POSTHOOK: Lineage: json_serde1_1.b EXPRESSION []
+PREHOOK: query: select * from json_serde1_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@json_serde1_1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from json_serde1_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@json_serde1_1
+#### A masked pattern was here ####
+["aaa"] {"aaa":1}
+["aaa"] {"aaa":1}
+PREHOOK: query: create table json_serde1_2 (
+ a array<int>,
+ b map<int,date>,
+ c struct<c1:int, c2:string, c3:array<string>, c4:map<string, int>, c5:struct<c5_1:string, c5_2:int>>
+) row format serde 'org.apache.hadoop.hive.serde2.JsonSerDe'
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@json_serde1_2
+POSTHOOK: query: create table json_serde1_2 (
+ a array<int>,
+ b map<int,date>,
+ c struct<c1:int, c2:string, c3:array<string>, c4:map<string, int>, c5:struct<c5_1:string, c5_2:int>>
+) row format serde 'org.apache.hadoop.hive.serde2.JsonSerDe'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@json_serde1_2
+PREHOOK: query: insert into table json_serde1_2
+ select
+ array(3, 2, 1),
+ map(1, date '2001-01-01', 2, null),
+ named_struct(
+ 'c1', 123456,
+ 'c2', 'hello',
+ 'c3', array('aa', 'bb', 'cc'),
+ 'c4', map('abc', 123, 'xyz', 456),
+ 'c5', named_struct('c5_1', 'bye', 'c5_2', 88))
+ from src limit 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@json_serde1_2
+POSTHOOK: query: insert into table json_serde1_2
+ select
+ array(3, 2, 1),
+ map(1, date '2001-01-01', 2, null),
+ named_struct(
+ 'c1', 123456,
+ 'c2', 'hello',
+ 'c3', array('aa', 'bb', 'cc'),
+ 'c4', map('abc', 123, 'xyz', 456),
+ 'c5', named_struct('c5_1', 'bye', 'c5_2', 88))
+ from src limit 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@json_serde1_2
+POSTHOOK: Lineage: json_serde1_2.a EXPRESSION []
+POSTHOOK: Lineage: json_serde1_2.b EXPRESSION []
+POSTHOOK: Lineage: json_serde1_2.c EXPRESSION []
+PREHOOK: query: select * from json_serde1_2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@json_serde1_2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from json_serde1_2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@json_serde1_2
+#### A masked pattern was here ####
+[3,2,1] {1:"2001-01-01",2:null} {"c1":123456,"c2":"hello","c3":["aa","bb","cc"],"c4":{"abc":123,"xyz":456},"c5":{"c5_1":"bye","c5_2":88}}
+[3,2,1] {1:"2001-01-01",2:null} {"c1":123456,"c2":"hello","c3":["aa","bb","cc"],"c4":{"abc":123,"xyz":456},"c5":{"c5_1":"bye","c5_2":88}}
+PREHOOK: query: drop table json_serde1_1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@json_serde1_1
+PREHOOK: Output: default@json_serde1_1
+POSTHOOK: query: drop table json_serde1_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@json_serde1_1
+POSTHOOK: Output: default@json_serde1_1
+PREHOOK: query: drop table json_serde1_2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@json_serde1_2
+PREHOOK: Output: default@json_serde1_2
+POSTHOOK: query: drop table json_serde1_2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@json_serde1_2
+POSTHOOK: Output: default@json_serde1_2
http://git-wip-us.apache.org/repos/asf/hive/blob/1105ef39/ql/src/test/results/clientpositive/show_functions.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/show_functions.q.out b/ql/src/test/results/clientpositive/show_functions.q.out
index e91cffd..629781a 100644
--- a/ql/src/test/results/clientpositive/show_functions.q.out
+++ b/ql/src/test/results/clientpositive/show_functions.q.out
@@ -130,6 +130,7 @@ isnottrue
isnull
istrue
java_method
+json_read
json_tuple
lag
last_day
http://git-wip-us.apache.org/repos/asf/hive/blob/1105ef39/ql/src/test/results/clientpositive/udf_json_read.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_json_read.q.out b/ql/src/test/results/clientpositive/udf_json_read.q.out
new file mode 100644
index 0000000..05b1eb8
--- /dev/null
+++ b/ql/src/test/results/clientpositive/udf_json_read.q.out
@@ -0,0 +1,107 @@
+PREHOOK: query: DESCRIBE FUNCTION java_read
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: DESCRIBE FUNCTION java_read
+POSTHOOK: type: DESCFUNCTION
+Function 'java_read' does not exist.
+PREHOOK: query: DESCRIBE FUNCTION EXTENDED java_read
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: DESCRIBE FUNCTION EXTENDED java_read
+POSTHOOK: type: DESCFUNCTION
+Function 'java_read' does not exist.
+PREHOOK: query: select json_read('[{"name":"john","alias":"j","address":{"city":"LA"}},{"name":"kinga","alias":"binga","age":2}]',
+ 'array<struct<name:string,age:int,alias:string,address:struct<city:string,street:string>>>')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select json_read('[{"name":"john","alias":"j","address":{"city":"LA"}},{"name":"kinga","alias":"binga","age":2}]',
+ 'array<struct<name:string,age:int,alias:string,address:struct<city:string,street:string>>>')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+[{"name":"john","age":null,"alias":"j","address":{"city":"LA","street":null}},{"name":"kinga","age":2,"alias":"binga","address":null}]
+PREHOOK: query: create table t (info array<struct<name:string,age:int,alias:string,address:struct<city:string,street:string>>>)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t
+POSTHOOK: query: create table t (info array<struct<name:string,age:int,alias:string,address:struct<city:string,street:string>>>)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t
+PREHOOK: query: insert into t
+ select json_read('[{"name":"john","alias":"j","address":{"city":"LA"}},{"name":"kinga","alias":"binga","age":2}]',
+ 'array<struct<name:string,age:int,alias:string,address:struct<city:string,street:string>>>')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@t
+POSTHOOK: query: insert into t
+ select json_read('[{"name":"john","alias":"j","address":{"city":"LA"}},{"name":"kinga","alias":"binga","age":2}]',
+ 'array<struct<name:string,age:int,alias:string,address:struct<city:string,street:string>>>')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@t
+POSTHOOK: Lineage: t.info EXPRESSION []
+PREHOOK: query: select json_read('[
+{
+ "business_id": "vcNAWiLM4dR7D2nwwJ7nCA",
+ "hours": {
+ "Tuesday": {
+ "close": "17:00",
+ "open": "08:00"
+ },
+ "Friday": {
+ "close": "17:00",
+ "open": "08:00"
+ }
+ },
+ "open": true,
+ "categories": [
+ "Doctors",
+ "Health & Medical"
+ ],
+ "review_count": 9,
+ "name": "Eric Goldberg, MD",
+ "neighborhoods": [],
+ "attributes": {
+ "By Appointment Only": true,
+ "Accepts Credit Cards": true,
+ "Good For Groups": 1
+ },
+ "type": "business"
+}
+]','array<struct<attributes:struct<accepts credit cards:boolean,by appointment only:boolean,good for groups:int>,business_id:string,categories:array<string>,hours:map<string,struct<close:string,open:string>>,name:string,neighborhoods:array<string>,open:boolean,review_count:int,type:string>>')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select json_read('[
+{
+ "business_id": "vcNAWiLM4dR7D2nwwJ7nCA",
+ "hours": {
+ "Tuesday": {
+ "close": "17:00",
+ "open": "08:00"
+ },
+ "Friday": {
+ "close": "17:00",
+ "open": "08:00"
+ }
+ },
+ "open": true,
+ "categories": [
+ "Doctors",
+ "Health & Medical"
+ ],
+ "review_count": 9,
+ "name": "Eric Goldberg, MD",
+ "neighborhoods": [],
+ "attributes": {
+ "By Appointment Only": true,
+ "Accepts Credit Cards": true,
+ "Good For Groups": 1
+ },
+ "type": "business"
+}
+]','array<struct<attributes:struct<accepts credit cards:boolean,by appointment only:boolean,good for groups:int>,business_id:string,categories:array<string>,hours:map<string,struct<close:string,open:string>>,name:string,neighborhoods:array<string>,open:boolean,review_count:int,type:string>>')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+[{"attributes":{"accepts credit cards":true,"by appointment only":true,"good for groups":1},"business_id":"vcNAWiLM4dR7D2nwwJ7nCA","categories":["Doctors","Health & Medical"],"hours":{"Tuesday":{"close":"17:00","open":"08:00"},"Friday":{"close":"17:00","open":"08:00"}},"name":"Eric Goldberg, MD","neighborhoods":[],"open":true,"review_count":9,"type":"business"}]
[05/32] hive git commit: HIVE-20067 : fix InsertEvent on mm tables to
not cause failing capability checks (Zoltan Haindrich via Sankar H)
Posted by se...@apache.org.
HIVE-20067 : fix InsertEvent on mm tables to not cause failing capability checks (Zoltan Haindrich via Sankar H)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ef53d69c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ef53d69c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ef53d69c
Branch: refs/heads/master-txnstats
Commit: ef53d69cca57f46ed23b8ceb867fe18bd6206cf6
Parents: cbcfab9
Author: Zoltan Haindrich <ki...@rxd.hu>
Authored: Thu Jul 5 16:39:15 2018 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Thu Jul 5 16:39:15 2018 -0700
----------------------------------------------------------------------
ql/src/test/queries/clientpositive/mm_all.q | 1 +
.../org/apache/hadoop/hive/metastore/HiveMetaStore.java | 2 +-
.../apache/hadoop/hive/metastore/events/InsertEvent.java | 9 +++++----
3 files changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ef53d69c/ql/src/test/queries/clientpositive/mm_all.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/mm_all.q b/ql/src/test/queries/clientpositive/mm_all.q
index 61dd3e7..a524c29 100644
--- a/ql/src/test/queries/clientpositive/mm_all.q
+++ b/ql/src/test/queries/clientpositive/mm_all.q
@@ -3,6 +3,7 @@
-- MASK_LINEAGE
+set hive.metastore.dml.events=true;
set hive.mapred.mode=nonstrict;
set hive.explain.user=false;
set hive.fetch.task.conversion=none;
http://git-wip-us.apache.org/repos/asf/hive/blob/ef53d69c/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 31bf615..8d88749 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -7262,7 +7262,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
private Table getTblObject(String db, String table) throws MetaException, NoSuchObjectException {
GetTableRequest req = new GetTableRequest(db, table);
- req.setCapabilities(new ClientCapabilities(Lists.newArrayList(ClientCapability.TEST_CAPABILITY)));
+ req.setCapabilities(new ClientCapabilities(Lists.newArrayList(ClientCapability.TEST_CAPABILITY, ClientCapability.INSERT_ONLY_TABLES)));
return get_table_req(req).getTable();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ef53d69c/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java
index aa014e9..60ad7db 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java
@@ -18,7 +18,9 @@
package org.apache.hadoop.hive.metastore.events;
-import com.google.common.collect.Lists;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hive.metastore.IHMSHandler;
@@ -33,8 +35,7 @@ import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.thrift.TException;
-import java.util.ArrayList;
-import java.util.List;
+import com.google.common.collect.Lists;
@InterfaceAudience.Public
@InterfaceStability.Stable
@@ -65,7 +66,7 @@ public class InsertEvent extends ListenerEvent {
// TODO MS-SPLIT Switch this back once HiveMetaStoreClient is moved.
//req.setCapabilities(HiveMetaStoreClient.TEST_VERSION);
req.setCapabilities(new ClientCapabilities(
- Lists.newArrayList(ClientCapability.TEST_CAPABILITY)));
+ Lists.newArrayList(ClientCapability.TEST_CAPABILITY, ClientCapability.INSERT_ONLY_TABLES)));
try {
this.tableObj = handler.get_table_req(req).getTable();
if (partVals != null) {
[26/32] hive git commit: HIVE-20126: OrcInputFormat does not pass
conf to orc reader options (Prasanth Jayachandran reviewed by Sergey
Shelukhin)
Posted by se...@apache.org.
HIVE-20126: OrcInputFormat does not pass conf to orc reader options
(Prasanth Jayachandran reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/18fb1b3f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/18fb1b3f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/18fb1b3f
Branch: refs/heads/master-txnstats
Commit: 18fb1b3f33bfbb02cb5612459c266ea20d914f6c
Parents: ddf7e25
Author: Prasanth Jayachandran <pr...@apache.org>
Authored: Tue Jul 10 00:45:21 2018 -0700
Committer: Prasanth Jayachandran <pr...@apache.org>
Committed: Tue Jul 10 00:45:26 2018 -0700
----------------------------------------------------------------------
data/files/studenttab10k | 10000 +++++++++++++++++
.../test/resources/testconfiguration.properties | 1 +
.../hadoop/hive/ql/io/orc/OrcInputFormat.java | 6 +-
.../ql/io/orc/VectorizedOrcInputFormat.java | 2 +-
.../clientpositive/orc_schema_evol_quoted.q | 42 +
.../llap/orc_schema_evol_quoted.q.out | 258 +
.../clientpositive/orc_schema_evol_quoted.q.out | 258 +
7 files changed, 10563 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
[23/32] hive git commit: HIVE-20100 : OpTraits : Select Optraits
should stop when a mismatch is detected (Deepak Jaiswal,
reviewed by Jason Dere)
Posted by se...@apache.org.
HIVE-20100 : OpTraits : Select Optraits should stop when a mismatch is detected (Deepak Jaiswal, reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ddf7e25d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ddf7e25d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ddf7e25d
Branch: refs/heads/master-txnstats
Commit: ddf7e25d3ee94224c17e353e334728241515fffb
Parents: 5016d6f
Author: Deepak Jaiswal <dj...@apache.org>
Authored: Mon Jul 9 15:31:13 2018 -0700
Committer: Deepak Jaiswal <dj...@apache.org>
Committed: Mon Jul 9 15:31:13 2018 -0700
----------------------------------------------------------------------
.../annotation/OpTraitsRulesProcFactory.java | 59 +++++-----
.../clientpositive/llap/subquery_notin.q.out | 118 ++++++++++++-------
.../results/clientpositive/llap/tez_join.q.out | 51 +++++---
3 files changed, 138 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ddf7e25d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java
index 263770e..89db530 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java
@@ -308,44 +308,40 @@ public class OpTraitsRulesProcFactory {
public static class SelectRule implements NodeProcessor {
- boolean processSortCols = false;
-
// For bucket columns
// If all the columns match to the parent, put them in the bucket cols
// else, add empty list.
// For sort columns
// Keep the subset of all the columns as long as order is maintained.
public List<List<String>> getConvertedColNames(
- List<List<String>> parentColNames, SelectOperator selOp) {
+ List<List<String>> parentColNames, SelectOperator selOp, boolean processSortCols) {
List<List<String>> listBucketCols = new ArrayList<>();
- if (selOp.getColumnExprMap() != null) {
- if (parentColNames != null) {
- for (List<String> colNames : parentColNames) {
- List<String> bucketColNames = new ArrayList<>();
- boolean found = false;
- for (String colName : colNames) {
- for (Entry<String, ExprNodeDesc> entry : selOp.getColumnExprMap().entrySet()) {
- if ((entry.getValue() instanceof ExprNodeColumnDesc) &&
- (((ExprNodeColumnDesc) (entry.getValue())).getColumn().equals(colName))) {
- bucketColNames.add(entry.getKey());
- found = true;
- break;
- }
- }
- if (!found) {
- // Bail out on first missed column.
- break;
- }
- }
- if (!processSortCols && !found) {
- // While processing bucket columns, atleast one bucket column
- // missed. This results in a different bucketing scheme.
- // Add empty list
- listBucketCols.add(new ArrayList<>());
- } else {
- listBucketCols.add(bucketColNames);
+ for (List<String> colNames : parentColNames) {
+ List<String> bucketColNames = new ArrayList<>();
+ boolean found = false;
+ for (String colName : colNames) {
+ // Reset found
+ found = false;
+ for (Entry<String, ExprNodeDesc> entry : selOp.getColumnExprMap().entrySet()) {
+ if ((entry.getValue() instanceof ExprNodeColumnDesc) &&
+ (((ExprNodeColumnDesc) (entry.getValue())).getColumn().equals(colName))) {
+ bucketColNames.add(entry.getKey());
+ found = true;
+ break;
}
}
+ if (!found) {
+ // Bail out on first missed column.
+ break;
+ }
+ }
+ if (!processSortCols && !found) {
+ // While processing bucket columns, atleast one bucket column
+ // missed. This results in a different bucketing scheme.
+ // Add empty list
+ listBucketCols.add(new ArrayList<>());
+ } else {
+ listBucketCols.add(bucketColNames);
}
}
@@ -363,13 +359,12 @@ public class OpTraitsRulesProcFactory {
List<List<String>> listSortCols = null;
if (selOp.getColumnExprMap() != null) {
if (parentBucketColNames != null) {
- listBucketCols = getConvertedColNames(parentBucketColNames, selOp);
+ listBucketCols = getConvertedColNames(parentBucketColNames, selOp, false);
}
List<List<String>> parentSortColNames =
selOp.getParentOperators().get(0).getOpTraits().getSortCols();
if (parentSortColNames != null) {
- processSortCols = true;
- listSortCols = getConvertedColNames(parentSortColNames, selOp);
+ listSortCols = getConvertedColNames(parentSortColNames, selOp, true);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ddf7e25d/ql/src/test/results/clientpositive/llap/subquery_notin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out
index c16b143..469ec69 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out
@@ -6102,9 +6102,11 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE), Reducer 7 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE)
Reducer 4 <- Map 3 (SIMPLE_EDGE)
- Reducer 7 <- Map 3 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE)
+ Reducer 5 <- Map 3 (SIMPLE_EDGE)
+ Reducer 6 <- Reducer 5 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE)
+ Reducer 8 <- Map 7 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -6160,7 +6162,7 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: vectorized, llap
LLAP IO: no inputs
- Map 6
+ Map 7
Map Operator Tree:
TableScan
alias: fixob
@@ -6223,7 +6225,8 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint), _col2 (type: bigint)
- Reducer 7
+ Reducer 5
+ Execution mode: vectorized, llap
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: int), KEY._col1 (type: int)
@@ -6234,27 +6237,42 @@ STAGE PLANS:
expressions: _col0 (type: int), true (type: boolean)
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: boolean)
+ Reducer 6
Execution mode: llap
Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col2
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col2 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col2 (type: int)
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: boolean)
+ Reducer 8
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: int)
mode: mergepartial
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col1, _col2
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col2 (type: int)
- sort order: +
- Map-reduce partition columns: _col2 (type: int)
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: boolean)
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
Stage: Stage-0
Fetch Operator
@@ -6320,9 +6338,11 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 3 (ONE_TO_ONE_EDGE), Reducer 4 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 3 (ONE_TO_ONE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE)
Reducer 3 <- Map 1 (SIMPLE_EDGE)
- Reducer 4 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE)
+ Reducer 4 <- Map 1 (SIMPLE_EDGE)
+ Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE)
+ Reducer 7 <- Map 6 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -6370,7 +6390,7 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: vectorized, llap
LLAP IO: no inputs
- Map 5
+ Map 6
Map Operator Tree:
TableScan
alias: t_n0
@@ -6434,13 +6454,7 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint), _col2 (type: bigint)
Reducer 4
- Reduce Operator Tree:
- Group By Operator
- keys: KEY._col0 (type: int)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
- Execution mode: llap
+ Execution mode: vectorized, llap
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: int), KEY._col1 (type: int)
@@ -6451,20 +6465,42 @@ STAGE PLANS:
expressions: _col0 (type: int), true (type: boolean)
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col1, _col2
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col2 (type: int)
- sort order: +
- Map-reduce partition columns: _col2 (type: int)
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: boolean)
+ value expressions: _col1 (type: boolean)
+ Reducer 5
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col2
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col2 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col2 (type: int)
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: boolean)
+ Reducer 7
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: int)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/ddf7e25d/ql/src/test/results/clientpositive/llap/tez_join.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/tez_join.q.out b/ql/src/test/results/clientpositive/llap/tez_join.q.out
index 53f8895..ecf9299 100644
--- a/ql/src/test/results/clientpositive/llap/tez_join.q.out
+++ b/ql/src/test/results/clientpositive/llap/tez_join.q.out
@@ -49,7 +49,9 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE)
+ Reducer 5 <- Map 4 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -71,7 +73,7 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized, llap
LLAP IO: no inputs
- Map 3
+ Map 4
Map Operator Tree:
TableScan
alias: t2_n26
@@ -91,32 +93,47 @@ STAGE PLANS:
Execution mode: vectorized, llap
LLAP IO: no inputs
Reducer 2
+ Execution mode: vectorized, llap
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string)
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE
+ Reducer 3
Execution mode: llap
Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 5
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string)
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE
- Merge Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: string)
- 1 _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 404 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 404 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Stage: Stage-0
Fetch Operator