You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ay...@apache.org on 2022/03/14 04:49:27 UTC
[hive] branch master updated: HIVE-25990: Optimise multiple copies in case of CTAS in external tables for Object stores. (#3058). (Ayush Saxena, reviewed by Rajesh Balamohan)
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 243b2ad HIVE-25990: Optimise multiple copies in case of CTAS in external tables for Object stores. (#3058). (Ayush Saxena, reviewed by Rajesh Balamohan)
243b2ad is described below
commit 243b2ad89fe6552fe3c92f2cf6df06806dcba673
Author: Ayush Saxena <ay...@apache.org>
AuthorDate: Mon Mar 14 10:18:05 2022 +0530
HIVE-25990: Optimise multiple copies in case of CTAS in external tables for Object stores. (#3058). (Ayush Saxena, reviewed by Rajesh Balamohan)
---
.../org/apache/hadoop/hive/ql/exec/MoveTask.java | 27 ++
.../org/apache/hadoop/hive/ql/exec/Utilities.java | 32 ++-
.../apache/hadoop/hive/ql/parse/TaskCompiler.java | 4 +-
.../org/apache/hadoop/hive/ql/plan/MoveWork.java | 10 +
ql/src/test/queries/clientpositive/ctas_blob.q | 67 +++++
.../results/clientpositive/llap/ctas_blob.q.out | 272 +++++++++++++++++++++
6 files changed, 406 insertions(+), 6 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index f0e8950..d4514f4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@ -19,10 +19,13 @@
package org.apache.hadoop.hive.ql.exec;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.BlobStorageUtils;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.common.HiveStatsUtils;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -77,14 +80,18 @@ import org.slf4j.LoggerFactory;
import java.io.Closeable;
import java.io.IOException;
import java.io.Serializable;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
+import static org.apache.hadoop.hive.ql.exec.Utilities.BLOB_MANIFEST_FILE;
+
/**
* MoveTask implementation.
**/
@@ -109,6 +116,26 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
console.printInfo(mesg, mesg_detail);
FileSystem fs = sourcePath.getFileSystem(conf);
+
+ if (work.isCTAS() && BlobStorageUtils.isBlobStorageFileSystem(conf, fs)) {
+ if (fs.exists(new Path(sourcePath, BLOB_MANIFEST_FILE))) {
+ LOG.debug("Attempting to copy using the paths available in {}", new Path(sourcePath, BLOB_MANIFEST_FILE));
+ ArrayList<String> filesKept;
+ try (FSDataInputStream inStream = fs.open(new Path(sourcePath, BLOB_MANIFEST_FILE))) {
+ String paths = IOUtils.toString(inStream, Charset.defaultCharset());
+ filesKept = new ArrayList(Arrays.asList(paths.split(System.lineSeparator())));
+ }
+ // Remove the first entry from the list, it is the source path.
+ Path srcPath = new Path(filesKept.remove(0));
+ LOG.info("Copying files {} from {} to {}", filesKept, srcPath, targetPath);
+ // Do the move using the filesKept now directly to the target dir.
+ Utilities.moveSpecifiedFilesInParallel(conf, fs, srcPath, targetPath, new HashSet<>(filesKept));
+ perfLogger.perfLogEnd("MoveTask", PerfLogger.FILE_MOVES);
+ return;
+ }
+ // Fallback case, in any case the _blob_files_kept isn't created, we can do the normal logic. The file won't
+ // be created in case of empty source table as well
+ }
if (isDfsDir) {
moveFileInDfs (sourcePath, targetPath, conf);
} else {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index ffb83b4..1f12d0c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -74,6 +74,7 @@ import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.stream.Collectors;
import java.util.zip.Deflater;
import java.util.zip.DeflaterOutputStream;
import java.util.zip.InflaterInputStream;
@@ -105,6 +106,7 @@ import org.apache.hadoop.hive.common.TableName;
import org.apache.hadoop.hive.common.ValidWriteIdList;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Order;
@@ -267,6 +269,7 @@ public final class Utilities {
private static final Object INPUT_SUMMARY_LOCK = new Object();
private static final Object ROOT_HDFS_DIR_LOCK = new Object();
+ public static final String BLOB_MANIFEST_FILE = "_blob_manifest_file";
@FunctionalInterface
public interface SupplierWithCheckedException<T, X extends Exception> {
@@ -1496,7 +1499,13 @@ public final class Utilities {
// for CTAS or Create MV statements
perfLogger.perfLogBegin("FileSinkOperator", "moveSpecifiedFileStatus");
LOG.debug("CTAS/Create MV: Files being renamed: " + filesKept.toString());
- moveSpecifiedFilesInParallel(hconf, fs, tmpPath, specPath, filesKept);
+ if (conf.getTable() != null && conf.getTable().getTableType().equals(TableType.EXTERNAL_TABLE)) {
+ // Do this optimisation only for External tables.
+ createFileList(filesKept, tmpPath, specPath, fs);
+ } else {
+ Set<String> filesKeptPaths = filesKept.stream().map(x -> x.getPath().toString()).collect(Collectors.toSet());
+ moveSpecifiedFilesInParallel(hconf, fs, tmpPath, specPath, filesKeptPaths);
+ }
perfLogger.perfLogEnd("FileSinkOperator", "moveSpecifiedFileStatus");
} else {
// for rest of the statement e.g. INSERT, LOAD etc
@@ -1514,6 +1523,19 @@ public final class Utilities {
fs.delete(taskTmpPath, true);
}
+ private static void createFileList(Set<FileStatus> filesKept, Path srcPath, Path targetPath, FileSystem fs)
+ throws IOException {
+ try (FSDataOutputStream outStream = fs.create(new Path(targetPath, BLOB_MANIFEST_FILE))) {
+ // Adding the first entry in the manifest file as the source path, the entries post that are the files to be
+ // copied.
+ outStream.writeBytes(srcPath.toString() + System.lineSeparator());
+ for (FileStatus file : filesKept) {
+ outStream.writeBytes(file.getPath().toString() + System.lineSeparator());
+ }
+ }
+ LOG.debug("Created path list at path: {}", new Path(targetPath, BLOB_MANIFEST_FILE));
+ }
+
/**
* move specified files to destination in parallel mode.
* Spins up multiple threads, schedules transfer and shuts down the pool.
@@ -1526,8 +1548,8 @@ public final class Utilities {
* @throws HiveException
* @throws IOException
*/
- private static void moveSpecifiedFilesInParallel(Configuration conf, FileSystem fs,
- Path srcPath, Path destPath, Set<FileStatus> filesToMove)
+ public static void moveSpecifiedFilesInParallel(Configuration conf, FileSystem fs,
+ Path srcPath, Path destPath, Set<String> filesToMove)
throws HiveException, IOException {
LOG.info("rename {} files from {} to dest {}",
@@ -1557,7 +1579,7 @@ public final class Utilities {
* @throws IOException
*/
private static void moveSpecifiedFilesInParallel(FileSystem fs,
- Path src, Path dst, Set<FileStatus> filesToMove, List<Future<Void>> futures,
+ Path src, Path dst, Set<String> filesToMove, List<Future<Void>> futures,
ExecutorService pool) throws IOException {
if (!fs.exists(dst)) {
LOG.info("Creating {}", dst);
@@ -1566,7 +1588,7 @@ public final class Utilities {
FileStatus[] files = fs.listStatus(src);
for (FileStatus fileStatus : files) {
- if (filesToMove.contains(fileStatus)) {
+ if (filesToMove.contains(fileStatus.getPath().toString())) {
futures.add(pool.submit(new Callable<Void>() {
@Override
public Void call() throws HiveException {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
index 28be7a8..21bbb95 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
@@ -284,7 +284,9 @@ public abstract class TaskCompiler {
setLoadFileLocation(pCtx, lfd);
oneLoadFileForCtas = false;
}
- mvTask.add(TaskFactory.get(new MoveWork(null, null, null, lfd, false)));
+ mvTask.add(TaskFactory.get(
+ new MoveWork(pCtx.getQueryProperties().isCTAS() && pCtx.getCreateTable().isExternal(), null, null, null,
+ lfd, false)));
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
index 4fd3768..be0e2fb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
@@ -45,6 +45,7 @@ public class MoveWork implements Serializable {
private boolean isReplication;
private String dumpDirectory;
private transient ReplicationMetricCollector metricCollector;
+ private boolean isCTAS;
/**
* ReadEntitites that are passed to the hooks.
@@ -91,6 +92,12 @@ public class MoveWork implements Serializable {
this(inputs, outputs, loadTableWork, loadFileWork, checkFileFormat, false);
}
+ public MoveWork(boolean isCTAS, Set<ReadEntity> inputs, Set<WriteEntity> outputs, final LoadTableDesc loadTableWork,
+ final LoadFileDesc loadFileWork, boolean checkFileFormat) {
+ this(inputs, outputs, loadTableWork, loadFileWork, checkFileFormat);
+ this.isCTAS = isCTAS;
+ }
+
public MoveWork(Set<ReadEntity> inputs, Set<WriteEntity> outputs,
final LoadTableDesc loadTableWork, final LoadFileDesc loadFileWork,
boolean checkFileFormat, String dumpRoot, ReplicationMetricCollector metricCollector,
@@ -111,6 +118,9 @@ public class MoveWork implements Serializable {
outputs = o.getOutputs();
needCleanTarget = o.needCleanTarget;
}
+ public boolean isCTAS() {
+ return isCTAS;
+ }
@Explain(displayName = "tables", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public LoadTableDesc getLoadTableWork() {
diff --git a/ql/src/test/queries/clientpositive/ctas_blob.q b/ql/src/test/queries/clientpositive/ctas_blob.q
new file mode 100644
index 0000000..7d87131
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/ctas_blob.q
@@ -0,0 +1,67 @@
+--! qt:dataset:src
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+set hive.strict.checks.cartesian.product=false;
+set hive.stats.fetch.column.stats=true;
+
+-- set the blob scheme as hdfs & file, as one of the test spins up the cluster with file scheme & one with hdfs, so
+-- should cover both
+set hive.blobstore.supported.schemes=hdfs,file;
+
+
+-- try CTAS with non-empty external table
+create external table t1 (key int, value string);
+
+insert into t1 values (1,'ABCD'),(2, 'BCDE');
+
+insert into t1 values (3,'FGHI'),(4, 'JKLM');
+
+insert into t1 values (5,'NOPQ'),(6, 'RSTUV');
+
+create external table t1_ctas as select * from t1;
+
+select * from t1_ctas order by key;
+
+-- try CTAS with empty external table.
+
+create external table t2 (key int, value string);
+
+create external table t2_ctas as select * from t2;
+
+select * from t2_ctas order by key;
+
+--sanity check, the managed tables CTAS shouldn't get affected by the optimisation
+create table t3 (key int, value string) stored as orc tblproperties ('transactional'='true');
+
+insert into t3 values (1,'ABC'),(2, 'BCD');
+
+insert into t3 values (3,'ABC'),(4, 'BCD');
+
+insert into t3 values (5,'ABC'),(6, 'BCD');
+
+create table t3_ctas stored as orc tblproperties ('transactional'='true') as select * from t3;
+
+select * from t3_ctas order by key;
+
+-- try CTAS with target as external and source as managed.
+
+create external table texternal as select * from t3;
+
+select * from texternal order by key;
+
+-- sanity check: try CTAS with target as managed and source as external.
+
+create table tmanaged stored as orc tblproperties ('transactional'='true') as select * from t1;
+
+select * from tmanaged order by key;
+
+-- cleanup
+
+drop table t1;
+drop table t2;
+drop table t3;
+drop table t1_ctas;
+drop table t2_ctas;
+drop table t3_ctas;
+drop table texternal;
+drop table tmanaged;
\ No newline at end of file
diff --git a/ql/src/test/results/clientpositive/llap/ctas_blob.q.out b/ql/src/test/results/clientpositive/llap/ctas_blob.q.out
new file mode 100644
index 0000000..9387c97
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/ctas_blob.q.out
@@ -0,0 +1,272 @@
+PREHOOK: query: create external table t1 (key int, value string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t1
+POSTHOOK: query: create external table t1 (key int, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t1
+PREHOOK: query: insert into t1 values (1,'ABCD'),(2, 'BCDE')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@t1
+POSTHOOK: query: insert into t1 values (1,'ABCD'),(2, 'BCDE')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@t1
+POSTHOOK: Lineage: t1.key SCRIPT []
+POSTHOOK: Lineage: t1.value SCRIPT []
+PREHOOK: query: insert into t1 values (3,'FGHI'),(4, 'JKLM')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@t1
+POSTHOOK: query: insert into t1 values (3,'FGHI'),(4, 'JKLM')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@t1
+POSTHOOK: Lineage: t1.key SCRIPT []
+POSTHOOK: Lineage: t1.value SCRIPT []
+PREHOOK: query: insert into t1 values (5,'NOPQ'),(6, 'RSTUV')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@t1
+POSTHOOK: query: insert into t1 values (5,'NOPQ'),(6, 'RSTUV')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@t1
+POSTHOOK: Lineage: t1.key SCRIPT []
+POSTHOOK: Lineage: t1.value SCRIPT []
+PREHOOK: query: create external table t1_ctas as select * from t1
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@t1
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t1_ctas
+POSTHOOK: query: create external table t1_ctas as select * from t1
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@t1
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t1_ctas
+POSTHOOK: Lineage: t1_ctas.key SIMPLE [(t1)t1.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: t1_ctas.value SIMPLE [(t1)t1.FieldSchema(name:value, type:string, comment:null), ]
+PREHOOK: query: select * from t1_ctas order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1_ctas
+#### A masked pattern was here ####
+POSTHOOK: query: select * from t1_ctas order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1_ctas
+#### A masked pattern was here ####
+1 ABCD
+2 BCDE
+3 FGHI
+4 JKLM
+5 NOPQ
+6 RSTUV
+PREHOOK: query: create external table t2 (key int, value string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t2
+POSTHOOK: query: create external table t2 (key int, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t2
+PREHOOK: query: create external table t2_ctas as select * from t2
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@t2
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t2_ctas
+POSTHOOK: query: create external table t2_ctas as select * from t2
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@t2
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t2_ctas
+POSTHOOK: Lineage: t2_ctas.key SIMPLE [(t2)t2.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: t2_ctas.value SIMPLE [(t2)t2.FieldSchema(name:value, type:string, comment:null), ]
+PREHOOK: query: select * from t2_ctas order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t2_ctas
+#### A masked pattern was here ####
+POSTHOOK: query: select * from t2_ctas order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t2_ctas
+#### A masked pattern was here ####
+PREHOOK: query: create table t3 (key int, value string) stored as orc tblproperties ('transactional'='true')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t3
+POSTHOOK: query: create table t3 (key int, value string) stored as orc tblproperties ('transactional'='true')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t3
+PREHOOK: query: insert into t3 values (1,'ABC'),(2, 'BCD')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@t3
+POSTHOOK: query: insert into t3 values (1,'ABC'),(2, 'BCD')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@t3
+POSTHOOK: Lineage: t3.key SCRIPT []
+POSTHOOK: Lineage: t3.value SCRIPT []
+PREHOOK: query: insert into t3 values (3,'ABC'),(4, 'BCD')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@t3
+POSTHOOK: query: insert into t3 values (3,'ABC'),(4, 'BCD')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@t3
+POSTHOOK: Lineage: t3.key SCRIPT []
+POSTHOOK: Lineage: t3.value SCRIPT []
+PREHOOK: query: insert into t3 values (5,'ABC'),(6, 'BCD')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@t3
+POSTHOOK: query: insert into t3 values (5,'ABC'),(6, 'BCD')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@t3
+POSTHOOK: Lineage: t3.key SCRIPT []
+POSTHOOK: Lineage: t3.value SCRIPT []
+PREHOOK: query: create table t3_ctas stored as orc tblproperties ('transactional'='true') as select * from t3
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@t3
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t3_ctas
+POSTHOOK: query: create table t3_ctas stored as orc tblproperties ('transactional'='true') as select * from t3
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@t3
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t3_ctas
+POSTHOOK: Lineage: t3_ctas.key SIMPLE [(t3)t3.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: t3_ctas.value SIMPLE [(t3)t3.FieldSchema(name:value, type:string, comment:null), ]
+PREHOOK: query: select * from t3_ctas order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t3_ctas
+#### A masked pattern was here ####
+POSTHOOK: query: select * from t3_ctas order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t3_ctas
+#### A masked pattern was here ####
+1 ABC
+2 BCD
+3 ABC
+4 BCD
+5 ABC
+6 BCD
+PREHOOK: query: create external table texternal as select * from t3
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@t3
+PREHOOK: Output: database:default
+PREHOOK: Output: default@texternal
+POSTHOOK: query: create external table texternal as select * from t3
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@t3
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@texternal
+POSTHOOK: Lineage: texternal.key SIMPLE [(t3)t3.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: texternal.value SIMPLE [(t3)t3.FieldSchema(name:value, type:string, comment:null), ]
+PREHOOK: query: select * from texternal order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@texternal
+#### A masked pattern was here ####
+POSTHOOK: query: select * from texternal order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@texternal
+#### A masked pattern was here ####
+1 ABC
+2 BCD
+3 ABC
+4 BCD
+5 ABC
+6 BCD
+PREHOOK: query: create table tmanaged stored as orc tblproperties ('transactional'='true') as select * from t1
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@t1
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tmanaged
+POSTHOOK: query: create table tmanaged stored as orc tblproperties ('transactional'='true') as select * from t1
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@t1
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tmanaged
+POSTHOOK: Lineage: tmanaged.key SIMPLE [(t1)t1.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: tmanaged.value SIMPLE [(t1)t1.FieldSchema(name:value, type:string, comment:null), ]
+PREHOOK: query: select * from tmanaged order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tmanaged
+#### A masked pattern was here ####
+POSTHOOK: query: select * from tmanaged order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tmanaged
+#### A masked pattern was here ####
+1 ABCD
+2 BCDE
+3 FGHI
+4 JKLM
+5 NOPQ
+6 RSTUV
+PREHOOK: query: drop table t1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@t1
+PREHOOK: Output: default@t1
+POSTHOOK: query: drop table t1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@t1
+POSTHOOK: Output: default@t1
+PREHOOK: query: drop table t2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@t2
+PREHOOK: Output: default@t2
+POSTHOOK: query: drop table t2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@t2
+POSTHOOK: Output: default@t2
+PREHOOK: query: drop table t3
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@t3
+PREHOOK: Output: default@t3
+POSTHOOK: query: drop table t3
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@t3
+POSTHOOK: Output: default@t3
+PREHOOK: query: drop table t1_ctas
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@t1_ctas
+PREHOOK: Output: default@t1_ctas
+POSTHOOK: query: drop table t1_ctas
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@t1_ctas
+POSTHOOK: Output: default@t1_ctas
+PREHOOK: query: drop table t2_ctas
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@t2_ctas
+PREHOOK: Output: default@t2_ctas
+POSTHOOK: query: drop table t2_ctas
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@t2_ctas
+POSTHOOK: Output: default@t2_ctas
+PREHOOK: query: drop table t3_ctas
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@t3_ctas
+PREHOOK: Output: default@t3_ctas
+POSTHOOK: query: drop table t3_ctas
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@t3_ctas
+POSTHOOK: Output: default@t3_ctas
+PREHOOK: query: drop table texternal
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@texternal
+PREHOOK: Output: default@texternal
+POSTHOOK: query: drop table texternal
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@texternal
+POSTHOOK: Output: default@texternal
+PREHOOK: query: drop table tmanaged
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@tmanaged
+PREHOOK: Output: default@tmanaged
+POSTHOOK: query: drop table tmanaged
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@tmanaged
+POSTHOOK: Output: default@tmanaged