You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ct...@apache.org on 2017/04/01 15:19:30 UTC
[1/2] hive git commit: HIVE-15880: Allow insert overwrite and
truncate table query to use auto.purge table property (Vihang Karajgaonkar
via Chaoyu Tang)
Repository: hive
Updated Branches:
refs/heads/master 79d51230e -> 3c29f8ff5
HIVE-15880: Allow insert overwrite and truncate table query to use auto.purge table property (Vihang Karajgaonkar via Chaoyu Tang)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d822003d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d822003d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d822003d
Branch: refs/heads/master
Commit: d822003d7611c45fed4e24c6f3244a3a99fef5e7
Parents: 79d5123
Author: Chaoyu Tang <ct...@cloudera.com>
Authored: Sat Apr 1 10:49:43 2017 -0400
Committer: Chaoyu Tang <ct...@cloudera.com>
Committed: Sat Apr 1 10:49:43 2017 -0400
----------------------------------------------------------------------
.../apache/hadoop/hive/common/FileUtils.java | 15 +-
.../hadoop/hive/ql/TestAutoPurgeTables.java | 436 +++++++++++++++++++
.../test/resources/testconfiguration.properties | 3 +-
.../hive/metastore/HiveMetaStoreFsImpl.java | 21 +-
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 6 +-
.../apache/hadoop/hive/ql/metadata/Hive.java | 19 +-
.../encryption_auto_purge_tables.q | 38 ++
.../encryption_auto_purge_tables.q.out | 157 +++++++
8 files changed, 661 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d822003d/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
index 9a0521c..e586015 100644
--- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
+++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
@@ -616,15 +616,19 @@ public final class FileUtils {
* @return true if move successful
* @throws IOException
*/
- public static boolean moveToTrash(FileSystem fs, Path f, Configuration conf)
+ public static boolean moveToTrash(FileSystem fs, Path f, Configuration conf, boolean purge)
throws IOException {
LOG.debug("deleting " + f);
boolean result = false;
try {
- result = Trash.moveToAppropriateTrash(fs, f, conf);
- if (result) {
- LOG.trace("Moved to trash: " + f);
- return true;
+ if(purge) {
+ LOG.debug("purge is set to true. Not moving to Trash " + f);
+ } else {
+ result = Trash.moveToAppropriateTrash(fs, f, conf);
+ if (result) {
+ LOG.trace("Moved to trash: " + f);
+ return true;
+ }
}
} catch (IOException ioe) {
// for whatever failure reason including that trash has lower encryption zone
@@ -636,7 +640,6 @@ public final class FileUtils {
if (!result) {
LOG.error("Failed to delete " + f);
}
-
return result;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d822003d/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java
new file mode 100644
index 0000000..abf9769
--- /dev/null
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java
@@ -0,0 +1,436 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.HashMap;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hive.jdbc.miniHS2.MiniHS2;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestAutoPurgeTables {
+ private static final String driverName = "org.apache.hive.jdbc.HiveDriver";
+ private static final String testDbName = "auto_purge_test_db";
+ //private static final String testTableName = "auto_purge_test_table";
+ private static final String INSERT_OVERWRITE_COMMAND_FORMAT =
+ "insert overwrite table " + testDbName + ".%s select 1, \"test\"";
+ private static final String TRUNCATE_TABLE_COMMAND_FORMAT =
+ "truncate table " + testDbName + ".%s";
+ private static final String partitionedColumnName = "partCol";
+ private static final String partitionedColumnValue1 = "20090619";
+ private static final String INSERT_OVERWRITE_COMMAND_PARTITIONED_FORMAT =
+ "insert overwrite table " + testDbName + ".%s PARTITION ("
+ + partitionedColumnName + "=" + partitionedColumnValue1 + ")" + " select 1, \"test\"";
+ private static final String partitionedColumnValue2 = "20100720";
+ private static HiveConf conf;
+ private static Connection con;
+ private static MiniHS2 miniHS2;
+ static final private Logger LOG = LoggerFactory.getLogger("TestAutoPurgeTables");
+
+ @Rule
+ public TestName name = new TestName();
+
+ private static Connection getConnection(String url) throws SQLException {
+ Connection con1;
+ con1 = DriverManager.getConnection(url, "", "");
+ Assert.assertNotNull("Connection is null", con1);
+ Assert.assertFalse("Connection should not be closed", con1.isClosed());
+ return con1;
+ }
+
+ private static void createTestTable(Statement stmt, String isAutopurge, boolean isExternal,
+ boolean isPartitioned, String testTableName) throws SQLException {
+ String createTablePrefix;
+ if (isExternal) {
+ createTablePrefix = "create external table ";
+ } else {
+ createTablePrefix = "create table ";
+ }
+ if (isPartitioned) {
+ // create a partitioned table
+ stmt.execute(createTablePrefix + testDbName + "." + testTableName + " (id int, value string) "
+ + " partitioned by (" + partitionedColumnName + " STRING)");
+ // load data
+ stmt.execute("insert into " + testDbName + "." + testTableName + " PARTITION ("
+ + partitionedColumnName + "=" + partitionedColumnValue1
+ + ") values (1, \"dummy1\"), (2, \"dummy2\"), (3, \"dummy3\")");
+ stmt.execute("insert into " + testDbName + "." + testTableName + " PARTITION ("
+ + partitionedColumnName + "=" + partitionedColumnValue2
+ + ") values (4, \"dummy4\"), (5, \"dummy5\"), (6, \"dummy6\")");
+ } else {
+ // create a table
+ stmt.execute(createTablePrefix + testDbName + "." + testTableName + " (id int, value string)");
+ // load data
+ stmt.execute("insert into " + testDbName + "." + testTableName
+ + " values (1, \"dummy1\"), (2, \"dummy2\"), (3, \"dummy3\")");
+ }
+ if (isAutopurge != null) {
+ stmt.execute("alter table " + testDbName + "." + testTableName
+ + " set tblproperties (\"auto.purge\"=\"" + isAutopurge + "\")");
+ }
+ }
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ conf = new HiveConf(TestAutoPurgeTables.class);
+ // enable trash so it can be tested
+ conf.setFloat("fs.trash.checkpoint.interval", 30);
+ conf.setFloat("fs.trash.interval", 30);
+ // Create test database and base tables once for all the test
+ miniHS2 = new MiniHS2.Builder().withConf(conf).build();
+ miniHS2.start(new HashMap<String, String>());
+ Class.forName(driverName);
+ con = getConnection(miniHS2.getBaseJdbcURL() + ";create=true");
+ try (Statement stmt = con.createStatement()) {
+ Assert.assertNotNull("Statement is null", stmt);
+ stmt.execute("set hive.support.concurrency = false");
+ stmt.execute("drop database if exists " + testDbName + " cascade");
+ stmt.execute("create database " + testDbName);
+ }
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() {
+ Statement stmt = null;
+ try {
+ stmt = con.createStatement();
+ // drop test db and its tables and views
+ stmt.execute("set hive.support.concurrency = false");
+ stmt.execute("drop database if exists " + testDbName + " cascade");
+ FileSystem fs = FileSystem.get(conf);
+ fs.deleteOnExit(ShimLoader.getHadoopShims().getCurrentTrashPath(conf, fs));
+ } catch (SQLException | IOException e) {
+ e.printStackTrace();
+ } finally {
+ if (stmt != null) {
+ try {
+ stmt.close();
+ } catch (SQLException e) {
+ //
+ }
+ }
+ if (con != null) {
+ try {
+ con.close();
+ } catch (SQLException e) {
+ //
+ }
+ }
+ if (miniHS2 != null) {
+ miniHS2.cleanup();
+ miniHS2.stop();
+ miniHS2 = null;
+ }
+ }
+ }
+
+ @Before
+ public void afterTest() throws Exception {
+ FileSystem fs = FileSystem.get(conf);
+ Path trashDir = ShimLoader.getHadoopShims().getCurrentTrashPath(conf, fs);
+ fs.delete(trashDir, true);
+ }
+
+ /**
+ * Tests if previous table data skips trash when insert overwrite table .. is run against a table
+ * which has auto.purge property set
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testAutoPurge() throws Exception {
+ LOG.info("Running " + name.getMethodName());
+ testUtil("true", false, false, false, name.getMethodName());
+ }
+
+ /**
+ * Tests when auto.purge is set to a invalid string, trash should be used for insert overwrite
+ * queries
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testAutoPurgeInvalid() throws Exception {
+ LOG.info("Running " + name.getMethodName());
+ testUtil("invalid", false, false, false, name.getMethodName());
+ }
+
+ /**
+ * Test when auto.purge property is not set. Data should be moved to trash for insert overwrite
+ * queries
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testAutoPurgeUnset() throws Exception {
+ LOG.info("Running " + name.getMethodName());
+ testUtil(null, false, false, false, name.getMethodName());
+ }
+
+ /**
+ * Tests if the auto.purge property works correctly for external tables. Old data should skip
+ * trash when insert overwrite table .. is run when auto.purge is set to true
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testExternalTable() throws Exception {
+ LOG.info("Running " + name.getMethodName());
+ testUtil("true", true, false, false, name.getMethodName());
+ }
+
+ /**
+ * Tests auto.purge when managed table is partitioned. Old data should skip trash when insert
+ * overwrite table .. is run and auto.purge property is set to true
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testPartitionedTable() throws Exception {
+ LOG.info("Running " + name.getMethodName());
+ testUtil("true", false, true, false, name.getMethodName());
+ }
+
+ /**
+ * Tests auto.purge for an external, partitioned table. Old partition data should skip trash when
+ * auto.purge is set to true
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testExternalPartitionedTable() throws Exception {
+ LOG.info("Running " + name.getMethodName());
+ testUtil("true", true, true, false, name.getMethodName());
+ }
+
+ /**
+ * Tests when auto.purge is set to false, older data is moved to Trash when insert overwrite table
+ * .. is run
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testNoAutoPurge() throws Exception {
+ LOG.info("Running " + name.getMethodName());
+ testUtil("false", false, false, false, name.getMethodName());
+ }
+
+ /**
+ * Tests when auto.purge is set to false on a external table, older data is moved to Trash when
+ * insert overwrite table .. is run
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testExternalNoAutoPurge() throws Exception {
+ LOG.info("Running " + name.getMethodName());
+ testUtil("false", true, false, false, name.getMethodName());
+ }
+
+ /**
+ * Tests when auto.purge is set to false on a partitioned table, older data is moved to Trash when
+ * insert overwrite table .. is run
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testPartitionedNoAutoPurge() throws Exception {
+ LOG.info("Running " + name.getMethodName());
+ testUtil("false", false, true, false, name.getMethodName());
+ }
+
+ /**
+ * Tests when auto.purge is set to false on a partitioned external table, older data is moved to
+ * Trash when insert overwrite table .. is run
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testPartitionedExternalNoAutoPurge() throws Exception {
+ LOG.info("Running " + name.getMethodName());
+ testUtil("false", true, true, false, name.getMethodName());
+ }
+
+ //truncate on external table is not allowed
+ @Test(expected = SQLException.class)
+ public void testTruncatePartitionedExternalNoAutoPurge() throws Exception {
+ LOG.info("Running " + name.getMethodName());
+ testUtil(String.valueOf(false), true, true, true, name.getMethodName());
+ }
+
+ //truncate on external table is not allowed
+ @Test(expected = SQLException.class)
+ public void testTruncateExternalNoAutoPurge() throws Exception {
+ LOG.info("Running " + name.getMethodName());
+ testUtil(String.valueOf(false), true, false, true, name.getMethodName());
+ }
+
+ @Test
+ public void testTruncatePartitionedNoAutoPurge() throws Exception {
+ LOG.info("Running " + name.getMethodName());
+ testUtil(String.valueOf(false), false, true, true, name.getMethodName());
+ }
+
+ @Test
+ public void testTruncateNoAutoPurge() throws Exception {
+ LOG.info("Running " + name.getMethodName());
+ testUtil(String.valueOf(false), false, false, true, name.getMethodName());
+ }
+
+ @Test
+ public void testTruncateInvalidAutoPurge() throws Exception {
+ LOG.info("Running " + name.getMethodName());
+ testUtil("invalid", false, false, true, name.getMethodName());
+ }
+
+ @Test
+ public void testTruncateUnsetAutoPurge() throws Exception {
+ LOG.info("Running " + name.getMethodName());
+ testUtil(null, false, false, true, name.getMethodName());
+ }
+
+ //truncate on external table is not allowed
+ @Test(expected = SQLException.class)
+ public void testTruncatePartitionedExternalAutoPurge() throws Exception {
+ LOG.info("Running " + name.getMethodName());
+ testUtil(String.valueOf(true), true, true, true, name.getMethodName());
+ }
+
+ //truncate on external table is not allowed
+ @Test(expected = SQLException.class)
+ public void testTruncateExternalAutoPurge() throws Exception {
+ LOG.info("Running " + name.getMethodName());
+ testUtil(String.valueOf(true), true, false, true, name.getMethodName());
+ }
+
+ @Test
+ public void testTruncatePartitionedAutoPurge() throws Exception {
+ LOG.info("Running " + name.getMethodName());
+ testUtil(String.valueOf(true), false, true, true, name.getMethodName());
+ }
+
+ /**
+ * Test util method to run the insert overwrite table or truncate table test on a table
+ *
+ * @param autoPurgePropValue - string value of the auto.purge property for the test table. Ignored
+ * if null
+ * @param isExternal - if set creates a external table for the test
+ * @param isPartitioned - if set creates a partitioned table for the test
+ * @param isTruncateTest - if set uses truncate table command for the test. Otherwise uses Insert
+ * overwrite table command for the test
+ * @param testTableName - test table name
+ * @throws Exception
+ */
+ private void testUtil(String autoPurgePropValue, boolean isExternal, boolean isPartitioned,
+ boolean isTruncateTest, String testTableName) throws Exception {
+ testUtil(autoPurgePropValue, isExternal, isPartitioned,
+ !"true".equalsIgnoreCase(autoPurgePropValue), isTruncateTest, testTableName);
+ }
+ /**
+ * Test util method to run the insert overwrite table or truncate table test on a table
+ *
+ * @param isAutoPurge - If set, creates a table with auto.purge with the given value
+ * @param isExternal - if set creates a external table for the test
+ * @param isPartitioned - if set creates a partitioned table for the test
+ * @param purgeExpected - if set the assert condition for the test is such that it expectes old
+ * table data to be moved to trash. If not creates a assert condition to make sure that
+ * data is not moved to trash
+ * @param isTruncateTest - if set uses truncate table command for the test. Otherwise uses Insert
+ * overwrite table command for the test
+ * @param testTableName - table name for the test table
+ * @throws Exception
+ */
+ private void testUtil(String isAutoPurge, boolean isExternal, boolean isPartitioned,
+ boolean purgeExpected, boolean isTruncateTest, String testTableName) throws Exception {
+ try (Statement stmt = con.createStatement()) {
+ // create a test table with auto.purge = true
+ createTestTable(stmt, isAutoPurge, isExternal, isPartitioned, testTableName);
+ int numFilesInTrashBefore = getTrashFileCount();
+ String command = getCommand(isTruncateTest, isPartitioned, testTableName);
+ stmt.execute(command);
+ int numFilesInTrashAfter = getTrashFileCount();
+ if (purgeExpected) {
+ Assert.assertTrue(
+ String.format(
+ "Data should have been moved to trash. Number of files in trash: before : %d after %d",
+ numFilesInTrashBefore, numFilesInTrashAfter),
+ numFilesInTrashBefore < numFilesInTrashAfter);
+ } else {
+ Assert.assertEquals(
+ String.format(
+ "Data should not have been moved to trash. Number of files in trash: before : %d after %d",
+ numFilesInTrashBefore, numFilesInTrashAfter),
+ numFilesInTrashBefore, numFilesInTrashAfter);
+ }
+ }
+ }
+
+ private static String getCommand(boolean isTruncateTest, boolean isPartitioned, String testTableName) {
+ if (isTruncateTest) {
+ return String.format(TRUNCATE_TABLE_COMMAND_FORMAT, testTableName);
+ } else if (isPartitioned) {
+ return String.format(INSERT_OVERWRITE_COMMAND_PARTITIONED_FORMAT, testTableName);
+ } else {
+ return String.format(INSERT_OVERWRITE_COMMAND_FORMAT, testTableName);
+ }
+ }
+
+ private int getTrashFileCount() throws Exception {
+ FileSystem fs = FileSystem.get(conf);
+ Path trashDir = ShimLoader.getHadoopShims().getCurrentTrashPath(conf, fs);
+ return getFileCount(fs, trashDir);
+ }
+
+ private int getFileCount(FileSystem fs, Path path) throws Exception {
+ try {
+ int count = 0;
+ if (!fs.exists(path)) {
+ return count;
+ }
+ RemoteIterator<LocatedFileStatus> lfs = fs.listFiles(path, true);
+ while (lfs.hasNext()) {
+ LocatedFileStatus lf = lfs.next();
+ LOG.info(lf.getPath().toString());
+ if (lf.isFile()) {
+ count++;
+ }
+ }
+ return count;
+ } catch (IOException e) {
+ throw new Exception("Exception while list files on " + path, e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/d822003d/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 3c98a57..7a70c9c 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -741,7 +741,8 @@ encrypted.query.files=encryption_join_unencrypted_tbl.q,\
encryption_drop_view.q \
encryption_drop_partition.q \
encryption_with_trash.q \
- encryption_ctas.q
+ encryption_ctas.q \
+ encryption_auto_purge_tables.q
beeline.positive.include=drop_with_concurrency.q,\
escape_comments.q
http://git-wip-us.apache.org/repos/asf/hive/blob/d822003d/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
index df698c8..b7d7b50 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
@@ -25,35 +25,23 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.Trash;
+import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.metastore.api.MetaException;
public class HiveMetaStoreFsImpl implements MetaStoreFS {
public static final Logger LOG = LoggerFactory
- .getLogger("hive.metastore.hivemetastoressimpl");
+ .getLogger("hive.metastore.hivemetastoreFsimpl");
@Override
public boolean deleteDir(FileSystem fs, Path f, boolean recursive,
boolean ifPurge, Configuration conf) throws MetaException {
- LOG.debug("deleting " + f);
-
try {
- if (ifPurge) {
- LOG.info("Not moving "+ f +" to trash");
- } else if (Trash.moveToAppropriateTrash(fs, f, conf)) {
- LOG.info("Moved to trash: " + f);
- return true;
- }
-
- if (fs.delete(f, true)) {
- LOG.debug("Deleted the diretory " + f);
- return true;
- }
-
+ FileUtils.moveToTrash(fs, f, conf, ifPurge);
if (fs.exists(f)) {
throw new MetaException("Unable to delete directory: " + f);
}
+ return true;
} catch (FileNotFoundException e) {
return true; // ok even if there is not data
} catch (Exception e) {
@@ -61,5 +49,4 @@ public class HiveMetaStoreFsImpl implements MetaStoreFS {
}
return false;
}
-
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d822003d/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 0769dad..917e565 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -4662,7 +4662,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
Map<String, String> partSpec = truncateTableDesc.getPartSpec();
Table table = db.getTable(tableName, true);
-
+ boolean isAutopurge = "true".equalsIgnoreCase(table.getProperty("auto.purge"));
try {
// this is not transactional
for (Path location : getLocations(db, table, partSpec)) {
@@ -4673,7 +4673,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
HdfsUtils.HadoopFileStatus status = new HdfsUtils.HadoopFileStatus(conf, fs, location);
FileStatus targetStatus = fs.getFileStatus(location);
String targetGroup = targetStatus == null ? null : targetStatus.getGroup();
- FileUtils.moveToTrash(fs, location, conf);
+ FileUtils.moveToTrash(fs, location, conf, isAutopurge);
fs.mkdirs(location);
HdfsUtils.setFullFileStatus(conf, status, targetGroup, fs, location, false);
} else {
@@ -4681,7 +4681,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
if (statuses == null || statuses.length == 0) {
continue;
}
- boolean success = Hive.trashFiles(fs, statuses, conf);
+ boolean success = Hive.trashFiles(fs, statuses, conf, isAutopurge);
if (!success) {
throw new HiveException("Error in deleting the contents of " + location.toString());
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d822003d/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index ed854bf..f64cfda 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -1646,8 +1646,9 @@ public class Hive {
PerfLogger perfLogger = SessionState.getPerfLogger();
perfLogger.PerfLogBegin("MoveTask", "FileMoves");
if (replace || (oldPart == null && !isAcid)) {
+ boolean isAutoPurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge"));
replaceFiles(tbl.getPath(), loadPath, newPartPath, oldPartPath, getConf(),
- isSrcLocal);
+ isSrcLocal, isAutoPurge);
} else {
if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary() && oldPart != null) {
newFiles = Collections.synchronizedList(new ArrayList<Path>());
@@ -2012,7 +2013,8 @@ private void constructOneLBLocationMap(FileStatus fSta,
}
if (replace) {
Path tableDest = tbl.getPath();
- replaceFiles(tableDest, loadPath, tableDest, tableDest, sessionConf, isSrcLocal);
+ boolean isAutopurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge"));
+ replaceFiles(tableDest, loadPath, tableDest, tableDest, sessionConf, isSrcLocal, isAutopurge);
} else {
FileSystem fs;
try {
@@ -3398,11 +3400,13 @@ private void constructOneLBLocationMap(FileStatus fSta,
* @param oldPath
* The directory where the old data location, need to be cleaned up. Most of time, will be the same
* as destf, unless its across FileSystem boundaries.
+ * @param purge
+ * When set to true files which needs to be deleted are not moved to Trash
* @param isSrcLocal
* If the source directory is LOCAL
*/
protected void replaceFiles(Path tablePath, Path srcf, Path destf, Path oldPath, HiveConf conf,
- boolean isSrcLocal) throws HiveException {
+ boolean isSrcLocal, boolean purge) throws HiveException {
try {
FileSystem destFs = destf.getFileSystem(conf);
@@ -3435,7 +3439,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
// existing content might result in incorrect (extra) data.
// But not sure why we changed not to delete the oldPath in HIVE-8750 if it is
// not the destf or its subdir?
- oldPathDeleted = trashFiles(oldFs, statuses, conf);
+ oldPathDeleted = trashFiles(oldFs, statuses, conf, purge);
}
} catch (IOException e) {
if (isOldPathUnderDestf) {
@@ -3495,7 +3499,8 @@ private void constructOneLBLocationMap(FileStatus fSta,
* @return true if deletion successful
* @throws IOException
*/
- public static boolean trashFiles(final FileSystem fs, final FileStatus[] statuses, final Configuration conf)
+ public static boolean trashFiles(final FileSystem fs, final FileStatus[] statuses,
+ final Configuration conf, final boolean purge)
throws IOException {
boolean result = true;
@@ -3509,13 +3514,13 @@ private void constructOneLBLocationMap(FileStatus fSta,
final SessionState parentSession = SessionState.get();
for (final FileStatus status : statuses) {
if (null == pool) {
- result &= FileUtils.moveToTrash(fs, status.getPath(), conf);
+ result &= FileUtils.moveToTrash(fs, status.getPath(), conf, purge);
} else {
futures.add(pool.submit(new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
SessionState.setCurrentSessionState(parentSession);
- return FileUtils.moveToTrash(fs, status.getPath(), conf);
+ return FileUtils.moveToTrash(fs, status.getPath(), conf, purge);
}
}));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d822003d/ql/src/test/queries/clientpositive/encryption_auto_purge_tables.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/encryption_auto_purge_tables.q b/ql/src/test/queries/clientpositive/encryption_auto_purge_tables.q
new file mode 100644
index 0000000..b96a0a0
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/encryption_auto_purge_tables.q
@@ -0,0 +1,38 @@
+-- SORT_QUERY_RESULTS;
+
+-- we're setting this so that TestNegaiveCliDriver.vm doesn't stop processing after DROP TABLE fails;
+
+DROP TABLE IF EXISTS encrypted_table PURGE;
+DROP TABLE IF EXISTS encrypted_ext_table PURGE;
+
+CREATE TABLE encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table';
+CRYPTO CREATE_KEY --keyName key_128 --bitLength 128;
+CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table;
+
+SHOW TABLES;
+
+ALTER TABLE encrypted_table SET TBLPROPERTIES("auto.purge"="true");
+
+INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src;
+SELECT COUNT(*) from encrypted_table;
+
+TRUNCATE TABLE encrypted_table;
+SELECT COUNT(*) FROM encrypted_table;
+
+INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src;
+SELECT COUNT(*) FROM encrypted_table;
+
+CREATE EXTERNAL TABLE encrypted_ext_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table';
+ALTER TABLE encrypted_ext_table SET TBLPROPERTIES("auto.purge"="true");
+
+INSERT OVERWRITE TABLE encrypted_ext_table SELECT * FROM src;
+SELECT COUNT(*) from encrypted_ext_table;
+
+DROP TABLE encrypted_table;
+DROP TABLE encrypted_ext_table;
+SHOW TABLES;
+
+-- cleanup
+DROP TABLE IF EXISTS encrypted_table PURGE;
+DROP TABLE IF EXISTS encrypted_ext_table PURGE;
+CRYPTO DELETE_KEY --keyName key_128;
http://git-wip-us.apache.org/repos/asf/hive/blob/d822003d/ql/src/test/results/clientpositive/encrypted/encryption_auto_purge_tables.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_auto_purge_tables.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_auto_purge_tables.q.out
new file mode 100644
index 0000000..a2d7dd2
--- /dev/null
+++ b/ql/src/test/results/clientpositive/encrypted/encryption_auto_purge_tables.q.out
@@ -0,0 +1,157 @@
+PREHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE IF EXISTS encrypted_ext_table PURGE
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS encrypted_ext_table PURGE
+POSTHOOK: type: DROPTABLE
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@encrypted_table
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@encrypted_table
+Encryption key created: 'key_128'
+Encryption zone created: '/build/ql/test/data/warehouse/default/encrypted_table' using key: 'key_128'
+PREHOOK: query: SHOW TABLES
+PREHOOK: type: SHOWTABLES
+PREHOOK: Input: database:default
+POSTHOOK: query: SHOW TABLES
+POSTHOOK: type: SHOWTABLES
+POSTHOOK: Input: database:default
+encrypted_table
+src
+PREHOOK: query: ALTER TABLE encrypted_table SET TBLPROPERTIES("auto.purge"="true")
+PREHOOK: type: ALTERTABLE_PROPERTIES
+PREHOOK: Input: default@encrypted_table
+PREHOOK: Output: default@encrypted_table
+POSTHOOK: query: ALTER TABLE encrypted_table SET TBLPROPERTIES("auto.purge"="true")
+POSTHOOK: type: ALTERTABLE_PROPERTIES
+POSTHOOK: Input: default@encrypted_table
+POSTHOOK: Output: default@encrypted_table
+PREHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@encrypted_table
+POSTHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@encrypted_table
+POSTHOOK: Lineage: encrypted_table.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: encrypted_table.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT COUNT(*) from encrypted_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@encrypted_table
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging
+POSTHOOK: query: SELECT COUNT(*) from encrypted_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@encrypted_table
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging
+500
+PREHOOK: query: TRUNCATE TABLE encrypted_table
+PREHOOK: type: TRUNCATETABLE
+PREHOOK: Output: default@encrypted_table
+POSTHOOK: query: TRUNCATE TABLE encrypted_table
+POSTHOOK: type: TRUNCATETABLE
+POSTHOOK: Output: default@encrypted_table
+PREHOOK: query: SELECT COUNT(*) FROM encrypted_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@encrypted_table
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging
+POSTHOOK: query: SELECT COUNT(*) FROM encrypted_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@encrypted_table
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging
+0
+PREHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@encrypted_table
+POSTHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@encrypted_table
+POSTHOOK: Lineage: encrypted_table.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: encrypted_table.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT COUNT(*) FROM encrypted_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@encrypted_table
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging
+POSTHOOK: query: SELECT COUNT(*) FROM encrypted_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@encrypted_table
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging
+500
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@encrypted_ext_table
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@encrypted_ext_table
+PREHOOK: query: ALTER TABLE encrypted_ext_table SET TBLPROPERTIES("auto.purge"="true")
+PREHOOK: type: ALTERTABLE_PROPERTIES
+PREHOOK: Input: default@encrypted_ext_table
+PREHOOK: Output: default@encrypted_ext_table
+POSTHOOK: query: ALTER TABLE encrypted_ext_table SET TBLPROPERTIES("auto.purge"="true")
+POSTHOOK: type: ALTERTABLE_PROPERTIES
+POSTHOOK: Input: default@encrypted_ext_table
+POSTHOOK: Output: default@encrypted_ext_table
+PREHOOK: query: INSERT OVERWRITE TABLE encrypted_ext_table SELECT * FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@encrypted_ext_table
+POSTHOOK: query: INSERT OVERWRITE TABLE encrypted_ext_table SELECT * FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@encrypted_ext_table
+POSTHOOK: Lineage: encrypted_ext_table.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: encrypted_ext_table.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT COUNT(*) from encrypted_ext_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@encrypted_ext_table
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging
+POSTHOOK: query: SELECT COUNT(*) from encrypted_ext_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@encrypted_ext_table
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging
+500
+PREHOOK: query: DROP TABLE encrypted_table
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@encrypted_table
+PREHOOK: Output: default@encrypted_table
+POSTHOOK: query: DROP TABLE encrypted_table
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@encrypted_table
+POSTHOOK: Output: default@encrypted_table
+PREHOOK: query: DROP TABLE encrypted_ext_table
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@encrypted_ext_table
+PREHOOK: Output: default@encrypted_ext_table
+POSTHOOK: query: DROP TABLE encrypted_ext_table
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@encrypted_ext_table
+POSTHOOK: Output: default@encrypted_ext_table
+PREHOOK: query: SHOW TABLES
+PREHOOK: type: SHOWTABLES
+PREHOOK: Input: database:default
+POSTHOOK: query: SHOW TABLES
+POSTHOOK: type: SHOWTABLES
+POSTHOOK: Input: database:default
+src
+PREHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE IF EXISTS encrypted_ext_table PURGE
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS encrypted_ext_table PURGE
+POSTHOOK: type: DROPTABLE
[2/2] hive git commit: HIVE-16308: PreExecutePrinter and
PostExecutePrinter should log to INFO level instead of ERROR (Sahil Takiar
via Chaoyu Tang)
Posted by ct...@apache.org.
HIVE-16308: PreExecutePrinter and PostExecutePrinter should log to INFO level instead of ERROR (Sahil Takiar via Chaoyu Tang)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3c29f8ff
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3c29f8ff
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3c29f8ff
Branch: refs/heads/master
Commit: 3c29f8ff5c23031e3e7b5cc0c9b099964162c899
Parents: d822003
Author: Chaoyu Tang <ct...@cloudera.com>
Authored: Sat Apr 1 10:51:39 2017 -0400
Committer: Chaoyu Tang <ct...@cloudera.com>
Committed: Sat Apr 1 10:51:39 2017 -0400
----------------------------------------------------------------------
.../hive/ql/hooks/PostExecOrcRowGroupCountPrinter.java | 4 ++--
.../hadoop/hive/ql/hooks/PostExecTezSummaryPrinter.java | 12 ++++++------
.../apache/hadoop/hive/ql/hooks/PostExecutePrinter.java | 6 +++---
.../apache/hadoop/hive/ql/hooks/PreExecutePrinter.java | 6 +++---
.../org/apache/hadoop/hive/ql/session/SessionState.java | 10 +++++++++-
5 files changed, 23 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3c29f8ff/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecOrcRowGroupCountPrinter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecOrcRowGroupCountPrinter.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecOrcRowGroupCountPrinter.java
index 18ef325..ac79ceb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecOrcRowGroupCountPrinter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecOrcRowGroupCountPrinter.java
@@ -60,10 +60,10 @@ public class PostExecOrcRowGroupCountPrinter implements ExecuteWithHookContext {
if (counters != null) {
for (CounterGroup group : counters) {
if (group.getName().equals(LlapIOCounters.class.getName())) {
- console.printError(tezTask.getId() + " LLAP IO COUNTERS:");
+ console.printInfo(tezTask.getId() + " LLAP IO COUNTERS:", false);
for (TezCounter counter : group) {
if (counter.getDisplayName().equals(LlapIOCounters.SELECTED_ROWGROUPS.name())) {
- console.printError(" " + counter.getDisplayName() + ": " + counter.getValue());
+ console.printInfo(" " + counter.getDisplayName() + ": " + counter.getValue(), false);
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3c29f8ff/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecTezSummaryPrinter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecTezSummaryPrinter.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecTezSummaryPrinter.java
index 412f45c..45bd6e0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecTezSummaryPrinter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecTezSummaryPrinter.java
@@ -62,25 +62,25 @@ public class PostExecTezSummaryPrinter implements ExecuteWithHookContext {
String hiveCountersGroup = HiveConf.getVar(conf, HiveConf.ConfVars.HIVECOUNTERGROUP);
for (CounterGroup group : counters) {
if (hiveCountersGroup.equals(group.getDisplayName())) {
- console.printError(tezTask.getId() + " HIVE COUNTERS:");
+ console.printInfo(tezTask.getId() + " HIVE COUNTERS:", false);
for (TezCounter counter : group) {
- console.printError(" " + counter.getDisplayName() + ": " + counter.getValue());
+ console.printInfo(" " + counter.getDisplayName() + ": " + counter.getValue(), false);
}
} else if (group.getName().equals(FileSystemCounter.class.getName())) {
- console.printError(tezTask.getId() + " FILE SYSTEM COUNTERS:");
+ console.printInfo(tezTask.getId() + " FILE SYSTEM COUNTERS:", false);
for (TezCounter counter : group) {
// HDFS counters should be relatively consistent across test runs when compared to
// local file system counters
if (counter.getName().contains("HDFS")) {
- console.printError(" " + counter.getDisplayName() + ": " + counter.getValue());
+ console.printInfo(" " + counter.getDisplayName() + ": " + counter.getValue(), false);
}
}
} else if (group.getName().equals(LlapIOCounters.class.getName())) {
- console.printError(tezTask.getId() + " LLAP IO COUNTERS:");
+ console.printInfo(tezTask.getId() + " LLAP IO COUNTERS:", false);
List<String> testSafeCounters = LlapIOCounters.testSafeCounterNames();
for (TezCounter counter : group) {
if (testSafeCounters.contains(counter.getDisplayName())) {
- console.printError(" " + counter.getDisplayName() + ": " + counter.getValue());
+ console.printInfo(" " + counter.getDisplayName() + ": " + counter.getValue(), false);
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3c29f8ff/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecutePrinter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecutePrinter.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecutePrinter.java
index b4fc125..3e74396 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecutePrinter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecutePrinter.java
@@ -116,8 +116,8 @@ public class PostExecutePrinter implements ExecuteWithHookContext {
}
if (queryState != null) {
- console.printError("POSTHOOK: query: " + queryState.getQueryString().trim());
- console.printError("POSTHOOK: type: " + queryState.getCommandType());
+ console.printInfo("POSTHOOK: query: " + queryState.getQueryString().trim(), false);
+ console.printInfo("POSTHOOK: type: " + queryState.getCommandType(), false);
}
PreExecutePrinter.printEntities(console, inputs, "POSTHOOK: Input: ");
@@ -167,7 +167,7 @@ public class PostExecutePrinter implements ExecuteWithHookContext {
}
sb.append("]");
- console.printError(sb.toString());
+ console.printInfo(sb.toString(), false);
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3c29f8ff/ql/src/java/org/apache/hadoop/hive/ql/hooks/PreExecutePrinter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/PreExecutePrinter.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/PreExecutePrinter.java
index 232c62d..20acfb1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/PreExecutePrinter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/PreExecutePrinter.java
@@ -65,8 +65,8 @@ public class PreExecutePrinter implements ExecuteWithHookContext {
}
if (queryState != null) {
- console.printError("PREHOOK: query: " + queryState.getQueryString().trim());
- console.printError("PREHOOK: type: " + queryState.getCommandType());
+ console.printInfo("PREHOOK: query: " + queryState.getQueryString().trim(), false);
+ console.printInfo("PREHOOK: type: " + queryState.getCommandType(), false);
}
printEntities(console, inputs, "PREHOOK: Input: ");
@@ -80,7 +80,7 @@ public class PreExecutePrinter implements ExecuteWithHookContext {
}
Collections.sort(strings);
for (String s : strings) {
- console.printError(prefix + s);
+ console.printInfo(prefix + s, false);
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3c29f8ff/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 1e9774f..ffce1d1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -1096,8 +1096,16 @@ public class SessionState {
printInfo(info, null);
}
+ public void printInfo(String info, boolean isSilent) {
+ printInfo(info, null, isSilent);
+ }
+
public void printInfo(String info, String detail) {
- if (!getIsSilent()) {
+ printInfo(info, detail, getIsSilent());
+ }
+
+ public void printInfo(String info, String detail, boolean isSilent) {
+ if (!isSilent) {
getInfoStream().println(info);
}
LOG.info(info + StringUtils.defaultString(detail));