You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by we...@apache.org on 2017/09/07 05:58:15 UTC
[01/50] [abbrv] hive git commit: HIVE-17318: Make Hikari CP
configurable using hive properties in hive-site.xml (Barna Zsombor Klara,
via Peter Vary)
Repository: hive
Updated Branches:
refs/heads/hive-14535 81853c12e -> 42a38577b
HIVE-17318: Make Hikari CP configurable using hive properties in hive-site.xml (Barna Zsombor Klara, via Peter Vary)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6b103828
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6b103828
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6b103828
Branch: refs/heads/hive-14535
Commit: 6b103828863dd00a8745311d50c26a1db09fae34
Parents: 835c066
Author: Peter Vary <pv...@cloudera.com>
Authored: Fri Aug 25 16:01:10 2017 +0200
Committer: Peter Vary <pv...@cloudera.com>
Committed: Fri Aug 25 16:01:10 2017 +0200
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 3 +-
.../datasource/DataSourceProviderFactory.java | 2 +-
.../datasource/HikariCPDataSourceProvider.java | 97 ++++++++++++++++++++
.../hadoop/hive/metastore/txn/TxnHandler.java | 14 +--
.../TestDataSourceProviderFactory.java | 60 ++++++++++++
5 files changed, 162 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/6b103828/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 5a2a436..0d8d7ae 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3464,7 +3464,8 @@ public class HiveConf extends Configuration {
"hive.spark.client.secret.bits," +
"hive.spark.client.rpc.server.address," +
"hive.spark.client.rpc.server.port," +
- "bonecp.",
+ "bonecp.,"+
+ "hikari.",
"Comma separated list of configuration options which are immutable at runtime"),
HIVE_CONF_HIDDEN_LIST("hive.conf.hidden.list",
METASTOREPWD.varname + "," + HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname
http://git-wip-us.apache.org/repos/asf/hive/blob/6b103828/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java b/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java
index fa6bb1c..1eb792c 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.conf.Configuration;
public abstract class DataSourceProviderFactory {
private static final ImmutableList<DataSourceProvider> FACTORIES =
- ImmutableList.<DataSourceProvider>builder().add(new BoneCPDataSourceProvider()).build();
+ ImmutableList.<DataSourceProvider>builder().add(new HikariCPDataSourceProvider(), new BoneCPDataSourceProvider()).build();
/**
* @param hdpConfig hadoop configuration
http://git-wip-us.apache.org/repos/asf/hive/blob/6b103828/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java b/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java
new file mode 100644
index 0000000..9b3d6d5
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.datasource;
+
+import com.zaxxer.hikari.HikariConfig;
+import com.zaxxer.hikari.HikariDataSource;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.sql.DataSource;
+import java.sql.SQLException;
+import java.util.Properties;
+
+/**
+ * DataSourceProvider for the HikariCP connection pool.
+ */
+public class HikariCPDataSourceProvider implements DataSourceProvider {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HikariCPDataSourceProvider.class);
+
+ public static final String HIKARI = "hikari";
+ private static final String CONNECTION_TIMEOUT_PROPERTY= "hikari.connectionTimeout";
+
+ @Override
+ public DataSource create(Configuration hdpConfig) throws SQLException {
+
+ LOG.debug("Creating Hikari connection pool for the MetaStore");
+
+ String driverUrl = DataSourceProvider.getMetastoreJdbcDriverUrl(hdpConfig);
+ String user = DataSourceProvider.getMetastoreJdbcUser(hdpConfig);
+ String passwd = DataSourceProvider.getMetastoreJdbcPasswd(hdpConfig);
+ int maxPoolSize = hdpConfig.getInt(
+ MetastoreConf.ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS.varname,
+ ((Long)MetastoreConf.ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS.defaultVal).intValue());
+
+ Properties properties = replacePrefix(
+ DataSourceProvider.getPrefixedProperties(hdpConfig, HIKARI));
+ long connectionTimeout = hdpConfig.getLong(CONNECTION_TIMEOUT_PROPERTY, 30000L);
+ HikariConfig config = null;
+ try {
+ config = new HikariConfig(properties);
+ } catch (Exception e) {
+ throw new SQLException("Cannot create HikariCP configuration: ", e);
+ }
+ config.setMaximumPoolSize(maxPoolSize);
+ config.setJdbcUrl(driverUrl);
+ config.setUsername(user);
+ config.setPassword(passwd);
+ //https://github.com/brettwooldridge/HikariCP
+ config.setConnectionTimeout(connectionTimeout);
+ return new HikariDataSource(config);
+ }
+
+ @Override
+ public boolean mayReturnClosedConnection() {
+ // Only BoneCP should return true
+ return false;
+ }
+
+ @Override
+ public boolean supports(Configuration configuration) {
+ String poolingType =
+ configuration.get(
+ MetastoreConf.ConfVars.CONNECTION_POOLING_TYPE.varname).toLowerCase();
+ if (HIKARI.equals(poolingType)) {
+ int hikariPropsNr = DataSourceProvider.getPrefixedProperties(configuration, HIKARI).size();
+ LOG.debug("Found " + hikariPropsNr + " nr. of hikari specific configurations");
+ return hikariPropsNr > 0;
+ }
+ LOG.debug("Configuration requested " + poolingType + " pooling, HikariCpDSProvider exiting");
+ return false;
+ }
+
+ private Properties replacePrefix(Properties props) {
+ Properties newProps = new Properties();
+ props.forEach((key,value) ->
+ newProps.put(key.toString().replaceFirst(HIKARI + ".", ""), value));
+ return newProps;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/6b103828/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
index a6d5613..71e7c0c 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.hive.metastore.txn;
import com.google.common.annotations.VisibleForTesting;
-import com.zaxxer.hikari.HikariConfig;
-import com.zaxxer.hikari.HikariDataSource;
import org.apache.commons.dbcp.ConnectionFactory;
import org.apache.commons.dbcp.DriverManagerConnectionFactory;
@@ -32,9 +30,9 @@ import org.apache.hadoop.hive.common.classification.RetrySemantics;
import org.apache.hadoop.hive.metastore.DatabaseProduct;
import org.apache.hadoop.hive.metastore.HouseKeeperService;
import org.apache.hadoop.hive.metastore.Warehouse;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.datasource.BoneCPDataSourceProvider;
import org.apache.hadoop.hive.metastore.datasource.DataSourceProvider;
+import org.apache.hadoop.hive.metastore.datasource.HikariCPDataSourceProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.commons.dbcp.PoolingDataSource;
@@ -3187,15 +3185,7 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
new PoolableConnectionFactory(connFactory, objectPool, null, null, false, true);
return new PoolingDataSource(objectPool);
} else if ("hikaricp".equals(connectionPooler)) {
- HikariConfig config = new HikariConfig();
- config.setMaximumPoolSize(maxPoolSize);
- config.setJdbcUrl(driverUrl);
- config.setUsername(user);
- config.setPassword(passwd);
- //https://github.com/brettwooldridge/HikariCP
- config.setConnectionTimeout(getConnectionTimeoutMs);
-
- return new HikariDataSource(config);
+ return new HikariCPDataSourceProvider().create(conf);
} else if ("none".equals(connectionPooler)) {
LOG.info("Choosing not to pool JDBC connections");
return new NoPoolConnectionPool(conf);
http://git-wip-us.apache.org/repos/asf/hive/blob/6b103828/metastore/src/test/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java b/metastore/src/test/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java
index 6284604..856ed67 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.metastore.datasource;
import com.jolbox.bonecp.BoneCPDataSource;
+import com.zaxxer.hikari.HikariDataSource;
import org.apache.hadoop.hive.conf.HiveConf;
import org.junit.Assert;
import org.junit.Before;
@@ -104,4 +105,63 @@ public class TestDataSourceProviderFactory {
Assert.assertTrue(ds instanceof BoneCPDataSource);
Assert.assertEquals(true, ((BoneCPDataSource)ds).isDisableJMX());
}
+
+ @Test
+ public void testCreateHikariCpDataSource() throws SQLException {
+
+ conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, HikariCPDataSourceProvider.HIKARI);
+ // This is needed to prevent the HikariDataSource from trying to connect to the DB
+ conf.set(HikariCPDataSourceProvider.HIKARI + ".initializationFailTimeout", "-1");
+
+ DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
+ Assert.assertNotNull(dsp);
+
+ DataSource ds = dsp.create(conf);
+ Assert.assertTrue(ds instanceof HikariDataSource);
+ }
+
+ @Test
+ public void testSetHikariCpStringProperty() throws SQLException {
+
+ conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, HikariCPDataSourceProvider.HIKARI);
+ conf.set(HikariCPDataSourceProvider.HIKARI + ".connectionInitSql", "select 1 from dual");
+ conf.set(HikariCPDataSourceProvider.HIKARI + ".initializationFailTimeout", "-1");
+
+ DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
+ Assert.assertNotNull(dsp);
+
+ DataSource ds = dsp.create(conf);
+ Assert.assertTrue(ds instanceof HikariDataSource);
+ Assert.assertEquals("select 1 from dual", ((HikariDataSource)ds).getConnectionInitSql());
+ }
+
+ @Test
+ public void testSetHikariCpNumberProperty() throws SQLException {
+
+ conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, HikariCPDataSourceProvider.HIKARI);
+ conf.set(HikariCPDataSourceProvider.HIKARI + ".idleTimeout", "59999");
+ conf.set(HikariCPDataSourceProvider.HIKARI + ".initializationFailTimeout", "-1");
+
+ DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
+ Assert.assertNotNull(dsp);
+
+ DataSource ds = dsp.create(conf);
+ Assert.assertTrue(ds instanceof HikariDataSource);
+ Assert.assertEquals(59999L, ((HikariDataSource)ds).getIdleTimeout());
+ }
+
+ @Test
+ public void testSetHikariCpBooleanProperty() throws SQLException {
+
+ conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, HikariCPDataSourceProvider.HIKARI);
+ conf.set(HikariCPDataSourceProvider.HIKARI + ".allowPoolSuspension", "false");
+ conf.set(HikariCPDataSourceProvider.HIKARI + ".initializationFailTimeout", "-1");
+
+ DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
+ Assert.assertNotNull(dsp);
+
+ DataSource ds = dsp.create(conf);
+ Assert.assertTrue(ds instanceof HikariDataSource);
+ Assert.assertEquals(false, ((HikariDataSource)ds).isAllowPoolSuspension());
+ }
}
[10/50] [abbrv] hive git commit: HIVE-17309 alter partition onto a
table not in current database throw InvalidOperationException (Wang Haihua
via Alan Gates)
Posted by we...@apache.org.
HIVE-17309 alter partition onto a table not in current database throw InvalidOperationException (Wang Haihua via Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e352ef40
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e352ef40
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e352ef40
Branch: refs/heads/hive-14535
Commit: e352ef409571383bcd546f399a17fc7c8a6e6bf0
Parents: 3fc131c
Author: Alan Gates <ga...@hortonworks.com>
Authored: Mon Aug 28 12:51:31 2017 -0700
Committer: Alan Gates <ga...@hortonworks.com>
Committed: Mon Aug 28 12:51:31 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 2 +-
.../alter_partition_onto_nocurrent_db.q | 15 ++++
.../alter_partition_onto_nocurrent_db.q.out | 78 ++++++++++++++++++++
3 files changed, 94 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e352ef40/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 364db27..acc2390 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -3646,7 +3646,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
if (allPartitions == null) {
db.alterTable(alterTbl.getOldName(), tbl, alterTbl.getIsCascade(), alterTbl.getEnvironmentContext());
} else {
- db.alterPartitions(alterTbl.getOldName(), allPartitions, alterTbl.getEnvironmentContext());
+ db.alterPartitions(Warehouse.getQualifiedName(tbl.getTTable()), allPartitions, alterTbl.getEnvironmentContext());
}
// Add constraints if necessary
addConstraints(db, alterTbl);
http://git-wip-us.apache.org/repos/asf/hive/blob/e352ef40/ql/src/test/queries/clientpositive/alter_partition_onto_nocurrent_db.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/alter_partition_onto_nocurrent_db.q b/ql/src/test/queries/clientpositive/alter_partition_onto_nocurrent_db.q
new file mode 100644
index 0000000..017fd44
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/alter_partition_onto_nocurrent_db.q
@@ -0,0 +1,15 @@
+CREATE DATABASE test_db_nocurr;
+
+CREATE TABLE test_db_nocurr.test_table_for_alter_partition_nocurrentdb (a INT) PARTITIONED BY (ds STRING) STORED AS SEQUENCEFILE;
+
+INSERT OVERWRITE TABLE test_db_nocurr.test_table_for_alter_partition_nocurrentdb PARTITION(ds='eleme_haihua') SELECT 1;
+
+desc extended test_db_nocurr.test_table_for_alter_partition_nocurrentdb partition(ds='eleme_haihua');
+
+ALTER TABLE test_db_nocurr.test_table_for_alter_partition_nocurrentdb partition(ds='eleme_haihua') CHANGE COLUMN a a_new BOOLEAN;
+
+desc extended test_db_nocurr.test_table_for_alter_partition_nocurrentdb partition(ds='eleme_haihua');
+
+DROP TABLE test_db_nocurr.test_table_for_alter_partition_nocurrentdb;
+
+DROP DATABASE test_db_nocurr;
http://git-wip-us.apache.org/repos/asf/hive/blob/e352ef40/ql/src/test/results/clientpositive/alter_partition_onto_nocurrent_db.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/alter_partition_onto_nocurrent_db.q.out b/ql/src/test/results/clientpositive/alter_partition_onto_nocurrent_db.q.out
new file mode 100644
index 0000000..f109bcc
--- /dev/null
+++ b/ql/src/test/results/clientpositive/alter_partition_onto_nocurrent_db.q.out
@@ -0,0 +1,78 @@
+PREHOOK: query: CREATE DATABASE test_db_nocurr
+PREHOOK: type: CREATEDATABASE
+PREHOOK: Output: database:test_db_nocurr
+POSTHOOK: query: CREATE DATABASE test_db_nocurr
+POSTHOOK: type: CREATEDATABASE
+POSTHOOK: Output: database:test_db_nocurr
+PREHOOK: query: CREATE TABLE test_db_nocurr.test_table_for_alter_partition_nocurrentdb (a INT) PARTITIONED BY (ds STRING) STORED AS SEQUENCEFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:test_db_nocurr
+PREHOOK: Output: test_db_nocurr@test_table_for_alter_partition_nocurrentdb
+POSTHOOK: query: CREATE TABLE test_db_nocurr.test_table_for_alter_partition_nocurrentdb (a INT) PARTITIONED BY (ds STRING) STORED AS SEQUENCEFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:test_db_nocurr
+POSTHOOK: Output: test_db_nocurr@test_table_for_alter_partition_nocurrentdb
+PREHOOK: query: INSERT OVERWRITE TABLE test_db_nocurr.test_table_for_alter_partition_nocurrentdb PARTITION(ds='eleme_haihua') SELECT 1
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: test_db_nocurr@test_table_for_alter_partition_nocurrentdb@ds=eleme_haihua
+POSTHOOK: query: INSERT OVERWRITE TABLE test_db_nocurr.test_table_for_alter_partition_nocurrentdb PARTITION(ds='eleme_haihua') SELECT 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: test_db_nocurr@test_table_for_alter_partition_nocurrentdb@ds=eleme_haihua
+POSTHOOK: Lineage: test_table_for_alter_partition_nocurrentdb PARTITION(ds=eleme_haihua).a SIMPLE []
+PREHOOK: query: desc extended test_db_nocurr.test_table_for_alter_partition_nocurrentdb partition(ds='eleme_haihua')
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: test_db_nocurr@test_table_for_alter_partition_nocurrentdb
+POSTHOOK: query: desc extended test_db_nocurr.test_table_for_alter_partition_nocurrentdb partition(ds='eleme_haihua')
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: test_db_nocurr@test_table_for_alter_partition_nocurrentdb
+a int
+ds string
+
+# Partition Information
+# col_name data_type comment
+
+ds string
+
+#### A masked pattern was here ####
+PREHOOK: query: ALTER TABLE test_db_nocurr.test_table_for_alter_partition_nocurrentdb partition(ds='eleme_haihua') CHANGE COLUMN a a_new BOOLEAN
+PREHOOK: type: ALTERTABLE_RENAMECOL
+PREHOOK: Input: test_db_nocurr@test_table_for_alter_partition_nocurrentdb
+PREHOOK: Output: test_db_nocurr@test_table_for_alter_partition_nocurrentdb@ds=eleme_haihua
+POSTHOOK: query: ALTER TABLE test_db_nocurr.test_table_for_alter_partition_nocurrentdb partition(ds='eleme_haihua') CHANGE COLUMN a a_new BOOLEAN
+POSTHOOK: type: ALTERTABLE_RENAMECOL
+POSTHOOK: Input: test_db_nocurr@test_table_for_alter_partition_nocurrentdb
+POSTHOOK: Input: test_db_nocurr@test_table_for_alter_partition_nocurrentdb@ds=eleme_haihua
+POSTHOOK: Output: test_db_nocurr@test_table_for_alter_partition_nocurrentdb@ds=eleme_haihua
+PREHOOK: query: desc extended test_db_nocurr.test_table_for_alter_partition_nocurrentdb partition(ds='eleme_haihua')
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: test_db_nocurr@test_table_for_alter_partition_nocurrentdb
+POSTHOOK: query: desc extended test_db_nocurr.test_table_for_alter_partition_nocurrentdb partition(ds='eleme_haihua')
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: test_db_nocurr@test_table_for_alter_partition_nocurrentdb
+a_new boolean
+ds string
+
+# Partition Information
+# col_name data_type comment
+
+ds string
+
+#### A masked pattern was here ####
+PREHOOK: query: DROP TABLE test_db_nocurr.test_table_for_alter_partition_nocurrentdb
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: test_db_nocurr@test_table_for_alter_partition_nocurrentdb
+PREHOOK: Output: test_db_nocurr@test_table_for_alter_partition_nocurrentdb
+POSTHOOK: query: DROP TABLE test_db_nocurr.test_table_for_alter_partition_nocurrentdb
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: test_db_nocurr@test_table_for_alter_partition_nocurrentdb
+POSTHOOK: Output: test_db_nocurr@test_table_for_alter_partition_nocurrentdb
+PREHOOK: query: DROP DATABASE test_db_nocurr
+PREHOOK: type: DROPDATABASE
+PREHOOK: Input: database:test_db_nocurr
+PREHOOK: Output: database:test_db_nocurr
+POSTHOOK: query: DROP DATABASE test_db_nocurr
+POSTHOOK: type: DROPDATABASE
+POSTHOOK: Input: database:test_db_nocurr
+POSTHOOK: Output: database:test_db_nocurr
[34/50] [abbrv] hive git commit: HIVE-17381: When we enable Parquet
Writer Version V2,
hive throws an exception: Unsupported encoding: DELTA_BYTE_ARRAY. (Colin Ma,
reviewed by Ferdinand Xu)
Posted by we...@apache.org.
HIVE-17381: When we enable Parquet Writer Version V2, hive throws an exception: Unsupported encoding: DELTA_BYTE_ARRAY. (Colin Ma, reviewed by Ferdinand Xu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/601c7649
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/601c7649
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/601c7649
Branch: refs/heads/hive-14535
Commit: 601c764911b850da97aefdc0f83e8efc5627e8b8
Parents: 0d0fa42
Author: Ferdinand Xu <ch...@intel.com>
Authored: Fri Sep 1 10:05:42 2017 +0800
Committer: Ferdinand Xu <ch...@intel.com>
Committed: Fri Sep 1 10:05:42 2017 +0800
----------------------------------------------------------------------
.../ql/io/parquet/vector/VectorizedPrimitiveColumnReader.java | 3 ---
1 file changed, 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/601c7649/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedPrimitiveColumnReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedPrimitiveColumnReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedPrimitiveColumnReader.java
index 3d5c6e6..e9543c6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedPrimitiveColumnReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedPrimitiveColumnReader.java
@@ -482,9 +482,6 @@ public class VectorizedPrimitiveColumnReader implements VectorizedColumnReader {
dataColumn = dataEncoding.getDictionaryBasedValuesReader(descriptor, VALUES, dictionary);
this.isCurrentPageDictionaryEncoded = true;
} else {
- if (dataEncoding != Encoding.PLAIN) {
- throw new UnsupportedOperationException("Unsupported encoding: " + dataEncoding);
- }
dataColumn = dataEncoding.getValuesReader(descriptor, VALUES);
this.isCurrentPageDictionaryEncoded = false;
}
[45/50] [abbrv] hive git commit: HIVE-14671 : merge master into
hive-14535 (Wei Zheng)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/test/results/clientpositive/mm_loaddata.q.out
----------------------------------------------------------------------
diff --cc ql/src/test/results/clientpositive/mm_loaddata.q.out
index 0000000,0000000..b849a88
new file mode 100644
--- /dev/null
+++ b/ql/src/test/results/clientpositive/mm_loaddata.q.out
@@@ -1,0 -1,0 +1,296 @@@
++PREHOOK: query: drop table load0_mm
++PREHOOK: type: DROPTABLE
++POSTHOOK: query: drop table load0_mm
++POSTHOOK: type: DROPTABLE
++PREHOOK: query: create table load0_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
++PREHOOK: type: CREATETABLE
++PREHOOK: Output: database:default
++PREHOOK: Output: default@load0_mm
++POSTHOOK: query: create table load0_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
++POSTHOOK: type: CREATETABLE
++POSTHOOK: Output: database:default
++POSTHOOK: Output: default@load0_mm
++PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table load0_mm
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@load0_mm
++POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table load0_mm
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@load0_mm
++PREHOOK: query: select count(1) from load0_mm
++PREHOOK: type: QUERY
++PREHOOK: Input: default@load0_mm
++#### A masked pattern was here ####
++POSTHOOK: query: select count(1) from load0_mm
++POSTHOOK: type: QUERY
++POSTHOOK: Input: default@load0_mm
++#### A masked pattern was here ####
++500
++PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table load0_mm
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@load0_mm
++POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table load0_mm
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@load0_mm
++PREHOOK: query: select count(1) from load0_mm
++PREHOOK: type: QUERY
++PREHOOK: Input: default@load0_mm
++#### A masked pattern was here ####
++POSTHOOK: query: select count(1) from load0_mm
++POSTHOOK: type: QUERY
++POSTHOOK: Input: default@load0_mm
++#### A masked pattern was here ####
++1000
++PREHOOK: query: load data local inpath '../../data/files/kv2.txt' overwrite into table load0_mm
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@load0_mm
++POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' overwrite into table load0_mm
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@load0_mm
++PREHOOK: query: select count(1) from load0_mm
++PREHOOK: type: QUERY
++PREHOOK: Input: default@load0_mm
++#### A masked pattern was here ####
++POSTHOOK: query: select count(1) from load0_mm
++POSTHOOK: type: QUERY
++POSTHOOK: Input: default@load0_mm
++#### A masked pattern was here ####
++500
++PREHOOK: query: drop table load0_mm
++PREHOOK: type: DROPTABLE
++PREHOOK: Input: default@load0_mm
++PREHOOK: Output: default@load0_mm
++POSTHOOK: query: drop table load0_mm
++POSTHOOK: type: DROPTABLE
++POSTHOOK: Input: default@load0_mm
++POSTHOOK: Output: default@load0_mm
++PREHOOK: query: drop table intermediate2
++PREHOOK: type: DROPTABLE
++POSTHOOK: query: drop table intermediate2
++POSTHOOK: type: DROPTABLE
++PREHOOK: query: create table intermediate2 (key string, value string) stored as textfile
++#### A masked pattern was here ####
++PREHOOK: type: CREATETABLE
++#### A masked pattern was here ####
++PREHOOK: Output: database:default
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: create table intermediate2 (key string, value string) stored as textfile
++#### A masked pattern was here ####
++POSTHOOK: type: CREATETABLE
++#### A masked pattern was here ####
++POSTHOOK: Output: database:default
++POSTHOOK: Output: default@intermediate2
++PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@intermediate2
++PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@intermediate2
++PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@intermediate2
++PREHOOK: query: drop table load1_mm
++PREHOOK: type: DROPTABLE
++POSTHOOK: query: drop table load1_mm
++POSTHOOK: type: DROPTABLE
++PREHOOK: query: create table load1_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
++PREHOOK: type: CREATETABLE
++PREHOOK: Output: database:default
++PREHOOK: Output: default@load1_mm
++POSTHOOK: query: create table load1_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
++POSTHOOK: type: CREATETABLE
++POSTHOOK: Output: database:default
++POSTHOOK: Output: default@load1_mm
++#### A masked pattern was here ####
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@load1_mm
++#### A masked pattern was here ####
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@load1_mm
++#### A masked pattern was here ####
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@load1_mm
++#### A masked pattern was here ####
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@load1_mm
++PREHOOK: query: select count(1) from load1_mm
++PREHOOK: type: QUERY
++PREHOOK: Input: default@load1_mm
++#### A masked pattern was here ####
++POSTHOOK: query: select count(1) from load1_mm
++POSTHOOK: type: QUERY
++POSTHOOK: Input: default@load1_mm
++#### A masked pattern was here ####
++1000
++PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@intermediate2
++PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@intermediate2
++PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@intermediate2
++#### A masked pattern was here ####
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@load1_mm
++#### A masked pattern was here ####
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@load1_mm
++PREHOOK: query: select count(1) from load1_mm
++PREHOOK: type: QUERY
++PREHOOK: Input: default@load1_mm
++#### A masked pattern was here ####
++POSTHOOK: query: select count(1) from load1_mm
++POSTHOOK: type: QUERY
++POSTHOOK: Input: default@load1_mm
++#### A masked pattern was here ####
++1050
++PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@intermediate2
++#### A masked pattern was here ####
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@load1_mm
++#### A masked pattern was here ####
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@load1_mm
++PREHOOK: query: select count(1) from load1_mm
++PREHOOK: type: QUERY
++PREHOOK: Input: default@load1_mm
++#### A masked pattern was here ####
++POSTHOOK: query: select count(1) from load1_mm
++POSTHOOK: type: QUERY
++POSTHOOK: Input: default@load1_mm
++#### A masked pattern was here ####
++500
++PREHOOK: query: drop table load1_mm
++PREHOOK: type: DROPTABLE
++PREHOOK: Input: default@load1_mm
++PREHOOK: Output: default@load1_mm
++POSTHOOK: query: drop table load1_mm
++POSTHOOK: type: DROPTABLE
++POSTHOOK: Input: default@load1_mm
++POSTHOOK: Output: default@load1_mm
++PREHOOK: query: drop table load2_mm
++PREHOOK: type: DROPTABLE
++POSTHOOK: query: drop table load2_mm
++POSTHOOK: type: DROPTABLE
++PREHOOK: query: create table load2_mm (key string, value string)
++ partitioned by (k int, l int) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
++PREHOOK: type: CREATETABLE
++PREHOOK: Output: database:default
++PREHOOK: Output: default@load2_mm
++POSTHOOK: query: create table load2_mm (key string, value string)
++ partitioned by (k int, l int) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
++POSTHOOK: type: CREATETABLE
++POSTHOOK: Output: database:default
++POSTHOOK: Output: default@load2_mm
++PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@intermediate2
++PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@intermediate2
++PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@intermediate2
++#### A masked pattern was here ####
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@load2_mm
++#### A masked pattern was here ####
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@load2_mm
++POSTHOOK: Output: default@load2_mm@k=5/l=5
++PREHOOK: query: select count(1) from load2_mm
++PREHOOK: type: QUERY
++PREHOOK: Input: default@load2_mm
++PREHOOK: Input: default@load2_mm@k=5/l=5
++#### A masked pattern was here ####
++POSTHOOK: query: select count(1) from load2_mm
++POSTHOOK: type: QUERY
++POSTHOOK: Input: default@load2_mm
++POSTHOOK: Input: default@load2_mm@k=5/l=5
++#### A masked pattern was here ####
++1025
++PREHOOK: query: drop table load2_mm
++PREHOOK: type: DROPTABLE
++PREHOOK: Input: default@load2_mm
++PREHOOK: Output: default@load2_mm
++POSTHOOK: query: drop table load2_mm
++POSTHOOK: type: DROPTABLE
++POSTHOOK: Input: default@load2_mm
++POSTHOOK: Output: default@load2_mm
++PREHOOK: query: drop table intermediate2
++PREHOOK: type: DROPTABLE
++PREHOOK: Input: default@intermediate2
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: drop table intermediate2
++POSTHOOK: type: DROPTABLE
++POSTHOOK: Input: default@intermediate2
++POSTHOOK: Output: default@intermediate2
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
----------------------------------------------------------------------
[03/50] [abbrv] hive git commit: HIVE-17341 -
DbTxnManger.startHeartbeat() - randomize initial delay (Eugene Koifman,
reviewed by Wei Zheng)
Posted by we...@apache.org.
HIVE-17341 - DbTxnManger.startHeartbeat() - randomize initial delay (Eugene Koifman, reviewed by Wei Zheng)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1938eeb5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1938eeb5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1938eeb5
Branch: refs/heads/hive-14535
Commit: 1938eeb5948bd1e2443d785cf4b1896cc2c3d4c5
Parents: 733bc5f
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Fri Aug 25 12:47:01 2017 -0700
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Fri Aug 25 12:47:01 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1938eeb5/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
index 03d5b09..5dec791 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
@@ -639,8 +639,12 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
if(conf.getBoolVar(HiveConf.ConfVars.HIVE_IN_TEST) && conf.getBoolVar(HiveConf.ConfVars.HIVETESTMODEFAILHEARTBEATER)) {
initialDelay = 0;
} else if (initialDelay == 0) {
- initialDelay = heartbeatInterval;
+ /*make initialDelay a random number in [0, 0.75*heartbeatInterval] so that if a lot
+ of queries land on the server at the same time and all get blocked on lack of
+ resources, that they all don't start heartbeating at the same time*/
+ initialDelay = (long)Math.floor(heartbeatInterval * 0.75 * Math.random());
}
+
heartbeatTask = heartbeatExecutorService.scheduleAtFixedRate(
heartbeater, initialDelay, heartbeatInterval, TimeUnit.MILLISECONDS);
LOG.info("Started heartbeat with delay/interval = " + initialDelay + "/" + heartbeatInterval +
[29/50] [abbrv] hive git commit: HIVE-17216: Additional qtests for
HoS DPP (Sahil Takiar, reviewed by Sergio Pena)
Posted by we...@apache.org.
HIVE-17216: Additional qtests for HoS DPP (Sahil Takiar, reviewed by Sergio Pena)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/642acdf7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/642acdf7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/642acdf7
Branch: refs/heads/hive-14535
Commit: 642acdf76a46d7bb5bd036510d5bcdaa85e9f613
Parents: 3573549
Author: Sahil Takiar <ta...@gmail.com>
Authored: Thu Aug 31 09:05:16 2017 -0700
Committer: Sahil Takiar <st...@cloudera.com>
Committed: Thu Aug 31 09:06:10 2017 -0700
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 1 +
.../clientpositive/spark_constprog_dpp.q | 17 +
.../spark_dynamic_partition_pruning.q | 40 +-
.../spark_dynamic_partition_pruning_3.q | 236 ++-
.../spark/spark_constprog_dpp.q.out | 138 ++
.../spark/spark_dynamic_partition_pruning.q.out | 351 ++--
.../spark_dynamic_partition_pruning_3.q.out | 1781 ++++++++++++++++--
7 files changed, 2272 insertions(+), 292 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/642acdf7/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index fa6a2aa..6f2efa7 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -1403,6 +1403,7 @@ miniSparkOnYarn.only.query.files=spark_combine_equivalent_work.q,\
spark_dynamic_partition_pruning_2.q,\
spark_dynamic_partition_pruning_3.q,\
spark_dynamic_partition_pruning_mapjoin_only.q,\
+ spark_constprog_dpp.q,\
dynamic_rdd_cache.q, \
spark_multi_insert_parallel_orderby.q,\
spark_explainuser_1.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/642acdf7/ql/src/test/queries/clientpositive/spark_constprog_dpp.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/spark_constprog_dpp.q b/ql/src/test/queries/clientpositive/spark_constprog_dpp.q
new file mode 100644
index 0000000..cbe6aa2
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/spark_constprog_dpp.q
@@ -0,0 +1,17 @@
+set hive.mapred.mode=nonstrict;
+set hive.optimize.constant.propagation=true;
+set hive.spark.dynamic.partition.pruning=true;
+
+drop table if exists tb1;
+create table tb1 (id int);
+
+drop table if exists tb2;
+create table tb2 (id smallint);
+
+explain
+select a.id from tb1 a
+left outer join
+(select id from tb2
+union all
+select 2 as id from tb2 limit 1) b
+on a.id=b.id;
http://git-wip-us.apache.org/repos/asf/hive/blob/642acdf7/ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning.q b/ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning.q
index 78e7515..71a7399 100644
--- a/ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning.q
+++ b/ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning.q
@@ -16,7 +16,7 @@ create table srcpart_hour as select hr as hr, hr as hour from srcpart group by h
create table srcpart_date_hour as select ds as ds, ds as `date`, hr as hr, hr as hour from srcpart group by ds, hr;
create table srcpart_double_hour as select (hr*2) as hr, hr as hour from srcpart group by hr;
--- single column, single key
+-- single column, single key -- join a partitioned table to a non-partitioned table, static filter on the non-partitioned table
EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) where srcpart_date.`date` = '2008-04-08';
select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) where srcpart_date.`date` = '2008-04-08';
set hive.spark.dynamic.partition.pruning=false;
@@ -41,8 +41,8 @@ select count(*) from srcpart join srcpart_date on abs(negative(cast(concat(cast(
EXPLAIN select count(*) from srcpart join srcpart_date on cast(day(srcpart.ds) as smallint) = cast(day(srcpart_date.ds) as decimal) where srcpart_date.`date` = '2008-04-08';
select count(*) from srcpart join srcpart_date on cast(day(srcpart.ds) as smallint) = cast(day(srcpart_date.ds) as decimal) where srcpart_date.`date` = '2008-04-08';
--- multiple sources, single key
-EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr)
+-- multiple sources, single key -- filter partitioned table on both partitioned columns via join with non-partitioned table
+EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr)
where srcpart_date.`date` = '2008-04-08' and srcpart_hour.hour = 11;
select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr)
where srcpart_date.`date` = '2008-04-08' and srcpart_hour.hour = 11;
@@ -54,7 +54,7 @@ where srcpart_date.`date` = '2008-04-08' and srcpart_hour.hour = 11;
set hive.spark.dynamic.partition.pruning=true;
select count(*) from srcpart where hr = 11 and ds = '2008-04-08';
--- multiple columns single source
+-- multiple columns single source -- filter partitioned table on both partitioned columns via join with non-partitioned table, filter non-partitioned table
EXPLAIN select count(*) from srcpart join srcpart_date_hour on (srcpart.ds = srcpart_date_hour.ds and srcpart.hr = srcpart_date_hour.hr) where srcpart_date_hour.`date` = '2008-04-08' and srcpart_date_hour.hour = 11;
select count(*) from srcpart join srcpart_date_hour on (srcpart.ds = srcpart_date_hour.ds and srcpart.hr = srcpart_date_hour.hr) where srcpart_date_hour.`date` = '2008-04-08' and srcpart_date_hour.hour = 11;
set hive.spark.dynamic.partition.pruning=false;
@@ -63,7 +63,7 @@ select count(*) from srcpart join srcpart_date_hour on (srcpart.ds = srcpart_dat
set hive.spark.dynamic.partition.pruning=true;
select count(*) from srcpart where ds = '2008-04-08' and hr = 11;
--- empty set
+-- empty set -- join a partitioned table to a non-partitioned table, static filter on the non-partitioned table that doesn't filter out anything
EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) where srcpart_date.`date` = 'I DONT EXIST';
select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) where srcpart_date.`date` = 'I DONT EXIST';
set hive.spark.dynamic.partition.pruning=false;
@@ -72,7 +72,7 @@ select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds)
set hive.spark.dynamic.partition.pruning=true;
select count(*) from srcpart where ds = 'I DONT EXIST';
--- expressions
+-- expressions -- triggers DPP with various expressions - e.g. cast, multiplication, division
EXPLAIN select count(*) from srcpart join srcpart_double_hour on (srcpart.hr = cast(srcpart_double_hour.hr/2 as int)) where srcpart_double_hour.hour = 11;
select count(*) from srcpart join srcpart_double_hour on (srcpart.hr = cast(srcpart_double_hour.hr/2 as int)) where srcpart_double_hour.hour = 11;
EXPLAIN select count(*) from srcpart join srcpart_double_hour on (srcpart.hr*2 = srcpart_double_hour.hr) where srcpart_double_hour.hour = 11;
@@ -89,8 +89,7 @@ select count(*) from srcpart join srcpart_double_hour on (cast(srcpart.hr*2 as s
set hive.spark.dynamic.partition.pruning=true;
select count(*) from srcpart where cast(hr as string) = 11;
-
--- parent is reduce tasks
+-- parent is reduce tasks -- join a partitioned table to a non-partitioned table, where the non-partitioned table is a subquery, static filter on the subquery
EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08';
select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08';
select count(*) from srcpart where ds = '2008-04-08';
@@ -155,8 +154,7 @@ select count(*) from srcpart where ds = '2008-04-08' and hr = 11;
-- empty set
EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) where srcpart_date.`date` = 'I DONT EXIST';
--- Disabled until TEZ-1486 is fixed
--- select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) where srcpart_date.`date` = 'I DONT EXIST';
+select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) where srcpart_date.`date` = 'I DONT EXIST';
-- expressions
EXPLAIN select count(*) from srcpart join srcpart_double_hour on (srcpart.hr = cast(srcpart_double_hour.hr/2 as int)) where srcpart_double_hour.hour = 11;
@@ -170,10 +168,6 @@ EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from sr
select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08';
select count(*) from srcpart where ds = '2008-04-08';
--- single column, single key, udf with typechange
-EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08';
-select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08';
-
-- left join
EXPLAIN select count(*) from srcpart left join srcpart_date on (srcpart.ds = srcpart_date.ds) where srcpart_date.`date` = '2008-04-08';
EXPLAIN select count(*) from srcpart_date left join srcpart on (srcpart.ds = srcpart_date.ds) where srcpart_date.`date` = '2008-04-08';
@@ -188,16 +182,26 @@ select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds)
where srcpart_date.`date` = '2008-04-08' and srcpart_hour.hour = 11 and srcpart.hr = 11;
EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr)
where srcpart_date.`date` = '2008-04-08' and srcpart.hr = 13;
--- Disabled until TEZ-1486 is fixed
--- select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr)
--- where srcpart_date.`date` = '2008-04-08' and srcpart.hr = 13;
+select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr)
+where srcpart_date.`date` = '2008-04-08' and srcpart.hr = 13;
-- union + subquery
EXPLAIN select distinct(ds) from srcpart where srcpart.ds in (select max(srcpart.ds) from srcpart union all select min(srcpart.ds) from srcpart);
select distinct(ds) from srcpart where srcpart.ds in (select max(srcpart.ds) from srcpart union all select min(srcpart.ds) from srcpart);
+-- different file format
+create table srcpart_parquet (key int, value string) partitioned by (ds string, hr int) stored as parquet;
+
+set hive.exec.dynamic.partition.mode=nonstrict;
+set hive.vectorized.execution.enabled=false;
+set hive.exec.max.dynamic.partitions=1000;
+
+insert into table srcpart_parquet partition (ds, hr) select key, value, ds, hr from srcpart;
+EXPLAIN select count(*) from srcpart_parquet join srcpart_date_hour on (srcpart_parquet.ds = srcpart_date_hour.ds and srcpart_parquet.hr = srcpart_date_hour.hr) where srcpart_date_hour.hour = 11 and (srcpart_date_hour.`date` = '2008-04-08' or srcpart_date_hour.`date` = '2008-04-09');
+select count(*) from srcpart_parquet join srcpart_date_hour on (srcpart_parquet.ds = srcpart_date_hour.ds and srcpart_parquet.hr = srcpart_date_hour.hr) where srcpart_date_hour.hour = 11 and (srcpart_date_hour.`date` = '2008-04-08' or srcpart_date_hour.`date` = '2008-04-09');
+select count(*) from srcpart where (ds = '2008-04-08' or ds = '2008-04-09') and hr = 11;
-drop table srcpart_orc;
+drop table srcpart_parquet;
drop table srcpart_date;
drop table srcpart_hour;
drop table srcpart_date_hour;
http://git-wip-us.apache.org/repos/asf/hive/blob/642acdf7/ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning_3.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning_3.q b/ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning_3.q
index ac4ff93..8863cf4 100644
--- a/ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning_3.q
+++ b/ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning_3.q
@@ -1,16 +1,228 @@
-set hive.spark.dynamic.partition.pruning=true;
-set hive.auto.convert.join=true;
+CREATE TABLE partitioned_table1 (col int) PARTITIONED BY (part_col int);
+CREATE TABLE partitioned_table2 (col int) PARTITIONED BY (part_col int);
+CREATE TABLE partitioned_table3 (col int) PARTITIONED BY (part_col int);
+CREATE TABLE partitioned_table4 (col int) PARTITIONED BY (part_col1 int, part_col2 int);
+CREATE TABLE partitioned_table5 (col int) PARTITIONED BY (part_col1 int, part_col2 int);
-create table partitioned_table1 (col int) partitioned by (part_col int);
-create table partitioned_table2 (col int) partitioned by (part_col int);
-create table regular_table (col int);
-insert into table regular_table values (1);
+CREATE TABLE regular_table1 (col1 int, col2 int);
+CREATE TABLE regular_table2 (col1 int, col2 int);
-alter table partitioned_table1 add partition (part_col = 1);
-insert into table partitioned_table1 partition (part_col = 1) values (1), (2), (3), (4), (5), (6), (7), (8), (9), (10);
+ALTER TABLE partitioned_table1 ADD PARTITION (part_col = 1);
+ALTER TABLE partitioned_table1 ADD PARTITION (part_col = 2);
+ALTER TABLE partitioned_table1 ADD PARTITION (part_col = 3);
-alter table partitioned_table2 add partition (part_col = 1);
-insert into table partitioned_table2 partition (part_col = 1) values (1), (2), (3), (4), (5), (6), (7), (8), (9), (10);
+ALTER TABLE partitioned_table2 ADD PARTITION (part_col = 1);
+ALTER TABLE partitioned_table2 ADD PARTITION (part_col = 2);
+ALTER TABLE partitioned_table2 ADD PARTITION (part_col = 3);
-explain select * from partitioned_table1, partitioned_table2 where partitioned_table1.part_col = partitioned_table2.part_col;
-explain select * from partitioned_table1 where partitioned_table1.part_col in (select regular_table.col from regular_table join partitioned_table2 on regular_table.col = partitioned_table2.part_col);
\ No newline at end of file
+ALTER TABLE partitioned_table3 ADD PARTITION (part_col = 1);
+ALTER TABLE partitioned_table3 ADD PARTITION (part_col = 2);
+ALTER TABLE partitioned_table3 ADD PARTITION (part_col = 3);
+
+ALTER TABLE partitioned_table4 ADD PARTITION (part_col1 = 1, part_col2 = 1);
+ALTER TABLE partitioned_table4 ADD PARTITION (part_col1 = 2, part_col2 = 2);
+ALTER TABLE partitioned_table4 ADD PARTITION (part_col1 = 3, part_col2 = 3);
+
+ALTER TABLE partitioned_table5 ADD PARTITION (part_col1 = 1, part_col2 = 1);
+ALTER TABLE partitioned_table5 ADD PARTITION (part_col1 = 2, part_col2 = 2);
+ALTER TABLE partitioned_table5 ADD PARTITION (part_col1 = 3, part_col2 = 3);
+
+INSERT INTO TABLE regular_table1 VALUES (0, 0), (1, 1), (2, 2);
+INSERT INTO TABLE regular_table2 VALUES (0, 0), (1, 1), (2, 2);
+
+INSERT INTO TABLE partitioned_table1 PARTITION (part_col = 1) VALUES (1), (2), (3), (4), (5), (6);
+INSERT INTO TABLE partitioned_table1 PARTITION (part_col = 2) VALUES (1), (2), (3), (4), (5), (6);
+INSERT INTO TABLE partitioned_table1 PARTITION (part_col = 3) VALUES (1), (2), (3), (4), (5), (6);
+
+INSERT INTO TABLE partitioned_table2 PARTITION (part_col = 1) VALUES (1), (2), (3), (4), (5), (6);
+INSERT INTO TABLE partitioned_table2 PARTITION (part_col = 2) VALUES (1), (2), (3), (4), (5), (6);
+INSERT INTO TABLE partitioned_table2 PARTITION (part_col = 3) VALUES (1), (2), (3), (4), (5), (6);
+
+INSERT INTO TABLE partitioned_table3 PARTITION (part_col = 1) VALUES (1), (2), (3), (4), (5), (6);
+INSERT INTO TABLE partitioned_table3 PARTITION (part_col = 2) VALUES (1), (2), (3), (4), (5), (6);
+INSERT INTO TABLE partitioned_table3 PARTITION (part_col = 3) VALUES (1), (2), (3), (4), (5), (6);
+
+INSERT INTO TABLE partitioned_table4 PARTITION (part_col1 = 1, part_col2 = 1) VALUES (1), (2), (3), (4), (5), (6);
+INSERT INTO TABLE partitioned_table4 PARTITION (part_col1 = 2, part_col2 = 2) VALUES (1), (2), (3), (4), (5), (6);
+INSERT INTO TABLE partitioned_table4 PARTITION (part_col1 = 3, part_col2 = 3) VALUES (1), (2), (3), (4), (5), (6);
+
+INSERT INTO TABLE partitioned_table5 PARTITION (part_col1 = 1, part_col2 = 1) VALUES (1), (2), (3), (4), (5), (6);
+INSERT INTO TABLE partitioned_table5 PARTITION (part_col1 = 2, part_col2 = 2) VALUES (1), (2), (3), (4), (5), (6);
+INSERT INTO TABLE partitioned_table5 PARTITION (part_col1 = 3, part_col2 = 3) VALUES (1), (2), (3), (4), (5), (6);
+
+SET hive.spark.dynamic.partition.pruning.map.join.only=true;
+SET hive.strict.checks.cartesian.product=false;
+SET hive.auto.convert.join=true;
+SET hive.auto.convert.join.noconditionaltask.size=15; -- ensure the partitioned tables are treated as big tables
+
+-- left semi join where the subquery is a join between a partitioned and a non-partitioned table
+EXPLAIN SELECT count(*) FROM partitioned_table1 WHERE partitioned_table1.part_col IN (
+SELECT regular_table1.col1 FROM regular_table1 JOIN partitioned_table2 ON
+regular_table1.col1 = partitioned_table2.part_col AND partitioned_table2.col > 3 AND regular_table1.col1 > 1);
+
+SELECT count(*)
+FROM partitioned_table1
+WHERE partitioned_table1.part_col IN (SELECT regular_table1.col1
+ FROM regular_table1
+ JOIN partitioned_table2
+ ON
+ regular_table1.col1 = partitioned_table2.part_col AND partitioned_table2.col > 3 AND regular_table1.col1 > 1);
+
+-- three-table join with dpp against one partitioned table
+EXPLAIN SELECT count(*) FROM partitioned_table1, regular_table1 rt1,
+regular_table1 rt2 WHERE rt1.col1 = partitioned_table1.part_col AND rt2.col1 =
+partitioned_table1.part_col AND rt1.col2 > 0 AND rt2.col2 > 1;
+
+SELECT count(*)
+FROM partitioned_table1,
+ regular_table1 rt1,
+ regular_table1 rt2
+WHERE rt1.col1 = partitioned_table1.part_col
+ AND rt2.col1 = partitioned_table1.part_col
+ AND rt1.col2 > 0
+ AND rt2.col2 > 1;
+
+-- four-table join with dpp against two separate partitioned tables
+EXPLAIN SELECT count(*) FROM partitioned_table1, partitioned_table2, regular_table1 rt1
+, regular_table1 rt2 WHERE rt1.col1 = partitioned_table1.part_col AND rt2.col1 =
+partitioned_table2.part_col AND rt1.col2 > 0 AND rt2.col2 > 1;
+
+SELECT count(*)
+FROM partitioned_table1,
+ partitioned_table2,
+ regular_table1 rt1,
+ regular_table1 rt2
+WHERE rt1.col1 = partitioned_table1.part_col
+ AND rt2.col1 = partitioned_table2.part_col
+ AND rt1.col2 > 0
+ AND rt2.col2 > 1;
+
+-- dpp with between filter
+EXPLAIN SELECT count(*) FROM regular_table1, partitioned_table1 WHERE regular_table1.col1
+= partitioned_table1.part_col AND regular_table1.col2 BETWEEN 1 AND 3;
+
+SELECT count(*)
+FROM regular_table1,
+ partitioned_table1
+WHERE regular_table1.col1 = partitioned_table1.part_col
+ AND regular_table1.col2 BETWEEN 1 AND 3;
+
+-- dpp with cte
+EXPLAIN WITH q1 AS (SELECT regular_table1.col1 AS col FROM regular_table1 WHERE
+regular_table1.col2 > 1), q2 AS (SELECT partitioned_table1.part_col AS col FROM
+partitioned_table1 WHERE partitioned_table1.col > 1) SELECT count(*) FROM q1 JOIN q2 ON
+q1.col = q2.col;
+
+WITH q1
+ AS (SELECT regular_table1.col1 AS col
+ FROM regular_table1
+ WHERE regular_table1.col2 > 1),
+ q2
+ AS (SELECT partitioned_table1.part_col AS col
+ FROM partitioned_table1
+ WHERE partitioned_table1.col > 1)
+SELECT count(*)
+FROM q1
+ JOIN q2
+ ON q1.col = q2.col;
+
+-- join two partitioned tables with a filter
+EXPLAIN SELECT count(*) FROM partitioned_table1, partitioned_table2 WHERE
+partitioned_table1.part_col = partitioned_table2.part_col AND partitioned_table2.col > 1;
+
+SELECT count(*)
+FROM partitioned_table1,
+ partitioned_table2
+WHERE partitioned_table1.part_col = partitioned_table2.part_col
+ AND partitioned_table2.col > 1;
+
+-- dpp betwen two partitioned tables, both with multiple partition columns
+SET hive.auto.convert.join.noconditionaltask.size=150; -- set auto convert size to a higher value so map-joins are triggered for the partitioned tables
+
+EXPLAIN SELECT count(*) FROM partitioned_table4, partitioned_table5 WHERE
+partitioned_table4.part_col1 = partitioned_table5.part_col1 AND
+partitioned_table4.part_col2 = partitioned_table5.part_col2;
+
+SELECT count(*)
+FROM partitioned_table4,
+ partitioned_table5
+WHERE partitioned_table4.part_col1 = partitioned_table5.part_col1
+ AND partitioned_table4.part_col2 = partitioned_table5.part_col2;
+
+-- dpp is pushed through multiple levels of joins
+EXPLAIN SELECT count(*) FROM partitioned_table1 JOIN regular_table1 ON
+partitioned_table1.part_col = regular_table1.col1 JOIN regular_table2 ON
+regular_table1.col1 = regular_table2.col1;
+
+SELECT count(*)
+FROM partitioned_table1
+ JOIN regular_table1
+ ON partitioned_table1.part_col = regular_table1.col1
+ JOIN regular_table2
+ ON regular_table1.col1 = regular_table2.col1;
+
+SET hive.auto.convert.join.noconditionaltask.size=15; -- reset auto convert size to previous value
+
+-- three-way join where the partitioned table is the smallest table
+-- disabled until HIVE-17225 is fixed
+-- EXPLAIN SELECT * FROM partitioned_table1, regular_table1 rt1, regular_table2 rt2
+-- WHERE rt1.col1 = partitioned_table1.part_col AND rt2.col1 =
+-- partitioned_table1.part_col AND partitioned_table1.col > 3;
+
+-- SELECT *
+-- FROM partitioned_table1,
+-- regular_table1 rt1,
+-- regular_table2 rt2
+-- WHERE rt1.col1 = partitioned_table1.part_col
+-- AND rt2.col1 = partitioned_table1.part_col AND partitioned_table1.col > 3;
+
+-- dpp is pushed to partition columns that are added to each other
+-- disabled until HIVE-17244 is fixed
+-- EXPLAIN SELECT count(*) FROM partitioned_table4 pt4 JOIN regular_table1 rt1
+-- ON pt4.part_col1 + pt4.part_col2 = rt1.col1 + 1;
+
+-- SELECT count(*)
+-- FROM partitioned_table4 pt4
+-- JOIN regular_table1 rt1
+-- ON pt4.part_col1 + pt4.part_col2 = rt1.col1 + 1;
+
+-- dpp pushed to all union operands
+-- disabled until HIVE-17239 is fixed
+-- EXPLAIN SELECT count(*) FROM (SELECT part_col FROM partitioned_table1 UNION ALL SELECT
+-- part_col FROM partitioned_table2) q1 JOIN regular_table1 JOIN regular_table2
+-- WHERE q1.part_col = regular_table1.col1 AND q1.part_col = regular_table2.col1;
+
+-- SELECT count(*)
+-- FROM (
+-- SELECT part_col
+-- FROM partitioned_table1
+-- UNION ALL
+-- SELECT part_col
+-- FROM partitioned_table2) q1
+-- JOIN regular_table1
+-- JOIN regular_table2
+-- where q1.part_col = regular_table1.col1
+-- AND q1.part_col = regular_table2.col1;
+
+-- left semi join where the subquery is a join between a partitioned and a partitioned table
+-- disabled until HIVE-17238 is fixed
+-- EXPLAIN SELECT count(*) FROM partitioned_table1 WHERE partitioned_table1.part_col IN (
+-- SELECT partitioned_table2.col FROM partitioned_table2 JOIN partitioned_table3 ON
+-- partitioned_table3.col = partitioned_table2.part_col AND partitioned_table2.col > 1);
+
+-- SELECT count(*)
+-- FROM partitioned_table1
+-- WHERE partitioned_table1.part_col IN (SELECT partitioned_table2.col
+-- FROM partitioned_table2
+-- JOIN partitioned_table3
+-- ON
+-- partitioned_table3.col = partitioned_table2.part_col
+-- AND partitioned_table2.col > 1);
+
+DROP TABLE partitioned_table1;
+DROP TABLE partitioned_table2;
+DROP TABLE partitioned_table3;
+DROP TABLE partitioned_table4;
+DROP TABLE partitioned_table5;
+DROP TABLE regular_table1;
+DROP TABLE regular_table2;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/642acdf7/ql/src/test/results/clientpositive/spark/spark_constprog_dpp.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/spark_constprog_dpp.q.out b/ql/src/test/results/clientpositive/spark/spark_constprog_dpp.q.out
new file mode 100644
index 0000000..79bbbdf
--- /dev/null
+++ b/ql/src/test/results/clientpositive/spark/spark_constprog_dpp.q.out
@@ -0,0 +1,138 @@
+PREHOOK: query: drop table if exists tb1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists tb1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table tb1 (id int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tb1
+POSTHOOK: query: create table tb1 (id int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tb1
+PREHOOK: query: drop table if exists tb2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists tb2
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table tb2 (id smallint)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tb2
+POSTHOOK: query: create table tb2 (id smallint)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tb2
+PREHOOK: query: explain
+select a.id from tb1 a
+left outer join
+(select id from tb2
+union all
+select 2 as id from tb2 limit 1) b
+on a.id=b.id
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select a.id from tb1 a
+left outer join
+(select id from tb2
+union all
+select 2 as id from tb2 limit 1) b
+on a.id=b.id
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Reducer 4 (PARTITION-LEVEL SORT, 4)
+ Reducer 4 <- Map 3 (GROUP, 1), Map 5 (GROUP, 1)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: id (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: tb2
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: UDFToInteger(id) (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ TopN Hash Memory Usage: 0.1
+ value expressions: _col0 (type: int)
+ Map 5
+ Map Operator Tree:
+ TableScan
+ alias: tb2
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Select Operator
+ expressions: 2 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ TopN Hash Memory Usage: 0.1
+ value expressions: _col0 (type: int)
+ Reducer 2
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Outer Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 4
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
http://git-wip-us.apache.org/repos/asf/hive/blob/642acdf7/ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out
index 63a9548..6c21f1a 100644
--- a/ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out
+++ b/ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out
@@ -976,10 +976,10 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
POSTHOOK: Input: default@srcpart_date
#### A masked pattern was here ####
1000
-PREHOOK: query: EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr)
+PREHOOK: query: EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr)
where srcpart_date.`date` = '2008-04-08' and srcpart_hour.hour = 11
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr)
+POSTHOOK: query: EXPLAIN select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr)
where srcpart_date.`date` = '2008-04-08' and srcpart_hour.hour = 11
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
@@ -5107,6 +5107,25 @@ STAGE PLANS:
Processor Tree:
ListSink
+PREHOOK: query: select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) where srcpart_date.`date` = 'I DONT EXIST'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+PREHOOK: Input: default@srcpart_date
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) where srcpart_date.`date` = 'I DONT EXIST'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+POSTHOOK: Input: default@srcpart_date
+#### A masked pattern was here ####
+0
PREHOOK: query: EXPLAIN select count(*) from srcpart join srcpart_double_hour on (srcpart.hr = cast(srcpart_double_hour.hr/2 as int)) where srcpart_double_hour.hour = 11
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN select count(*) from srcpart join srcpart_double_hour on (srcpart.hr = cast(srcpart_double_hour.hr/2 as int)) where srcpart_double_hour.hour = 11
@@ -5493,130 +5512,6 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@srcpart
#### A masked pattern was here ####
1000
-PREHOOK: query: EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08'
-PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08'
-POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
- Stage-2 is a root stage
- Stage-1 depends on stages: Stage-2
- Stage-0 depends on stages: Stage-1
-
-STAGE PLANS:
- Stage: Stage-2
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 3
- Map Operator Tree:
- TableScan
- alias: srcpart_date
- filterExpr: ((date = '2008-04-08') and ds is not null) (type: boolean)
- Statistics: Num rows: 2 Data size: 42 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((date = '2008-04-08') and ds is not null) (type: boolean)
- Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: ds (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE
- Spark HashTable Sink Operator
- keys:
- 0 day(_col0) (type: int)
- 1 day(_col0) (type: int)
- Select Operator
- expressions: day(_col0) (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- keys: _col0 (type: int)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE
- Spark Partition Pruning Sink Operator
- Target column: ds (string)
- partition key expr: day(ds)
- Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE
- target work: Map 1
- Local Work:
- Map Reduce Local Work
-
- Stage: Stage-1
- Spark
- Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
-#### A masked pattern was here ####
- Vertices:
- Map 1
- Map Operator Tree:
- TableScan
- alias: srcpart
- filterExpr: ds is not null (type: boolean)
- Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: ds (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 day(_col0) (type: int)
- 1 day(_col0) (type: int)
- input vertices:
- 1 Map 3
- Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count()
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint)
- Local Work:
- Map Reduce Local Work
- Reducer 2
- Reduce Operator Tree:
- Group By Operator
- aggregations: count(VALUE._col0)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
- Processor Tree:
- ListSink
-
-PREHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@srcpart
-PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
-PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
-PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
-PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
-PREHOOK: Input: default@srcpart_date
-#### A masked pattern was here ####
-POSTHOOK: query: select count(*) from srcpart join srcpart_date on (day(srcpart.ds) = day(srcpart_date.ds)) where srcpart_date.`date` = '2008-04-08'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@srcpart
-POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
-POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
-POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
-POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
-POSTHOOK: Input: default@srcpart_date
-#### A masked pattern was here ####
-1000
PREHOOK: query: EXPLAIN select count(*) from srcpart left join srcpart_date on (srcpart.ds = srcpart_date.ds) where srcpart_date.`date` = '2008-04-08'
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN select count(*) from srcpart left join srcpart_date on (srcpart.ds = srcpart_date.ds) where srcpart_date.`date` = '2008-04-08'
@@ -6201,6 +6096,21 @@ STAGE PLANS:
Processor Tree:
ListSink
+PREHOOK: query: select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr)
+where srcpart_date.`date` = '2008-04-08' and srcpart.hr = 13
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart_date
+PREHOOK: Input: default@srcpart_hour
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from srcpart join srcpart_date on (srcpart.ds = srcpart_date.ds) join srcpart_hour on (srcpart.hr = srcpart_hour.hr)
+where srcpart_date.`date` = '2008-04-08' and srcpart.hr = 13
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart_date
+POSTHOOK: Input: default@srcpart_hour
+#### A masked pattern was here ####
+0
PREHOOK: query: EXPLAIN select distinct(ds) from srcpart where srcpart.ds in (select max(srcpart.ds) from srcpart union all select min(srcpart.ds) from srcpart)
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN select distinct(ds) from srcpart where srcpart.ds in (select max(srcpart.ds) from srcpart union all select min(srcpart.ds) from srcpart)
@@ -6472,10 +6382,195 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
#### A masked pattern was here ####
2008-04-08
2008-04-09
-PREHOOK: query: drop table srcpart_orc
+PREHOOK: query: create table srcpart_parquet (key int, value string) partitioned by (ds string, hr int) stored as parquet
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcpart_parquet
+POSTHOOK: query: create table srcpart_parquet (key int, value string) partitioned by (ds string, hr int) stored as parquet
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcpart_parquet
+PREHOOK: query: insert into table srcpart_parquet partition (ds, hr) select key, value, ds, hr from srcpart
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+PREHOOK: Output: default@srcpart_parquet
+POSTHOOK: query: insert into table srcpart_parquet partition (ds, hr) select key, value, ds, hr from srcpart
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@srcpart_parquet@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_parquet@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_parquet@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_parquet@ds=2008-04-09/hr=12
+POSTHOOK: Lineage: srcpart_parquet PARTITION(ds=2008-04-08,hr=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_parquet PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_parquet PARTITION(ds=2008-04-08,hr=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_parquet PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_parquet PARTITION(ds=2008-04-09,hr=11).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_parquet PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_parquet PARTITION(ds=2008-04-09,hr=12).key EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_parquet PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: EXPLAIN select count(*) from srcpart_parquet join srcpart_date_hour on (srcpart_parquet.ds = srcpart_date_hour.ds and srcpart_parquet.hr = srcpart_date_hour.hr) where srcpart_date_hour.hour = 11 and (srcpart_date_hour.`date` = '2008-04-08' or srcpart_date_hour.`date` = '2008-04-09')
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN select count(*) from srcpart_parquet join srcpart_date_hour on (srcpart_parquet.ds = srcpart_date_hour.ds and srcpart_parquet.hr = srcpart_date_hour.hr) where srcpart_date_hour.hour = 11 and (srcpart_date_hour.`date` = '2008-04-08' or srcpart_date_hour.`date` = '2008-04-09')
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-2
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_date_hour
+ filterExpr: (((date = '2008-04-08') or (date = '2008-04-09')) and (UDFToDouble(hour) = 11.0) and ds is not null and hr is not null) (type: boolean)
+ Statistics: Num rows: 4 Data size: 108 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((date = '2008-04-08') or (date = '2008-04-09')) and (UDFToDouble(hour) = 11.0) and ds is not null and hr is not null) (type: boolean)
+ Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ds (type: string), hr (type: string)
+ outputColumnNames: _col0, _col2
+ Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string), UDFToDouble(_col1) (type: double)
+ 1 _col0 (type: string), UDFToDouble(_col2) (type: double)
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE
+ Spark Partition Pruning Sink Operator
+ Target column: ds (string)
+ partition key expr: ds
+ Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE
+ target work: Map 1
+ Select Operator
+ expressions: UDFToDouble(_col2) (type: double)
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: double)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE
+ Spark Partition Pruning Sink Operator
+ Target column: hr (int)
+ partition key expr: UDFToDouble(hr)
+ Statistics: Num rows: 2 Data size: 54 Basic stats: COMPLETE Column stats: NONE
+ target work: Map 1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 2 <- Map 1 (GROUP, 1)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_parquet
+ Statistics: Num rows: 2000 Data size: 4000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ds (type: string), hr (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2000 Data size: 4000 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string), UDFToDouble(_col1) (type: double)
+ 1 _col0 (type: string), UDFToDouble(_col2) (type: double)
+ input vertices:
+ 1 Map 3
+ Statistics: Num rows: 2200 Data size: 4400 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Local Work:
+ Map Reduce Local Work
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from srcpart_parquet join srcpart_date_hour on (srcpart_parquet.ds = srcpart_date_hour.ds and srcpart_parquet.hr = srcpart_date_hour.hr) where srcpart_date_hour.hour = 11 and (srcpart_date_hour.`date` = '2008-04-08' or srcpart_date_hour.`date` = '2008-04-09')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_date_hour
+PREHOOK: Input: default@srcpart_parquet
+PREHOOK: Input: default@srcpart_parquet@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_parquet@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_parquet@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_parquet@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from srcpart_parquet join srcpart_date_hour on (srcpart_parquet.ds = srcpart_date_hour.ds and srcpart_parquet.hr = srcpart_date_hour.hr) where srcpart_date_hour.hour = 11 and (srcpart_date_hour.`date` = '2008-04-08' or srcpart_date_hour.`date` = '2008-04-09')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_date_hour
+POSTHOOK: Input: default@srcpart_parquet
+POSTHOOK: Input: default@srcpart_parquet@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_parquet@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_parquet@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_parquet@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+1000
+PREHOOK: query: select count(*) from srcpart where (ds = '2008-04-08' or ds = '2008-04-09') and hr = 11
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from srcpart where (ds = '2008-04-08' or ds = '2008-04-09') and hr = 11
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart
+#### A masked pattern was here ####
+1000
+PREHOOK: query: drop table srcpart_parquet
PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table srcpart_orc
+PREHOOK: Input: default@srcpart_parquet
+PREHOOK: Output: default@srcpart_parquet
+POSTHOOK: query: drop table srcpart_parquet
POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@srcpart_parquet
+POSTHOOK: Output: default@srcpart_parquet
PREHOOK: query: drop table srcpart_date
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@srcpart_date
[17/50] [abbrv] hive git commit: HIVE-17411 : LLAP IO may incorrectly
release a refcount in some rare cases (Sergey Shelukhin,
reviewed by Prasanth Jayachandran)
Posted by we...@apache.org.
HIVE-17411 : LLAP IO may incorrectly release a refcount in some rare cases (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/77c28af6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/77c28af6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/77c28af6
Branch: refs/heads/hive-14535
Commit: 77c28af605f6c3f3866ca9faf5f70a46ba41e37a
Parents: b67d52c
Author: sergey <se...@apache.org>
Authored: Wed Aug 30 10:33:46 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Wed Aug 30 10:33:46 2017 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/77c28af6/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
index 3f5f99c..690cce7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
@@ -1211,8 +1211,11 @@ class EncodedReaderImpl implements EncodedReader {
// as long as they are still in the same stream and are not already released.
DiskRangeList prev = cc.prev;
while (true) {
- if ((prev == null) || (prev.getEnd() <= streamStartOffset)
- || !(prev instanceof CacheChunk)) break;
+ // Do not release beyond current stream (we don't know which RGs that buffer is for).
+ if ((prev == null) || (prev.getEnd() <= streamStartOffset)) break;
+ // Only release cache chunks; do not release ProcCacheChunks - they may not yet have data.
+ if (prev.getClass() != CacheChunk.class) break;
+
CacheChunk prevCc = (CacheChunk)prev;
if (prevCc.buffer == null) break;
try {
[40/50] [abbrv] hive git commit: HIVE-14671 : merge master into
hive-14535 (Wei Zheng)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
----------------------------------------------------------------------
diff --cc standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
index 0000000,7570895..c21dfb0
mode 000000,100644..100644
--- a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
+++ b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
@@@ -1,0 -1,15160 +1,15163 @@@
+ #
+ # Autogenerated by Thrift Compiler (0.9.3)
+ #
+ # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ #
+ # options string: py
+ #
+
+ from thrift.Thrift import TType, TMessageType, TException, TApplicationException
+ import fb303.ttypes
+
+
+ from thrift.transport import TTransport
+ from thrift.protocol import TBinaryProtocol, TProtocol
+ try:
+ from thrift.protocol import fastbinary
+ except:
+ fastbinary = None
+
+
+ class HiveObjectType:
+ GLOBAL = 1
+ DATABASE = 2
+ TABLE = 3
+ PARTITION = 4
+ COLUMN = 5
+
+ _VALUES_TO_NAMES = {
+ 1: "GLOBAL",
+ 2: "DATABASE",
+ 3: "TABLE",
+ 4: "PARTITION",
+ 5: "COLUMN",
+ }
+
+ _NAMES_TO_VALUES = {
+ "GLOBAL": 1,
+ "DATABASE": 2,
+ "TABLE": 3,
+ "PARTITION": 4,
+ "COLUMN": 5,
+ }
+
+ class PrincipalType:
+ USER = 1
+ ROLE = 2
+ GROUP = 3
+
+ _VALUES_TO_NAMES = {
+ 1: "USER",
+ 2: "ROLE",
+ 3: "GROUP",
+ }
+
+ _NAMES_TO_VALUES = {
+ "USER": 1,
+ "ROLE": 2,
+ "GROUP": 3,
+ }
+
+ class PartitionEventType:
+ LOAD_DONE = 1
+
+ _VALUES_TO_NAMES = {
+ 1: "LOAD_DONE",
+ }
+
+ _NAMES_TO_VALUES = {
+ "LOAD_DONE": 1,
+ }
+
+ class TxnState:
+ COMMITTED = 1
+ ABORTED = 2
+ OPEN = 3
+
+ _VALUES_TO_NAMES = {
+ 1: "COMMITTED",
+ 2: "ABORTED",
+ 3: "OPEN",
+ }
+
+ _NAMES_TO_VALUES = {
+ "COMMITTED": 1,
+ "ABORTED": 2,
+ "OPEN": 3,
+ }
+
+ class LockLevel:
+ DB = 1
+ TABLE = 2
+ PARTITION = 3
+
+ _VALUES_TO_NAMES = {
+ 1: "DB",
+ 2: "TABLE",
+ 3: "PARTITION",
+ }
+
+ _NAMES_TO_VALUES = {
+ "DB": 1,
+ "TABLE": 2,
+ "PARTITION": 3,
+ }
+
+ class LockState:
+ ACQUIRED = 1
+ WAITING = 2
+ ABORT = 3
+ NOT_ACQUIRED = 4
+
+ _VALUES_TO_NAMES = {
+ 1: "ACQUIRED",
+ 2: "WAITING",
+ 3: "ABORT",
+ 4: "NOT_ACQUIRED",
+ }
+
+ _NAMES_TO_VALUES = {
+ "ACQUIRED": 1,
+ "WAITING": 2,
+ "ABORT": 3,
+ "NOT_ACQUIRED": 4,
+ }
+
+ class LockType:
+ SHARED_READ = 1
+ SHARED_WRITE = 2
+ EXCLUSIVE = 3
+
+ _VALUES_TO_NAMES = {
+ 1: "SHARED_READ",
+ 2: "SHARED_WRITE",
+ 3: "EXCLUSIVE",
+ }
+
+ _NAMES_TO_VALUES = {
+ "SHARED_READ": 1,
+ "SHARED_WRITE": 2,
+ "EXCLUSIVE": 3,
+ }
+
+ class CompactionType:
+ MINOR = 1
+ MAJOR = 2
+
+ _VALUES_TO_NAMES = {
+ 1: "MINOR",
+ 2: "MAJOR",
+ }
+
+ _NAMES_TO_VALUES = {
+ "MINOR": 1,
+ "MAJOR": 2,
+ }
+
+ class GrantRevokeType:
+ GRANT = 1
+ REVOKE = 2
+
+ _VALUES_TO_NAMES = {
+ 1: "GRANT",
+ 2: "REVOKE",
+ }
+
+ _NAMES_TO_VALUES = {
+ "GRANT": 1,
+ "REVOKE": 2,
+ }
+
+ class DataOperationType:
+ SELECT = 1
+ INSERT = 2
+ UPDATE = 3
+ DELETE = 4
+ UNSET = 5
+ NO_TXN = 6
+
+ _VALUES_TO_NAMES = {
+ 1: "SELECT",
+ 2: "INSERT",
+ 3: "UPDATE",
+ 4: "DELETE",
+ 5: "UNSET",
+ 6: "NO_TXN",
+ }
+
+ _NAMES_TO_VALUES = {
+ "SELECT": 1,
+ "INSERT": 2,
+ "UPDATE": 3,
+ "DELETE": 4,
+ "UNSET": 5,
+ "NO_TXN": 6,
+ }
+
+ class EventRequestType:
+ INSERT = 1
+ UPDATE = 2
+ DELETE = 3
+
+ _VALUES_TO_NAMES = {
+ 1: "INSERT",
+ 2: "UPDATE",
+ 3: "DELETE",
+ }
+
+ _NAMES_TO_VALUES = {
+ "INSERT": 1,
+ "UPDATE": 2,
+ "DELETE": 3,
+ }
+
+ class FunctionType:
+ JAVA = 1
+
+ _VALUES_TO_NAMES = {
+ 1: "JAVA",
+ }
+
+ _NAMES_TO_VALUES = {
+ "JAVA": 1,
+ }
+
+ class ResourceType:
+ JAR = 1
+ FILE = 2
+ ARCHIVE = 3
+
+ _VALUES_TO_NAMES = {
+ 1: "JAR",
+ 2: "FILE",
+ 3: "ARCHIVE",
+ }
+
+ _NAMES_TO_VALUES = {
+ "JAR": 1,
+ "FILE": 2,
+ "ARCHIVE": 3,
+ }
+
+ class FileMetadataExprType:
+ ORC_SARG = 1
+
+ _VALUES_TO_NAMES = {
+ 1: "ORC_SARG",
+ }
+
+ _NAMES_TO_VALUES = {
+ "ORC_SARG": 1,
+ }
+
+ class ClientCapability:
+ TEST_CAPABILITY = 1
++ INSERT_ONLY_TABLES = 2
+
+ _VALUES_TO_NAMES = {
+ 1: "TEST_CAPABILITY",
++ 2: "INSERT_ONLY_TABLES",
+ }
+
+ _NAMES_TO_VALUES = {
+ "TEST_CAPABILITY": 1,
++ "INSERT_ONLY_TABLES": 2,
+ }
+
+
+ class Version:
+ """
+ Attributes:
+ - version
+ - comments
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.STRING, 'version', None, None, ), # 1
+ (2, TType.STRING, 'comments', None, None, ), # 2
+ )
+
+ def __init__(self, version=None, comments=None,):
+ self.version = version
+ self.comments = comments
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.STRING:
+ self.version = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 2:
+ if ftype == TType.STRING:
+ self.comments = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('Version')
+ if self.version is not None:
+ oprot.writeFieldBegin('version', TType.STRING, 1)
+ oprot.writeString(self.version)
+ oprot.writeFieldEnd()
+ if self.comments is not None:
+ oprot.writeFieldBegin('comments', TType.STRING, 2)
+ oprot.writeString(self.comments)
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.version)
+ value = (value * 31) ^ hash(self.comments)
+ return value
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ class FieldSchema:
+ """
+ Attributes:
+ - name
+ - type
+ - comment
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.STRING, 'name', None, None, ), # 1
+ (2, TType.STRING, 'type', None, None, ), # 2
+ (3, TType.STRING, 'comment', None, None, ), # 3
+ )
+
+ def __init__(self, name=None, type=None, comment=None,):
+ self.name = name
+ self.type = type
+ self.comment = comment
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.STRING:
+ self.name = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 2:
+ if ftype == TType.STRING:
+ self.type = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 3:
+ if ftype == TType.STRING:
+ self.comment = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('FieldSchema')
+ if self.name is not None:
+ oprot.writeFieldBegin('name', TType.STRING, 1)
+ oprot.writeString(self.name)
+ oprot.writeFieldEnd()
+ if self.type is not None:
+ oprot.writeFieldBegin('type', TType.STRING, 2)
+ oprot.writeString(self.type)
+ oprot.writeFieldEnd()
+ if self.comment is not None:
+ oprot.writeFieldBegin('comment', TType.STRING, 3)
+ oprot.writeString(self.comment)
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.name)
+ value = (value * 31) ^ hash(self.type)
+ value = (value * 31) ^ hash(self.comment)
+ return value
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ class SQLPrimaryKey:
+ """
+ Attributes:
+ - table_db
+ - table_name
+ - column_name
+ - key_seq
+ - pk_name
+ - enable_cstr
+ - validate_cstr
+ - rely_cstr
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.STRING, 'table_db', None, None, ), # 1
+ (2, TType.STRING, 'table_name', None, None, ), # 2
+ (3, TType.STRING, 'column_name', None, None, ), # 3
+ (4, TType.I32, 'key_seq', None, None, ), # 4
+ (5, TType.STRING, 'pk_name', None, None, ), # 5
+ (6, TType.BOOL, 'enable_cstr', None, None, ), # 6
+ (7, TType.BOOL, 'validate_cstr', None, None, ), # 7
+ (8, TType.BOOL, 'rely_cstr', None, None, ), # 8
+ )
+
+ def __init__(self, table_db=None, table_name=None, column_name=None, key_seq=None, pk_name=None, enable_cstr=None, validate_cstr=None, rely_cstr=None,):
+ self.table_db = table_db
+ self.table_name = table_name
+ self.column_name = column_name
+ self.key_seq = key_seq
+ self.pk_name = pk_name
+ self.enable_cstr = enable_cstr
+ self.validate_cstr = validate_cstr
+ self.rely_cstr = rely_cstr
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.STRING:
+ self.table_db = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 2:
+ if ftype == TType.STRING:
+ self.table_name = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 3:
+ if ftype == TType.STRING:
+ self.column_name = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 4:
+ if ftype == TType.I32:
+ self.key_seq = iprot.readI32()
+ else:
+ iprot.skip(ftype)
+ elif fid == 5:
+ if ftype == TType.STRING:
+ self.pk_name = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 6:
+ if ftype == TType.BOOL:
+ self.enable_cstr = iprot.readBool()
+ else:
+ iprot.skip(ftype)
+ elif fid == 7:
+ if ftype == TType.BOOL:
+ self.validate_cstr = iprot.readBool()
+ else:
+ iprot.skip(ftype)
+ elif fid == 8:
+ if ftype == TType.BOOL:
+ self.rely_cstr = iprot.readBool()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('SQLPrimaryKey')
+ if self.table_db is not None:
+ oprot.writeFieldBegin('table_db', TType.STRING, 1)
+ oprot.writeString(self.table_db)
+ oprot.writeFieldEnd()
+ if self.table_name is not None:
+ oprot.writeFieldBegin('table_name', TType.STRING, 2)
+ oprot.writeString(self.table_name)
+ oprot.writeFieldEnd()
+ if self.column_name is not None:
+ oprot.writeFieldBegin('column_name', TType.STRING, 3)
+ oprot.writeString(self.column_name)
+ oprot.writeFieldEnd()
+ if self.key_seq is not None:
+ oprot.writeFieldBegin('key_seq', TType.I32, 4)
+ oprot.writeI32(self.key_seq)
+ oprot.writeFieldEnd()
+ if self.pk_name is not None:
+ oprot.writeFieldBegin('pk_name', TType.STRING, 5)
+ oprot.writeString(self.pk_name)
+ oprot.writeFieldEnd()
+ if self.enable_cstr is not None:
+ oprot.writeFieldBegin('enable_cstr', TType.BOOL, 6)
+ oprot.writeBool(self.enable_cstr)
+ oprot.writeFieldEnd()
+ if self.validate_cstr is not None:
+ oprot.writeFieldBegin('validate_cstr', TType.BOOL, 7)
+ oprot.writeBool(self.validate_cstr)
+ oprot.writeFieldEnd()
+ if self.rely_cstr is not None:
+ oprot.writeFieldBegin('rely_cstr', TType.BOOL, 8)
+ oprot.writeBool(self.rely_cstr)
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.table_db)
+ value = (value * 31) ^ hash(self.table_name)
+ value = (value * 31) ^ hash(self.column_name)
+ value = (value * 31) ^ hash(self.key_seq)
+ value = (value * 31) ^ hash(self.pk_name)
+ value = (value * 31) ^ hash(self.enable_cstr)
+ value = (value * 31) ^ hash(self.validate_cstr)
+ value = (value * 31) ^ hash(self.rely_cstr)
+ return value
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ class SQLForeignKey:
+ """
+ Attributes:
+ - pktable_db
+ - pktable_name
+ - pkcolumn_name
+ - fktable_db
+ - fktable_name
+ - fkcolumn_name
+ - key_seq
+ - update_rule
+ - delete_rule
+ - fk_name
+ - pk_name
+ - enable_cstr
+ - validate_cstr
+ - rely_cstr
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.STRING, 'pktable_db', None, None, ), # 1
+ (2, TType.STRING, 'pktable_name', None, None, ), # 2
+ (3, TType.STRING, 'pkcolumn_name', None, None, ), # 3
+ (4, TType.STRING, 'fktable_db', None, None, ), # 4
+ (5, TType.STRING, 'fktable_name', None, None, ), # 5
+ (6, TType.STRING, 'fkcolumn_name', None, None, ), # 6
+ (7, TType.I32, 'key_seq', None, None, ), # 7
+ (8, TType.I32, 'update_rule', None, None, ), # 8
+ (9, TType.I32, 'delete_rule', None, None, ), # 9
+ (10, TType.STRING, 'fk_name', None, None, ), # 10
+ (11, TType.STRING, 'pk_name', None, None, ), # 11
+ (12, TType.BOOL, 'enable_cstr', None, None, ), # 12
+ (13, TType.BOOL, 'validate_cstr', None, None, ), # 13
+ (14, TType.BOOL, 'rely_cstr', None, None, ), # 14
+ )
+
+ def __init__(self, pktable_db=None, pktable_name=None, pkcolumn_name=None, fktable_db=None, fktable_name=None, fkcolumn_name=None, key_seq=None, update_rule=None, delete_rule=None, fk_name=None, pk_name=None, enable_cstr=None, validate_cstr=None, rely_cstr=None,):
+ self.pktable_db = pktable_db
+ self.pktable_name = pktable_name
+ self.pkcolumn_name = pkcolumn_name
+ self.fktable_db = fktable_db
+ self.fktable_name = fktable_name
+ self.fkcolumn_name = fkcolumn_name
+ self.key_seq = key_seq
+ self.update_rule = update_rule
+ self.delete_rule = delete_rule
+ self.fk_name = fk_name
+ self.pk_name = pk_name
+ self.enable_cstr = enable_cstr
+ self.validate_cstr = validate_cstr
+ self.rely_cstr = rely_cstr
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.STRING:
+ self.pktable_db = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 2:
+ if ftype == TType.STRING:
+ self.pktable_name = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 3:
+ if ftype == TType.STRING:
+ self.pkcolumn_name = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 4:
+ if ftype == TType.STRING:
+ self.fktable_db = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 5:
+ if ftype == TType.STRING:
+ self.fktable_name = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 6:
+ if ftype == TType.STRING:
+ self.fkcolumn_name = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 7:
+ if ftype == TType.I32:
+ self.key_seq = iprot.readI32()
+ else:
+ iprot.skip(ftype)
+ elif fid == 8:
+ if ftype == TType.I32:
+ self.update_rule = iprot.readI32()
+ else:
+ iprot.skip(ftype)
+ elif fid == 9:
+ if ftype == TType.I32:
+ self.delete_rule = iprot.readI32()
+ else:
+ iprot.skip(ftype)
+ elif fid == 10:
+ if ftype == TType.STRING:
+ self.fk_name = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 11:
+ if ftype == TType.STRING:
+ self.pk_name = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 12:
+ if ftype == TType.BOOL:
+ self.enable_cstr = iprot.readBool()
+ else:
+ iprot.skip(ftype)
+ elif fid == 13:
+ if ftype == TType.BOOL:
+ self.validate_cstr = iprot.readBool()
+ else:
+ iprot.skip(ftype)
+ elif fid == 14:
+ if ftype == TType.BOOL:
+ self.rely_cstr = iprot.readBool()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('SQLForeignKey')
+ if self.pktable_db is not None:
+ oprot.writeFieldBegin('pktable_db', TType.STRING, 1)
+ oprot.writeString(self.pktable_db)
+ oprot.writeFieldEnd()
+ if self.pktable_name is not None:
+ oprot.writeFieldBegin('pktable_name', TType.STRING, 2)
+ oprot.writeString(self.pktable_name)
+ oprot.writeFieldEnd()
+ if self.pkcolumn_name is not None:
+ oprot.writeFieldBegin('pkcolumn_name', TType.STRING, 3)
+ oprot.writeString(self.pkcolumn_name)
+ oprot.writeFieldEnd()
+ if self.fktable_db is not None:
+ oprot.writeFieldBegin('fktable_db', TType.STRING, 4)
+ oprot.writeString(self.fktable_db)
+ oprot.writeFieldEnd()
+ if self.fktable_name is not None:
+ oprot.writeFieldBegin('fktable_name', TType.STRING, 5)
+ oprot.writeString(self.fktable_name)
+ oprot.writeFieldEnd()
+ if self.fkcolumn_name is not None:
+ oprot.writeFieldBegin('fkcolumn_name', TType.STRING, 6)
+ oprot.writeString(self.fkcolumn_name)
+ oprot.writeFieldEnd()
+ if self.key_seq is not None:
+ oprot.writeFieldBegin('key_seq', TType.I32, 7)
+ oprot.writeI32(self.key_seq)
+ oprot.writeFieldEnd()
+ if self.update_rule is not None:
+ oprot.writeFieldBegin('update_rule', TType.I32, 8)
+ oprot.writeI32(self.update_rule)
+ oprot.writeFieldEnd()
+ if self.delete_rule is not None:
+ oprot.writeFieldBegin('delete_rule', TType.I32, 9)
+ oprot.writeI32(self.delete_rule)
+ oprot.writeFieldEnd()
+ if self.fk_name is not None:
+ oprot.writeFieldBegin('fk_name', TType.STRING, 10)
+ oprot.writeString(self.fk_name)
+ oprot.writeFieldEnd()
+ if self.pk_name is not None:
+ oprot.writeFieldBegin('pk_name', TType.STRING, 11)
+ oprot.writeString(self.pk_name)
+ oprot.writeFieldEnd()
+ if self.enable_cstr is not None:
+ oprot.writeFieldBegin('enable_cstr', TType.BOOL, 12)
+ oprot.writeBool(self.enable_cstr)
+ oprot.writeFieldEnd()
+ if self.validate_cstr is not None:
+ oprot.writeFieldBegin('validate_cstr', TType.BOOL, 13)
+ oprot.writeBool(self.validate_cstr)
+ oprot.writeFieldEnd()
+ if self.rely_cstr is not None:
+ oprot.writeFieldBegin('rely_cstr', TType.BOOL, 14)
+ oprot.writeBool(self.rely_cstr)
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.pktable_db)
+ value = (value * 31) ^ hash(self.pktable_name)
+ value = (value * 31) ^ hash(self.pkcolumn_name)
+ value = (value * 31) ^ hash(self.fktable_db)
+ value = (value * 31) ^ hash(self.fktable_name)
+ value = (value * 31) ^ hash(self.fkcolumn_name)
+ value = (value * 31) ^ hash(self.key_seq)
+ value = (value * 31) ^ hash(self.update_rule)
+ value = (value * 31) ^ hash(self.delete_rule)
+ value = (value * 31) ^ hash(self.fk_name)
+ value = (value * 31) ^ hash(self.pk_name)
+ value = (value * 31) ^ hash(self.enable_cstr)
+ value = (value * 31) ^ hash(self.validate_cstr)
+ value = (value * 31) ^ hash(self.rely_cstr)
+ return value
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ class SQLUniqueConstraint:
+ """
+ Attributes:
+ - table_db
+ - table_name
+ - column_name
+ - key_seq
+ - uk_name
+ - enable_cstr
+ - validate_cstr
+ - rely_cstr
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.STRING, 'table_db', None, None, ), # 1
+ (2, TType.STRING, 'table_name', None, None, ), # 2
+ (3, TType.STRING, 'column_name', None, None, ), # 3
+ (4, TType.I32, 'key_seq', None, None, ), # 4
+ (5, TType.STRING, 'uk_name', None, None, ), # 5
+ (6, TType.BOOL, 'enable_cstr', None, None, ), # 6
+ (7, TType.BOOL, 'validate_cstr', None, None, ), # 7
+ (8, TType.BOOL, 'rely_cstr', None, None, ), # 8
+ )
+
+ def __init__(self, table_db=None, table_name=None, column_name=None, key_seq=None, uk_name=None, enable_cstr=None, validate_cstr=None, rely_cstr=None,):
+ self.table_db = table_db
+ self.table_name = table_name
+ self.column_name = column_name
+ self.key_seq = key_seq
+ self.uk_name = uk_name
+ self.enable_cstr = enable_cstr
+ self.validate_cstr = validate_cstr
+ self.rely_cstr = rely_cstr
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.STRING:
+ self.table_db = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 2:
+ if ftype == TType.STRING:
+ self.table_name = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 3:
+ if ftype == TType.STRING:
+ self.column_name = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 4:
+ if ftype == TType.I32:
+ self.key_seq = iprot.readI32()
+ else:
+ iprot.skip(ftype)
+ elif fid == 5:
+ if ftype == TType.STRING:
+ self.uk_name = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 6:
+ if ftype == TType.BOOL:
+ self.enable_cstr = iprot.readBool()
+ else:
+ iprot.skip(ftype)
+ elif fid == 7:
+ if ftype == TType.BOOL:
+ self.validate_cstr = iprot.readBool()
+ else:
+ iprot.skip(ftype)
+ elif fid == 8:
+ if ftype == TType.BOOL:
+ self.rely_cstr = iprot.readBool()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('SQLUniqueConstraint')
+ if self.table_db is not None:
+ oprot.writeFieldBegin('table_db', TType.STRING, 1)
+ oprot.writeString(self.table_db)
+ oprot.writeFieldEnd()
+ if self.table_name is not None:
+ oprot.writeFieldBegin('table_name', TType.STRING, 2)
+ oprot.writeString(self.table_name)
+ oprot.writeFieldEnd()
+ if self.column_name is not None:
+ oprot.writeFieldBegin('column_name', TType.STRING, 3)
+ oprot.writeString(self.column_name)
+ oprot.writeFieldEnd()
+ if self.key_seq is not None:
+ oprot.writeFieldBegin('key_seq', TType.I32, 4)
+ oprot.writeI32(self.key_seq)
+ oprot.writeFieldEnd()
+ if self.uk_name is not None:
+ oprot.writeFieldBegin('uk_name', TType.STRING, 5)
+ oprot.writeString(self.uk_name)
+ oprot.writeFieldEnd()
+ if self.enable_cstr is not None:
+ oprot.writeFieldBegin('enable_cstr', TType.BOOL, 6)
+ oprot.writeBool(self.enable_cstr)
+ oprot.writeFieldEnd()
+ if self.validate_cstr is not None:
+ oprot.writeFieldBegin('validate_cstr', TType.BOOL, 7)
+ oprot.writeBool(self.validate_cstr)
+ oprot.writeFieldEnd()
+ if self.rely_cstr is not None:
+ oprot.writeFieldBegin('rely_cstr', TType.BOOL, 8)
+ oprot.writeBool(self.rely_cstr)
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.table_db)
+ value = (value * 31) ^ hash(self.table_name)
+ value = (value * 31) ^ hash(self.column_name)
+ value = (value * 31) ^ hash(self.key_seq)
+ value = (value * 31) ^ hash(self.uk_name)
+ value = (value * 31) ^ hash(self.enable_cstr)
+ value = (value * 31) ^ hash(self.validate_cstr)
+ value = (value * 31) ^ hash(self.rely_cstr)
+ return value
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ class SQLNotNullConstraint:
+ """
+ Attributes:
+ - table_db
+ - table_name
+ - column_name
+ - nn_name
+ - enable_cstr
+ - validate_cstr
+ - rely_cstr
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.STRING, 'table_db', None, None, ), # 1
+ (2, TType.STRING, 'table_name', None, None, ), # 2
+ (3, TType.STRING, 'column_name', None, None, ), # 3
+ (4, TType.STRING, 'nn_name', None, None, ), # 4
+ (5, TType.BOOL, 'enable_cstr', None, None, ), # 5
+ (6, TType.BOOL, 'validate_cstr', None, None, ), # 6
+ (7, TType.BOOL, 'rely_cstr', None, None, ), # 7
+ )
+
+ def __init__(self, table_db=None, table_name=None, column_name=None, nn_name=None, enable_cstr=None, validate_cstr=None, rely_cstr=None,):
+ self.table_db = table_db
+ self.table_name = table_name
+ self.column_name = column_name
+ self.nn_name = nn_name
+ self.enable_cstr = enable_cstr
+ self.validate_cstr = validate_cstr
+ self.rely_cstr = rely_cstr
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.STRING:
+ self.table_db = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 2:
+ if ftype == TType.STRING:
+ self.table_name = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 3:
+ if ftype == TType.STRING:
+ self.column_name = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 4:
+ if ftype == TType.STRING:
+ self.nn_name = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 5:
+ if ftype == TType.BOOL:
+ self.enable_cstr = iprot.readBool()
+ else:
+ iprot.skip(ftype)
+ elif fid == 6:
+ if ftype == TType.BOOL:
+ self.validate_cstr = iprot.readBool()
+ else:
+ iprot.skip(ftype)
+ elif fid == 7:
+ if ftype == TType.BOOL:
+ self.rely_cstr = iprot.readBool()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('SQLNotNullConstraint')
+ if self.table_db is not None:
+ oprot.writeFieldBegin('table_db', TType.STRING, 1)
+ oprot.writeString(self.table_db)
+ oprot.writeFieldEnd()
+ if self.table_name is not None:
+ oprot.writeFieldBegin('table_name', TType.STRING, 2)
+ oprot.writeString(self.table_name)
+ oprot.writeFieldEnd()
+ if self.column_name is not None:
+ oprot.writeFieldBegin('column_name', TType.STRING, 3)
+ oprot.writeString(self.column_name)
+ oprot.writeFieldEnd()
+ if self.nn_name is not None:
+ oprot.writeFieldBegin('nn_name', TType.STRING, 4)
+ oprot.writeString(self.nn_name)
+ oprot.writeFieldEnd()
+ if self.enable_cstr is not None:
+ oprot.writeFieldBegin('enable_cstr', TType.BOOL, 5)
+ oprot.writeBool(self.enable_cstr)
+ oprot.writeFieldEnd()
+ if self.validate_cstr is not None:
+ oprot.writeFieldBegin('validate_cstr', TType.BOOL, 6)
+ oprot.writeBool(self.validate_cstr)
+ oprot.writeFieldEnd()
+ if self.rely_cstr is not None:
+ oprot.writeFieldBegin('rely_cstr', TType.BOOL, 7)
+ oprot.writeBool(self.rely_cstr)
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.table_db)
+ value = (value * 31) ^ hash(self.table_name)
+ value = (value * 31) ^ hash(self.column_name)
+ value = (value * 31) ^ hash(self.nn_name)
+ value = (value * 31) ^ hash(self.enable_cstr)
+ value = (value * 31) ^ hash(self.validate_cstr)
+ value = (value * 31) ^ hash(self.rely_cstr)
+ return value
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ class Type:
+ """
+ Attributes:
+ - name
+ - type1
+ - type2
+ - fields
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.STRING, 'name', None, None, ), # 1
+ (2, TType.STRING, 'type1', None, None, ), # 2
+ (3, TType.STRING, 'type2', None, None, ), # 3
+ (4, TType.LIST, 'fields', (TType.STRUCT,(FieldSchema, FieldSchema.thrift_spec)), None, ), # 4
+ )
+
+ def __init__(self, name=None, type1=None, type2=None, fields=None,):
+ self.name = name
+ self.type1 = type1
+ self.type2 = type2
+ self.fields = fields
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.STRING:
+ self.name = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 2:
+ if ftype == TType.STRING:
+ self.type1 = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 3:
+ if ftype == TType.STRING:
+ self.type2 = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 4:
+ if ftype == TType.LIST:
+ self.fields = []
+ (_etype3, _size0) = iprot.readListBegin()
+ for _i4 in xrange(_size0):
+ _elem5 = FieldSchema()
+ _elem5.read(iprot)
+ self.fields.append(_elem5)
+ iprot.readListEnd()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('Type')
+ if self.name is not None:
+ oprot.writeFieldBegin('name', TType.STRING, 1)
+ oprot.writeString(self.name)
+ oprot.writeFieldEnd()
+ if self.type1 is not None:
+ oprot.writeFieldBegin('type1', TType.STRING, 2)
+ oprot.writeString(self.type1)
+ oprot.writeFieldEnd()
+ if self.type2 is not None:
+ oprot.writeFieldBegin('type2', TType.STRING, 3)
+ oprot.writeString(self.type2)
+ oprot.writeFieldEnd()
+ if self.fields is not None:
+ oprot.writeFieldBegin('fields', TType.LIST, 4)
+ oprot.writeListBegin(TType.STRUCT, len(self.fields))
+ for iter6 in self.fields:
+ iter6.write(oprot)
+ oprot.writeListEnd()
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.name)
+ value = (value * 31) ^ hash(self.type1)
+ value = (value * 31) ^ hash(self.type2)
+ value = (value * 31) ^ hash(self.fields)
+ return value
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ class HiveObjectRef:
+ """
+ Attributes:
+ - objectType
+ - dbName
+ - objectName
+ - partValues
+ - columnName
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.I32, 'objectType', None, None, ), # 1
+ (2, TType.STRING, 'dbName', None, None, ), # 2
+ (3, TType.STRING, 'objectName', None, None, ), # 3
+ (4, TType.LIST, 'partValues', (TType.STRING,None), None, ), # 4
+ (5, TType.STRING, 'columnName', None, None, ), # 5
+ )
+
+ def __init__(self, objectType=None, dbName=None, objectName=None, partValues=None, columnName=None,):
+ self.objectType = objectType
+ self.dbName = dbName
+ self.objectName = objectName
+ self.partValues = partValues
+ self.columnName = columnName
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.I32:
+ self.objectType = iprot.readI32()
+ else:
+ iprot.skip(ftype)
+ elif fid == 2:
+ if ftype == TType.STRING:
+ self.dbName = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 3:
+ if ftype == TType.STRING:
+ self.objectName = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 4:
+ if ftype == TType.LIST:
+ self.partValues = []
+ (_etype10, _size7) = iprot.readListBegin()
+ for _i11 in xrange(_size7):
+ _elem12 = iprot.readString()
+ self.partValues.append(_elem12)
+ iprot.readListEnd()
+ else:
+ iprot.skip(ftype)
+ elif fid == 5:
+ if ftype == TType.STRING:
+ self.columnName = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('HiveObjectRef')
+ if self.objectType is not None:
+ oprot.writeFieldBegin('objectType', TType.I32, 1)
+ oprot.writeI32(self.objectType)
+ oprot.writeFieldEnd()
+ if self.dbName is not None:
+ oprot.writeFieldBegin('dbName', TType.STRING, 2)
+ oprot.writeString(self.dbName)
+ oprot.writeFieldEnd()
+ if self.objectName is not None:
+ oprot.writeFieldBegin('objectName', TType.STRING, 3)
+ oprot.writeString(self.objectName)
+ oprot.writeFieldEnd()
+ if self.partValues is not None:
+ oprot.writeFieldBegin('partValues', TType.LIST, 4)
+ oprot.writeListBegin(TType.STRING, len(self.partValues))
+ for iter13 in self.partValues:
+ oprot.writeString(iter13)
+ oprot.writeListEnd()
+ oprot.writeFieldEnd()
+ if self.columnName is not None:
+ oprot.writeFieldBegin('columnName', TType.STRING, 5)
+ oprot.writeString(self.columnName)
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.objectType)
+ value = (value * 31) ^ hash(self.dbName)
+ value = (value * 31) ^ hash(self.objectName)
+ value = (value * 31) ^ hash(self.partValues)
+ value = (value * 31) ^ hash(self.columnName)
+ return value
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ class PrivilegeGrantInfo:
+ """
+ Attributes:
+ - privilege
+ - createTime
+ - grantor
+ - grantorType
+ - grantOption
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.STRING, 'privilege', None, None, ), # 1
+ (2, TType.I32, 'createTime', None, None, ), # 2
+ (3, TType.STRING, 'grantor', None, None, ), # 3
+ (4, TType.I32, 'grantorType', None, None, ), # 4
+ (5, TType.BOOL, 'grantOption', None, None, ), # 5
+ )
+
+ def __init__(self, privilege=None, createTime=None, grantor=None, grantorType=None, grantOption=None,):
+ self.privilege = privilege
+ self.createTime = createTime
+ self.grantor = grantor
+ self.grantorType = grantorType
+ self.grantOption = grantOption
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.STRING:
+ self.privilege = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 2:
+ if ftype == TType.I32:
+ self.createTime = iprot.readI32()
+ else:
+ iprot.skip(ftype)
+ elif fid == 3:
+ if ftype == TType.STRING:
+ self.grantor = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 4:
+ if ftype == TType.I32:
+ self.grantorType = iprot.readI32()
+ else:
+ iprot.skip(ftype)
+ elif fid == 5:
+ if ftype == TType.BOOL:
+ self.grantOption = iprot.readBool()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('PrivilegeGrantInfo')
+ if self.privilege is not None:
+ oprot.writeFieldBegin('privilege', TType.STRING, 1)
+ oprot.writeString(self.privilege)
+ oprot.writeFieldEnd()
+ if self.createTime is not None:
+ oprot.writeFieldBegin('createTime', TType.I32, 2)
+ oprot.writeI32(self.createTime)
+ oprot.writeFieldEnd()
+ if self.grantor is not None:
+ oprot.writeFieldBegin('grantor', TType.STRING, 3)
+ oprot.writeString(self.grantor)
+ oprot.writeFieldEnd()
+ if self.grantorType is not None:
+ oprot.writeFieldBegin('grantorType', TType.I32, 4)
+ oprot.writeI32(self.grantorType)
+ oprot.writeFieldEnd()
+ if self.grantOption is not None:
+ oprot.writeFieldBegin('grantOption', TType.BOOL, 5)
+ oprot.writeBool(self.grantOption)
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.privilege)
+ value = (value * 31) ^ hash(self.createTime)
+ value = (value * 31) ^ hash(self.grantor)
+ value = (value * 31) ^ hash(self.grantorType)
+ value = (value * 31) ^ hash(self.grantOption)
+ return value
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ class HiveObjectPrivilege:
+ """
+ Attributes:
+ - hiveObject
+ - principalName
+ - principalType
+ - grantInfo
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.STRUCT, 'hiveObject', (HiveObjectRef, HiveObjectRef.thrift_spec), None, ), # 1
+ (2, TType.STRING, 'principalName', None, None, ), # 2
+ (3, TType.I32, 'principalType', None, None, ), # 3
+ (4, TType.STRUCT, 'grantInfo', (PrivilegeGrantInfo, PrivilegeGrantInfo.thrift_spec), None, ), # 4
+ )
+
+ def __init__(self, hiveObject=None, principalName=None, principalType=None, grantInfo=None,):
+ self.hiveObject = hiveObject
+ self.principalName = principalName
+ self.principalType = principalType
+ self.grantInfo = grantInfo
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.STRUCT:
+ self.hiveObject = HiveObjectRef()
+ self.hiveObject.read(iprot)
+ else:
+ iprot.skip(ftype)
+ elif fid == 2:
+ if ftype == TType.STRING:
+ self.principalName = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 3:
+ if ftype == TType.I32:
+ self.principalType = iprot.readI32()
+ else:
+ iprot.skip(ftype)
+ elif fid == 4:
+ if ftype == TType.STRUCT:
+ self.grantInfo = PrivilegeGrantInfo()
+ self.grantInfo.read(iprot)
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('HiveObjectPrivilege')
+ if self.hiveObject is not None:
+ oprot.writeFieldBegin('hiveObject', TType.STRUCT, 1)
+ self.hiveObject.write(oprot)
+ oprot.writeFieldEnd()
+ if self.principalName is not None:
+ oprot.writeFieldBegin('principalName', TType.STRING, 2)
+ oprot.writeString(self.principalName)
+ oprot.writeFieldEnd()
+ if self.principalType is not None:
+ oprot.writeFieldBegin('principalType', TType.I32, 3)
+ oprot.writeI32(self.principalType)
+ oprot.writeFieldEnd()
+ if self.grantInfo is not None:
+ oprot.writeFieldBegin('grantInfo', TType.STRUCT, 4)
+ self.grantInfo.write(oprot)
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.hiveObject)
+ value = (value * 31) ^ hash(self.principalName)
+ value = (value * 31) ^ hash(self.principalType)
+ value = (value * 31) ^ hash(self.grantInfo)
+ return value
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ class PrivilegeBag:
+ """
+ Attributes:
+ - privileges
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.LIST, 'privileges', (TType.STRUCT,(HiveObjectPrivilege, HiveObjectPrivilege.thrift_spec)), None, ), # 1
+ )
+
+ def __init__(self, privileges=None,):
+ self.privileges = privileges
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.LIST:
+ self.privileges = []
+ (_etype17, _size14) = iprot.readListBegin()
+ for _i18 in xrange(_size14):
+ _elem19 = HiveObjectPrivilege()
+ _elem19.read(iprot)
+ self.privileges.append(_elem19)
+ iprot.readListEnd()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('PrivilegeBag')
+ if self.privileges is not None:
+ oprot.writeFieldBegin('privileges', TType.LIST, 1)
+ oprot.writeListBegin(TType.STRUCT, len(self.privileges))
+ for iter20 in self.privileges:
+ iter20.write(oprot)
+ oprot.writeListEnd()
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.privileges)
+ return value
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ class PrincipalPrivilegeSet:
+ """
+ Attributes:
+ - userPrivileges
+ - groupPrivileges
+ - rolePrivileges
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.MAP, 'userPrivileges', (TType.STRING,None,TType.LIST,(TType.STRUCT,(PrivilegeGrantInfo, PrivilegeGrantInfo.thrift_spec))), None, ), # 1
+ (2, TType.MAP, 'groupPrivileges', (TType.STRING,None,TType.LIST,(TType.STRUCT,(PrivilegeGrantInfo, PrivilegeGrantInfo.thrift_spec))), None, ), # 2
+ (3, TType.MAP, 'rolePrivileges', (TType.STRING,None,TType.LIST,(TType.STRUCT,(PrivilegeGrantInfo, PrivilegeGrantInfo.thrift_spec))), None, ), # 3
+ )
+
+ def __init__(self, userPrivileges=None, groupPrivileges=None, rolePrivileges=None,):
+ self.userPrivileges = userPrivileges
+ self.groupPrivileges = groupPrivileges
+ self.rolePrivileges = rolePrivileges
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.MAP:
+ self.userPrivileges = {}
+ (_ktype22, _vtype23, _size21 ) = iprot.readMapBegin()
+ for _i25 in xrange(_size21):
+ _key26 = iprot.readString()
+ _val27 = []
+ (_etype31, _size28) = iprot.readListBegin()
+ for _i32 in xrange(_size28):
+ _elem33 = PrivilegeGrantInfo()
+ _elem33.read(iprot)
+ _val27.append(_elem33)
+ iprot.readListEnd()
+ self.userPrivileges[_key26] = _val27
+ iprot.readMapEnd()
+ else:
+ iprot.skip(ftype)
+ elif fid == 2:
+ if ftype == TType.MAP:
+ self.groupPrivileges = {}
+ (_ktype35, _vtype36, _size34 ) = iprot.readMapBegin()
+ for _i38 in xrange(_size34):
+ _key39 = iprot.readString()
+ _val40 = []
+ (_etype44, _size41) = iprot.readListBegin()
+ for _i45 in xrange(_size41):
+ _elem46 = PrivilegeGrantInfo()
+ _elem46.read(iprot)
+ _val40.append(_elem46)
+ iprot.readListEnd()
+ self.groupPrivileges[_key39] = _val40
+ iprot.readMapEnd()
+ else:
+ iprot.skip(ftype)
+ elif fid == 3:
+ if ftype == TType.MAP:
+ self.rolePrivileges = {}
+ (_ktype48, _vtype49, _size47 ) = iprot.readMapBegin()
+ for _i51 in xrange(_size47):
+ _key52 = iprot.readString()
+ _val53 = []
+ (_etype57, _size54) = iprot.readListBegin()
+ for _i58 in xrange(_size54):
+ _elem59 = PrivilegeGrantInfo()
+ _elem59.read(iprot)
+ _val53.append(_elem59)
+ iprot.readListEnd()
+ self.rolePrivileges[_key52] = _val53
+ iprot.readMapEnd()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('PrincipalPrivilegeSet')
+ if self.userPrivileges is not None:
+ oprot.writeFieldBegin('userPrivileges', TType.MAP, 1)
+ oprot.writeMapBegin(TType.STRING, TType.LIST, len(self.userPrivileges))
+ for kiter60,viter61 in self.userPrivileges.items():
+ oprot.writeString(kiter60)
+ oprot.writeListBegin(TType.STRUCT, len(viter61))
+ for iter62 in viter61:
+ iter62.write(oprot)
+ oprot.writeListEnd()
+ oprot.writeMapEnd()
+ oprot.writeFieldEnd()
+ if self.groupPrivileges is not None:
+ oprot.writeFieldBegin('groupPrivileges', TType.MAP, 2)
+ oprot.writeMapBegin(TType.STRING, TType.LIST, len(self.groupPrivileges))
+ for kiter63,viter64 in self.groupPrivileges.items():
+ oprot.writeString(kiter63)
+ oprot.writeListBegin(TType.STRUCT, len(viter64))
+ for iter65 in viter64:
+ iter65.write(oprot)
+ oprot.writeListEnd()
+ oprot.writeMapEnd()
+ oprot.writeFieldEnd()
+ if self.rolePrivileges is not None:
+ oprot.writeFieldBegin('rolePrivileges', TType.MAP, 3)
+ oprot.writeMapBegin(TType.STRING, TType.LIST, len(self.rolePrivileges))
+ for kiter66,viter67 in self.rolePrivileges.items():
+ oprot.writeString(kiter66)
+ oprot.writeListBegin(TType.STRUCT, len(viter67))
+ for iter68 in viter67:
+ iter68.write(oprot)
+ oprot.writeListEnd()
+ oprot.writeMapEnd()
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.userPrivileges)
+ value = (value * 31) ^ hash(self.groupPrivileges)
+ value = (value * 31) ^ hash(self.rolePrivileges)
+ return value
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ class GrantRevokePrivilegeRequest:
+ """
+ Attributes:
+ - requestType
+ - privileges
+ - revokeGrantOption
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.I32, 'requestType', None, None, ), # 1
+ (2, TType.STRUCT, 'privileges', (PrivilegeBag, PrivilegeBag.thrift_spec), None, ), # 2
+ (3, TType.BOOL, 'revokeGrantOption', None, None, ), # 3
+ )
+
+ def __init__(self, requestType=None, privileges=None, revokeGrantOption=None,):
+ self.requestType = requestType
+ self.privileges = privileges
+ self.revokeGrantOption = revokeGrantOption
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.I32:
+ self.requestType = iprot.readI32()
+ else:
+ iprot.skip(ftype)
+ elif fid == 2:
+ if ftype == TType.STRUCT:
+ self.privileges = PrivilegeBag()
+ self.privileges.read(iprot)
+ else:
+ iprot.skip(ftype)
+ elif fid == 3:
+ if ftype == TType.BOOL:
+ self.revokeGrantOption = iprot.readBool()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('GrantRevokePrivilegeRequest')
+ if self.requestType is not None:
+ oprot.writeFieldBegin('requestType', TType.I32, 1)
+ oprot.writeI32(self.requestType)
+ oprot.writeFieldEnd()
+ if self.privileges is not None:
+ oprot.writeFieldBegin('privileges', TType.STRUCT, 2)
+ self.privileges.write(oprot)
+ oprot.writeFieldEnd()
+ if self.revokeGrantOption is not None:
+ oprot.writeFieldBegin('revokeGrantOption', TType.BOOL, 3)
+ oprot.writeBool(self.revokeGrantOption)
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.requestType)
+ value = (value * 31) ^ hash(self.privileges)
+ value = (value * 31) ^ hash(self.revokeGrantOption)
+ return value
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ class GrantRevokePrivilegeResponse:
+ """
+ Attributes:
+ - success
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.BOOL, 'success', None, None, ), # 1
+ )
+
+ def __init__(self, success=None,):
+ self.success = success
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.BOOL:
+ self.success = iprot.readBool()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('GrantRevokePrivilegeResponse')
+ if self.success is not None:
+ oprot.writeFieldBegin('success', TType.BOOL, 1)
+ oprot.writeBool(self.success)
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.success)
+ return value
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ class Role:
+ """
+ Attributes:
+ - roleName
+ - createTime
+ - ownerName
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.STRING, 'roleName', None, None, ), # 1
+ (2, TType.I32, 'createTime', None, None, ), # 2
+ (3, TType.STRING, 'ownerName', None, None, ), # 3
+ )
+
+ def __init__(self, roleName=None, createTime=None, ownerName=None,):
+ self.roleName = roleName
+ self.createTime = createTime
+ self.ownerName = ownerName
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.STRING:
+ self.roleName = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 2:
+ if ftype == TType.I32:
+ self.createTime = iprot.readI32()
+ else:
+ iprot.skip(ftype)
+ elif fid == 3:
+ if ftype == TType.STRING:
+ self.ownerName = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('Role')
+ if self.roleName is not None:
+ oprot.writeFieldBegin('roleName', TType.STRING, 1)
+ oprot.writeString(self.roleName)
+ oprot.writeFieldEnd()
+ if self.createTime is not None:
+ oprot.writeFieldBegin('createTime', TType.I32, 2)
+ oprot.writeI32(self.createTime)
+ oprot.writeFieldEnd()
+ if self.ownerName is not None:
+ oprot.writeFieldBegin('ownerName', TType.STRING, 3)
+ oprot.writeString(self.ownerName)
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.roleName)
+ value = (value * 31) ^ hash(self.createTime)
+ value = (value * 31) ^ hash(self.ownerName)
+ return value
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ class RolePrincipalGrant:
+ """
+ Attributes:
+ - roleName
+ - principalName
+ - principalType
+ - grantOption
+ - grantTime
+ - grantorName
+ - grantorPrincipalType
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.STRING, 'roleName', None, None, ), # 1
+ (2, TType.STRING, 'principalName', None, None, ), # 2
+ (3, TType.I32, 'principalType', None, None, ), # 3
+ (4, TType.BOOL, 'grantOption', None, None, ), # 4
+ (5, TType.I32, 'grantTime', None, None, ), # 5
+ (6, TType.STRING, 'grantorName', None, None, ), # 6
+ (7, TType.I32, 'grantorPrincipalType', None, None, ), # 7
+ )
+
+ def __init__(self, roleName=None, principalName=None, principalType=None, grantOption=None, grantTime=None, grantorName=None, grantorPrincipalType=None,):
+ self.roleName = roleName
+ self.principalName = principalName
+ self.principalType = principalType
+ self.grantOption = grantOption
+ self.grantTime = grantTime
+ self.grantorName = grantorName
+ self.grantorPrincipalType = grantorPrincipalType
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.STRING:
+ self.roleName = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 2:
+ if ftype == TType.STRING:
+ self.principalName = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 3:
+ if ftype == TType.I32:
+ self.principalType = iprot.readI32()
+ else:
+ iprot.skip(ftype)
+ elif fid == 4:
+ if ftype == TType.BOOL:
+ self.grantOption = iprot.readBool()
+ else:
+ iprot.skip(ftype)
+ elif fid == 5:
+ if ftype == TType.I32:
+ self.grantTime = iprot.readI32()
+ else:
+ iprot.skip(ftype)
+ elif fid == 6:
+ if ftype == TType.STRING:
+ self.grantorName = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 7:
+ if ftype == TType.I32:
+ self.grantorPrincipalType = iprot.readI32()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('RolePrincipalGrant')
+ if self.roleName is not None:
+ oprot.writeFieldBegin('roleName', TType.STRING, 1)
+ oprot.writeString(self.roleName)
+ oprot.writeFieldEnd()
+ if self.principalName is not None:
+ oprot.writeFieldBegin('principalName', TType.STRING, 2)
+ oprot.writeString(self.principalName)
+ oprot.writeFieldEnd()
+ if self.principalType is not None:
+ oprot.writeFieldBegin('principalType', TType.I32, 3)
+ oprot.writeI32(self.principalType)
+ oprot.writeFieldEnd()
+ if self.grantOption is not None:
+ oprot.writeFieldBegin('grantOption', TType.BOOL, 4)
+ oprot.writeBool(self.grantOption)
+ oprot.writeFieldEnd()
+ if self.grantTime is not None:
+ oprot.writeFieldBegin('grantTime', TType.I32, 5)
+ oprot.writeI32(self.grantTime)
+ oprot.writeFieldEnd()
+ if self.grantorName is not None:
+ oprot.writeFieldBegin('grantorName', TType.STRING, 6)
+ oprot.writeString(self.grantorName)
+ oprot.writeFieldEnd()
+ if self.grantorPrincipalType is not None:
+ oprot.writeFieldBegin('grantorPrincipalType', TType.I32, 7)
+ oprot.writeI32(self.grantorPrincipalType)
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.roleName)
+ value = (value * 31) ^ hash(self.principalName)
+ value = (value * 31) ^ hash(self.principalType)
+ value = (value * 31) ^ hash(self.grantOption)
+ value = (value * 31) ^ hash(self.grantTime)
+ value = (value * 31) ^ hash(self.grantorName)
+ value = (value * 31) ^ hash(self.grantorPrincipalType)
+ return value
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ class GetRoleGrantsForPrincipalRequest:
+ """
+ Attributes:
+ - principal_name
+ - principal_type
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.STRING, 'principal_name', None, None, ), # 1
+ (2, TType.I32, 'principal_type', None, None, ), # 2
+ )
+
+ def __init__(self, principal_name=None, principal_type=None,):
+ self.principal_name = principal_name
+ self.principal_type = principal_type
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.STRING:
+ self.principal_name = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 2:
+ if ftype == TType.I32:
+ self.principal_type = iprot.readI32()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('GetRoleGrantsForPrincipalRequest')
+ if self.principal_name is not None:
+ oprot.writeFieldBegin('principal_name', TType.STRING, 1)
+ oprot.writeString(self.principal_name)
+ oprot.writeFieldEnd()
+ if self.principal_type is not None:
+ oprot.writeFieldBegin('principal_type', TType.I32, 2)
+ oprot.writeI32(self.principal_type)
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ if self.principal_name is None:
+ raise TProtocol.TProtocolException(message='Required field principal_name is unset!')
+ if self.principal_type is None:
+ raise TProtocol.TProtocolException(message='Required field principal_type is unset!')
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.principal_name)
+ value = (value * 31) ^ hash(self.principal_type)
+ return value
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ class GetRoleGrantsForPrincipalResponse:
+ """
+ Attributes:
+ - principalGrants
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.LIST, 'principalGrants', (TType.STRUCT,(RolePrincipalGrant, RolePrincipalGrant.thrift_spec)), None, ), # 1
+ )
+
+ def __init__(self, principalGrants=None,):
+ self.principalGrants = principalGrants
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.LIST:
+ self.principalGrants = []
+ (_etype72, _size69) = iprot.readListBegin()
+ for _i73 in xrange(_size69):
+ _elem74 = RolePrincipalGrant()
+ _elem74.read(iprot)
+ self.principalGrants.append(_elem74)
+ iprot.readListEnd()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('GetRoleGrantsForPrincipalResponse')
+ if self.principalGrants is not None:
+ oprot.writeFieldBegin('principalGrants', TType.LIST, 1)
+ oprot.writeListBegin(TType.STRUCT, len(self.principalGrants))
+ for iter75 in self.principalGrants:
+ iter75.write(oprot)
+ oprot.writeListEnd()
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ if self.principalGrants is None:
+ raise TProtocol.TProtocolException(message='Required field principalGrants is unset!')
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.principalGrants)
+ return value
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ class GetPrincipalsInRoleRequest:
+ """
+ Attributes:
+ - roleName
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.STRING, 'roleName', None, None, ), # 1
+ )
+
+ def __init__(self, roleName=None,):
+ self.roleName = roleName
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.STRING:
+ self.roleName = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('GetPrincipalsInRoleRequest')
+ if self.roleName is not None:
+ oprot.writeFieldBegin('roleName', TType.STRING, 1)
+ oprot.writeString(self.roleName)
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ if self.roleName is None:
+ raise TProtocol.TProtocolException(message='Required field roleName is unset!')
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.roleName)
+ return value
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ class GetPrincipalsInRoleResponse:
+ """
+ Attributes:
+ - principalGrants
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.LIST, 'principalGrants', (TType.STRUCT,(RolePrincipalGrant, RolePrincipalGrant.thrift_spec)), None, ), # 1
+ )
+
+ def __init__(self, principalGrants=None,):
+ self.principalGrants = principalGrants
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.LIST:
+ self.principalGrants = []
+ (_etype79, _size76) = iprot.readListBegin()
+ for _i80 in xrange(_size76):
+ _elem81 = RolePrincipalGrant()
+ _elem81.read(iprot)
+ self.principalGrants.append(_elem81)
+ iprot.readListEnd()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('GetPrincipalsInRoleResponse')
+ if self.principalGrants is not None:
+ oprot.writeFieldBegin('principalGrants', TType.LIST, 1)
+ oprot.writeListBegin(TType.STRUCT, len(self.principalGrants))
+ for iter82 in self.principalGrants:
+ iter82.write(oprot)
+ oprot.writeListEnd()
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ if self.principalGrants is None:
+ raise TProtocol.TProtocolException(message='Required field principalGrants is unset!')
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.principalGrants)
+ return value
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ class GrantRevokeRoleRequest:
+ """
+ Attributes:
+ - requestType
+ - roleName
+ - principalName
+ - principalType
+ - grantor
+ - grantorType
+ - grantOption
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.I32, 'requestType', None, None, ), # 1
+ (2, TType.STRING, 'roleName', None, None, ), # 2
+ (3, TType.STRING, 'principalName', None, None, ), # 3
+ (4, TType.I32, 'principalType', None, None, ), # 4
+ (5, TType.STRING, 'grantor', None, None, ), # 5
+ (6, TType.I32, 'grantorType', None, None, ), # 6
+ (7, TType.BOOL, 'grantOption', None, None, ), # 7
+ )
+
+ def __init__(self, requestType=None, roleName=None, principalName=None, principalType=None, grantor=None, grantorType=None, grantOption=None,):
+ self.requestType = requestType
+ self.roleName = roleName
+ self.principalName = principalName
+ self.principalType = principalType
+ self.grantor = grantor
+ self.grantorType = grantorType
+ self.grantOption = grantOption
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.I32:
+ self.requestType = iprot.readI32()
+ else:
+ iprot.skip(ftype)
+ elif fid == 2:
+ if ftype == TType.STRING:
+ self.roleName = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 3:
+ if ftype == TType.STRING:
+ self.principalName = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 4:
+ if ftype == TType.I32:
+ self.principalType = iprot.readI32()
+ else:
+ iprot.skip(ftype)
+ elif fid == 5:
+ if ftype == TType.STRING:
+ self.grantor = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ elif fid == 6:
+ if ftype == TType.I32:
+ self.grantorType = iprot.readI32()
+ else:
+ iprot.skip(ftype)
+ elif fid == 7:
+ if ftype == TType.BOOL:
+ self.grantOption = iprot.readBool()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('GrantRevokeRoleRequest')
+ if self.requestType is not None:
+ oprot.writeFieldBegin('requestType', TType.I32, 1)
+ oprot.writeI32(self.requestType)
+ oprot.writeFieldEnd()
+ if self.roleName is not None:
+ oprot.writeFieldBegin('roleName', TType.STRING, 2)
+ oprot.writeString(self.roleName)
+ oprot.writeFieldEnd()
+ if self.principalName is not None:
+ oprot.writeFieldBegin('principalName', TType.STRING, 3)
+ oprot.writeString(self.principalName)
+ oprot.writeFieldEnd()
+ if self.principalType is not None:
+ oprot.writeFieldBegin('principalType', TType.I32, 4)
+ oprot.writeI32(self.principalType)
+ oprot.writeFieldEnd()
+ if self.grantor is not None:
+ oprot.writeFieldBegin('grantor', TType.STRING, 5)
+ oprot.writeString(self.grantor)
+ oprot.writeFieldEnd()
+ if self.grantorType is not None:
+ oprot.writeFieldBegin('grantorType', TType.I32, 6)
+ oprot.writeI32(self.grantorType)
+ oprot.writeFieldEnd()
+ if self.grantOption is not None:
+ oprot.writeFieldBegin('grantOption', TType.BOOL, 7)
+ oprot.writeBool(self.grantOption)
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.requestType)
+ value = (value * 31) ^ hash(self.roleName)
+ value = (value * 31) ^ hash(self.principalName)
+ value = (value * 31) ^ hash(self.principalType)
+ value = (value * 31) ^ hash(self.grantor)
+ value = (value * 31) ^ hash(self.grantorType)
+ value = (value * 31) ^ hash(self.grantOption)
+ return value
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ class GrantRevokeRoleResponse:
+ """
+ Attributes:
+ - success
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.BOOL, 'success', None, None, ), # 1
+ )
+
+ def __init__(self, success=None,):
+ self.success = success
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.BOOL:
+ self.success = iprot.readBool()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('GrantRevokeRoleResponse')
+ if self.success is not None:
+ oprot.writeFieldBegin('success', TType.BOOL, 1)
+ oprot.writeBool(self.success)
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.
<TRUNCATED>
[47/50] [abbrv] hive git commit: HIVE-14671 : merge master into
hive-14535 (Wei Zheng)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/test/queries/clientpositive/mm_loaddata.q
----------------------------------------------------------------------
diff --cc ql/src/test/queries/clientpositive/mm_loaddata.q
index 0000000,0000000..7e5787f
new file mode 100644
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/mm_loaddata.q
@@@ -1,0 -1,0 +1,53 @@@
++set hive.mapred.mode=nonstrict;
++set hive.explain.user=false;
++set hive.fetch.task.conversion=none;
++set tez.grouping.min-size=1;
++set tez.grouping.max-size=2;
++set hive.exec.dynamic.partition.mode=nonstrict;
++set hive.support.concurrency=true;
++set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
++
++
++drop table load0_mm;
++create table load0_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only");
++load data local inpath '../../data/files/kv1.txt' into table load0_mm;
++select count(1) from load0_mm;
++load data local inpath '../../data/files/kv2.txt' into table load0_mm;
++select count(1) from load0_mm;
++load data local inpath '../../data/files/kv2.txt' overwrite into table load0_mm;
++select count(1) from load0_mm;
++drop table load0_mm;
++
++
++drop table intermediate2;
++create table intermediate2 (key string, value string) stored as textfile
++location 'file:${system:test.tmp.dir}/intermediate2';
++load data local inpath '../../data/files/kv1.txt' into table intermediate2;
++load data local inpath '../../data/files/kv2.txt' into table intermediate2;
++load data local inpath '../../data/files/kv3.txt' into table intermediate2;
++
++drop table load1_mm;
++create table load1_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only");
++load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv2.txt' into table load1_mm;
++load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv1.txt' into table load1_mm;
++select count(1) from load1_mm;
++load data local inpath '../../data/files/kv1.txt' into table intermediate2;
++load data local inpath '../../data/files/kv2.txt' into table intermediate2;
++load data local inpath '../../data/files/kv3.txt' into table intermediate2;
++load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv*.txt' overwrite into table load1_mm;
++select count(1) from load1_mm;
++load data local inpath '../../data/files/kv2.txt' into table intermediate2;
++load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv2.txt' overwrite into table load1_mm;
++select count(1) from load1_mm;
++drop table load1_mm;
++
++drop table load2_mm;
++create table load2_mm (key string, value string)
++ partitioned by (k int, l int) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only");
++load data local inpath '../../data/files/kv1.txt' into table intermediate2;
++load data local inpath '../../data/files/kv2.txt' into table intermediate2;
++load data local inpath '../../data/files/kv3.txt' into table intermediate2;
++load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv*.txt' into table load2_mm partition(k=5, l=5);
++select count(1) from load2_mm;
++drop table load2_mm;
++drop table intermediate2;
[26/50] [abbrv] hive git commit: HIVE-17408 : replication distcp
should only be invoked if number of files AND file size cross configured
limits (Anishek Agarwal reviewed by Thejas Nair)
Posted by we...@apache.org.
HIVE-17408 : replication distcp should only be invoked if number of files AND file size cross configured limits (Anishek Agarwal reviewed by Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2e5ead14
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2e5ead14
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2e5ead14
Branch: refs/heads/hive-14535
Commit: 2e5ead142f88333899223aaf810976c4dcf02efb
Parents: 70cb7f0
Author: Anishek Agarwal <an...@gmail.com>
Authored: Wed Aug 30 17:43:01 2017 -0700
Committer: Thejas M Nair <th...@hortonworks.com>
Committed: Wed Aug 30 17:43:01 2017 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/parse/repl/CopyUtils.java | 4 +-
.../hive/ql/parse/repl/CopyUtilsTest.java | 47 ++++++++++++++++++++
2 files changed, 49 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/2e5ead14/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java
index db923e3..28e7bcb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java
@@ -108,8 +108,8 @@ public class CopyUtils {
return true;
}
- private boolean limitReachedForLocalCopy(long size, long numberOfFiles) {
- boolean result = size > maxCopyFileSize || numberOfFiles > maxNumberOfFiles;
+ boolean limitReachedForLocalCopy(long size, long numberOfFiles) {
+ boolean result = size > maxCopyFileSize && numberOfFiles > maxNumberOfFiles;
if (result) {
LOG.info("Source is {} bytes. (MAX: {})", size, maxCopyFileSize);
LOG.info("Source is {} files. (MAX: {})", numberOfFiles, maxNumberOfFiles);
http://git-wip-us.apache.org/repos/asf/hive/blob/2e5ead14/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/CopyUtilsTest.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/CopyUtilsTest.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/CopyUtilsTest.java
new file mode 100644
index 0000000..e643d8f
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/CopyUtilsTest.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse.repl;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+
+public class CopyUtilsTest {
+ /*
+ Distcp currently does not copy a single file in a distributed manner hence we dont care about
+ the size of file, if there is only file, we dont want to launch distcp.
+ */
+ @Test
+ public void distcpShouldNotBeCalledOnlyForOneFile() {
+ HiveConf conf = new HiveConf();
+ conf.setLongVar(HiveConf.ConfVars.HIVE_EXEC_COPYFILE_MAXSIZE, 1);
+ CopyUtils copyUtils = new CopyUtils("", conf);
+ long MB_128 = 128 * 1024 * 1024;
+ assertFalse(copyUtils.limitReachedForLocalCopy(MB_128, 1L));
+ }
+
+ @Test
+ public void distcpShouldNotBeCalledForSmallerFileSize() {
+ HiveConf conf = new HiveConf();
+ CopyUtils copyUtils = new CopyUtils("", conf);
+ long MB_16 = 16 * 1024 * 1024;
+ assertFalse(copyUtils.limitReachedForLocalCopy(MB_16, 100L));
+ }
+}
\ No newline at end of file
[06/50] [abbrv] hive git commit: HIVE-17205 - add functional support
for unbucketed tables (Eugene Koifman, reviewed by Wei Zheng)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out b/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out
new file mode 100644
index 0000000..34dd487
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out
@@ -0,0 +1,1976 @@
+PREHOOK: query: drop table if exists srcpart_acid
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists srcpart_acid
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE srcpart_acid (key STRING, value STRING) PARTITIONED BY (ds STRING, hr STRING) stored as ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcpart_acid
+POSTHOOK: query: CREATE TABLE srcpart_acid (key STRING, value STRING) PARTITIONED BY (ds STRING, hr STRING) stored as ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcpart_acid
+PREHOOK: query: insert into srcpart_acid PARTITION (ds, hr) select * from srcpart
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+PREHOOK: Output: default@srcpart_acid
+POSTHOOK: query: insert into srcpart_acid PARTITION (ds, hr) select * from srcpart
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=12
+POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select ds, hr, key, value from srcpart_acid where cast(key as integer) in(413,43) and hr='11' order by ds, hr, cast(key as integer)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acid
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+#### A masked pattern was here ####
+POSTHOOK: query: select ds, hr, key, value from srcpart_acid where cast(key as integer) in(413,43) and hr='11' order by ds, hr, cast(key as integer)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acid
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+#### A masked pattern was here ####
+2008-04-08 11 43 val_43
+2008-04-08 11 413 val_413
+2008-04-08 11 413 val_413
+2008-04-09 11 43 val_43
+2008-04-09 11 413 val_413
+2008-04-09 11 413 val_413
+PREHOOK: query: analyze table srcpart_acid PARTITION(ds, hr) compute statistics
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acid
+PREHOOK: Output: default@srcpart_acid
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=12
+POSTHOOK: query: analyze table srcpart_acid PARTITION(ds, hr) compute statistics
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acid
+POSTHOOK: Output: default@srcpart_acid
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=12
+PREHOOK: query: analyze table srcpart_acid PARTITION(ds, hr) compute statistics for columns
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acid
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=12
+PREHOOK: Output: default@srcpart_acid
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table srcpart_acid PARTITION(ds, hr) compute statistics for columns
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acid
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@srcpart_acid
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+PREHOOK: query: explain update srcpart_acid set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11'
+PREHOOK: type: QUERY
+POSTHOOK: query: explain update srcpart_acid set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11'
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_acid
+ Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: PARTIAL
+ Filter Operator
+ predicate: (UDFToInteger(key)) IN (413, 43) (type: boolean)
+ Statistics: Num rows: 500 Data size: 181000 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: ROW__ID (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), key (type: string), concat(value, 'updated') (type: string), ds (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 500 Data size: 308500 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 308500 Basic stats: COMPLETE Column stats: PARTIAL
+ value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string)
+ Execution mode: llap
+ LLAP IO: may be used (ACID table)
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), '11' (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 500 Data size: 308500 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 500 Data size: 308500 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.srcpart_acid
+ Write Type: UPDATE
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ ds
+ hr
+ replace: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.srcpart_acid
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+PREHOOK: query: update srcpart_acid set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acid
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=11
+POSTHOOK: query: update srcpart_acid set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acid
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=11
+PREHOOK: query: select ds, hr, key, value from srcpart_acid where value like '%updated' order by ds, hr, cast(key as integer)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acid
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: select ds, hr, key, value from srcpart_acid where value like '%updated' order by ds, hr, cast(key as integer)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acid
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+2008-04-08 11 43 val_43updated
+2008-04-08 11 413 val_413updated
+2008-04-08 11 413 val_413updated
+2008-04-09 11 43 val_43updated
+2008-04-09 11 413 val_413updated
+2008-04-09 11 413 val_413updated
+PREHOOK: query: insert into srcpart_acid PARTITION (ds='2008-04-08', hr=='11') values ('1001','val1001'),('1002','val1002'),('1003','val1003')
+PREHOOK: type: QUERY
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: query: insert into srcpart_acid PARTITION (ds='2008-04-08', hr=='11') values ('1001','val1001'),('1002','val1002'),('1003','val1003')
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+PREHOOK: query: select ds, hr, key, value from srcpart_acid where cast(key as integer) > 1000 order by ds, hr, cast(key as integer)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acid
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: select ds, hr, key, value from srcpart_acid where cast(key as integer) > 1000 order by ds, hr, cast(key as integer)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acid
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+2008-04-08 11 1001 val1001
+2008-04-08 11 1002 val1002
+2008-04-08 11 1003 val1003
+PREHOOK: query: analyze table srcpart_acid PARTITION(ds, hr) compute statistics
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acid
+PREHOOK: Output: default@srcpart_acid
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=12
+POSTHOOK: query: analyze table srcpart_acid PARTITION(ds, hr) compute statistics
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acid
+POSTHOOK: Output: default@srcpart_acid
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=12
+PREHOOK: query: analyze table srcpart_acid PARTITION(ds, hr) compute statistics for columns
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acid
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=12
+PREHOOK: Output: default@srcpart_acid
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table srcpart_acid PARTITION(ds, hr) compute statistics for columns
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acid
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@srcpart_acid
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+PREHOOK: query: explain delete from srcpart_acid where key in( '1001', '213', '43')
+PREHOOK: type: QUERY
+POSTHOOK: query: explain delete from srcpart_acid where key in( '1001', '213', '43')
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_acid
+ Statistics: Num rows: 2015 Data size: 916825 Basic stats: COMPLETE Column stats: PARTIAL
+ Filter Operator
+ predicate: (key) IN ('1001', '213', '43') (type: boolean)
+ Statistics: Num rows: 20 Data size: 9100 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: ROW__ID (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), ds (type: string), hr (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 20 Data size: 8880 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ sort order: +
+ Statistics: Num rows: 20 Data size: 8880 Basic stats: COMPLETE Column stats: PARTIAL
+ value expressions: _col1 (type: string), _col2 (type: string)
+ Execution mode: llap
+ LLAP IO: may be used (ACID table)
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), VALUE._col0 (type: string), VALUE._col1 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 20 Data size: 8880 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 20 Data size: 8880 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.srcpart_acid
+ Write Type: DELETE
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ ds
+ hr
+ replace: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.srcpart_acid
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+PREHOOK: query: delete from srcpart_acid where key in( '1001', '213', '43')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acid
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=12
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=12
+POSTHOOK: query: delete from srcpart_acid where key in( '1001', '213', '43')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acid
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=12
+PREHOOK: query: select count(*) from srcpart_acid where key in( '1001', '213', '43')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acid
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from srcpart_acid where key in( '1001', '213', '43')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acid
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+0
+PREHOOK: query: select count(*) from srcpart_acid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acid
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from srcpart_acid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acid
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+1990
+PREHOOK: query: merge into srcpart_acid t using (select distinct ds, hr, key, value from srcpart_acid) s
+on s.ds=t.ds and s.hr=t.hr and s.key=t.key and s.value=t.value
+when matched and s.ds='2008-04-08' and s.hr=='11' and s.key='44' then update set value=concat(s.value,'updated by merge')
+when matched and s.ds='2008-04-08' and s.hr=='12' then delete
+when not matched then insert values('this','should','not','be there')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acid
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=12
+PREHOOK: Output: default@merge_tmp_table
+PREHOOK: Output: default@srcpart_acid
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=12
+PREHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=12
+POSTHOOK: query: merge into srcpart_acid t using (select distinct ds, hr, key, value from srcpart_acid) s
+on s.ds=t.ds and s.hr=t.hr and s.key=t.key and s.value=t.value
+when matched and s.ds='2008-04-08' and s.hr=='11' and s.key='44' then update set value=concat(s.value,'updated by merge')
+when matched and s.ds='2008-04-08' and s.hr=='12' then delete
+when not matched then insert values('this','should','not','be there')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acid
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@merge_tmp_table
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=12
+POSTHOOK: Lineage: merge_tmp_table.val EXPRESSION [(srcpart_acid)t.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), (srcpart_acid)t.FieldSchema(name:ds, type:string, comment:null), (srcpart_acid)t.FieldSchema(name:hr, type:string, comment:null), ]
+PREHOOK: query: select count(*) from srcpart_acid where ds='2008-04-08' and hr=='12'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acid
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from srcpart_acid where ds='2008-04-08' and hr=='12'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acid
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=12
+#### A masked pattern was here ####
+0
+PREHOOK: query: select ds, hr, key, value from srcpart_acid where value like '%updated by merge'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acid
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: select ds, hr, key, value from srcpart_acid where value like '%updated by merge'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acid
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acid@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+2008-04-08 11 44 val_44updated by merge
+PREHOOK: query: select count(*) from srcpart_acid where ds = 'this' and hr = 'should' and key = 'not' and value = 'be there'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acid
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from srcpart_acid where ds = 'this' and hr = 'should' and key = 'not' and value = 'be there'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acid
+#### A masked pattern was here ####
+0
+PREHOOK: query: drop table if exists srcpart_acid
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@srcpart_acid
+PREHOOK: Output: default@srcpart_acid
+POSTHOOK: query: drop table if exists srcpart_acid
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@srcpart_acid
+POSTHOOK: Output: default@srcpart_acid
+PREHOOK: query: drop table if exists srcpart_acidb
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists srcpart_acidb
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE srcpart_acidb (key STRING, value STRING) PARTITIONED BY (ds STRING, hr STRING) CLUSTERED BY(key) INTO 2 BUCKETS stored as ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcpart_acidb
+POSTHOOK: query: CREATE TABLE srcpart_acidb (key STRING, value STRING) PARTITIONED BY (ds STRING, hr STRING) CLUSTERED BY(key) INTO 2 BUCKETS stored as ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcpart_acidb
+PREHOOK: query: insert into srcpart_acidb PARTITION (ds, hr) select * from srcpart
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+PREHOOK: Output: default@srcpart_acidb
+POSTHOOK: query: insert into srcpart_acidb PARTITION (ds, hr) select * from srcpart
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=12
+POSTHOOK: Lineage: srcpart_acidb PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acidb PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acidb PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acidb PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acidb PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acidb PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acidb PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acidb PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select ds, hr, key, value from srcpart_acidb where cast(key as integer) in(413,43) and hr='11' order by ds, hr, cast(key as integer)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidb
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+#### A masked pattern was here ####
+POSTHOOK: query: select ds, hr, key, value from srcpart_acidb where cast(key as integer) in(413,43) and hr='11' order by ds, hr, cast(key as integer)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidb
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+#### A masked pattern was here ####
+2008-04-08 11 43 val_43
+2008-04-08 11 413 val_413
+2008-04-08 11 413 val_413
+2008-04-09 11 43 val_43
+2008-04-09 11 413 val_413
+2008-04-09 11 413 val_413
+PREHOOK: query: analyze table srcpart_acidb PARTITION(ds, hr) compute statistics
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidb
+PREHOOK: Output: default@srcpart_acidb
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=12
+POSTHOOK: query: analyze table srcpart_acidb PARTITION(ds, hr) compute statistics
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidb
+POSTHOOK: Output: default@srcpart_acidb
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=12
+PREHOOK: query: analyze table srcpart_acidb PARTITION(ds, hr) compute statistics for columns
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidb
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=12
+PREHOOK: Output: default@srcpart_acidb
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table srcpart_acidb PARTITION(ds, hr) compute statistics for columns
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidb
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@srcpart_acidb
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+PREHOOK: query: explain update srcpart_acidb set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11'
+PREHOOK: type: QUERY
+POSTHOOK: query: explain update srcpart_acidb set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11'
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_acidb
+ Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: PARTIAL
+ Filter Operator
+ predicate: (UDFToInteger(key)) IN (413, 43) (type: boolean)
+ Statistics: Num rows: 500 Data size: 181000 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: ROW__ID (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), key (type: string), concat(value, 'updated') (type: string), ds (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 500 Data size: 308500 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ sort order: +
+ Map-reduce partition columns: UDFToInteger(_col0) (type: int)
+ Statistics: Num rows: 500 Data size: 308500 Basic stats: COMPLETE Column stats: PARTIAL
+ value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string)
+ Execution mode: llap
+ LLAP IO: may be used (ACID table)
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), '11' (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 500 Data size: 308500 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 500 Data size: 308500 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.srcpart_acidb
+ Write Type: UPDATE
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ ds
+ hr
+ replace: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.srcpart_acidb
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+PREHOOK: query: update srcpart_acidb set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidb
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=11
+POSTHOOK: query: update srcpart_acidb set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidb
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=11
+PREHOOK: query: select ds, hr, key, value from srcpart_acidb where value like '%updated' order by ds, hr, cast(key as integer)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidb
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: select ds, hr, key, value from srcpart_acidb where value like '%updated' order by ds, hr, cast(key as integer)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidb
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+2008-04-08 11 43 val_43updated
+2008-04-08 11 413 val_413updated
+2008-04-08 11 413 val_413updated
+2008-04-09 11 43 val_43updated
+2008-04-09 11 413 val_413updated
+2008-04-09 11 413 val_413updated
+PREHOOK: query: insert into srcpart_acidb PARTITION (ds='2008-04-08', hr=='11') values ('1001','val1001'),('1002','val1002'),('1003','val1003')
+PREHOOK: type: QUERY
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=11
+POSTHOOK: query: insert into srcpart_acidb PARTITION (ds='2008-04-08', hr=='11') values ('1001','val1001'),('1002','val1002'),('1003','val1003')
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=11
+POSTHOOK: Lineage: srcpart_acidb PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: srcpart_acidb PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+PREHOOK: query: select ds, hr, key, value from srcpart_acidb where cast(key as integer) > 1000 order by ds, hr, cast(key as integer)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidb
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: select ds, hr, key, value from srcpart_acidb where cast(key as integer) > 1000 order by ds, hr, cast(key as integer)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidb
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+2008-04-08 11 1001 val1001
+2008-04-08 11 1002 val1002
+2008-04-08 11 1003 val1003
+PREHOOK: query: analyze table srcpart_acidb PARTITION(ds, hr) compute statistics
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidb
+PREHOOK: Output: default@srcpart_acidb
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=12
+POSTHOOK: query: analyze table srcpart_acidb PARTITION(ds, hr) compute statistics
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidb
+POSTHOOK: Output: default@srcpart_acidb
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=12
+PREHOOK: query: analyze table srcpart_acidb PARTITION(ds, hr) compute statistics for columns
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidb
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=12
+PREHOOK: Output: default@srcpart_acidb
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table srcpart_acidb PARTITION(ds, hr) compute statistics for columns
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidb
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@srcpart_acidb
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+PREHOOK: query: explain delete from srcpart_acidb where key in( '1001', '213', '43')
+PREHOOK: type: QUERY
+POSTHOOK: query: explain delete from srcpart_acidb where key in( '1001', '213', '43')
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_acidb
+ Statistics: Num rows: 2015 Data size: 916825 Basic stats: COMPLETE Column stats: PARTIAL
+ Filter Operator
+ predicate: (key) IN ('1001', '213', '43') (type: boolean)
+ Statistics: Num rows: 20 Data size: 9100 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: ROW__ID (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), ds (type: string), hr (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 20 Data size: 8880 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ sort order: +
+ Map-reduce partition columns: UDFToInteger(_col0) (type: int)
+ Statistics: Num rows: 20 Data size: 8880 Basic stats: COMPLETE Column stats: PARTIAL
+ value expressions: _col1 (type: string), _col2 (type: string)
+ Execution mode: llap
+ LLAP IO: may be used (ACID table)
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), VALUE._col0 (type: string), VALUE._col1 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 20 Data size: 8880 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 20 Data size: 8880 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.srcpart_acidb
+ Write Type: DELETE
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ ds
+ hr
+ replace: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.srcpart_acidb
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+PREHOOK: query: delete from srcpart_acidb where key in( '1001', '213', '43')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidb
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=12
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=12
+POSTHOOK: query: delete from srcpart_acidb where key in( '1001', '213', '43')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidb
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=12
+PREHOOK: query: select count(*) from srcpart_acidb where key in( '1001', '213', '43')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidb
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from srcpart_acidb where key in( '1001', '213', '43')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidb
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+0
+PREHOOK: query: select count(*) from srcpart_acidb
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidb
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from srcpart_acidb
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidb
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+1990
+PREHOOK: query: merge into srcpart_acidb t using (select distinct ds, hr, key, value from srcpart_acidb) s
+on s.ds=t.ds and s.hr=t.hr and s.key=t.key and s.value=t.value
+when matched and s.ds='2008-04-08' and s.hr=='11' and s.key='44' then update set value=concat(s.value,'updated by merge')
+when matched and s.ds='2008-04-08' and s.hr=='12' then delete
+when not matched then insert values('this','should','not','be there')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidb
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=12
+PREHOOK: Output: default@merge_tmp_table
+PREHOOK: Output: default@srcpart_acidb
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=12
+PREHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=12
+POSTHOOK: query: merge into srcpart_acidb t using (select distinct ds, hr, key, value from srcpart_acidb) s
+on s.ds=t.ds and s.hr=t.hr and s.key=t.key and s.value=t.value
+when matched and s.ds='2008-04-08' and s.hr=='11' and s.key='44' then update set value=concat(s.value,'updated by merge')
+when matched and s.ds='2008-04-08' and s.hr=='12' then delete
+when not matched then insert values('this','should','not','be there')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidb
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@merge_tmp_table
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@srcpart_acidb@ds=2008-04-09/hr=12
+POSTHOOK: Lineage: merge_tmp_table.val EXPRESSION [(srcpart_acidb)t.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), (srcpart_acidb)t.FieldSchema(name:ds, type:string, comment:null), (srcpart_acidb)t.FieldSchema(name:hr, type:string, comment:null), ]
+PREHOOK: query: select count(*) from srcpart_acidb where ds='2008-04-08' and hr=='12'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidb
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from srcpart_acidb where ds='2008-04-08' and hr=='12'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidb
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=12
+#### A masked pattern was here ####
+0
+PREHOOK: query: select ds, hr, key, value from srcpart_acidb where value like '%updated by merge'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidb
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: select ds, hr, key, value from srcpart_acidb where value like '%updated by merge'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidb
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acidb@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+2008-04-08 11 44 val_44updated by merge
+PREHOOK: query: select count(*) from srcpart_acidb where ds = 'this' and hr = 'should' and key = 'not' and value = 'be there'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidb
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from srcpart_acidb where ds = 'this' and hr = 'should' and key = 'not' and value = 'be there'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidb
+#### A masked pattern was here ####
+0
+PREHOOK: query: drop table if exists srcpart_acidb
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@srcpart_acidb
+PREHOOK: Output: default@srcpart_acidb
+POSTHOOK: query: drop table if exists srcpart_acidb
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@srcpart_acidb
+POSTHOOK: Output: default@srcpart_acidb
+PREHOOK: query: drop table if exists srcpart_acidv
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists srcpart_acidv
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE srcpart_acidv (key STRING, value STRING) PARTITIONED BY (ds STRING, hr STRING) stored as ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcpart_acidv
+POSTHOOK: query: CREATE TABLE srcpart_acidv (key STRING, value STRING) PARTITIONED BY (ds STRING, hr STRING) stored as ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcpart_acidv
+PREHOOK: query: insert into srcpart_acidv PARTITION (ds, hr) select * from srcpart
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+PREHOOK: Output: default@srcpart_acidv
+POSTHOOK: query: insert into srcpart_acidv PARTITION (ds, hr) select * from srcpart
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=12
+POSTHOOK: Lineage: srcpart_acidv PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acidv PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acidv PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acidv PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acidv PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acidv PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acidv PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acidv PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select ds, hr, key, value from srcpart_acidv where cast(key as integer) in(413,43) and hr='11' order by ds, hr, cast(key as integer)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidv
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+#### A masked pattern was here ####
+POSTHOOK: query: select ds, hr, key, value from srcpart_acidv where cast(key as integer) in(413,43) and hr='11' order by ds, hr, cast(key as integer)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidv
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+#### A masked pattern was here ####
+2008-04-08 11 43 val_43
+2008-04-08 11 413 val_413
+2008-04-08 11 413 val_413
+2008-04-09 11 43 val_43
+2008-04-09 11 413 val_413
+2008-04-09 11 413 val_413
+PREHOOK: query: analyze table srcpart_acidv PARTITION(ds, hr) compute statistics
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidv
+PREHOOK: Output: default@srcpart_acidv
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=12
+POSTHOOK: query: analyze table srcpart_acidv PARTITION(ds, hr) compute statistics
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidv
+POSTHOOK: Output: default@srcpart_acidv
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=12
+PREHOOK: query: analyze table srcpart_acidv PARTITION(ds, hr) compute statistics for columns
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidv
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=12
+PREHOOK: Output: default@srcpart_acidv
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table srcpart_acidv PARTITION(ds, hr) compute statistics for columns
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidv
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@srcpart_acidv
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+PREHOOK: query: explain update srcpart_acidv set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11'
+PREHOOK: type: QUERY
+POSTHOOK: query: explain update srcpart_acidv set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11'
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_acidv
+ Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: PARTIAL
+ Filter Operator
+ predicate: (UDFToInteger(key)) IN (413, 43) (type: boolean)
+ Statistics: Num rows: 500 Data size: 181000 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: ROW__ID (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), key (type: string), concat(value, 'updated') (type: string), ds (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 500 Data size: 308500 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 308500 Basic stats: COMPLETE Column stats: PARTIAL
+ value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string)
+ Execution mode: llap
+ LLAP IO: may be used (ACID table)
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), '11' (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 500 Data size: 308500 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 500 Data size: 308500 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.srcpart_acidv
+ Write Type: UPDATE
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ ds
+ hr
+ replace: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.srcpart_acidv
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+PREHOOK: query: update srcpart_acidv set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidv
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=11
+POSTHOOK: query: update srcpart_acidv set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidv
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=11
+PREHOOK: query: select ds, hr, key, value from srcpart_acidv where value like '%updated' order by ds, hr, cast(key as integer)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidv
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: select ds, hr, key, value from srcpart_acidv where value like '%updated' order by ds, hr, cast(key as integer)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidv
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+2008-04-08 11 43 val_43updated
+2008-04-08 11 413 val_413updated
+2008-04-08 11 413 val_413updated
+2008-04-09 11 43 val_43updated
+2008-04-09 11 413 val_413updated
+2008-04-09 11 413 val_413updated
+PREHOOK: query: insert into srcpart_acidv PARTITION (ds='2008-04-08', hr=='11') values ('1001','val1001'),('1002','val1002'),('1003','val1003')
+PREHOOK: type: QUERY
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=11
+POSTHOOK: query: insert into srcpart_acidv PARTITION (ds='2008-04-08', hr=='11') values ('1001','val1001'),('1002','val1002'),('1003','val1003')
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=11
+POSTHOOK: Lineage: srcpart_acidv PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: srcpart_acidv PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+PREHOOK: query: select ds, hr, key, value from srcpart_acidv where cast(key as integer) > 1000 order by ds, hr, cast(key as integer)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidv
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: select ds, hr, key, value from srcpart_acidv where cast(key as integer) > 1000 order by ds, hr, cast(key as integer)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidv
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+2008-04-08 11 1001 val1001
+2008-04-08 11 1002 val1002
+2008-04-08 11 1003 val1003
+PREHOOK: query: analyze table srcpart_acidv PARTITION(ds, hr) compute statistics
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidv
+PREHOOK: Output: default@srcpart_acidv
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=12
+POSTHOOK: query: analyze table srcpart_acidv PARTITION(ds, hr) compute statistics
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidv
+POSTHOOK: Output: default@srcpart_acidv
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=12
+PREHOOK: query: analyze table srcpart_acidv PARTITION(ds, hr) compute statistics for columns
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidv
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=12
+PREHOOK: Output: default@srcpart_acidv
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table srcpart_acidv PARTITION(ds, hr) compute statistics for columns
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidv
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@srcpart_acidv
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+PREHOOK: query: explain delete from srcpart_acidv where key in( '1001', '213', '43')
+PREHOOK: type: QUERY
+POSTHOOK: query: explain delete from srcpart_acidv where key in( '1001', '213', '43')
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_acidv
+ Statistics: Num rows: 2015 Data size: 916825 Basic stats: COMPLETE Column stats: PARTIAL
+ Filter Operator
+ predicate: (key) IN ('1001', '213', '43') (type: boolean)
+ Statistics: Num rows: 20 Data size: 9100 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: ROW__ID (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), ds (type: string), hr (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 20 Data size: 8880 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ sort order: +
+ Statistics: Num rows: 20 Data size: 8880 Basic stats: COMPLETE Column stats: PARTIAL
+ value expressions: _col1 (type: string), _col2 (type: string)
+ Execution mode: llap
+ LLAP IO: may be used (ACID table)
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), VALUE._col0 (type: string), VALUE._col1 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 20 Data size: 8880 Basic stats: COMPLETE Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 20 Data size: 8880 Basic stats: COMPLETE Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.srcpart_acidv
+ Write Type: DELETE
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ ds
+ hr
+ replace: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.srcpart_acidv
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+PREHOOK: query: delete from srcpart_acidv where key in( '1001', '213', '43')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidv
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=12
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=12
+POSTHOOK: query: delete from srcpart_acidv where key in( '1001', '213', '43')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidv
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=12
+PREHOOK: query: select count(*) from srcpart_acidv where key in( '1001', '213', '43')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidv
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from srcpart_acidv where key in( '1001', '213', '43')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidv
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+0
+PREHOOK: query: select count(*) from srcpart_acidv
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidv
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from srcpart_acidv
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidv
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+1990
+PREHOOK: query: merge into srcpart_acidv t using (select distinct ds, hr, key, value from srcpart_acidv) s
+on s.ds=t.ds and s.hr=t.hr and s.key=t.key and s.value=t.value
+when matched and s.ds='2008-04-08' and s.hr=='11' and s.key='44' then update set value=concat(s.value,'updated by merge')
+when matched and s.ds='2008-04-08' and s.hr=='12' then delete
+when not matched then insert values('this','should','not','be there')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidv
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=12
+PREHOOK: Output: default@merge_tmp_table
+PREHOOK: Output: default@srcpart_acidv
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=12
+PREHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=12
+POSTHOOK: query: merge into srcpart_acidv t using (select distinct ds, hr, key, value from srcpart_acidv) s
+on s.ds=t.ds and s.hr=t.hr and s.key=t.key and s.value=t.value
+when matched and s.ds='2008-04-08' and s.hr=='11' and s.key='44' then update set value=concat(s.value,'updated by merge')
+when matched and s.ds='2008-04-08' and s.hr=='12' then delete
+when not matched then insert values('this','should','not','be there')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidv
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@merge_tmp_table
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@srcpart_acidv@ds=2008-04-09/hr=12
+POSTHOOK: Lineage: merge_tmp_table.val EXPRESSION [(srcpart_acidv)t.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), (srcpart_acidv)t.FieldSchema(name:ds, type:string, comment:null), (srcpart_acidv)t.FieldSchema(name:hr, type:string, comment:null), ]
+PREHOOK: query: select count(*) from srcpart_acidv where ds='2008-04-08' and hr=='12'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidv
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from srcpart_acidv where ds='2008-04-08' and hr=='12'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidv
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=12
+#### A masked pattern was here ####
+0
+PREHOOK: query: select ds, hr, key, value from srcpart_acidv where value like '%updated by merge'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidv
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: select ds, hr, key, value from srcpart_acidv where value like '%updated by merge'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidv
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acidv@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+2008-04-08 11 44 val_44updated by merge
+PREHOOK: query: select count(*) from srcpart_acidv where ds = 'this' and hr = 'should' and key = 'not' and value = 'be there'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidv
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from srcpart_acidv where ds = 'this' and hr = 'should' and key = 'not' and value = 'be there'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidv
+#### A masked pattern was here ####
+0
+PREHOOK: query: drop table if exists srcpart_acidv
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@srcpart_acidv
+PREHOOK: Output: default@srcpart_acidv
+POSTHOOK: query: drop table if exists srcpart_acidv
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@srcpart_acidv
+POSTHOOK: Output: default@srcpart_acidv
+PREHOOK: query: drop table if exists srcpart_acidvb
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists srcpart_acidvb
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE srcpart_acidvb (key STRING, value STRING) PARTITIONED BY (ds STRING, hr STRING) CLUSTERED BY(key) INTO 2 BUCKETS stored as ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcpart_acidvb
+POSTHOOK: query: CREATE TABLE srcpart_acidvb (key STRING, value STRING) PARTITIONED BY (ds STRING, hr STRING) CLUSTERED BY(key) INTO 2 BUCKETS stored as ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcpart_acidvb
+PREHOOK: query: insert into srcpart_acidvb PARTITION (ds, hr) select * from srcpart
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+PREHOOK: Output: default@srcpart_acidvb
+POSTHOOK: query: insert into srcpart_acidvb PARTITION (ds, hr) select * from srcpart
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@srcpart_acidvb@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acidvb@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acidvb@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acidvb@ds=2008-04-09/hr=12
+POSTHOOK: Lineage: srcpart_acidvb PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acidvb PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acidvb PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acidvb PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acidvb PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acidvb PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acidvb PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_acidvb PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select ds, hr, key, value from srcpart_acidvb where cast(key as integer) in(413,43) and hr='11' order by ds, hr, cast(key as integer)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidvb
+PREHOOK: Input: default@srcpart_acidvb@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidvb@ds=2008-04-09/hr=11
+#### A masked pattern was here ####
+POSTHOOK: query: select ds, hr, key, value from srcpart_acidvb where cast(key as integer) in(413,43) and hr='11' order by ds, hr, cast(key as integer)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidvb
+POSTHOOK: Input: default@srcpart_acidvb@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidvb@ds=2008-04-09/hr=11
+#### A masked pattern was here ####
+2008-04-08 11 43 val_43
+2008-04-08 11 413 val_413
+2008-04-08 11 413 val_413
+2008-04-09 11 43 val_43
+2008-04-09 11 413 val_413
+2008-04-09 11 413 val_413
+PREHOOK: query: analyze table srcpart_acidvb PARTITION(ds, hr) compute statistics
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidvb
+PREHOOK: Output: default@srcpart_acidvb
+PREHOOK: Output: default@srcpart_acidvb@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acidvb@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_acidvb@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acidvb@ds=2008-04-09/hr=12
+POSTHOOK: query: analyze table srcpart_acidvb PARTITION(ds, hr) compute statistics
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidvb
+POSTHOOK: Output: default@srcpart_acidvb
+POSTHOOK: Output: default@srcpart_acidvb@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acidvb@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acidvb@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acidvb@ds=2008-04-09/hr=12
+PREHOOK: query: analyze table srcpart_acidvb PARTITION(ds, hr) compute statistics for columns
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_acidvb
+PREHOOK: Input: default@srcpart_acidvb@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart_acidvb@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart_acidvb@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart_acidvb@ds=2008-04-09/hr=12
+PREHOOK: Output: default@srcpart_acidvb
+PREHOOK: Output: default@srcpart_acidvb@ds=2008-04-08/hr=11
+PREHOOK: Output: default@srcpart_acidvb@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_acidvb@ds=2008-04-09/hr=11
+PREHOOK: Output: default@srcpart_acidvb@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table srcpart_acidvb PARTITION(ds, hr) compute statistics for columns
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_acidvb
+POSTHOOK: Input: default@srcpart_acidvb@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart_acidvb@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart_acidvb@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart_acidvb@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@srcpart_acidvb
+POSTHOOK: Output: default@srcpart_acidvb@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@srcpart_acidvb@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_acidvb@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@srcpart_acidvb@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+PREHOOK: query: explain update srcpart_acidvb set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11'
+PREHOOK: type: QUERY
+POSTHOOK: query: explain update srcpart_acidvb set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11'
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_acidvb
+ Statistics: Num rows: 1000 Data size: 362000 Basic stats: COMPLETE Column stats: PARTIAL
+ Filter Operator
+ predicate: (UDFToInteger(key)) IN (413, 43) (type: boolean)
+ Statistics: Num rows: 500 Data size: 181000 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: ROW__ID (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), key (type: string), concat(value, 'updated') (type: string), ds (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 500 D
<TRUNCATED>
[32/50] [abbrv] hive git commit: HIVE-17367 : IMPORT table doesn't
load from data dump if a metadata-only dump was already imported. (Sankar
Hariappan, reviewed by Anishek Agarwal, Thejas Nair)
Posted by we...@apache.org.
HIVE-17367 : IMPORT table doesn't load from data dump if a metadata-only dump was already imported. (Sankar Hariappan, reviewed by Anishek Agarwal, Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6c8f14b7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6c8f14b7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6c8f14b7
Branch: refs/heads/hive-14535
Commit: 6c8f14b78afc0a6721433af6554b0ab45b8e163d
Parents: 9e673a7
Author: Sankar Hariappan <ma...@gmail.com>
Authored: Thu Aug 31 15:51:01 2017 -0700
Committer: Thejas M Nair <th...@hortonworks.com>
Committed: Thu Aug 31 15:51:01 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/parse/TestExport.java | 78 ---------------
.../hadoop/hive/ql/parse/TestExportImport.java | 100 +++++++++++++++++++
.../hadoop/hive/ql/exec/repl/ReplDumpTask.java | 5 +-
.../hive/ql/parse/ExportSemanticAnalyzer.java | 8 ++
.../hive/ql/parse/ImportSemanticAnalyzer.java | 8 +-
.../hadoop/hive/ql/parse/ReplicationSpec.java | 60 +++++++----
.../dump/BootStrapReplicationSpecFunction.java | 1 -
.../hive/ql/parse/repl/dump/TableExport.java | 2 -
.../parse/repl/dump/io/PartitionSerializer.java | 3 +-
.../ql/parse/repl/dump/io/TableSerializer.java | 3 +-
.../parse/repl/load/message/TableHandler.java | 2 +-
11 files changed, 159 insertions(+), 111 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/6c8f14b7/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExport.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExport.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExport.java
deleted file mode 100644
index 1c7c844..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExport.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.parse;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestName;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-public class TestExport {
-
- protected static final Logger LOG = LoggerFactory.getLogger(TestExport.class);
- private static WarehouseInstance hiveWarehouse;
-
- @Rule
- public final TestName testName = new TestName();
- private String dbName;
-
- @BeforeClass
- public static void classLevelSetup() throws Exception {
- Configuration conf = new Configuration();
- conf.set("dfs.client.use.datanode.hostname", "true");
- MiniDFSCluster miniDFSCluster =
- new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build();
- hiveWarehouse = new WarehouseInstance(LOG, miniDFSCluster, false);
- }
-
- @AfterClass
- public static void classLevelTearDown() throws IOException {
- hiveWarehouse.close();
- }
-
- @Before
- public void setup() throws Throwable {
- dbName = testName.getMethodName() + "_" + +System.currentTimeMillis();
- hiveWarehouse.run("create database " + dbName);
- }
-
- @Test
- public void shouldExportImportATemporaryTable() throws Throwable {
- String path = "hdfs:///tmp/" + dbName + "/";
- String exportPath = "'" + path + "'";
- String importDataPath = path + "/data";
- hiveWarehouse
- .run("use " + dbName)
- .run("create temporary table t1 (i int)")
- .run("insert into table t1 values (1),(2)")
- .run("export table t1 to " + exportPath)
- .run("create temporary table t2 like t1")
- .run("load data inpath '" + importDataPath + "' overwrite into table t2")
- .run("select * from t2")
- .verifyResults(new String[] { "1", "2" });
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/6c8f14b7/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java
new file mode 100644
index 0000000..1f19dfd
--- /dev/null
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+public class TestExportImport {
+
+ protected static final Logger LOG = LoggerFactory.getLogger(TestExportImport.class);
+ private static WarehouseInstance srcHiveWarehouse;
+ private static WarehouseInstance destHiveWarehouse;
+
+ @Rule
+ public final TestName testName = new TestName();
+ private String dbName;
+ private String replDbName;
+
+ @BeforeClass
+ public static void classLevelSetup() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set("dfs.client.use.datanode.hostname", "true");
+ MiniDFSCluster miniDFSCluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build();
+ srcHiveWarehouse = new WarehouseInstance(LOG, miniDFSCluster, false);
+ destHiveWarehouse = new WarehouseInstance(LOG, miniDFSCluster, false);
+ }
+
+ @AfterClass
+ public static void classLevelTearDown() throws IOException {
+ srcHiveWarehouse.close();
+ destHiveWarehouse.close();
+ }
+
+ @Before
+ public void setup() throws Throwable {
+ dbName = testName.getMethodName() + "_" + +System.currentTimeMillis();
+ replDbName = dbName + "_dupe";
+ srcHiveWarehouse.run("create database " + dbName);
+ destHiveWarehouse.run("create database " + replDbName);
+ }
+
+ @Test
+ public void shouldExportImportATemporaryTable() throws Throwable {
+ String path = "hdfs:///tmp/" + dbName + "/";
+ String exportPath = "'" + path + "'";
+ String importDataPath = path + "/data";
+ srcHiveWarehouse
+ .run("use " + dbName)
+ .run("create temporary table t1 (i int)")
+ .run("insert into table t1 values (1),(2)")
+ .run("export table t1 to " + exportPath)
+ .run("create temporary table t2 like t1")
+ .run("load data inpath '" + importDataPath + "' overwrite into table t2")
+ .run("select * from t2")
+ .verifyResults(new String[] { "1", "2" });
+ }
+
+ @Test
+ public void dataImportAfterMetadataOnlyImport() throws Throwable {
+ String path = "hdfs:///tmp/" + dbName + "/";
+ String exportMDPath = "'" + path + "1/'";
+ String exportDataPath = "'" + path + "2/'";
+ srcHiveWarehouse.run("create table " + dbName + ".t1 (i int)")
+ .run("insert into table " + dbName + ".t1 values (1),(2)")
+ .run("export table " + dbName + ".t1 to " + exportMDPath + " for metadata replication('1')")
+ .run("export table " + dbName + ".t1 to " + exportDataPath + " for replication('2')");
+
+ destHiveWarehouse.run("import table " + replDbName + ".t1 from " + exportMDPath)
+ .run("import table " + replDbName + ".t1 from " + exportDataPath)
+ .run("select * from " + replDbName + ".t1")
+ .verifyResults(new String[] { "1", "2" });
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/6c8f14b7/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
index d3af0ed..3ebd3cc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
@@ -170,7 +170,7 @@ public class ReplDumpTask extends Task<ReplDumpWork> implements Serializable {
private ReplicationSpec getNewEventOnlyReplicationSpec(Long eventId) throws SemanticException {
ReplicationSpec rspec = getNewReplicationSpec(eventId.toString(), eventId.toString());
- rspec.setIsIncrementalDump(true);
+ rspec.setReplSpecType(ReplicationSpec.Type.INCREMENTAL_DUMP);
return rspec;
}
@@ -260,8 +260,7 @@ public class ReplDumpTask extends Task<ReplDumpWork> implements Serializable {
}
private ReplicationSpec getNewReplicationSpec(String evState, String objState) {
- return new ReplicationSpec(true, false, false, evState, objState,
- false, true, true);
+ return new ReplicationSpec(true, false, evState, objState, false, true, true);
}
private String getNextDumpDir() {
http://git-wip-us.apache.org/repos/asf/hive/blob/6c8f14b7/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
index 74144ac..b8c6ea9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
@@ -46,6 +46,14 @@ public class ExportSemanticAnalyzer extends BaseSemanticAnalyzer {
} else {
replicationSpec = new ReplicationSpec();
}
+ if (replicationSpec.getCurrentReplicationState() == null) {
+ try {
+ long currentEventId = db.getMSC().getCurrentNotificationEventId().getEventId();
+ replicationSpec.setCurrentReplicationState(String.valueOf(currentEventId));
+ } catch (Exception e) {
+ throw new SemanticException("Error when getting current notification event ID", e);
+ }
+ }
// initialize source table/partition
TableSpec ts;
http://git-wip-us.apache.org/repos/asf/hive/blob/6c8f14b7/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
index 606a414..f8d5c8d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
@@ -139,7 +139,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
}
// parsing statement is now done, on to logic.
- tableExists = prepareImport(
+ tableExists = prepareImport(true,
isLocationSet, isExternalSet, isPartSpecSet, waitOnPrecursor,
parsedLocation, parsedTableName, parsedDbName, parsedPartSpec, fromTree.getText(),
new EximUtil.SemanticAnalyzerWrapperContext(conf, db, inputs, outputs, rootTasks, LOG, ctx),
@@ -175,7 +175,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
}
}
- public static boolean prepareImport(
+ public static boolean prepareImport(boolean isImportCmd,
boolean isLocationSet, boolean isExternalSet, boolean isPartSpecSet, boolean waitOnPrecursor,
String parsedLocation, String parsedTableName, String parsedDbName,
LinkedHashMap<String, String> parsedPartSpec,
@@ -205,6 +205,10 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
return false;
}
+ if (isImportCmd) {
+ replicationSpec.setReplSpecType(ReplicationSpec.Type.IMPORT);
+ }
+
String dbname = SessionState.get().getCurrentDatabase();
if ((parsedDbName !=null) && (!parsedDbName.isEmpty())){
// If the parsed statement contained a db.tablename specification, prefer that.
http://git-wip-us.apache.org/repos/asf/hive/blob/6c8f14b7/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java
index 235a44c..f257933 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java
@@ -38,12 +38,12 @@ public class ReplicationSpec {
private boolean isInReplicationScope = false; // default is that it's not in a repl scope
private boolean isMetadataOnly = false; // default is full export/import, not metadata-only
- private boolean isIncrementalDump = false; // default is replv2 bootstrap dump or replv1 export or import/load.
private String eventId = null;
private String currStateId = null;
private boolean isNoop = false;
private boolean isLazy = false; // lazy mode => we only list files, and expect that the eventual copy will pull data in.
private boolean isReplace = true; // default is that the import mode is insert overwrite
+ private Type specType = Type.DEFAULT; // DEFAULT means REPL_LOAD or BOOTSTRAP_DUMP or EXPORT
// Key definitions related to replication
public enum KEY {
@@ -66,9 +66,9 @@ public class ReplicationSpec {
}
}
- public enum SCOPE { NO_REPL, MD_ONLY, REPL };
+ public enum SCOPE { NO_REPL, MD_ONLY, REPL }
- static private Collator collator = Collator.getInstance();
+ public enum Type { DEFAULT, INCREMENTAL_DUMP, IMPORT }
/**
* Constructor to construct spec based on either the ASTNode that
@@ -106,28 +106,27 @@ public class ReplicationSpec {
}
public ReplicationSpec(String fromId, String toId) {
- this(true, false, false, fromId, toId, false, true, false);
+ this(true, false, fromId, toId, false, true, false);
}
public ReplicationSpec(boolean isInReplicationScope, boolean isMetadataOnly,
- boolean isIncrementalDump,
String eventReplicationState, String currentReplicationState,
boolean isNoop, boolean isLazy, boolean isReplace) {
this.isInReplicationScope = isInReplicationScope;
this.isMetadataOnly = isMetadataOnly;
- this.isIncrementalDump = isIncrementalDump;
this.eventId = eventReplicationState;
this.currStateId = currentReplicationState;
this.isNoop = isNoop;
this.isLazy = isLazy;
this.isReplace = isReplace;
+ this.specType = Type.DEFAULT;
}
public ReplicationSpec(Function<String, String> keyFetcher) {
String scope = keyFetcher.apply(ReplicationSpec.KEY.REPL_SCOPE.toString());
this.isInReplicationScope = false;
this.isMetadataOnly = false;
- this.isIncrementalDump = false;
+ this.specType = Type.DEFAULT;
if (scope != null) {
if (scope.equalsIgnoreCase("metadata")) {
this.isMetadataOnly = true;
@@ -155,7 +154,7 @@ public class ReplicationSpec {
* @param replacementReplState Replacement-candidate state
* @return whether or not a provided replacement candidate is newer(or equal) to the existing object state or not
*/
- public static boolean allowReplacement(String currReplState, String replacementReplState){
+ public boolean allowReplacement(String currReplState, String replacementReplState){
if ((currReplState == null) || (currReplState.isEmpty())) {
// if we have no replication state on record for the obj, allow replacement.
return true;
@@ -171,7 +170,16 @@ public class ReplicationSpec {
long currReplStateLong = Long.parseLong(currReplState.replaceAll("\\D",""));
long replacementReplStateLong = Long.parseLong(replacementReplState.replaceAll("\\D",""));
- return ((currReplStateLong - replacementReplStateLong) < 0);
+ // Failure handling of IMPORT command and REPL LOAD commands are different.
+ // IMPORT will set the last repl ID before copying data files and hence need to allow
+ // replacement if loaded from same dump twice after failing to copy in previous attempt.
+ // But, REPL LOAD will set the last repl ID only after the successful copy of data files and
+ // hence need not allow if same event is applied twice.
+ if (specType == Type.IMPORT) {
+ return (currReplStateLong <= replacementReplStateLong);
+ } else {
+ return (currReplStateLong < replacementReplStateLong);
+ }
}
/**
@@ -211,9 +219,17 @@ public class ReplicationSpec {
// -> ^(TOK_REPLICATION $replId $isMetadataOnly)
isInReplicationScope = true;
eventId = PlanUtils.stripQuotes(node.getChild(0).getText());
- if (node.getChildCount() > 1){
- if (node.getChild(1).getText().toLowerCase().equals("metadata")) {
- isMetadataOnly= true;
+ if ((node.getChildCount() > 1)
+ && node.getChild(1).getText().toLowerCase().equals("metadata")) {
+ isMetadataOnly= true;
+ try {
+ if (Long.parseLong(eventId) >= 0) {
+ // If metadata-only dump, then the state of the dump shouldn't be the latest event id as
+ // the data is not yet dumped and shall be dumped in future export.
+ currStateId = eventId;
+ }
+ } catch (Exception ex) {
+ // Ignore the exception and fall through the default currentStateId
}
}
}
@@ -226,21 +242,21 @@ public class ReplicationSpec {
}
/**
- * @return true if this statement is being run for the purposes of replication
+ * @return true if this statement refers to incremental dump operation.
*/
- public boolean isInReplicationScope(){
- return isInReplicationScope;
+ public Type getReplSpecType(){
+ return this.specType;
}
- /**
- * @return true if this statement refers to incremental dump operation.
- */
- public boolean isIncrementalDump(){
- return isIncrementalDump;
+ public void setReplSpecType(Type specType){
+ this.specType = specType;
}
- public void setIsIncrementalDump(boolean isIncrementalDump){
- this.isIncrementalDump = isIncrementalDump;
+ /**
+ * @return true if this statement is being run for the purposes of replication
+ */
+ public boolean isInReplicationScope(){
+ return isInReplicationScope;
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/6c8f14b7/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/BootStrapReplicationSpecFunction.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/BootStrapReplicationSpecFunction.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/BootStrapReplicationSpecFunction.java
index 6a05ea4..8b43110 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/BootStrapReplicationSpecFunction.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/BootStrapReplicationSpecFunction.java
@@ -38,7 +38,6 @@ class BootStrapReplicationSpecFunction implements HiveWrapper.Tuple.Function<Rep
new ReplicationSpec(
true,
false,
- false,
"replv2",
"will-be-set",
false,
http://git-wip-us.apache.org/repos/asf/hive/blob/6c8f14b7/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
index 5eae35a..7533a39 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
@@ -93,8 +93,6 @@ public class TableExport {
private PartitionIterable partitions() throws SemanticException {
try {
- long currentEventId = db.getMSC().getCurrentNotificationEventId().getEventId();
- replicationSpec.setCurrentReplicationState(String.valueOf(currentEventId));
if (tableSpec.tableHandle.isPartitioned()) {
if (tableSpec.specType == TableSpec.SpecType.TABLE_ONLY) {
// TABLE-ONLY, fetch partitions if regular export, don't if metadata-only
http://git-wip-us.apache.org/repos/asf/hive/blob/6c8f14b7/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/PartitionSerializer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/PartitionSerializer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/PartitionSerializer.java
index 2c7414f..ce83523 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/PartitionSerializer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/PartitionSerializer.java
@@ -44,7 +44,8 @@ public class PartitionSerializer implements JsonWriter.Serializer {
if (additionalPropertiesProvider.isInReplicationScope()) {
// Current replication state must be set on the Partition object only for bootstrap dump.
// Event replication State will be null in case of bootstrap dump.
- if (!additionalPropertiesProvider.isIncrementalDump()) {
+ if (additionalPropertiesProvider.getReplSpecType()
+ != ReplicationSpec.Type.INCREMENTAL_DUMP) {
partition.putToParameters(
ReplicationSpec.KEY.CURR_STATE_ID.toString(),
additionalPropertiesProvider.getCurrentReplicationState());
http://git-wip-us.apache.org/repos/asf/hive/blob/6c8f14b7/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java
index c443e53..92408a4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java
@@ -70,7 +70,8 @@ public class TableSerializer implements JsonWriter.Serializer {
if (additionalPropertiesProvider.isInReplicationScope()) {
// Current replication state must be set on the Table object only for bootstrap dump.
// Event replication State will be null in case of bootstrap dump.
- if (!additionalPropertiesProvider.isIncrementalDump()) {
+ if (additionalPropertiesProvider.getReplSpecType()
+ != ReplicationSpec.Type.INCREMENTAL_DUMP) {
table.putToParameters(
ReplicationSpec.KEY.CURR_STATE_ID.toString(),
additionalPropertiesProvider.getCurrentReplicationState());
http://git-wip-us.apache.org/repos/asf/hive/blob/6c8f14b7/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TableHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TableHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TableHandler.java
index 2c5c2d9..4ba6256 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TableHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TableHandler.java
@@ -45,7 +45,7 @@ public class TableHandler extends AbstractMessageHandler {
// REPL LOAD is not partition level. It is always DB or table level. So, passing null for partition specs.
// Also, REPL LOAD doesn't support external table and hence no location set as well.
- ImportSemanticAnalyzer.prepareImport(false, false, false,
+ ImportSemanticAnalyzer.prepareImport(false, false, false, false,
(context.precursor != null), null, context.tableName, context.dbName,
null, context.location, x, updatedMetadata);
[28/50] [abbrv] hive git commit: HIVE-17216: Additional qtests for
HoS DPP (Sahil Takiar, reviewed by Sergio Pena)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/642acdf7/ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_3.q.out b/ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_3.q.out
index 82e746a..3ae922f 100644
--- a/ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_3.q.out
+++ b/ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning_3.q.out
@@ -1,65 +1,1307 @@
-PREHOOK: query: create table partitioned_table1 (col int) partitioned by (part_col int)
+PREHOOK: query: CREATE TABLE partitioned_table1 (col int) PARTITIONED BY (part_col int)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@partitioned_table1
-POSTHOOK: query: create table partitioned_table1 (col int) partitioned by (part_col int)
+POSTHOOK: query: CREATE TABLE partitioned_table1 (col int) PARTITIONED BY (part_col int)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@partitioned_table1
-PREHOOK: query: create table partitioned_table2 (col int) partitioned by (part_col int)
+PREHOOK: query: CREATE TABLE partitioned_table2 (col int) PARTITIONED BY (part_col int)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@partitioned_table2
-POSTHOOK: query: create table partitioned_table2 (col int) partitioned by (part_col int)
+POSTHOOK: query: CREATE TABLE partitioned_table2 (col int) PARTITIONED BY (part_col int)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@partitioned_table2
-PREHOOK: query: create table regular_table (col int)
+PREHOOK: query: CREATE TABLE partitioned_table3 (col int) PARTITIONED BY (part_col int)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
-PREHOOK: Output: default@regular_table
-POSTHOOK: query: create table regular_table (col int)
+PREHOOK: Output: default@partitioned_table3
+POSTHOOK: query: CREATE TABLE partitioned_table3 (col int) PARTITIONED BY (part_col int)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
-POSTHOOK: Output: default@regular_table
-PREHOOK: query: insert into table regular_table values (1)
-PREHOOK: type: QUERY
-PREHOOK: Output: default@regular_table
-POSTHOOK: query: insert into table regular_table values (1)
-POSTHOOK: type: QUERY
-POSTHOOK: Output: default@regular_table
-POSTHOOK: Lineage: regular_table.col EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-PREHOOK: query: alter table partitioned_table1 add partition (part_col = 1)
+POSTHOOK: Output: default@partitioned_table3
+PREHOOK: query: CREATE TABLE partitioned_table4 (col int) PARTITIONED BY (part_col1 int, part_col2 int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@partitioned_table4
+POSTHOOK: query: CREATE TABLE partitioned_table4 (col int) PARTITIONED BY (part_col1 int, part_col2 int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@partitioned_table4
+PREHOOK: query: CREATE TABLE partitioned_table5 (col int) PARTITIONED BY (part_col1 int, part_col2 int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@partitioned_table5
+POSTHOOK: query: CREATE TABLE partitioned_table5 (col int) PARTITIONED BY (part_col1 int, part_col2 int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@partitioned_table5
+PREHOOK: query: CREATE TABLE regular_table1 (col1 int, col2 int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@regular_table1
+POSTHOOK: query: CREATE TABLE regular_table1 (col1 int, col2 int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@regular_table1
+PREHOOK: query: CREATE TABLE regular_table2 (col1 int, col2 int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@regular_table2
+POSTHOOK: query: CREATE TABLE regular_table2 (col1 int, col2 int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@regular_table2
+PREHOOK: query: ALTER TABLE partitioned_table1 ADD PARTITION (part_col = 1)
PREHOOK: type: ALTERTABLE_ADDPARTS
PREHOOK: Output: default@partitioned_table1
-POSTHOOK: query: alter table partitioned_table1 add partition (part_col = 1)
+POSTHOOK: query: ALTER TABLE partitioned_table1 ADD PARTITION (part_col = 1)
POSTHOOK: type: ALTERTABLE_ADDPARTS
POSTHOOK: Output: default@partitioned_table1
POSTHOOK: Output: default@partitioned_table1@part_col=1
-PREHOOK: query: insert into table partitioned_table1 partition (part_col = 1) values (1), (2), (3), (4), (5), (6), (7), (8), (9), (10)
-PREHOOK: type: QUERY
-PREHOOK: Output: default@partitioned_table1@part_col=1
-POSTHOOK: query: insert into table partitioned_table1 partition (part_col = 1) values (1), (2), (3), (4), (5), (6), (7), (8), (9), (10)
-POSTHOOK: type: QUERY
-POSTHOOK: Output: default@partitioned_table1@part_col=1
-POSTHOOK: Lineage: partitioned_table1 PARTITION(part_col=1).col EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-PREHOOK: query: alter table partitioned_table2 add partition (part_col = 1)
+PREHOOK: query: ALTER TABLE partitioned_table1 ADD PARTITION (part_col = 2)
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@partitioned_table1
+POSTHOOK: query: ALTER TABLE partitioned_table1 ADD PARTITION (part_col = 2)
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@partitioned_table1
+POSTHOOK: Output: default@partitioned_table1@part_col=2
+PREHOOK: query: ALTER TABLE partitioned_table1 ADD PARTITION (part_col = 3)
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@partitioned_table1
+POSTHOOK: query: ALTER TABLE partitioned_table1 ADD PARTITION (part_col = 3)
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@partitioned_table1
+POSTHOOK: Output: default@partitioned_table1@part_col=3
+PREHOOK: query: ALTER TABLE partitioned_table2 ADD PARTITION (part_col = 1)
PREHOOK: type: ALTERTABLE_ADDPARTS
PREHOOK: Output: default@partitioned_table2
-POSTHOOK: query: alter table partitioned_table2 add partition (part_col = 1)
+POSTHOOK: query: ALTER TABLE partitioned_table2 ADD PARTITION (part_col = 1)
POSTHOOK: type: ALTERTABLE_ADDPARTS
POSTHOOK: Output: default@partitioned_table2
POSTHOOK: Output: default@partitioned_table2@part_col=1
-PREHOOK: query: insert into table partitioned_table2 partition (part_col = 1) values (1), (2), (3), (4), (5), (6), (7), (8), (9), (10)
+PREHOOK: query: ALTER TABLE partitioned_table2 ADD PARTITION (part_col = 2)
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@partitioned_table2
+POSTHOOK: query: ALTER TABLE partitioned_table2 ADD PARTITION (part_col = 2)
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@partitioned_table2
+POSTHOOK: Output: default@partitioned_table2@part_col=2
+PREHOOK: query: ALTER TABLE partitioned_table2 ADD PARTITION (part_col = 3)
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@partitioned_table2
+POSTHOOK: query: ALTER TABLE partitioned_table2 ADD PARTITION (part_col = 3)
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@partitioned_table2
+POSTHOOK: Output: default@partitioned_table2@part_col=3
+PREHOOK: query: ALTER TABLE partitioned_table3 ADD PARTITION (part_col = 1)
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@partitioned_table3
+POSTHOOK: query: ALTER TABLE partitioned_table3 ADD PARTITION (part_col = 1)
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@partitioned_table3
+POSTHOOK: Output: default@partitioned_table3@part_col=1
+PREHOOK: query: ALTER TABLE partitioned_table3 ADD PARTITION (part_col = 2)
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@partitioned_table3
+POSTHOOK: query: ALTER TABLE partitioned_table3 ADD PARTITION (part_col = 2)
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@partitioned_table3
+POSTHOOK: Output: default@partitioned_table3@part_col=2
+PREHOOK: query: ALTER TABLE partitioned_table3 ADD PARTITION (part_col = 3)
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@partitioned_table3
+POSTHOOK: query: ALTER TABLE partitioned_table3 ADD PARTITION (part_col = 3)
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@partitioned_table3
+POSTHOOK: Output: default@partitioned_table3@part_col=3
+PREHOOK: query: ALTER TABLE partitioned_table4 ADD PARTITION (part_col1 = 1, part_col2 = 1)
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@partitioned_table4
+POSTHOOK: query: ALTER TABLE partitioned_table4 ADD PARTITION (part_col1 = 1, part_col2 = 1)
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@partitioned_table4
+POSTHOOK: Output: default@partitioned_table4@part_col1=1/part_col2=1
+PREHOOK: query: ALTER TABLE partitioned_table4 ADD PARTITION (part_col1 = 2, part_col2 = 2)
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@partitioned_table4
+POSTHOOK: query: ALTER TABLE partitioned_table4 ADD PARTITION (part_col1 = 2, part_col2 = 2)
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@partitioned_table4
+POSTHOOK: Output: default@partitioned_table4@part_col1=2/part_col2=2
+PREHOOK: query: ALTER TABLE partitioned_table4 ADD PARTITION (part_col1 = 3, part_col2 = 3)
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@partitioned_table4
+POSTHOOK: query: ALTER TABLE partitioned_table4 ADD PARTITION (part_col1 = 3, part_col2 = 3)
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@partitioned_table4
+POSTHOOK: Output: default@partitioned_table4@part_col1=3/part_col2=3
+PREHOOK: query: ALTER TABLE partitioned_table5 ADD PARTITION (part_col1 = 1, part_col2 = 1)
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@partitioned_table5
+POSTHOOK: query: ALTER TABLE partitioned_table5 ADD PARTITION (part_col1 = 1, part_col2 = 1)
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@partitioned_table5
+POSTHOOK: Output: default@partitioned_table5@part_col1=1/part_col2=1
+PREHOOK: query: ALTER TABLE partitioned_table5 ADD PARTITION (part_col1 = 2, part_col2 = 2)
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@partitioned_table5
+POSTHOOK: query: ALTER TABLE partitioned_table5 ADD PARTITION (part_col1 = 2, part_col2 = 2)
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@partitioned_table5
+POSTHOOK: Output: default@partitioned_table5@part_col1=2/part_col2=2
+PREHOOK: query: ALTER TABLE partitioned_table5 ADD PARTITION (part_col1 = 3, part_col2 = 3)
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@partitioned_table5
+POSTHOOK: query: ALTER TABLE partitioned_table5 ADD PARTITION (part_col1 = 3, part_col2 = 3)
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@partitioned_table5
+POSTHOOK: Output: default@partitioned_table5@part_col1=3/part_col2=3
+PREHOOK: query: INSERT INTO TABLE regular_table1 VALUES (0, 0), (1, 1), (2, 2)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@regular_table1
+POSTHOOK: query: INSERT INTO TABLE regular_table1 VALUES (0, 0), (1, 1), (2, 2)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@regular_table1
+POSTHOOK: Lineage: regular_table1.col1 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: regular_table1.col2 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+PREHOOK: query: INSERT INTO TABLE regular_table2 VALUES (0, 0), (1, 1), (2, 2)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@regular_table2
+POSTHOOK: query: INSERT INTO TABLE regular_table2 VALUES (0, 0), (1, 1), (2, 2)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@regular_table2
+POSTHOOK: Lineage: regular_table2.col1 EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: regular_table2.col2 EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+PREHOOK: query: INSERT INTO TABLE partitioned_table1 PARTITION (part_col = 1) VALUES (1), (2), (3), (4), (5), (6)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@partitioned_table1@part_col=1
+POSTHOOK: query: INSERT INTO TABLE partitioned_table1 PARTITION (part_col = 1) VALUES (1), (2), (3), (4), (5), (6)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@partitioned_table1@part_col=1
+POSTHOOK: Lineage: partitioned_table1 PARTITION(part_col=1).col EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: INSERT INTO TABLE partitioned_table1 PARTITION (part_col = 2) VALUES (1), (2), (3), (4), (5), (6)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@partitioned_table1@part_col=2
+POSTHOOK: query: INSERT INTO TABLE partitioned_table1 PARTITION (part_col = 2) VALUES (1), (2), (3), (4), (5), (6)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@partitioned_table1@part_col=2
+POSTHOOK: Lineage: partitioned_table1 PARTITION(part_col=2).col EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: INSERT INTO TABLE partitioned_table1 PARTITION (part_col = 3) VALUES (1), (2), (3), (4), (5), (6)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@partitioned_table1@part_col=3
+POSTHOOK: query: INSERT INTO TABLE partitioned_table1 PARTITION (part_col = 3) VALUES (1), (2), (3), (4), (5), (6)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@partitioned_table1@part_col=3
+POSTHOOK: Lineage: partitioned_table1 PARTITION(part_col=3).col EXPRESSION [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: INSERT INTO TABLE partitioned_table2 PARTITION (part_col = 1) VALUES (1), (2), (3), (4), (5), (6)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@partitioned_table2@part_col=1
+POSTHOOK: query: INSERT INTO TABLE partitioned_table2 PARTITION (part_col = 1) VALUES (1), (2), (3), (4), (5), (6)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@partitioned_table2@part_col=1
+POSTHOOK: Lineage: partitioned_table2 PARTITION(part_col=1).col EXPRESSION [(values__tmp__table__6)values__tmp__table__6.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: INSERT INTO TABLE partitioned_table2 PARTITION (part_col = 2) VALUES (1), (2), (3), (4), (5), (6)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@partitioned_table2@part_col=2
+POSTHOOK: query: INSERT INTO TABLE partitioned_table2 PARTITION (part_col = 2) VALUES (1), (2), (3), (4), (5), (6)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@partitioned_table2@part_col=2
+POSTHOOK: Lineage: partitioned_table2 PARTITION(part_col=2).col EXPRESSION [(values__tmp__table__7)values__tmp__table__7.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: INSERT INTO TABLE partitioned_table2 PARTITION (part_col = 3) VALUES (1), (2), (3), (4), (5), (6)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@partitioned_table2@part_col=3
+POSTHOOK: query: INSERT INTO TABLE partitioned_table2 PARTITION (part_col = 3) VALUES (1), (2), (3), (4), (5), (6)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@partitioned_table2@part_col=3
+POSTHOOK: Lineage: partitioned_table2 PARTITION(part_col=3).col EXPRESSION [(values__tmp__table__8)values__tmp__table__8.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: INSERT INTO TABLE partitioned_table3 PARTITION (part_col = 1) VALUES (1), (2), (3), (4), (5), (6)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@partitioned_table3@part_col=1
+POSTHOOK: query: INSERT INTO TABLE partitioned_table3 PARTITION (part_col = 1) VALUES (1), (2), (3), (4), (5), (6)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@partitioned_table3@part_col=1
+POSTHOOK: Lineage: partitioned_table3 PARTITION(part_col=1).col EXPRESSION [(values__tmp__table__9)values__tmp__table__9.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: INSERT INTO TABLE partitioned_table3 PARTITION (part_col = 2) VALUES (1), (2), (3), (4), (5), (6)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@partitioned_table3@part_col=2
+POSTHOOK: query: INSERT INTO TABLE partitioned_table3 PARTITION (part_col = 2) VALUES (1), (2), (3), (4), (5), (6)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@partitioned_table3@part_col=2
+POSTHOOK: Lineage: partitioned_table3 PARTITION(part_col=2).col EXPRESSION [(values__tmp__table__10)values__tmp__table__10.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: INSERT INTO TABLE partitioned_table3 PARTITION (part_col = 3) VALUES (1), (2), (3), (4), (5), (6)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@partitioned_table3@part_col=3
+POSTHOOK: query: INSERT INTO TABLE partitioned_table3 PARTITION (part_col = 3) VALUES (1), (2), (3), (4), (5), (6)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@partitioned_table3@part_col=3
+POSTHOOK: Lineage: partitioned_table3 PARTITION(part_col=3).col EXPRESSION [(values__tmp__table__11)values__tmp__table__11.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: INSERT INTO TABLE partitioned_table4 PARTITION (part_col1 = 1, part_col2 = 1) VALUES (1), (2), (3), (4), (5), (6)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@partitioned_table4@part_col1=1/part_col2=1
+POSTHOOK: query: INSERT INTO TABLE partitioned_table4 PARTITION (part_col1 = 1, part_col2 = 1) VALUES (1), (2), (3), (4), (5), (6)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@partitioned_table4@part_col1=1/part_col2=1
+POSTHOOK: Lineage: partitioned_table4 PARTITION(part_col1=1,part_col2=1).col EXPRESSION [(values__tmp__table__12)values__tmp__table__12.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: INSERT INTO TABLE partitioned_table4 PARTITION (part_col1 = 2, part_col2 = 2) VALUES (1), (2), (3), (4), (5), (6)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@partitioned_table4@part_col1=2/part_col2=2
+POSTHOOK: query: INSERT INTO TABLE partitioned_table4 PARTITION (part_col1 = 2, part_col2 = 2) VALUES (1), (2), (3), (4), (5), (6)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@partitioned_table4@part_col1=2/part_col2=2
+POSTHOOK: Lineage: partitioned_table4 PARTITION(part_col1=2,part_col2=2).col EXPRESSION [(values__tmp__table__13)values__tmp__table__13.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: INSERT INTO TABLE partitioned_table4 PARTITION (part_col1 = 3, part_col2 = 3) VALUES (1), (2), (3), (4), (5), (6)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@partitioned_table4@part_col1=3/part_col2=3
+POSTHOOK: query: INSERT INTO TABLE partitioned_table4 PARTITION (part_col1 = 3, part_col2 = 3) VALUES (1), (2), (3), (4), (5), (6)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@partitioned_table4@part_col1=3/part_col2=3
+POSTHOOK: Lineage: partitioned_table4 PARTITION(part_col1=3,part_col2=3).col EXPRESSION [(values__tmp__table__14)values__tmp__table__14.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: INSERT INTO TABLE partitioned_table5 PARTITION (part_col1 = 1, part_col2 = 1) VALUES (1), (2), (3), (4), (5), (6)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@partitioned_table5@part_col1=1/part_col2=1
+POSTHOOK: query: INSERT INTO TABLE partitioned_table5 PARTITION (part_col1 = 1, part_col2 = 1) VALUES (1), (2), (3), (4), (5), (6)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@partitioned_table5@part_col1=1/part_col2=1
+POSTHOOK: Lineage: partitioned_table5 PARTITION(part_col1=1,part_col2=1).col EXPRESSION [(values__tmp__table__15)values__tmp__table__15.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: INSERT INTO TABLE partitioned_table5 PARTITION (part_col1 = 2, part_col2 = 2) VALUES (1), (2), (3), (4), (5), (6)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@partitioned_table5@part_col1=2/part_col2=2
+POSTHOOK: query: INSERT INTO TABLE partitioned_table5 PARTITION (part_col1 = 2, part_col2 = 2) VALUES (1), (2), (3), (4), (5), (6)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@partitioned_table5@part_col1=2/part_col2=2
+POSTHOOK: Lineage: partitioned_table5 PARTITION(part_col1=2,part_col2=2).col EXPRESSION [(values__tmp__table__16)values__tmp__table__16.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: INSERT INTO TABLE partitioned_table5 PARTITION (part_col1 = 3, part_col2 = 3) VALUES (1), (2), (3), (4), (5), (6)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@partitioned_table5@part_col1=3/part_col2=3
+POSTHOOK: query: INSERT INTO TABLE partitioned_table5 PARTITION (part_col1 = 3, part_col2 = 3) VALUES (1), (2), (3), (4), (5), (6)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@partitioned_table5@part_col1=3/part_col2=3
+POSTHOOK: Lineage: partitioned_table5 PARTITION(part_col1=3,part_col2=3).col EXPRESSION [(values__tmp__table__17)values__tmp__table__17.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: -- ensure the partitioned tables are treated as big tables
+
+
+EXPLAIN SELECT count(*) FROM partitioned_table1 WHERE partitioned_table1.part_col IN (
+SELECT regular_table1.col1 FROM regular_table1 JOIN partitioned_table2 ON
+regular_table1.col1 = partitioned_table2.part_col AND partitioned_table2.col > 3 AND regular_table1.col1 > 1)
+PREHOOK: type: QUERY
+POSTHOOK: query: -- ensure the partitioned tables are treated as big tables
+
+
+EXPLAIN SELECT count(*) FROM partitioned_table1 WHERE partitioned_table1.part_col IN (
+SELECT regular_table1.col1 FROM regular_table1 JOIN partitioned_table2 ON
+regular_table1.col1 = partitioned_table2.part_col AND partitioned_table2.col > 3 AND regular_table1.col1 > 1)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-3 is a root stage
+ Stage-2 depends on stages: Stage-3
+ Stage-1 depends on stages: Stage-2
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-3
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: regular_table1
+ Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (col1 > 1) (type: boolean)
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: col1 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col0 (type: int)
+ 1 _col1 (type: int)
+ Select Operator
+ expressions: _col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: int)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Spark Partition Pruning Sink Operator
+ Target column: part_col (int)
+ partition key expr: part_col
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ target work: Map 4
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-2
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: partitioned_table2
+ Statistics: Num rows: 12 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (col > 3) (type: boolean)
+ Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: part_col (type: int)
+ outputColumnNames: _col1
+ Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col1 (type: int)
+ outputColumnNames: _col0
+ input vertices:
+ 0 Map 3
+ Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: int)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ Select Operator
+ expressions: _col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: int)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ Spark Partition Pruning Sink Operator
+ Target column: part_col (int)
+ partition key expr: part_col
+ Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ target work: Map 1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 2 <- Map 1 (GROUP, 1)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: partitioned_table1
+ Statistics: Num rows: 12 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: part_col (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 12 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Semi Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ input vertices:
+ 1 Map 4
+ Statistics: Num rows: 13 Data size: 13 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Local Work:
+ Map Reduce Local Work
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT count(*)
+FROM partitioned_table1
+WHERE partitioned_table1.part_col IN (SELECT regular_table1.col1
+ FROM regular_table1
+ JOIN partitioned_table2
+ ON
+ regular_table1.col1 = partitioned_table2.part_col AND partitioned_table2.col > 3 AND regular_table1.col1 > 1)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@partitioned_table1
+PREHOOK: Input: default@partitioned_table1@part_col=2
+PREHOOK: Input: default@partitioned_table1@part_col=3
+PREHOOK: Input: default@partitioned_table2
+PREHOOK: Input: default@partitioned_table2@part_col=2
+PREHOOK: Input: default@partitioned_table2@part_col=3
+PREHOOK: Input: default@regular_table1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT count(*)
+FROM partitioned_table1
+WHERE partitioned_table1.part_col IN (SELECT regular_table1.col1
+ FROM regular_table1
+ JOIN partitioned_table2
+ ON
+ regular_table1.col1 = partitioned_table2.part_col AND partitioned_table2.col > 3 AND regular_table1.col1 > 1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@partitioned_table1
+POSTHOOK: Input: default@partitioned_table1@part_col=2
+POSTHOOK: Input: default@partitioned_table1@part_col=3
+POSTHOOK: Input: default@partitioned_table2
+POSTHOOK: Input: default@partitioned_table2@part_col=2
+POSTHOOK: Input: default@partitioned_table2@part_col=3
+POSTHOOK: Input: default@regular_table1
+#### A masked pattern was here ####
+6
+PREHOOK: query: EXPLAIN SELECT count(*) FROM partitioned_table1, regular_table1 rt1,
+regular_table1 rt2 WHERE rt1.col1 = partitioned_table1.part_col AND rt2.col1 =
+partitioned_table1.part_col AND rt1.col2 > 0 AND rt2.col2 > 1
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT count(*) FROM partitioned_table1, regular_table1 rt1,
+regular_table1 rt2 WHERE rt1.col1 = partitioned_table1.part_col AND rt2.col1 =
+partitioned_table1.part_col AND rt1.col2 > 0 AND rt2.col2 > 1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-2
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: rt1
+ Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((col2 > 0) and col1 is not null) (type: boolean)
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: col1 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ 2 _col0 (type: int)
+ Select Operator
+ expressions: _col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: int)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Spark Partition Pruning Sink Operator
+ Target column: part_col (int)
+ partition key expr: part_col
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ target work: Map 1
+ Local Work:
+ Map Reduce Local Work
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: rt2
+ Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((col2 > 1) and col1 is not null) (type: boolean)
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: col1 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ 2 _col0 (type: int)
+ Select Operator
+ expressions: _col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: int)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Spark Partition Pruning Sink Operator
+ Target column: part_col (int)
+ partition key expr: part_col
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ target work: Map 1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 2 <- Map 1 (GROUP, 1)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: partitioned_table1
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: part_col (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ 2 _col0 (type: int)
+ input vertices:
+ 1 Map 3
+ 2 Map 4
+ Statistics: Num rows: 39 Data size: 39 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Local Work:
+ Map Reduce Local Work
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT count(*)
+FROM partitioned_table1,
+ regular_table1 rt1,
+ regular_table1 rt2
+WHERE rt1.col1 = partitioned_table1.part_col
+ AND rt2.col1 = partitioned_table1.part_col
+ AND rt1.col2 > 0
+ AND rt2.col2 > 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@partitioned_table1
+PREHOOK: Input: default@partitioned_table1@part_col=1
+PREHOOK: Input: default@partitioned_table1@part_col=2
+PREHOOK: Input: default@partitioned_table1@part_col=3
+PREHOOK: Input: default@regular_table1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT count(*)
+FROM partitioned_table1,
+ regular_table1 rt1,
+ regular_table1 rt2
+WHERE rt1.col1 = partitioned_table1.part_col
+ AND rt2.col1 = partitioned_table1.part_col
+ AND rt1.col2 > 0
+ AND rt2.col2 > 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@partitioned_table1
+POSTHOOK: Input: default@partitioned_table1@part_col=1
+POSTHOOK: Input: default@partitioned_table1@part_col=2
+POSTHOOK: Input: default@partitioned_table1@part_col=3
+POSTHOOK: Input: default@regular_table1
+#### A masked pattern was here ####
+6
+Warning: Shuffle Join JOIN[14][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product
+PREHOOK: query: EXPLAIN SELECT count(*) FROM partitioned_table1, partitioned_table2, regular_table1 rt1
+, regular_table1 rt2 WHERE rt1.col1 = partitioned_table1.part_col AND rt2.col1 =
+partitioned_table2.part_col AND rt1.col2 > 0 AND rt2.col2 > 1
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT count(*) FROM partitioned_table1, partitioned_table2, regular_table1 rt1
+, regular_table1 rt2 WHERE rt1.col1 = partitioned_table1.part_col AND rt2.col1 =
+partitioned_table2.part_col AND rt1.col2 > 0 AND rt2.col2 > 1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-2
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 5
+ Map Operator Tree:
+ TableScan
+ alias: rt1
+ Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((col2 > 0) and col1 is not null) (type: boolean)
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: col1 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ Select Operator
+ expressions: _col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: int)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Spark Partition Pruning Sink Operator
+ Target column: part_col (int)
+ partition key expr: part_col
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ target work: Map 1
+ Local Work:
+ Map Reduce Local Work
+ Map 6
+ Map Operator Tree:
+ TableScan
+ alias: rt2
+ Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((col2 > 1) and col1 is not null) (type: boolean)
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: col1 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ Select Operator
+ expressions: _col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: int)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Spark Partition Pruning Sink Operator
+ Target column: part_col (int)
+ partition key expr: part_col
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ target work: Map 4
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 3 <- Reducer 2 (GROUP, 1)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: partitioned_table1
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: part_col (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int)
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: partitioned_table2
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: part_col (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int)
+ Reducer 2
+ Local Work:
+ Map Reduce Local Work
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0
+ 1
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 324 Data size: 972 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1
+ input vertices:
+ 1 Map 5
+ Statistics: Num rows: 356 Data size: 1069 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ input vertices:
+ 1 Map 6
+ Statistics: Num rows: 391 Data size: 1175 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reducer 3
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+Warning: Shuffle Join JOIN[14][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product
+PREHOOK: query: SELECT count(*)
+FROM partitioned_table1,
+ partitioned_table2,
+ regular_table1 rt1,
+ regular_table1 rt2
+WHERE rt1.col1 = partitioned_table1.part_col
+ AND rt2.col1 = partitioned_table2.part_col
+ AND rt1.col2 > 0
+ AND rt2.col2 > 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@partitioned_table1
+PREHOOK: Input: default@partitioned_table1@part_col=1
+PREHOOK: Input: default@partitioned_table1@part_col=2
+PREHOOK: Input: default@partitioned_table1@part_col=3
+PREHOOK: Input: default@partitioned_table2
+PREHOOK: Input: default@partitioned_table2@part_col=1
+PREHOOK: Input: default@partitioned_table2@part_col=2
+PREHOOK: Input: default@partitioned_table2@part_col=3
+PREHOOK: Input: default@regular_table1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT count(*)
+FROM partitioned_table1,
+ partitioned_table2,
+ regular_table1 rt1,
+ regular_table1 rt2
+WHERE rt1.col1 = partitioned_table1.part_col
+ AND rt2.col1 = partitioned_table2.part_col
+ AND rt1.col2 > 0
+ AND rt2.col2 > 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@partitioned_table1
+POSTHOOK: Input: default@partitioned_table1@part_col=1
+POSTHOOK: Input: default@partitioned_table1@part_col=2
+POSTHOOK: Input: default@partitioned_table1@part_col=3
+POSTHOOK: Input: default@partitioned_table2
+POSTHOOK: Input: default@partitioned_table2@part_col=1
+POSTHOOK: Input: default@partitioned_table2@part_col=2
+POSTHOOK: Input: default@partitioned_table2@part_col=3
+POSTHOOK: Input: default@regular_table1
+#### A masked pattern was here ####
+72
+PREHOOK: query: EXPLAIN SELECT count(*) FROM regular_table1, partitioned_table1 WHERE regular_table1.col1
+= partitioned_table1.part_col AND regular_table1.col2 BETWEEN 1 AND 3
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT count(*) FROM regular_table1, partitioned_table1 WHERE regular_table1.col1
+= partitioned_table1.part_col AND regular_table1.col2 BETWEEN 1 AND 3
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-2
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: regular_table1
+ Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (col2 BETWEEN 1 AND 3 and col1 is not null) (type: boolean)
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: col1 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ Select Operator
+ expressions: _col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: int)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Spark Partition Pruning Sink Operator
+ Target column: part_col (int)
+ partition key expr: part_col
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ target work: Map 2
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 3 <- Map 2 (GROUP, 1)
+#### A masked pattern was here ####
+ Vertices:
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: partitioned_table1
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: part_col (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ input vertices:
+ 0 Map 1
+ Statistics: Num rows: 19 Data size: 19 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Local Work:
+ Map Reduce Local Work
+ Reducer 3
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT count(*)
+FROM regular_table1,
+ partitioned_table1
+WHERE regular_table1.col1 = partitioned_table1.part_col
+ AND regular_table1.col2 BETWEEN 1 AND 3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@partitioned_table1
+PREHOOK: Input: default@partitioned_table1@part_col=1
+PREHOOK: Input: default@partitioned_table1@part_col=2
+PREHOOK: Input: default@partitioned_table1@part_col=3
+PREHOOK: Input: default@regular_table1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT count(*)
+FROM regular_table1,
+ partitioned_table1
+WHERE regular_table1.col1 = partitioned_table1.part_col
+ AND regular_table1.col2 BETWEEN 1 AND 3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@partitioned_table1
+POSTHOOK: Input: default@partitioned_table1@part_col=1
+POSTHOOK: Input: default@partitioned_table1@part_col=2
+POSTHOOK: Input: default@partitioned_table1@part_col=3
+POSTHOOK: Input: default@regular_table1
+#### A masked pattern was here ####
+12
+PREHOOK: query: EXPLAIN WITH q1 AS (SELECT regular_table1.col1 AS col FROM regular_table1 WHERE
+regular_table1.col2 > 1), q2 AS (SELECT partitioned_table1.part_col AS col FROM
+partitioned_table1 WHERE partitioned_table1.col > 1) SELECT count(*) FROM q1 JOIN q2 ON
+q1.col = q2.col
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN WITH q1 AS (SELECT regular_table1.col1 AS col FROM regular_table1 WHERE
+regular_table1.col2 > 1), q2 AS (SELECT partitioned_table1.part_col AS col FROM
+partitioned_table1 WHERE partitioned_table1.col > 1) SELECT count(*) FROM q1 JOIN q2 ON
+q1.col = q2.col
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-2
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: regular_table1
+ Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((col2 > 1) and col1 is not null) (type: boolean)
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: col1 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ Select Operator
+ expressions: _col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: int)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Spark Partition Pruning Sink Operator
+ Target column: part_col (int)
+ partition key expr: part_col
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ target work: Map 2
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 3 <- Map 2 (GROUP, 1)
+#### A masked pattern was here ####
+ Vertices:
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: partitioned_table1
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (col > 1) (type: boolean)
+ Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: part_col (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ input vertices:
+ 0 Map 1
+ Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Local Work:
+ Map Reduce Local Work
+ Reducer 3
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: WITH q1
+ AS (SELECT regular_table1.col1 AS col
+ FROM regular_table1
+ WHERE regular_table1.col2 > 1),
+ q2
+ AS (SELECT partitioned_table1.part_col AS col
+ FROM partitioned_table1
+ WHERE partitioned_table1.col > 1)
+SELECT count(*)
+FROM q1
+ JOIN q2
+ ON q1.col = q2.col
+PREHOOK: type: QUERY
+PREHOOK: Input: default@partitioned_table1
+PREHOOK: Input: default@partitioned_table1@part_col=1
+PREHOOK: Input: default@partitioned_table1@part_col=2
+PREHOOK: Input: default@partitioned_table1@part_col=3
+PREHOOK: Input: default@regular_table1
+#### A masked pattern was here ####
+POSTHOOK: query: WITH q1
+ AS (SELECT regular_table1.col1 AS col
+ FROM regular_table1
+ WHERE regular_table1.col2 > 1),
+ q2
+ AS (SELECT partitioned_table1.part_col AS col
+ FROM partitioned_table1
+ WHERE partitioned_table1.col > 1)
+SELECT count(*)
+FROM q1
+ JOIN q2
+ ON q1.col = q2.col
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@partitioned_table1
+POSTHOOK: Input: default@partitioned_table1@part_col=1
+POSTHOOK: Input: default@partitioned_table1@part_col=2
+POSTHOOK: Input: default@partitioned_table1@part_col=3
+POSTHOOK: Input: default@regular_table1
+#### A masked pattern was here ####
+5
+PREHOOK: query: EXPLAIN SELECT count(*) FROM partitioned_table1, partitioned_table2 WHERE
+partitioned_table1.part_col = partitioned_table2.part_col AND partitioned_table2.col > 1
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT count(*) FROM partitioned_table1, partitioned_table2 WHERE
+partitioned_table1.part_col = partitioned_table2.part_col AND partitioned_table2.col > 1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-2
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: partitioned_table2
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (col > 1) (type: boolean)
+ Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: part_col (type: int)
+ outputColumnNames: _col1
+ Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col0 (type: int)
+ 1 _col1 (type: int)
+ Select Operator
+ expressions: _col1 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: int)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ Spark Partition Pruning Sink Operator
+ Target column: part_col (int)
+ partition key expr: part_col
+ Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ target work: Map 1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 2 <- Map 1 (GROUP, 1)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: partitioned_table1
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: part_col (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col1 (type: int)
+ input vertices:
+ 1 Map 3
+ Statistics: Num rows: 19 Data size: 19 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Local Work:
+ Map Reduce Local Work
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT count(*)
+FROM partitioned_table1,
+ partitioned_table2
+WHERE partitioned_table1.part_col = partitioned_table2.part_col
+ AND partitioned_table2.col > 1
PREHOOK: type: QUERY
-PREHOOK: Output: default@partitioned_table2@part_col=1
-POSTHOOK: query: insert into table partitioned_table2 partition (part_col = 1) values (1), (2), (3), (4), (5), (6), (7), (8), (9), (10)
+PREHOOK: Input: default@partitioned_table1
+PREHOOK: Input: default@partitioned_table1@part_col=1
+PREHOOK: Input: default@partitioned_table1@part_col=2
+PREHOOK: Input: default@partitioned_table1@part_col=3
+PREHOOK: Input: default@partitioned_table2
+PREHOOK: Input: default@partitioned_table2@part_col=1
+PREHOOK: Input: default@partitioned_table2@part_col=2
+PREHOOK: Input: default@partitioned_table2@part_col=3
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT count(*)
+FROM partitioned_table1,
+ partitioned_table2
+WHERE partitioned_table1.part_col = partitioned_table2.part_col
+ AND partitioned_table2.col > 1
POSTHOOK: type: QUERY
-POSTHOOK: Output: default@partitioned_table2@part_col=1
-POSTHOOK: Lineage: partitioned_table2 PARTITION(part_col=1).col EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-PREHOOK: query: explain select * from partitioned_table1, partitioned_table2 where partitioned_table1.part_col = partitioned_table2.part_col
+POSTHOOK: Input: default@partitioned_table1
+POSTHOOK: Input: default@partitioned_table1@part_col=1
+POSTHOOK: Input: default@partitioned_table1@part_col=2
+POSTHOOK: Input: default@partitioned_table1@part_col=3
+POSTHOOK: Input: default@partitioned_table2
+POSTHOOK: Input: default@partitioned_table2@part_col=1
+POSTHOOK: Input: default@partitioned_table2@part_col=2
+POSTHOOK: Input: default@partitioned_table2@part_col=3
+#### A masked pattern was here ####
+90
+PREHOOK: query: -- set auto convert size to a higher value so map-joins are triggered for the partitioned tables
+
+EXPLAIN SELECT count(*) FROM partitioned_table4, partitioned_table5 WHERE
+partitioned_table4.part_col1 = partitioned_table5.part_col1 AND
+partitioned_table4.part_col2 = partitioned_table5.part_col2
PREHOOK: type: QUERY
-POSTHOOK: query: explain select * from partitioned_table1, partitioned_table2 where partitioned_table1.part_col = partitioned_table2.part_col
+POSTHOOK: query: -- set auto convert size to a higher value so map-joins are triggered for the partitioned tables
+
+EXPLAIN SELECT count(*) FROM partitioned_table4, partitioned_table5 WHERE
+partitioned_table4.part_col1 = partitioned_table5.part_col1 AND
+partitioned_table4.part_col2 = partitioned_table5.part_col2
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
@@ -71,68 +1313,99 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 2
+ Map 3
Map Operator Tree:
TableScan
- alias: partitioned_table2
- Statistics: Num rows: 10 Data size: 11 Basic stats: COMPLETE Column stats: NONE
+ alias: partitioned_table5
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: col (type: int), part_col (type: int)
+ expressions: part_col1 (type: int), part_col2 (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 10 Data size: 11 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col1 (type: int)
- 1 _col1 (type: int)
+ 0 _col0 (type: int), _col1 (type: int)
+ 1 _col0 (type: int), _col1 (type: int)
+ Select Operator
+ expressions: _col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: int)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ Spark Partition Pruning Sink Operator
+ Target column: part_col1 (int)
+ partition key expr: part_col1
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ target work: Map 1
Select Operator
expressions: _col1 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 10 Data size: 11 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: int)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 10 Data size: 11 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
Spark Partition Pruning Sink Operator
- Target column: part_col (int)
- partition key expr: part_col
- Statistics: Num rows: 10 Data size: 11 Basic stats: COMPLETE Column stats: NONE
+ Target column: part_col2 (int)
+ partition key expr: part_col2
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
target work: Map 1
Local Work:
Map Reduce Local Work
Stage: Stage-1
Spark
+ Edges:
+ Reducer 2 <- Map 1 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
- alias: partitioned_table1
- Statistics: Num rows: 10 Data size: 11 Basic stats: COMPLETE Column stats: NONE
+ alias: partitioned_table4
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: col (type: int), part_col (type: int)
+ expressions: part_col1 (type: int), part_col2 (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 10 Data size: 11 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
- 1 _col1 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3
+ 0 _col0 (type: int), _col1 (type: int)
+ 1 _col0 (type: int), _col1 (type: int)
input vertices:
- 1 Map 2
- Statistics: Num rows: 11 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 11 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ 1 Map 3
+ Statistics: Num rows: 19 Data size: 19 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
Local Work:
Map Reduce Local Work
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -140,135 +1413,174 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: explain select * from partitioned_table1 where partitioned_table1.part_col in (select regular_table.col from regular_table join partitioned_table2 on regular_table.col = partitioned_table2.part_col)
+PREHOOK: query: SELECT count(*)
+FROM partitioned_table4,
+ partitioned_table5
+WHERE partitioned_table4.part_col1 = partitioned_table5.part_col1
+ AND partitioned_table4.part_col2 = partitioned_table5.part_col2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@partitioned_table4
+PREHOOK: Input: default@partitioned_table4@part_col1=1/part_col2=1
+PREHOOK: Input: default@partitioned_table4@part_col1=2/part_col2=2
+PREHOOK: Input: default@partitioned_table4@part_col1=3/part_col2=3
+PREHOOK: Input: default@partitioned_table5
+PREHOOK: Input: default@partitioned_table5@part_col1=1/part_col2=1
+PREHOOK: Input: default@partitioned_table5@part_col1=2/part_col2=2
+PREHOOK: Input: default@partitioned_table5@part_col1=3/part_col2=3
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT count(*)
+FROM partitioned_table4,
+ partitioned_table5
+WHERE partitioned_table4.part_col1 = partitioned_table5.part_col1
+ AND partitioned_table4.part_col2 = partitioned_table5.part_col2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@partitioned_table4
+POSTHOOK: Input: default@partitioned_table4@part_col1=1/part_col2=1
+POSTHOOK: Input: default@partitioned_table4@part_col1=2/part_col2=2
+POSTHOOK: Input: default@partitioned_table4@part_col1=3/part_col2=3
+POSTHOOK: Input: default@partitioned_table5
+POSTHOOK: Input: default@partitioned_table5@part_col1=1/part_col2=1
+POSTHOOK: Input: default@partitioned_table5@part_col1=2/part_col2=2
+POSTHOOK: Input: default@partitioned_table5@part_col1=3/part_col2=3
+#### A masked pattern was here ####
+108
+PREHOOK: query: EXPLAIN SELECT count(*) FROM partitioned_table1 JOIN regular_table1 ON
+partitioned_table1.part_col = regular_table1.col1 JOIN regular_table2 ON
+regular_table1.col1 = regular_table2.col1
PREHOOK: type: QUERY
-POSTHOOK: query: explain select * from partitioned_table1 where partitioned_table1.part_col in (select regular_table.col from regular_table join partitioned_table2 on regular_table.col = partitioned_table2.part_col)
+POSTHOOK: query: EXPLAIN SELECT count(*) FROM partitioned_table1 JOIN regular_table1 ON
+partitioned_table1.part_col = regular_table1.col1 JOIN regular_table2 ON
+regular_table1.col1 = regular_table2.col1
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-3 is a root stage
- Stage-2 depends on stages: Stage-3
+ Stage-2 is a root stage
Stage-1 depends on stages: Stage-2
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-3
+ Stage: Stage-2
Spark
#### A masked pattern was here ####
Vertices:
- Map 2
+ Map 3
Map Operator Tree:
TableScan
- alias: regular_table
- Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE
+ alias: regular_table1
+ Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: col is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE
+ predicate: col1 is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: col (type: int)
+ expressions: col1 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
0 _col0 (type: int)
1 _col0 (type: int)
+ 2 _col0 (type: int)
Select Operator
expressions: _col0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: int)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE
Spark Partition Pruning Sink Operator
Target column: part_col (int)
partition key expr: part_col
- Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE
- target work: Map 3
+ Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE
+ target work: Map 1
Local Work:
Map Reduce Local Work
-
- Stage: Stage-2
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 3
+ Map 4
Map Operator Tree:
TableScan
- alias: partitioned_table2
- Statistics: Num rows: 10 Data size: 11 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: part_col (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 10 Data size: 11 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
+ alias: regular_table2
+ Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: col1 is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: col1 (type: int)
outputColumnNames: _col0
- input vertices:
- 0 Map 2
- Statistics: Num rows: 11 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- keys: _col0 (type: int)
- mode: hash
+ Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ 2 _col0 (type: int)
+ Select Operator
+ expressions: _col0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 11 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Spark HashTable Sink Operator
- keys:
- 0 _col1 (type: int)
- 1 _col0 (type: int)
- Select Operator
- expressions: _col0 (type: int)
+ Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: int)
+ mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 11 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- keys: _col0 (type: int)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 11 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Spark Partition Pruning Sink Operator
- Target column: part_col (int)
- partition key expr: part_col
- Statistics: Num rows: 11 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- target work: Map 1
+ Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE
+ Spark Partition Pruning Sink Operator
+ Target column: part_col (int)
+ partition key expr: part_col
+ Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE
+ target work: Map 1
Local Work:
Map Reduce Local Work
Stage: Stage-1
Spark
+ Edges:
+ Reducer 2 <- Map 1 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
alias: partitioned_table1
- Statistics: Num rows: 10 Data size: 11 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: col (type: int), part_col (type: int)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 10 Data size: 11 Basic stats: COMPLETE Column stats: NONE
+ expressions: part_col (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 18 Data size: 18 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
- Left Semi Join 0 to 1
+ Inner Join 0 to 1
+ Inner Join 1 to 2
keys:
- 0 _col1 (type: int)
+ 0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col1
+ 2 _col0 (type: int)
input vertices:
1 Map 3
- Statistics: Num rows: 12 Data size: 13 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 12 Data size: 13 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ 2 Map 4
+ Statistics: Num rows: 39 Data size: 39 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
Local Work:
Map Reduce Local Work
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _
<TRUNCATED>
[19/50] [abbrv] hive git commit: HIVE-17307 Change the metastore to
not use the metrics code in hive/common. This closes #235. (Alan Gates,
reviewed by Vihang Karajgaonkar)
Posted by we...@apache.org.
HIVE-17307 Change the metastore to not use the metrics code in hive/common. This closes #235. (Alan Gates, reviewed by Vihang Karajgaonkar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a6b9cd52
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a6b9cd52
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a6b9cd52
Branch: refs/heads/hive-14535
Commit: a6b9cd527b095fc77e4adc839206455bd937dd07
Parents: 77c28af
Author: Alan Gates <ga...@hortonworks.com>
Authored: Wed Aug 30 12:55:54 2017 -0700
Committer: Alan Gates <ga...@hortonworks.com>
Committed: Wed Aug 30 12:55:54 2017 -0700
----------------------------------------------------------------------
.../common/metrics/common/MetricsConstant.java | 14 --
.../hive/metastore/TestMetaStoreMetrics.java | 78 +++---
.../hive/metastore/HMSMetricsListener.java | 47 ++--
.../hadoop/hive/metastore/HiveMetaStore.java | 124 +++++-----
.../hadoop/hive/metastore/ObjectStore.java | 24 +-
.../hive/metastore/RetryingHMSHandler.java | 4 +-
.../hadoop/hive/metastore/txn/TxnHandler.java | 22 +-
.../hadoop/hive/metastore/TestObjectStore.java | 25 +-
standalone-metastore/pom.xml | 20 ++
.../hadoop/hive/metastore/ThreadPool.java | 63 +++++
.../hive/metastore/metrics/JsonReporter.java | 166 +++++++++++++
.../hive/metastore/metrics/JvmPauseMonitor.java | 222 +++++++++++++++++
.../hadoop/hive/metastore/metrics/Metrics.java | 243 +++++++++++++++++++
.../metastore/metrics/MetricsConstants.java | 46 ++++
.../hive/metastore/metrics/PerfLogger.java | 194 +++++++++++++++
.../hive/metastore/metrics/TestMetrics.java | 200 +++++++++++++++
16 files changed, 1312 insertions(+), 180 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a6b9cd52/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java b/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
index b2e78c3..5d28e2d 100644
--- a/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
@@ -47,20 +47,6 @@ public class MetricsConstant {
public static final String SQL_OPERATION_PREFIX = "hs2_sql_operation_";
public static final String COMPLETED_SQL_OPERATION_PREFIX = "hs2_completed_sql_operation_";
- public static final String INIT_TOTAL_DATABASES = "init_total_count_dbs";
- public static final String INIT_TOTAL_TABLES = "init_total_count_tables";
- public static final String INIT_TOTAL_PARTITIONS = "init_total_count_partitions";
-
- public static final String CREATE_TOTAL_DATABASES = "create_total_count_dbs";
- public static final String CREATE_TOTAL_TABLES = "create_total_count_tables";
- public static final String CREATE_TOTAL_PARTITIONS = "create_total_count_partitions";
-
- public static final String DELETE_TOTAL_DATABASES = "delete_total_count_dbs";
- public static final String DELETE_TOTAL_TABLES = "delete_total_count_tables";
- public static final String DELETE_TOTAL_PARTITIONS = "delete_total_count_partitions";
-
- public static final String DIRECTSQL_ERRORS = "directsql_errors";
-
// The number of Hive operations that are waiting to enter the compile block
public static final String WAITING_COMPILE_OPS = "waiting_compile_ops";
http://git-wip-us.apache.org/repos/asf/hive/blob/a6b9cd52/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
index 9b6cab3..e1cb1f3 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
@@ -18,14 +18,13 @@
package org.apache.hadoop.hive.metastore;
import org.apache.hadoop.hive.cli.CliSessionState;
-import org.apache.hadoop.hive.common.metrics.MetricsTestUtils;
-import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
-import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
-import org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.metrics.Metrics;
+import org.apache.hadoop.hive.metastore.metrics.MetricsConstants;
import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.session.SessionState;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -38,7 +37,6 @@ public class TestMetaStoreMetrics {
private static HiveConf hiveConf;
private static Driver driver;
- private static CodahaleMetrics metrics;
@BeforeClass
public static void before() throws Exception {
@@ -53,10 +51,6 @@ public class TestMetaStoreMetrics {
.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
"org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory");
- MetricsFactory.close();
- MetricsFactory.init(hiveConf);
- metrics = (CodahaleMetrics) MetricsFactory.getInstance();
-
//Increments one HMS connection
MetaStoreUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), hiveConf);
@@ -69,23 +63,19 @@ public class TestMetaStoreMetrics {
@Test
public void testMethodCounts() throws Exception {
driver.run("show databases");
- String json = metrics.dumpJson();
//one call by init, one called here.
- MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.TIMER, "api_get_all_databases", 2);
+ Assert.assertEquals(2, Metrics.getRegistry().getTimers().get("api_get_all_databases").getCount());
}
@Test
public void testMetaDataCounts() throws Exception {
- CodahaleMetrics metrics = (CodahaleMetrics) MetricsFactory.getInstance();
- String json = metrics.dumpJson();
-
- int initDbCount = (new Integer((MetricsTestUtils.getJsonNode(json, MetricsTestUtils.GAUGE,
- MetricsConstant.INIT_TOTAL_DATABASES)).asText())).intValue();
- int initTblCount = (new Integer((MetricsTestUtils.getJsonNode(json, MetricsTestUtils.GAUGE,
- MetricsConstant.INIT_TOTAL_TABLES)).asText())).intValue();
- int initPartCount = (new Integer((MetricsTestUtils.getJsonNode(json, MetricsTestUtils.GAUGE,
- MetricsConstant.INIT_TOTAL_PARTITIONS)).asText())).intValue();
+ int initDbCount =
+ (Integer)Metrics.getRegistry().getGauges().get(MetricsConstants.TOTAL_DATABASES).getValue();
+ int initTblCount =
+ (Integer)Metrics.getRegistry().getGauges().get(MetricsConstants.TOTAL_TABLES).getValue();
+ int initPartCount =
+ (Integer)Metrics.getRegistry().getGauges().get(MetricsConstants.TOTAL_PARTITIONS).getValue();
//1 databases created
driver.run("create database testdb1");
@@ -123,25 +113,22 @@ public class TestMetaStoreMetrics {
driver.run("use default");
driver.run("drop database tempdb cascade");
- json = metrics.dumpJson();
- MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.CREATE_TOTAL_DATABASES, 2);
- MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.CREATE_TOTAL_TABLES, 7);
- MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.CREATE_TOTAL_PARTITIONS, 9);
+ Assert.assertEquals(2, Metrics.getRegistry().getCounters().get(MetricsConstants.CREATE_TOTAL_DATABASES).getCount());
+ Assert.assertEquals(7, Metrics.getRegistry().getCounters().get(MetricsConstants.CREATE_TOTAL_TABLES).getCount());
+ Assert.assertEquals(9, Metrics.getRegistry().getCounters().get(MetricsConstants.CREATE_TOTAL_PARTITIONS).getCount());
- MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.DELETE_TOTAL_DATABASES, 1);
- MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.DELETE_TOTAL_TABLES, 3);
- MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.DELETE_TOTAL_PARTITIONS, 3);
+ Assert.assertEquals(1, Metrics.getRegistry().getCounters().get(MetricsConstants.DELETE_TOTAL_DATABASES).getCount());
+ Assert.assertEquals(3, Metrics.getRegistry().getCounters().get(MetricsConstants.DELETE_TOTAL_TABLES).getCount());
+ Assert.assertEquals(3, Metrics.getRegistry().getCounters().get(MetricsConstants.DELETE_TOTAL_PARTITIONS).getCount());
//to test initial metadata count metrics.
- hiveConf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL, ObjectStore.class.getName());
- HiveMetaStore.HMSHandler baseHandler = new HiveMetaStore.HMSHandler("test", hiveConf, false);
- baseHandler.init();
- baseHandler.updateMetrics();
-
- json = metrics.dumpJson();
- MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.GAUGE, MetricsConstant.INIT_TOTAL_DATABASES, initDbCount + 1);
- MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.GAUGE, MetricsConstant.INIT_TOTAL_TABLES, initTblCount + 4);
- MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.GAUGE, MetricsConstant.INIT_TOTAL_PARTITIONS, initPartCount + 6);
+ Assert.assertEquals(initDbCount + 1,
+ Metrics.getRegistry().getGauges().get(MetricsConstants.TOTAL_DATABASES).getValue());
+ Assert.assertEquals(initTblCount + 4,
+ Metrics.getRegistry().getGauges().get(MetricsConstants.TOTAL_TABLES).getValue());
+ Assert.assertEquals(initPartCount + 6,
+ Metrics.getRegistry().getGauges().get(MetricsConstants.TOTAL_PARTITIONS).getValue());
+
}
@@ -149,25 +136,26 @@ public class TestMetaStoreMetrics {
public void testConnections() throws Exception {
//initial state is one connection
- String json = metrics.dumpJson();
- int initialCount = (new Integer((MetricsTestUtils.getJsonNode(json, MetricsTestUtils.COUNTER,
- MetricsConstant.OPEN_CONNECTIONS)).asText())).intValue();
+ int initialCount =
+ (Integer)Metrics.getRegistry().getGauges().get(MetricsConstants.OPEN_CONNECTIONS).getValue();
//create two connections
HiveMetaStoreClient msc = new HiveMetaStoreClient(hiveConf);
HiveMetaStoreClient msc2 = new HiveMetaStoreClient(hiveConf);
- json = metrics.dumpJson();
- MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.OPEN_CONNECTIONS, initialCount + 2);
+ Assert.assertEquals(initialCount + 2,
+ Metrics.getRegistry().getGauges().get(MetricsConstants.OPEN_CONNECTIONS).getValue());
//close one connection, verify still two left
msc.close();
- json = metrics.dumpJson();
- MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.OPEN_CONNECTIONS, initialCount + 1);
+ Thread.sleep(500); // TODO Evil! Need to figure out a way to remove this sleep.
+ Assert.assertEquals(initialCount + 1,
+ Metrics.getRegistry().getGauges().get(MetricsConstants.OPEN_CONNECTIONS).getValue());
//close one connection, verify still one left
msc2.close();
- json = metrics.dumpJson();
- MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.OPEN_CONNECTIONS, initialCount);
+ Thread.sleep(500); // TODO Evil! Need to figure out a way to remove this sleep.
+ Assert.assertEquals(initialCount,
+ Metrics.getRegistry().getGauges().get(MetricsConstants.OPEN_CONNECTIONS).getValue());
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a6b9cd52/metastore/src/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java
index 98288a0..a2ad06f 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -17,9 +17,9 @@
*/
package org.apache.hadoop.hive.metastore;
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.MetricRegistry;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.common.metrics.common.Metrics;
-import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent;
@@ -27,6 +27,8 @@ import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
import org.apache.hadoop.hive.metastore.events.DropPartitionEvent;
import org.apache.hadoop.hive.metastore.events.DropTableEvent;
+import org.apache.hadoop.hive.metastore.metrics.Metrics;
+import org.apache.hadoop.hive.metastore.metrics.MetricsConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,47 +37,54 @@ import org.slf4j.LoggerFactory;
*/
public class HMSMetricsListener extends MetaStoreEventListener {
- public static final Logger LOGGER = LoggerFactory.getLogger(HMSMetricsListener.class);
- private Metrics metrics;
+ private static final Logger LOGGER = LoggerFactory.getLogger(HMSMetricsListener.class);
+
+ private Counter createdDatabases, deletedDatabases, createdTables, deletedTables, createdParts,
+ deletedParts;
- public HMSMetricsListener(Configuration config, Metrics metrics) {
+ public HMSMetricsListener(Configuration config) {
super(config);
- this.metrics = metrics;
+ createdDatabases = Metrics.getOrCreateCounter(MetricsConstants.CREATE_TOTAL_DATABASES);
+ deletedDatabases = Metrics.getOrCreateCounter(MetricsConstants.DELETE_TOTAL_DATABASES);
+ createdTables = Metrics.getOrCreateCounter(MetricsConstants.CREATE_TOTAL_TABLES);
+ deletedTables = Metrics.getOrCreateCounter(MetricsConstants.DELETE_TOTAL_TABLES);
+ createdParts = Metrics.getOrCreateCounter(MetricsConstants.CREATE_TOTAL_PARTITIONS);
+ deletedParts = Metrics.getOrCreateCounter(MetricsConstants.DELETE_TOTAL_PARTITIONS);
}
@Override
public void onCreateDatabase(CreateDatabaseEvent dbEvent) throws MetaException {
- incrementCounterInternal(MetricsConstant.CREATE_TOTAL_DATABASES);
+ HiveMetaStore.HMSHandler.databaseCount.incrementAndGet();
+ createdDatabases.inc();
}
@Override
public void onDropDatabase(DropDatabaseEvent dbEvent) throws MetaException {
- incrementCounterInternal(MetricsConstant.DELETE_TOTAL_DATABASES);
+ HiveMetaStore.HMSHandler.databaseCount.decrementAndGet();
+ deletedDatabases.inc();
}
@Override
public void onCreateTable(CreateTableEvent tableEvent) throws MetaException {
- incrementCounterInternal(MetricsConstant.CREATE_TOTAL_TABLES);
+ HiveMetaStore.HMSHandler.tableCount.incrementAndGet();
+ createdTables.inc();
}
@Override
public void onDropTable(DropTableEvent tableEvent) throws MetaException {
- incrementCounterInternal(MetricsConstant.DELETE_TOTAL_TABLES);
+ HiveMetaStore.HMSHandler.tableCount.decrementAndGet();
+ deletedTables.inc();
}
@Override
public void onDropPartition(DropPartitionEvent partitionEvent) throws MetaException {
- incrementCounterInternal(MetricsConstant.DELETE_TOTAL_PARTITIONS);
+ HiveMetaStore.HMSHandler.partCount.decrementAndGet();
+ deletedParts.inc();
}
@Override
public void onAddPartition(AddPartitionEvent partitionEvent) throws MetaException {
- incrementCounterInternal(MetricsConstant.CREATE_TOTAL_PARTITIONS);
- }
-
- private void incrementCounterInternal(String name) {
- if (metrics != null) {
- metrics.incrementCounter(name);
- }
+ HiveMetaStore.HMSHandler.partCount.incrementAndGet();
+ createdParts.inc();
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a6b9cd52/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index df01b25..55def04 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -1,5 +1,4 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
+/** * Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
@@ -32,7 +31,6 @@ import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
-import java.util.Formatter;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -52,6 +50,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
@@ -59,6 +58,7 @@ import java.util.regex.Pattern;
import javax.jdo.JDOException;
+import com.codahale.metrics.Counter;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableListMultimap;
import com.google.common.collect.Lists;
@@ -69,7 +69,6 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
-import org.apache.hadoop.hive.common.JvmPauseMonitor;
import org.apache.hadoop.hive.common.LogUtils;
import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
import org.apache.hadoop.hive.common.StatsSetupConst;
@@ -77,10 +76,6 @@ import org.apache.hadoop.hive.common.auth.HiveAuthUtils;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hadoop.hive.common.classification.InterfaceStability;
import org.apache.hadoop.hive.common.cli.CommonCliOptions;
-import org.apache.hadoop.hive.common.metrics.common.Metrics;
-import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
-import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
-import org.apache.hadoop.hive.common.metrics.common.MetricsVariable;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.io.HdfsUtils;
@@ -127,6 +122,10 @@ import org.apache.hadoop.hive.metastore.events.PreReadDatabaseEvent;
import org.apache.hadoop.hive.metastore.events.PreReadTableEvent;
import org.apache.hadoop.hive.metastore.filemeta.OrcFileMetadataHandler;
import org.apache.hadoop.hive.metastore.messaging.EventMessage.EventType;
+import org.apache.hadoop.hive.metastore.metrics.JvmPauseMonitor;
+import org.apache.hadoop.hive.metastore.metrics.Metrics;
+import org.apache.hadoop.hive.metastore.metrics.MetricsConstants;
+import org.apache.hadoop.hive.metastore.metrics.PerfLogger;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
import org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager;
@@ -244,8 +243,9 @@ public class HiveMetaStore extends ThriftHiveMetastore {
private FileMetadataManager fileMetadataManager;
private PartitionExpressionProxy expressionProxy;
- //For Metrics
- private int initDatabaseCount, initTableCount, initPartCount;
+ // Variables for metrics
+ // Package visible so that HMSMetricsListener can see them.
+ static AtomicInteger databaseCount, tableCount, partCount;
private Warehouse wh; // hdfs warehouse
private static final ThreadLocal<RawStore> threadLocalMS =
@@ -263,6 +263,14 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
};
+ private static final ThreadLocal<Map<String, com.codahale.metrics.Timer.Context>> timerContexts =
+ new ThreadLocal<Map<String, com.codahale.metrics.Timer.Context>>() {
+ @Override
+ protected Map<String, com.codahale.metrics.Timer.Context> initialValue() {
+ return new HashMap<>();
+ }
+ };
+
public static RawStore getRawStore() {
return threadLocalMS.get();
}
@@ -487,41 +495,14 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
}
- //Start Metrics for Embedded mode
+ //Start Metrics
if (hiveConf.getBoolVar(ConfVars.METASTORE_METRICS)) {
- try {
- MetricsFactory.init(hiveConf);
- } catch (Exception e) {
- // log exception, but ignore inability to start
- LOG.error("error in Metrics init: " + e.getClass().getName() + " "
- + e.getMessage(), e);
- }
- }
-
- Metrics metrics = MetricsFactory.getInstance();
- if (metrics != null && hiveConf.getBoolVar(ConfVars.METASTORE_INIT_METADATA_COUNT_ENABLED)) {
LOG.info("Begin calculating metadata count metrics.");
+ Metrics.initialize(hiveConf);
+ databaseCount = Metrics.getOrCreateGauge(MetricsConstants.TOTAL_DATABASES);
+ tableCount = Metrics.getOrCreateGauge(MetricsConstants.TOTAL_TABLES);
+ partCount = Metrics.getOrCreateGauge(MetricsConstants.TOTAL_PARTITIONS);
updateMetrics();
- LOG.info("Finished metadata count metrics: " + initDatabaseCount + " databases, " + initTableCount +
- " tables, " + initPartCount + " partitions.");
- metrics.addGauge(MetricsConstant.INIT_TOTAL_DATABASES, new MetricsVariable() {
- @Override
- public Object getValue() {
- return initDatabaseCount;
- }
- });
- metrics.addGauge(MetricsConstant.INIT_TOTAL_TABLES, new MetricsVariable() {
- @Override
- public Object getValue() {
- return initTableCount;
- }
- });
- metrics.addGauge(MetricsConstant.INIT_TOTAL_PARTITIONS, new MetricsVariable() {
- @Override
- public Object getValue() {
- return initPartCount;
- }
- });
}
preListeners = MetaStoreUtils.getMetaStoreListeners(MetaStorePreEventListener.class,
@@ -534,8 +515,8 @@ public class HiveMetaStore extends ThriftHiveMetastore {
listeners.add(new AcidEventListener(hiveConf));
transactionalListeners = MetaStoreUtils.getMetaStoreListeners(TransactionalMetaStoreEventListener.class,hiveConf,
hiveConf.getVar(ConfVars.METASTORE_TRANSACTIONAL_EVENT_LISTENERS));
- if (metrics != null) {
- listeners.add(new HMSMetricsListener(hiveConf, metrics));
+ if (Metrics.getRegistry() != null) {
+ listeners.add(new HMSMetricsListener(hiveConf));
}
endFunctionListeners = MetaStoreUtils.getMetaStoreListeners(
@@ -851,9 +832,14 @@ public class HiveMetaStore extends ThriftHiveMetastore {
incrementCounter(function);
logInfo((getThreadLocalIpAddress() == null ? "" : "source:" + getThreadLocalIpAddress() + " ") +
function + extraLogInfo);
- if (MetricsFactory.getInstance() != null) {
- MetricsFactory.getInstance().startStoredScope(MetricsConstant.API_PREFIX + function);
- }
+ com.codahale.metrics.Timer timer =
+ Metrics.getOrCreateTimer(MetricsConstants.API_PREFIX + function);
+ if (timer != null) {
+ // Timer will be null we aren't using the metrics
+ timerContexts.get().put(function, timer.time());
+ }
+ Counter counter = Metrics.getOrCreateCounter(MetricsConstants.ACTIVE_CALLS + function);
+ if (counter != null) counter.inc();
return function;
}
@@ -890,9 +876,12 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
private void endFunction(String function, MetaStoreEndFunctionContext context) {
- if (MetricsFactory.getInstance() != null) {
- MetricsFactory.getInstance().endStoredScope(MetricsConstant.API_PREFIX + function);
+ com.codahale.metrics.Timer.Context timerContext = timerContexts.get().remove(function);
+ if (timerContext != null) {
+ timerContext.close();
}
+ Counter counter = Metrics.getOrCreateCounter(MetricsConstants.ACTIVE_CALLS + function);
+ if (counter != null) counter.dec();
for (MetaStoreEndFunctionListener listener : endFunctionListeners) {
listener.onEndFunction(function, context);
@@ -907,6 +896,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
@Override
public void shutdown() {
cleanupRawStore();
+ PerfLogger.getPerfLogger(false).cleanupPerfLogMetrics();
}
@Override
@@ -7254,9 +7244,11 @@ public class HiveMetaStore extends ThriftHiveMetastore {
@VisibleForTesting
public void updateMetrics() throws MetaException {
- initTableCount = getMS().getTableCount();
- initPartCount = getMS().getPartitionCount();
- initDatabaseCount = getMS().getDatabaseCount();
+ if (databaseCount != null) {
+ tableCount.set(getMS().getTableCount());
+ partCount.set(getMS().getPartitionCount());
+ databaseCount.set(getMS().getDatabaseCount());
+ }
}
@Override
@@ -7538,7 +7530,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
if (conf.getBoolVar(ConfVars.METASTORE_METRICS)) {
try {
- MetricsFactory.close();
+ Metrics.shutdown();
} catch (Exception e) {
LOG.error("error in Metrics deinit: " + e.getClass().getName() + " "
+ e.getMessage(), e);
@@ -7550,7 +7542,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
//Start Metrics for Standalone (Remote) Mode
if (conf.getBoolVar(ConfVars.METASTORE_METRICS)) {
try {
- MetricsFactory.init(conf);
+ Metrics.initialize(conf);
} catch (Exception e) {
// log exception, but ignore inability to start
LOG.error("error in Metrics init: " + e.getClass().getName() + " "
@@ -7572,6 +7564,8 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
}
+ private static AtomicInteger openConnections;
+
/**
* Start Metastore based on a passed {@link HadoopThriftAuthBridge}
*
@@ -7700,6 +7694,10 @@ public class HiveMetaStore extends ThriftHiveMetastore {
serverSocket = new TServerSocketKeepAlive(serverSocket);
}
+ // Metrics will have already been initialized if we're using them since HMSHandler
+ // initializes them.
+ openConnections = Metrics.getOrCreateGauge(MetricsConstants.OPEN_CONNECTIONS);
+
TThreadPoolServer.Args args = new TThreadPoolServer.Args(serverSocket)
.processor(processor)
.transportFactory(transFactory)
@@ -7716,27 +7714,13 @@ public class HiveMetaStore extends ThriftHiveMetastore {
@Override
public ServerContext createContext(TProtocol tProtocol, TProtocol tProtocol1) {
- try {
- Metrics metrics = MetricsFactory.getInstance();
- if (metrics != null) {
- metrics.incrementCounter(MetricsConstant.OPEN_CONNECTIONS);
- }
- } catch (Exception e) {
- LOG.warn("Error Reporting Metastore open connection to Metrics system", e);
- }
+ openConnections.incrementAndGet();
return null;
}
@Override
public void deleteContext(ServerContext serverContext, TProtocol tProtocol, TProtocol tProtocol1) {
- try {
- Metrics metrics = MetricsFactory.getInstance();
- if (metrics != null) {
- metrics.decrementCounter(MetricsConstant.OPEN_CONNECTIONS);
- }
- } catch (Exception e) {
- LOG.warn("Error Reporting Metastore close connection to Metrics system", e);
- }
+ openConnections.decrementAndGet();
// If the IMetaStoreClient#close was called, HMSHandler#shutdown would have already
// cleaned up thread local RawStore. Otherwise, do it now.
cleanupRawStore();
http://git-wip-us.apache.org/repos/asf/hive/blob/a6b9cd52/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 0a80241..efbdb8f 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -61,6 +61,8 @@ import javax.jdo.datastore.DataStoreCache;
import javax.jdo.identity.IntIdentity;
import javax.sql.DataSource;
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.MetricRegistry;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configurable;
@@ -71,9 +73,6 @@ import org.apache.hadoop.hive.common.ObjectPair;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hadoop.hive.common.classification.InterfaceStability;
-import org.apache.hadoop.hive.common.metrics.common.Metrics;
-import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
-import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.MetaStoreDirectSql.SqlFilterForPushdown;
@@ -126,6 +125,8 @@ import org.apache.hadoop.hive.metastore.api.UnknownTableException;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.datasource.DataSourceProvider;
import org.apache.hadoop.hive.metastore.datasource.DataSourceProviderFactory;
+import org.apache.hadoop.hive.metastore.metrics.Metrics;
+import org.apache.hadoop.hive.metastore.metrics.MetricsConstants;
import org.apache.hadoop.hive.metastore.model.MColumnDescriptor;
import org.apache.hadoop.hive.metastore.model.MConstraint;
import org.apache.hadoop.hive.metastore.model.MDBPrivilege;
@@ -245,6 +246,7 @@ public class ObjectStore implements RawStore, Configurable {
private Transaction currentTransaction = null;
private TXN_STATUS transactionStatus = TXN_STATUS.NO_STATE;
private Pattern partitionValidationPattern;
+ private Counter directSqlErrors;
/**
* A Autocloseable wrapper around Query class to pass the Query object to the caller and let the caller release
@@ -325,6 +327,13 @@ public class ObjectStore implements RawStore, Configurable {
partitionValidationPattern = null;
}
+ // Note, if metrics have not been initialized this will return null, which means we aren't
+ // using metrics. Thus we should always check whether this is non-null before using.
+ MetricRegistry registry = Metrics.getRegistry();
+ if (registry != null) {
+ directSqlErrors = Metrics.getOrCreateCounter(MetricsConstants.DIRECTSQL_ERRORS);
+ }
+
if (!isInitialized) {
throw new RuntimeException(
"Unable to create persistence manager. Check dss.log for details");
@@ -2845,13 +2854,8 @@ public class ObjectStore implements RawStore, Configurable {
start = doTrace ? System.nanoTime() : 0;
}
- Metrics metrics = MetricsFactory.getInstance();
- if (metrics != null) {
- try {
- metrics.incrementCounter(MetricsConstant.DIRECTSQL_ERRORS);
- } catch (Exception e) {
- LOG.warn("Error reporting Direct SQL errors to metrics system", e);
- }
+ if (directSqlErrors != null) {
+ directSqlErrors.inc();
}
doUseDirectSql = false;
http://git-wip-us.apache.org/repos/asf/hive/blob/a6b9cd52/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
index a08c5bd..affb38f 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
@@ -26,6 +26,7 @@ import java.lang.reflect.UndeclaredThrowableException;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.hadoop.hive.metastore.metrics.PerfLogger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -34,7 +35,6 @@ import org.apache.hadoop.hive.common.classification.InterfaceStability;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.datanucleus.exceptions.NucleusException;
@InterfaceAudience.Private
@@ -101,7 +101,7 @@ public class RetryingHMSHandler implements InvocationHandler {
int retryCount = -1;
int threadId = HiveMetaStore.HMSHandler.get();
boolean error = true;
- PerfLogger perfLogger = PerfLogger.getPerfLogger(origConf, false);
+ PerfLogger perfLogger = PerfLogger.getPerfLogger(false);
perfLogger.PerfLogBegin(CLASS_NAME, method.getName());
try {
Result result = invokeInternal(proxy, method, args);
http://git-wip-us.apache.org/repos/asf/hive/blob/a6b9cd52/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
index 5bfc029..1887c05 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
@@ -33,6 +33,8 @@ import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.datasource.BoneCPDataSourceProvider;
import org.apache.hadoop.hive.metastore.datasource.DataSourceProvider;
import org.apache.hadoop.hive.metastore.datasource.HikariCPDataSourceProvider;
+import org.apache.hadoop.hive.metastore.metrics.Metrics;
+import org.apache.hadoop.hive.metastore.metrics.MetricsConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.commons.dbcp.PoolingDataSource;
@@ -54,6 +56,7 @@ import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.Pattern;
@@ -188,8 +191,6 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
// Maximum number of open transactions that's allowed
private static volatile int maxOpenTxns = 0;
- // Current number of open txns
- private static volatile long numOpenTxns = 0;
// Whether number of open transactions reaches the threshold
private static volatile boolean tooManyOpenTxns = false;
// The AcidHouseKeeperService for counting open transactions
@@ -211,6 +212,9 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
private long retryInterval;
private int retryLimit;
private int retryNum;
+ // Current number of open txns
+ private AtomicInteger numOpenTxns;
+
/**
* Derby specific concurrency control
*/
@@ -276,6 +280,8 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
}
}
+ numOpenTxns = Metrics.getOrCreateGauge(MetricsConstants.NUM_OPEN_TXNS);
+
timeout = HiveConf.getTimeVar(conf, HiveConf.ConfVars.HIVE_TXN_TIMEOUT, TimeUnit.MILLISECONDS);
buildJumpTable();
retryInterval = HiveConf.getTimeVar(conf, HiveConf.ConfVars.HMSHANDLERINTERVAL,
@@ -460,11 +466,11 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
}
}
- if (!tooManyOpenTxns && numOpenTxns >= maxOpenTxns) {
+ if (!tooManyOpenTxns && numOpenTxns.get() >= maxOpenTxns) {
tooManyOpenTxns = true;
}
if (tooManyOpenTxns) {
- if (numOpenTxns < maxOpenTxns * 0.9) {
+ if (numOpenTxns.get() < maxOpenTxns * 0.9) {
tooManyOpenTxns = false;
} else {
LOG.warn("Maximum allowed number of open transactions (" + maxOpenTxns + ") has been " +
@@ -3159,7 +3165,13 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
LOG.error("Transaction database not properly configured, " +
"can't find txn_state from TXNS.");
} else {
- numOpenTxns = rs.getLong(1);
+ Long numOpen = rs.getLong(1);
+ if (numOpen > Integer.MAX_VALUE) {
+ LOG.error("Open transaction count above " + Integer.MAX_VALUE +
+ ", can't count that high!");
+ } else {
+ numOpenTxns.set(numOpen.intValue());
+ }
}
} catch (SQLException e) {
LOG.debug("Going to rollback");
http://git-wip-us.apache.org/repos/asf/hive/blob/a6b9cd52/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
index b28ea73..08228af 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
@@ -23,11 +23,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
-import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
-import org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics;
-import org.apache.hadoop.hive.common.metrics.metrics2.MetricsReporting;
-import org.apache.hadoop.hive.common.metrics.MetricsTestUtils;
+import com.codahale.metrics.Counter;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
import org.apache.hadoop.hive.metastore.api.Database;
@@ -51,6 +47,8 @@ import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.messaging.EventMessage;
+import org.apache.hadoop.hive.metastore.metrics.Metrics;
+import org.apache.hadoop.hive.metastore.metrics.MetricsConstants;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
@@ -364,11 +362,12 @@ public class TestObjectStore {
public void testDirectSqlErrorMetrics() throws Exception {
HiveConf conf = new HiveConf();
conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_METRICS_ENABLED, true);
- conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, MetricsReporting.JSON_FILE.name()
- + "," + MetricsReporting.JMX.name());
+ Metrics.initialize(conf);
- MetricsFactory.init(conf);
- CodahaleMetrics metrics = (CodahaleMetrics) MetricsFactory.getInstance();
+ // recall setup so that we get an object store with the metrics initalized
+ setUp();
+ Counter directSqlErrors =
+ Metrics.getRegistry().getCounters().get(MetricsConstants.DIRECTSQL_ERRORS);
objectStore.new GetDbHelper("foo", null, true, true) {
@Override
@@ -383,9 +382,7 @@ public class TestObjectStore {
}
}.run(false);
- String json = metrics.dumpJson();
- MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER,
- MetricsConstant.DIRECTSQL_ERRORS, "");
+ Assert.assertEquals(0, directSqlErrors.getCount());
objectStore.new GetDbHelper("foo", null, true, true) {
@Override
@@ -400,9 +397,7 @@ public class TestObjectStore {
}
}.run(false);
- json = metrics.dumpJson();
- MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER,
- MetricsConstant.DIRECTSQL_ERRORS, 1);
+ Assert.assertEquals(1, directSqlErrors.getCount());
}
public static void dropAllStoreObjects(RawStore store) throws MetaException, InvalidObjectException, InvalidInputException {
http://git-wip-us.apache.org/repos/asf/hive/blob/a6b9cd52/standalone-metastore/pom.xml
----------------------------------------------------------------------
diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml
index 2e0c51e..e38ef37 100644
--- a/standalone-metastore/pom.xml
+++ b/standalone-metastore/pom.xml
@@ -33,11 +33,31 @@
<dependencies>
<dependency>
+ <groupId>com.github.joshelser</groupId>
+ <artifactId>dropwizard-metrics-hadoop-metrics2-reporter</artifactId>
+ <version>${dropwizard-metrics-hadoop-metrics2-reporter.version}</version>
+ </dependency>
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
<dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${dropwizard.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-jvm</artifactId>
+ <version>${dropwizard.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-json</artifactId>
+ <version>${dropwizard.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
http://git-wip-us.apache.org/repos/asf/hive/blob/a6b9cd52/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ThreadPool.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ThreadPool.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ThreadPool.java
new file mode 100644
index 0000000..d0fcd25
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ThreadPool.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+
+/**
+ * Utility singleton class to manage all the threads.
+ */
+public class ThreadPool {
+
+ static final private Logger LOG = LoggerFactory.getLogger(ThreadPool.class);
+ private static ThreadPool self = null;
+ private static ScheduledExecutorService pool;
+
+ public static synchronized ThreadPool initialize(Configuration conf) {
+ if (self == null) {
+ self = new ThreadPool(conf);
+ LOG.debug("ThreadPool initialized");
+ }
+ return self;
+ }
+
+ private ThreadPool(Configuration conf) {
+ pool = Executors.newScheduledThreadPool(MetastoreConf.getIntVar(conf,
+ MetastoreConf.ConfVars.THREAD_POOL_SIZE));
+ }
+
+ public static ScheduledExecutorService getPool() {
+ if (self == null) {
+ throw new RuntimeException("ThreadPool accessed before initialized");
+ }
+ return pool;
+ }
+
+ public static synchronized void shutdown() {
+ if (self != null) {
+ pool.shutdown();
+ self = null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/a6b9cd52/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/metrics/JsonReporter.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/metrics/JsonReporter.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/metrics/JsonReporter.java
new file mode 100644
index 0000000..b804cda
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/metrics/JsonReporter.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.metrics;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricFilter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.ScheduledReporter;
+import com.codahale.metrics.Timer;
+import com.codahale.metrics.json.MetricsModule;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.net.URI;
+import java.util.SortedMap;
+import java.util.concurrent.TimeUnit;
+
+public class JsonReporter extends ScheduledReporter {
+ private static final Logger LOG = LoggerFactory.getLogger(JsonReporter.class);
+
+ private final Configuration conf;
+ private final MetricRegistry registry;
+ private ObjectWriter jsonWriter;
+ private Path path;
+ private Path tmpPath;
+ private FileSystem fs;
+
+ private JsonReporter(MetricRegistry registry, String name, MetricFilter filter,
+ TimeUnit rateUnit, TimeUnit durationUnit, Configuration conf) {
+ super(registry, name, filter, rateUnit, durationUnit);
+ this.conf = conf;
+ this.registry = registry;
+ }
+
+ @Override
+ public void start(long period, TimeUnit unit) {
+ jsonWriter = new ObjectMapper().registerModule(new MetricsModule(TimeUnit.MILLISECONDS,
+ TimeUnit.MILLISECONDS, false)).writerWithDefaultPrettyPrinter();
+ String pathString = MetastoreConf.getVar(conf, MetastoreConf.ConfVars .METRICS_JSON_FILE_LOCATION);
+ path = new Path(pathString);
+
+ tmpPath = new Path(pathString + ".tmp");
+ URI tmpPathURI = tmpPath.toUri();
+ try {
+ if (tmpPathURI.getScheme() == null && tmpPathURI.getAuthority() == null) {
+ //default local
+ fs = FileSystem.getLocal(conf);
+ } else {
+ fs = FileSystem.get(tmpPathURI, conf);
+ }
+ }
+ catch (IOException e) {
+ LOG.error("Unable to access filesystem for path " + tmpPath + ". Aborting reporting", e);
+ return;
+ }
+ super.start(period, unit);
+ }
+
+ @Override
+ public void report(SortedMap<String, Gauge> sortedMap, SortedMap<String, Counter> sortedMap1,
+ SortedMap<String, Histogram> sortedMap2, SortedMap<String, Meter> sortedMap3,
+ SortedMap<String, Timer> sortedMap4) {
+
+ String json;
+ try {
+ json = jsonWriter.writeValueAsString(registry);
+ } catch (JsonProcessingException e) {
+ LOG.error("Unable to convert json to string ", e);
+ return;
+ }
+
+ BufferedWriter bw = null;
+ try {
+ fs.delete(tmpPath, true);
+ bw = new BufferedWriter(new OutputStreamWriter(fs.create(tmpPath, true)));
+ bw.write(json);
+ fs.setPermission(tmpPath, FsPermission.createImmutable((short) 0644));
+ } catch (IOException e) {
+ LOG.error("Unable to write to temp file " + tmpPath, e);
+ return;
+ } finally {
+ if (bw != null) {
+ try {
+ bw.close();
+ } catch (IOException e) {
+ // Not much we can do
+ LOG.error("Caught error closing json metric reporter file", e);
+ }
+ }
+ }
+
+ try {
+ fs.rename(tmpPath, path);
+ fs.setPermission(path, FsPermission.createImmutable((short) 0644));
+ } catch (IOException e) {
+ LOG.error("Unable to rename temp file " + tmpPath + " to " + path, e);
+ }
+ }
+
+ public static Builder forRegistry(MetricRegistry registry, Configuration conf) {
+ return new Builder(registry, conf);
+ }
+
+ public static class Builder {
+ private final MetricRegistry registry;
+ private final Configuration conf;
+ private TimeUnit rate = TimeUnit.SECONDS;
+ private TimeUnit duration = TimeUnit.MILLISECONDS;
+ private MetricFilter filter = MetricFilter.ALL;
+
+ private Builder(MetricRegistry registry, Configuration conf) {
+ this.registry = registry;
+ this.conf = conf;
+ }
+
+ public Builder convertRatesTo(TimeUnit rate) {
+ this.rate = rate;
+ return this;
+ }
+
+ public Builder convertDurationsTo(TimeUnit duration) {
+ this.duration = duration;
+ return this;
+ }
+
+ public Builder filter(MetricFilter filter) {
+ this.filter = filter;
+ return this;
+ }
+
+ public JsonReporter build() {
+ return new JsonReporter(registry, "json-reporter", filter, rate, duration, conf);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/a6b9cd52/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/metrics/JvmPauseMonitor.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/metrics/JvmPauseMonitor.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/metrics/JvmPauseMonitor.java
new file mode 100644
index 0000000..93414fc
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/metrics/JvmPauseMonitor.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.metrics;
+
+import com.codahale.metrics.Counter;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.Daemon;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Based on the JvmPauseMonitor from Hadoop.
+ */
+public class JvmPauseMonitor {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ JvmPauseMonitor.class);
+
+ /** The target sleep time */
+ private static final long SLEEP_INTERVAL_MS = 500;
+
+ /** log WARN if we detect a pause longer than this threshold */
+ private final long warnThresholdMs;
+ private static final String WARN_THRESHOLD_KEY =
+ "jvm.pause.warn-threshold.ms";
+ private static final long WARN_THRESHOLD_DEFAULT = 10000;
+
+ /** log INFO if we detect a pause longer than this threshold */
+ private final long infoThresholdMs;
+ private static final String INFO_THRESHOLD_KEY =
+ "jvm.pause.info-threshold.ms";
+ private static final long INFO_THRESHOLD_DEFAULT = 1000;
+
+ private long numGcWarnThresholdExceeded = 0;
+ private long numGcInfoThresholdExceeded = 0;
+ private long totalGcExtraSleepTime = 0;
+
+ private Thread monitorThread;
+ private volatile boolean shouldRun = true;
+
+ public JvmPauseMonitor(Configuration conf) {
+ this.warnThresholdMs = conf.getLong(WARN_THRESHOLD_KEY, WARN_THRESHOLD_DEFAULT);
+ this.infoThresholdMs = conf.getLong(INFO_THRESHOLD_KEY, INFO_THRESHOLD_DEFAULT);
+ }
+
+ public void start() {
+ Preconditions.checkState(monitorThread == null,
+ "JvmPauseMonitor thread is Already started");
+ monitorThread = new Daemon(new Monitor());
+ monitorThread.start();
+ }
+
+ public void stop() {
+ shouldRun = false;
+ if (isStarted()) {
+ monitorThread.interrupt();
+ try {
+ monitorThread.join();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ public boolean isStarted() {
+ return monitorThread != null;
+ }
+
+ public long getNumGcWarnThreadholdExceeded() {
+ return numGcWarnThresholdExceeded;
+ }
+
+ public long getNumGcInfoThresholdExceeded() {
+ return numGcInfoThresholdExceeded;
+ }
+
+ public long getTotalGcExtraSleepTime() {
+ return totalGcExtraSleepTime;
+ }
+
+ private String formatMessage(long extraSleepTime,
+ Map<String, GcTimes> gcTimesAfterSleep,
+ Map<String, GcTimes> gcTimesBeforeSleep) {
+
+ Set<String> gcBeanNames = Sets.intersection(
+ gcTimesAfterSleep.keySet(),
+ gcTimesBeforeSleep.keySet());
+ List<String> gcDiffs = Lists.newArrayList();
+ for (String name : gcBeanNames) {
+ GcTimes diff = gcTimesAfterSleep.get(name).subtract(
+ gcTimesBeforeSleep.get(name));
+ if (diff.gcCount != 0) {
+ gcDiffs.add("GC pool '" + name + "' had collection(s): " +
+ diff.toString());
+ }
+ }
+
+ String ret = "Detected pause in JVM or host machine (eg GC): " +
+ "pause of approximately " + extraSleepTime + "ms\n";
+ if (gcDiffs.isEmpty()) {
+ ret += "No GCs detected";
+ } else {
+ ret += Joiner.on("\n").join(gcDiffs);
+ }
+ return ret;
+ }
+
+ private Map<String, GcTimes> getGcTimes() {
+ Map<String, GcTimes> map = Maps.newHashMap();
+ List<GarbageCollectorMXBean> gcBeans =
+ ManagementFactory.getGarbageCollectorMXBeans();
+ for (GarbageCollectorMXBean gcBean : gcBeans) {
+ map.put(gcBean.getName(), new GcTimes(gcBean));
+ }
+ return map;
+ }
+
+ private static class GcTimes {
+ private GcTimes(GarbageCollectorMXBean gcBean) {
+ gcCount = gcBean.getCollectionCount();
+ gcTimeMillis = gcBean.getCollectionTime();
+ }
+
+ private GcTimes(long count, long time) {
+ this.gcCount = count;
+ this.gcTimeMillis = time;
+ }
+
+ private GcTimes subtract(GcTimes other) {
+ return new GcTimes(this.gcCount - other.gcCount,
+ this.gcTimeMillis - other.gcTimeMillis);
+ }
+
+ @Override
+ public String toString() {
+ return "count=" + gcCount + " time=" + gcTimeMillis + "ms";
+ }
+
+ private final long gcCount;
+ private final long gcTimeMillis;
+ }
+
+ private class Monitor implements Runnable {
+ @Override
+ public void run() {
+ Stopwatch sw = new Stopwatch();
+ Map<String, GcTimes> gcTimesBeforeSleep = getGcTimes();
+ Counter jvmPauseWarnCnt = Metrics.getOrCreateCounter(MetricsConstants.JVM_PAUSE_WARN);
+ Counter jvmPauseInfoCnt = Metrics.getOrCreateCounter(MetricsConstants.JVM_PAUSE_INFO);
+ Counter jvmExtraSleepCnt = Metrics.getOrCreateCounter(MetricsConstants.JVM_EXTRA_SLEEP);
+ while (shouldRun) {
+ sw.reset().start();
+ try {
+ Thread.sleep(SLEEP_INTERVAL_MS);
+ } catch (InterruptedException ie) {
+ return;
+ }
+ long extraSleepTime = sw.elapsed(TimeUnit.MILLISECONDS) - SLEEP_INTERVAL_MS;
+ Map<String, GcTimes> gcTimesAfterSleep = getGcTimes();
+
+ if (extraSleepTime > warnThresholdMs) {
+ ++numGcWarnThresholdExceeded;
+ LOG.warn(formatMessage(
+ extraSleepTime, gcTimesAfterSleep, gcTimesBeforeSleep));
+ if (jvmPauseWarnCnt != null) jvmPauseWarnCnt.inc();
+ } else if (extraSleepTime > infoThresholdMs) {
+ ++numGcInfoThresholdExceeded;
+ LOG.info(formatMessage(
+ extraSleepTime, gcTimesAfterSleep, gcTimesBeforeSleep));
+ if (jvmPauseInfoCnt != null) jvmPauseInfoCnt.inc();
+ }
+ if (jvmExtraSleepCnt != null) jvmExtraSleepCnt.inc(extraSleepTime);
+ totalGcExtraSleepTime += extraSleepTime;
+ gcTimesBeforeSleep = gcTimesAfterSleep;
+ }
+ }
+ }
+
+ /**
+ * Simple 'main' to facilitate manual testing of the pause monitor.
+ *
+ * This main function just leaks memory into a list. Running this class
+ * with a 1GB heap will very quickly go into "GC hell" and result in
+ * log messages about the GC pauses.
+ */
+ public static void main(String []args) throws Exception {
+ new JvmPauseMonitor(new Configuration()).start();
+ List<String> list = Lists.newArrayList();
+ int i = 0;
+ while (true) {
+ list.add(String.valueOf(i++));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/a6b9cd52/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/metrics/Metrics.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/metrics/Metrics.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/metrics/Metrics.java
new file mode 100644
index 0000000..c0b1ebc
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/metrics/Metrics.java
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.metrics;
+
+import com.codahale.metrics.ConsoleReporter;
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.JmxReporter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Reporter;
+import com.codahale.metrics.ScheduledReporter;
+import com.codahale.metrics.Timer;
+import com.codahale.metrics.jvm.BufferPoolMetricSet;
+import com.codahale.metrics.jvm.ClassLoadingGaugeSet;
+import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
+import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
+import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
+import com.github.joshelser.dropwizard.metrics.hadoop.HadoopMetrics2Reporter;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class Metrics {
+ private static final Logger LOGGER = LoggerFactory.getLogger(Metrics.class);
+
+ private static Metrics self;
+ private static final AtomicInteger singletonAtomicInteger = new AtomicInteger();
+
+ private final MetricRegistry registry;
+ private List<Reporter> reporters;
+ private List<ScheduledReporter> scheduledReporters;
+ private Map<String, AtomicInteger> gaugeAtomics;
+ private boolean hadoopMetricsStarted;
+
+ public static synchronized Metrics initialize(Configuration conf) {
+ if (self == null) {
+ self = new Metrics(conf);
+ }
+ return self;
+ }
+
+ public static MetricRegistry getRegistry() {
+ if (self == null) return null;
+ return self.registry;
+ }
+
+ public static void shutdown() {
+ if (self != null) {
+ for (ScheduledReporter reporter : self.scheduledReporters) {
+ reporter.stop();
+ reporter.close();
+ }
+ if (self.hadoopMetricsStarted) DefaultMetricsSystem.shutdown();
+ self = null;
+ }
+ }
+
+ /**
+ * Get an existing counter or create a new one if the requested one does not yet exist. Creation
+ * is synchronized to assure that only one instance of the counter is created.
+ * @param name name of the counter
+ * @return new Counter, or existing one if it already exists. This will return null if the
+ * metrics have not been initialized.
+ */
+ public static Counter getOrCreateCounter(String name) {
+ if (self == null) return null;
+ Map<String, Counter> counters = self.registry.getCounters();
+ Counter counter = counters.get(name);
+ if (counter != null) return counter;
+ // Looks like it doesn't exist. Lock so that two threads don't create it at once.
+ synchronized (Metrics.class) {
+ // Recheck to make sure someone didn't create it while we waited.
+ counters = self.registry.getCounters();
+ counter = counters.get(name);
+ if (counter != null) return counter;
+ return self.registry.counter(name);
+ }
+ }
+
+ /**
+ * Get an existing timer or create a new one if the requested one does not yet exist. Creation
+ * is synchronized to assure that only one instance of the counter is created.
+ * @param name timer name
+ * @return new Timer, or existing one if it already exists, null if the metrics have not been
+ * initialized.
+ */
+ public static Timer getOrCreateTimer(String name) {
+ if (self == null) return null;
+ Map<String, Timer> timers = self.registry.getTimers();
+ Timer timer = timers.get(name);
+ if (timer != null) return timer;
+ synchronized (Metrics.class) {
+ timers = self.registry.getTimers();
+ timer = timers.get(name);
+ if (timer != null) return timer;
+ return self.registry.timer(name);
+ }
+ }
+
+ /**
+ * Get the AtomicInteger behind an existing gauge, or create a new gauge if it does not already
+ * exist.
+ * @param name Name of gauge. This should come from MetricConstants
+ * @return AtomicInteger underlying this gauge.
+ */
+ public static AtomicInteger getOrCreateGauge(String name) {
+ // We return a garbage value if metrics haven't been initialized so that callers don't have
+ // to keep checking if the resulting value is null.
+ if (self == null) return singletonAtomicInteger;
+ AtomicInteger ai = self.gaugeAtomics.get(name);
+ if (ai != null) return ai;
+ synchronized (Metrics.class) {
+ ai = self.gaugeAtomics.get(name);
+ if (ai != null) return ai;
+ ai = new AtomicInteger();
+ final AtomicInteger forGauge = ai;
+ self.gaugeAtomics.put(name, ai);
+ self.registry.register(name, new Gauge<Integer>() {
+ @Override
+ public Integer getValue() {
+ return forGauge.get();
+ }
+ });
+ return ai;
+ }
+ }
+
+ @VisibleForTesting
+ static List<Reporter> getReporters() {
+ return self.reporters;
+ }
+
+ private Metrics(Configuration conf) {
+ registry = new MetricRegistry();
+
+ registry.registerAll(new GarbageCollectorMetricSet());
+ registry.registerAll(new BufferPoolMetricSet(ManagementFactory.getPlatformMBeanServer()));
+ registry.registerAll(new MemoryUsageGaugeSet());
+ registry.registerAll(new ThreadStatesGaugeSet());
+ registry.registerAll(new ClassLoadingGaugeSet());
+
+ /*
+ * This is little complicated. First we look for our own config values on this. If those
+ * aren't set we use the Hive ones. But Hive also has multiple ways to do this, so we need to
+ * look in both of theirs as well. We can't use theirs directly because they wrap the
+ * codahale reporters in their own and we do not.
+ */
+ // Check our config value first. I'm explicitly avoiding getting the default value for now,
+ // as I don't want our default to override a Hive set value.
+ String reportersToStart = conf.get(MetastoreConf.ConfVars.METRICS_REPORTERS.varname);
+ if (reportersToStart == null) {
+ // Now look in the current Hive config value. Again, avoiding getting defaults
+ reportersToStart =
+ conf.get(MetastoreConf.ConfVars.HIVE_CODAHALE_METRICS_REPORTER_CLASSES.hiveName);
+ if (reportersToStart == null) {
+ // Last chance, look in the old Hive config value. Still avoiding defaults.
+ reportersToStart =
+ conf.get(MetastoreConf.ConfVars.HIVE_METRICS_REPORTER.hiveName);
+ if (reportersToStart == null) {
+ // Alright fine, we'll use our defaults
+ reportersToStart = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.METRICS_REPORTERS);
+ }
+ }
+ }
+
+ reporters = new ArrayList<>();
+ scheduledReporters = new ArrayList<>();
+ if (reportersToStart != null && reportersToStart.length() > 0) {
+ String[] reporterNames = reportersToStart.toLowerCase().split(",");
+ for (String reporterName : reporterNames) {
+ if (reporterName.equals("console") || reporterName.endsWith("consolemetricsreporter")) {
+ ConsoleReporter reporter = ConsoleReporter.forRegistry(registry)
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .build();
+ reporter.start(15, TimeUnit.SECONDS);
+ reporters.add(reporter);
+ scheduledReporters.add(reporter);
+ } else if (reporterName.equals("jmx") || reporterName.endsWith("jmxmetricsreporter")) {
+ JmxReporter reporter = JmxReporter.forRegistry(registry)
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .build();
+ reporter.start();
+ reporters.add(reporter);
+ } else if (reporterName.startsWith("json") || reporterName.endsWith("jsonfilemetricsreporter")) {
+ // We have to initialize the thread pool before we start this one, as it uses it
+ JsonReporter reporter = JsonReporter.forRegistry(registry, conf)
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .build();
+ reporter.start(MetastoreConf.getTimeVar(conf,
+ MetastoreConf.ConfVars.METRICS_JSON_FILE_INTERVAL, TimeUnit.SECONDS), TimeUnit.SECONDS);
+ reporters.add(reporter);
+ scheduledReporters.add(reporter);
+ } else if (reporterName.startsWith("hadoop") || reporterName.endsWith("metrics2reporter")) {
+ HadoopMetrics2Reporter reporter = HadoopMetrics2Reporter.forRegistry(registry)
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .build(DefaultMetricsSystem.initialize("metastore"), "metastore", "Runtime metadata" +
+ " catalog", "general-metadata");
+ reporter.start(1, TimeUnit.MINUTES);
+ reporters.add(reporter);
+ scheduledReporters.add(reporter);
+ hadoopMetricsStarted = true;
+ } else {
+ throw new RuntimeException("Unknown metric type " + reporterName);
+ }
+ }
+ } else {
+ LOGGER.warn("No metrics reporters configured.");
+ }
+
+ // Create map for tracking gauges
+ gaugeAtomics = new HashMap<>();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/a6b9cd52/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/metrics/MetricsConstants.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/metrics/MetricsConstants.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/metrics/MetricsConstants.java
new file mode 100644
index 0000000..3b188f8
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/metrics/MetricsConstants.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.metrics;
+
+public class MetricsConstants {
+ public static final String ACTIVE_CALLS = "active_calls_";
+ public static final String API_PREFIX = "api_";
+
+ public static final String CREATE_TOTAL_DATABASES = "create_total_count_dbs";
+ public static final String CREATE_TOTAL_TABLES = "create_total_count_tables";
+ public static final String CREATE_TOTAL_PARTITIONS = "create_total_count_partitions";
+
+ public static final String DELETE_TOTAL_DATABASES = "delete_total_count_dbs";
+ public static final String DELETE_TOTAL_TABLES = "delete_total_count_tables";
+ public static final String DELETE_TOTAL_PARTITIONS = "delete_total_count_partitions";
+
+ public static final String DIRECTSQL_ERRORS = "directsql_errors";
+
+ public static final String JVM_PAUSE_INFO = "jvm.pause.info-threshold";
+ public static final String JVM_PAUSE_WARN = "jvm.pause.warn-threshold";
+ public static final String JVM_EXTRA_SLEEP = "jvm.pause.extraSleepTime";
+
+ public static final String NUM_OPEN_TXNS = "num_open_transactions";
+ public static final String NUM_TIMED_OUT_TXNS = "num_timed_out_transactions";
+
+ public static final String OPEN_CONNECTIONS = "open_connections";
+
+ public static final String TOTAL_DATABASES = "total_count_dbs";
+ public static final String TOTAL_TABLES = "total_count_tables";
+ public static final String TOTAL_PARTITIONS = "total_count_partitions";
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/a6b9cd52/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/metrics/PerfLogger.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/metrics/PerfLogger.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/metrics/PerfLogger.java
new file mode 100644
index 0000000..a2def26
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/metrics/PerfLogger.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.metrics;
+
+import com.codahale.metrics.Timer;
+import com.google.common.collect.ImmutableMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * PerfLogger.
+ *
+ * Can be used to measure and log the time spent by a piece of code.
+ */
+public class PerfLogger {
+ protected final Map<String, Long> startTimes = new HashMap<>();
+ protected final Map<String, Long> endTimes = new HashMap<>();
+
+ static final private Logger LOG = LoggerFactory.getLogger(PerfLogger.class.getName());
+ protected static final ThreadLocal<PerfLogger> perfLogger = new ThreadLocal<>();
+
+
+ private PerfLogger() {
+ // Use getPerfLogger to get an instance of PerfLogger
+ }
+
+ /**
+ * Get the singleton PerfLogger instance.
+ * @param resetPerfLogger if false, get the current PerfLogger, or create a new one if a
+ * current one does not exist. If true, a new instance of PerfLogger
+ * will be returned rather than the existing one. Note that the
+ * existing PerfLogger is not shutdown and any object which already has a
+ * reference to it may continue to use it. But all future calls to this
+ * method with this set to false will get the new PerfLogger instance.
+ * @return a PerfLogger
+ */
+ public static PerfLogger getPerfLogger(boolean resetPerfLogger) {
+ PerfLogger result = perfLogger.get();
+ if (resetPerfLogger || result == null) {
+ result = new PerfLogger();
+ perfLogger.set(result);
+ }
+ return result;
+ }
+
+ public static void setPerfLogger(PerfLogger resetPerfLogger) {
+ perfLogger.set(resetPerfLogger);
+ }
+
+ /**
+ * Call this function when you start to measure time spent by a piece of code.
+ * @param callerName the logging object to be used.
+ * @param method method or ID that identifies this perf log element.
+ */
+ public void PerfLogBegin(String callerName, String method) {
+ long startTime = System.currentTimeMillis();
+ startTimes.put(method, new Long(startTime));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<PERFLOG method=" + method + " from=" + callerName + ">");
+ }
+ beginMetrics(method);
+ }
+ /**
+ * Call this function in correspondence of PerfLogBegin to mark the end of the measurement.
+ * @param callerName
+ * @param method
+ * @return long duration the difference between now and startTime, or -1 if startTime is null
+ */
+ public long PerfLogEnd(String callerName, String method) {
+ return PerfLogEnd(callerName, method, null);
+ }
+
+ /**
+ * Call this function in correspondence of PerfLogBegin to mark the end of the measurement.
+ * @param callerName
+ * @param method
+ * @return long duration the difference between now and startTime, or -1 if startTime is null
+ */
+ public long PerfLogEnd(String callerName, String method, String additionalInfo) {
+ Long startTime = startTimes.get(method);
+ long endTime = System.currentTimeMillis();
+ endTimes.put(method, new Long(endTime));
+ long duration = startTime == null ? -1 : endTime - startTime.longValue();
+
+ if (LOG.isDebugEnabled()) {
+ StringBuilder sb = new StringBuilder("</PERFLOG method=").append(method);
+ if (startTime != null) {
+ sb.append(" start=").append(startTime);
+ }
+ sb.append(" end=").append(endTime);
+ if (startTime != null) {
+ sb.append(" duration=").append(duration);
+ }
+ sb.append(" from=").append(callerName);
+ if (additionalInfo != null) {
+ sb.append(" ").append(additionalInfo);
+ }
+ sb.append(">");
+ LOG.debug(sb.toString());
+ }
+ endMetrics(method);
+ return duration;
+ }
+
+ public Long getStartTime(String method) {
+ long startTime = 0L;
+
+ if (startTimes.containsKey(method)) {
+ startTime = startTimes.get(method);
+ }
+ return startTime;
+ }
+
+ public Long getEndTime(String method) {
+ long endTime = 0L;
+
+ if (endTimes.containsKey(method)) {
+ endTime = endTimes.get(method);
+ }
+ return endTime;
+ }
+
+ public boolean startTimeHasMethod(String method) {
+ return startTimes.containsKey(method);
+ }
+
+ public boolean endTimeHasMethod(String method) {
+ return endTimes.containsKey(method);
+ }
+
+ public Long getDuration(String method) {
+ long duration = 0;
+ if (startTimes.containsKey(method) && endTimes.containsKey(method)) {
+ duration = endTimes.get(method) - startTimes.get(method);
+ }
+ return duration;
+ }
+
+
+ public ImmutableMap<String, Long> getStartTimes() {
+ return ImmutableMap.copyOf(startTimes);
+ }
+
+ public ImmutableMap<String, Long> getEndTimes() {
+ return ImmutableMap.copyOf(endTimes);
+ }
+
+ //Methods for metrics integration. Each thread-local PerfLogger will open/close scope during each perf-log method.
+ protected transient Map<String, Timer.Context> timerContexts = new HashMap<>();
+
+ private void beginMetrics(String method) {
+ Timer timer = Metrics.getOrCreateTimer(MetricsConstants.API_PREFIX + method);
+ if (timer != null) {
+ timerContexts.put(method, timer.time());
+ }
+
+ }
+
+ private void endMetrics(String method) {
+ Timer.Context context = timerContexts.remove(method);
+ if (context != null) {
+ context.close();
+ }
+ }
+
+ /**
+ * Cleans up any dangling perfLog metric call scopes.
+ */
+ public void cleanupPerfLogMetrics() {
+ for (Timer.Context context : timerContexts.values()) {
+ context.close();
+ }
+ timerContexts.clear();
+ }
+}
[27/50] [abbrv] hive git commit: HIVE-17412: Add "--
SORT_QUERY_RESULTS" for spark_vectorized_dynamic_partition_pruning.q (Liyun
Zhang, reviewed by Sahil and Xuefu)
Posted by we...@apache.org.
HIVE-17412: Add "-- SORT_QUERY_RESULTS" for spark_vectorized_dynamic_partition_pruning.q (Liyun Zhang, reviewed by Sahil and Xuefu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3573549b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3573549b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3573549b
Branch: refs/heads/hive-14535
Commit: 3573549bc6c134569e3aaa61530c448d34ae360f
Parents: 2e5ead1
Author: Ferdinand Xu <ch...@intel.com>
Authored: Thu Aug 31 16:43:09 2017 +0800
Committer: Ferdinand Xu <ch...@intel.com>
Committed: Thu Aug 31 16:43:09 2017 +0800
----------------------------------------------------------------------
.../clientpositive/spark_vectorized_dynamic_partition_pruning.q | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3573549b/ql/src/test/queries/clientpositive/spark_vectorized_dynamic_partition_pruning.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/spark_vectorized_dynamic_partition_pruning.q b/ql/src/test/queries/clientpositive/spark_vectorized_dynamic_partition_pruning.q
index 791c3a9..8868a34 100644
--- a/ql/src/test/queries/clientpositive/spark_vectorized_dynamic_partition_pruning.q
+++ b/ql/src/test/queries/clientpositive/spark_vectorized_dynamic_partition_pruning.q
@@ -6,7 +6,7 @@ set hive.optimize.index.filter=true;
set hive.vectorized.execution.enabled=true;
set hive.strict.checks.cartesian.product=false;
-
+-- SORT_QUERY_RESULTS
select distinct ds from srcpart;
select distinct hr from srcpart;
[43/50] [abbrv] hive git commit: HIVE-14671 : merge master into
hive-14535 (Wei Zheng)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
----------------------------------------------------------------------
diff --cc standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
index 0000000,d5963f3..cfbc488
mode 000000,100644..100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
@@@ -1,0 -1,9033 +1,9034 @@@
+ /**
+ * Autogenerated by Thrift Compiler (0.9.3)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+ #ifndef hive_metastore_TYPES_H
+ #define hive_metastore_TYPES_H
+
+ #include <iosfwd>
+
+ #include <thrift/Thrift.h>
+ #include <thrift/TApplicationException.h>
+ #include <thrift/protocol/TProtocol.h>
+ #include <thrift/transport/TTransport.h>
+
+ #include <thrift/cxxfunctional.h>
+ #include "fb303_types.h"
+
+
+ namespace Apache { namespace Hadoop { namespace Hive {
+
+ struct HiveObjectType {
+ enum type {
+ GLOBAL = 1,
+ DATABASE = 2,
+ TABLE = 3,
+ PARTITION = 4,
+ COLUMN = 5
+ };
+ };
+
+ extern const std::map<int, const char*> _HiveObjectType_VALUES_TO_NAMES;
+
+ struct PrincipalType {
+ enum type {
+ USER = 1,
+ ROLE = 2,
+ GROUP = 3
+ };
+ };
+
+ extern const std::map<int, const char*> _PrincipalType_VALUES_TO_NAMES;
+
+ struct PartitionEventType {
+ enum type {
+ LOAD_DONE = 1
+ };
+ };
+
+ extern const std::map<int, const char*> _PartitionEventType_VALUES_TO_NAMES;
+
+ struct TxnState {
+ enum type {
+ COMMITTED = 1,
+ ABORTED = 2,
+ OPEN = 3
+ };
+ };
+
+ extern const std::map<int, const char*> _TxnState_VALUES_TO_NAMES;
+
+ struct LockLevel {
+ enum type {
+ DB = 1,
+ TABLE = 2,
+ PARTITION = 3
+ };
+ };
+
+ extern const std::map<int, const char*> _LockLevel_VALUES_TO_NAMES;
+
+ struct LockState {
+ enum type {
+ ACQUIRED = 1,
+ WAITING = 2,
+ ABORT = 3,
+ NOT_ACQUIRED = 4
+ };
+ };
+
+ extern const std::map<int, const char*> _LockState_VALUES_TO_NAMES;
+
+ struct LockType {
+ enum type {
+ SHARED_READ = 1,
+ SHARED_WRITE = 2,
+ EXCLUSIVE = 3
+ };
+ };
+
+ extern const std::map<int, const char*> _LockType_VALUES_TO_NAMES;
+
+ struct CompactionType {
+ enum type {
+ MINOR = 1,
+ MAJOR = 2
+ };
+ };
+
+ extern const std::map<int, const char*> _CompactionType_VALUES_TO_NAMES;
+
+ struct GrantRevokeType {
+ enum type {
+ GRANT = 1,
+ REVOKE = 2
+ };
+ };
+
+ extern const std::map<int, const char*> _GrantRevokeType_VALUES_TO_NAMES;
+
+ struct DataOperationType {
+ enum type {
+ SELECT = 1,
+ INSERT = 2,
+ UPDATE = 3,
+ DELETE = 4,
+ UNSET = 5,
+ NO_TXN = 6
+ };
+ };
+
+ extern const std::map<int, const char*> _DataOperationType_VALUES_TO_NAMES;
+
+ struct EventRequestType {
+ enum type {
+ INSERT = 1,
+ UPDATE = 2,
+ DELETE = 3
+ };
+ };
+
+ extern const std::map<int, const char*> _EventRequestType_VALUES_TO_NAMES;
+
+ struct FunctionType {
+ enum type {
+ JAVA = 1
+ };
+ };
+
+ extern const std::map<int, const char*> _FunctionType_VALUES_TO_NAMES;
+
+ struct ResourceType {
+ enum type {
+ JAR = 1,
+ FILE = 2,
+ ARCHIVE = 3
+ };
+ };
+
+ extern const std::map<int, const char*> _ResourceType_VALUES_TO_NAMES;
+
+ struct FileMetadataExprType {
+ enum type {
+ ORC_SARG = 1
+ };
+ };
+
+ extern const std::map<int, const char*> _FileMetadataExprType_VALUES_TO_NAMES;
+
+ struct ClientCapability {
+ enum type {
- TEST_CAPABILITY = 1
++ TEST_CAPABILITY = 1,
++ INSERT_ONLY_TABLES = 2
+ };
+ };
+
+ extern const std::map<int, const char*> _ClientCapability_VALUES_TO_NAMES;
+
+ class Version;
+
+ class FieldSchema;
+
+ class SQLPrimaryKey;
+
+ class SQLForeignKey;
+
+ class SQLUniqueConstraint;
+
+ class SQLNotNullConstraint;
+
+ class Type;
+
+ class HiveObjectRef;
+
+ class PrivilegeGrantInfo;
+
+ class HiveObjectPrivilege;
+
+ class PrivilegeBag;
+
+ class PrincipalPrivilegeSet;
+
+ class GrantRevokePrivilegeRequest;
+
+ class GrantRevokePrivilegeResponse;
+
+ class Role;
+
+ class RolePrincipalGrant;
+
+ class GetRoleGrantsForPrincipalRequest;
+
+ class GetRoleGrantsForPrincipalResponse;
+
+ class GetPrincipalsInRoleRequest;
+
+ class GetPrincipalsInRoleResponse;
+
+ class GrantRevokeRoleRequest;
+
+ class GrantRevokeRoleResponse;
+
+ class Database;
+
+ class SerDeInfo;
+
+ class Order;
+
+ class SkewedInfo;
+
+ class StorageDescriptor;
+
+ class Table;
+
+ class Partition;
+
+ class PartitionWithoutSD;
+
+ class PartitionSpecWithSharedSD;
+
+ class PartitionListComposingSpec;
+
+ class PartitionSpec;
+
+ class Index;
+
+ class BooleanColumnStatsData;
+
+ class DoubleColumnStatsData;
+
+ class LongColumnStatsData;
+
+ class StringColumnStatsData;
+
+ class BinaryColumnStatsData;
+
+ class Decimal;
+
+ class DecimalColumnStatsData;
+
+ class Date;
+
+ class DateColumnStatsData;
+
+ class ColumnStatisticsData;
+
+ class ColumnStatisticsObj;
+
+ class ColumnStatisticsDesc;
+
+ class ColumnStatistics;
+
+ class AggrStats;
+
+ class SetPartitionsStatsRequest;
+
+ class Schema;
+
+ class EnvironmentContext;
+
+ class PrimaryKeysRequest;
+
+ class PrimaryKeysResponse;
+
+ class ForeignKeysRequest;
+
+ class ForeignKeysResponse;
+
+ class UniqueConstraintsRequest;
+
+ class UniqueConstraintsResponse;
+
+ class NotNullConstraintsRequest;
+
+ class NotNullConstraintsResponse;
+
+ class DropConstraintRequest;
+
+ class AddPrimaryKeyRequest;
+
+ class AddForeignKeyRequest;
+
+ class AddUniqueConstraintRequest;
+
+ class AddNotNullConstraintRequest;
+
+ class PartitionsByExprResult;
+
+ class PartitionsByExprRequest;
+
+ class TableStatsResult;
+
+ class PartitionsStatsResult;
+
+ class TableStatsRequest;
+
+ class PartitionsStatsRequest;
+
+ class AddPartitionsResult;
+
+ class AddPartitionsRequest;
+
+ class DropPartitionsResult;
+
+ class DropPartitionsExpr;
+
+ class RequestPartsSpec;
+
+ class DropPartitionsRequest;
+
+ class ResourceUri;
+
+ class Function;
+
+ class TxnInfo;
+
+ class GetOpenTxnsInfoResponse;
+
+ class GetOpenTxnsResponse;
+
+ class OpenTxnRequest;
+
+ class OpenTxnsResponse;
+
+ class AbortTxnRequest;
+
+ class AbortTxnsRequest;
+
+ class CommitTxnRequest;
+
+ class LockComponent;
+
+ class LockRequest;
+
+ class LockResponse;
+
+ class CheckLockRequest;
+
+ class UnlockRequest;
+
+ class ShowLocksRequest;
+
+ class ShowLocksResponseElement;
+
+ class ShowLocksResponse;
+
+ class HeartbeatRequest;
+
+ class HeartbeatTxnRangeRequest;
+
+ class HeartbeatTxnRangeResponse;
+
+ class CompactionRequest;
+
+ class CompactionResponse;
+
+ class ShowCompactRequest;
+
+ class ShowCompactResponseElement;
+
+ class ShowCompactResponse;
+
+ class AddDynamicPartitions;
+
+ class NotificationEventRequest;
+
+ class NotificationEvent;
+
+ class NotificationEventResponse;
+
+ class CurrentNotificationEventId;
+
+ class NotificationEventsCountRequest;
+
+ class NotificationEventsCountResponse;
+
+ class InsertEventRequestData;
+
+ class FireEventRequestData;
+
+ class FireEventRequest;
+
+ class FireEventResponse;
+
+ class MetadataPpdResult;
+
+ class GetFileMetadataByExprResult;
+
+ class GetFileMetadataByExprRequest;
+
+ class GetFileMetadataResult;
+
+ class GetFileMetadataRequest;
+
+ class PutFileMetadataResult;
+
+ class PutFileMetadataRequest;
+
+ class ClearFileMetadataResult;
+
+ class ClearFileMetadataRequest;
+
+ class CacheFileMetadataResult;
+
+ class CacheFileMetadataRequest;
+
+ class GetAllFunctionsResponse;
+
+ class ClientCapabilities;
+
+ class GetTableRequest;
+
+ class GetTableResult;
+
+ class GetTablesRequest;
+
+ class GetTablesResult;
+
+ class CmRecycleRequest;
+
+ class CmRecycleResponse;
+
+ class TableMeta;
+
+ class MetaException;
+
+ class UnknownTableException;
+
+ class UnknownDBException;
+
+ class AlreadyExistsException;
+
+ class InvalidPartitionException;
+
+ class UnknownPartitionException;
+
+ class InvalidObjectException;
+
+ class NoSuchObjectException;
+
+ class IndexAlreadyExistsException;
+
+ class InvalidOperationException;
+
+ class ConfigValSecurityException;
+
+ class InvalidInputException;
+
+ class NoSuchTxnException;
+
+ class TxnAbortedException;
+
+ class TxnOpenException;
+
+ class NoSuchLockException;
+
+ typedef struct _Version__isset {
+ _Version__isset() : version(false), comments(false) {}
+ bool version :1;
+ bool comments :1;
+ } _Version__isset;
+
+ class Version {
+ public:
+
+ Version(const Version&);
+ Version& operator=(const Version&);
+ Version() : version(), comments() {
+ }
+
+ virtual ~Version() throw();
+ std::string version;
+ std::string comments;
+
+ _Version__isset __isset;
+
+ void __set_version(const std::string& val);
+
+ void __set_comments(const std::string& val);
+
+ bool operator == (const Version & rhs) const
+ {
+ if (!(version == rhs.version))
+ return false;
+ if (!(comments == rhs.comments))
+ return false;
+ return true;
+ }
+ bool operator != (const Version &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const Version & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(Version &a, Version &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const Version& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _FieldSchema__isset {
+ _FieldSchema__isset() : name(false), type(false), comment(false) {}
+ bool name :1;
+ bool type :1;
+ bool comment :1;
+ } _FieldSchema__isset;
+
+ class FieldSchema {
+ public:
+
+ FieldSchema(const FieldSchema&);
+ FieldSchema& operator=(const FieldSchema&);
+ FieldSchema() : name(), type(), comment() {
+ }
+
+ virtual ~FieldSchema() throw();
+ std::string name;
+ std::string type;
+ std::string comment;
+
+ _FieldSchema__isset __isset;
+
+ void __set_name(const std::string& val);
+
+ void __set_type(const std::string& val);
+
+ void __set_comment(const std::string& val);
+
+ bool operator == (const FieldSchema & rhs) const
+ {
+ if (!(name == rhs.name))
+ return false;
+ if (!(type == rhs.type))
+ return false;
+ if (!(comment == rhs.comment))
+ return false;
+ return true;
+ }
+ bool operator != (const FieldSchema &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const FieldSchema & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(FieldSchema &a, FieldSchema &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const FieldSchema& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _SQLPrimaryKey__isset {
+ _SQLPrimaryKey__isset() : table_db(false), table_name(false), column_name(false), key_seq(false), pk_name(false), enable_cstr(false), validate_cstr(false), rely_cstr(false) {}
+ bool table_db :1;
+ bool table_name :1;
+ bool column_name :1;
+ bool key_seq :1;
+ bool pk_name :1;
+ bool enable_cstr :1;
+ bool validate_cstr :1;
+ bool rely_cstr :1;
+ } _SQLPrimaryKey__isset;
+
+ class SQLPrimaryKey {
+ public:
+
+ SQLPrimaryKey(const SQLPrimaryKey&);
+ SQLPrimaryKey& operator=(const SQLPrimaryKey&);
+ SQLPrimaryKey() : table_db(), table_name(), column_name(), key_seq(0), pk_name(), enable_cstr(0), validate_cstr(0), rely_cstr(0) {
+ }
+
+ virtual ~SQLPrimaryKey() throw();
+ std::string table_db;
+ std::string table_name;
+ std::string column_name;
+ int32_t key_seq;
+ std::string pk_name;
+ bool enable_cstr;
+ bool validate_cstr;
+ bool rely_cstr;
+
+ _SQLPrimaryKey__isset __isset;
+
+ void __set_table_db(const std::string& val);
+
+ void __set_table_name(const std::string& val);
+
+ void __set_column_name(const std::string& val);
+
+ void __set_key_seq(const int32_t val);
+
+ void __set_pk_name(const std::string& val);
+
+ void __set_enable_cstr(const bool val);
+
+ void __set_validate_cstr(const bool val);
+
+ void __set_rely_cstr(const bool val);
+
+ bool operator == (const SQLPrimaryKey & rhs) const
+ {
+ if (!(table_db == rhs.table_db))
+ return false;
+ if (!(table_name == rhs.table_name))
+ return false;
+ if (!(column_name == rhs.column_name))
+ return false;
+ if (!(key_seq == rhs.key_seq))
+ return false;
+ if (!(pk_name == rhs.pk_name))
+ return false;
+ if (!(enable_cstr == rhs.enable_cstr))
+ return false;
+ if (!(validate_cstr == rhs.validate_cstr))
+ return false;
+ if (!(rely_cstr == rhs.rely_cstr))
+ return false;
+ return true;
+ }
+ bool operator != (const SQLPrimaryKey &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const SQLPrimaryKey & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(SQLPrimaryKey &a, SQLPrimaryKey &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const SQLPrimaryKey& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _SQLForeignKey__isset {
+ _SQLForeignKey__isset() : pktable_db(false), pktable_name(false), pkcolumn_name(false), fktable_db(false), fktable_name(false), fkcolumn_name(false), key_seq(false), update_rule(false), delete_rule(false), fk_name(false), pk_name(false), enable_cstr(false), validate_cstr(false), rely_cstr(false) {}
+ bool pktable_db :1;
+ bool pktable_name :1;
+ bool pkcolumn_name :1;
+ bool fktable_db :1;
+ bool fktable_name :1;
+ bool fkcolumn_name :1;
+ bool key_seq :1;
+ bool update_rule :1;
+ bool delete_rule :1;
+ bool fk_name :1;
+ bool pk_name :1;
+ bool enable_cstr :1;
+ bool validate_cstr :1;
+ bool rely_cstr :1;
+ } _SQLForeignKey__isset;
+
+ class SQLForeignKey {
+ public:
+
+ SQLForeignKey(const SQLForeignKey&);
+ SQLForeignKey& operator=(const SQLForeignKey&);
+ SQLForeignKey() : pktable_db(), pktable_name(), pkcolumn_name(), fktable_db(), fktable_name(), fkcolumn_name(), key_seq(0), update_rule(0), delete_rule(0), fk_name(), pk_name(), enable_cstr(0), validate_cstr(0), rely_cstr(0) {
+ }
+
+ virtual ~SQLForeignKey() throw();
+ std::string pktable_db;
+ std::string pktable_name;
+ std::string pkcolumn_name;
+ std::string fktable_db;
+ std::string fktable_name;
+ std::string fkcolumn_name;
+ int32_t key_seq;
+ int32_t update_rule;
+ int32_t delete_rule;
+ std::string fk_name;
+ std::string pk_name;
+ bool enable_cstr;
+ bool validate_cstr;
+ bool rely_cstr;
+
+ _SQLForeignKey__isset __isset;
+
+ void __set_pktable_db(const std::string& val);
+
+ void __set_pktable_name(const std::string& val);
+
+ void __set_pkcolumn_name(const std::string& val);
+
+ void __set_fktable_db(const std::string& val);
+
+ void __set_fktable_name(const std::string& val);
+
+ void __set_fkcolumn_name(const std::string& val);
+
+ void __set_key_seq(const int32_t val);
+
+ void __set_update_rule(const int32_t val);
+
+ void __set_delete_rule(const int32_t val);
+
+ void __set_fk_name(const std::string& val);
+
+ void __set_pk_name(const std::string& val);
+
+ void __set_enable_cstr(const bool val);
+
+ void __set_validate_cstr(const bool val);
+
+ void __set_rely_cstr(const bool val);
+
+ bool operator == (const SQLForeignKey & rhs) const
+ {
+ if (!(pktable_db == rhs.pktable_db))
+ return false;
+ if (!(pktable_name == rhs.pktable_name))
+ return false;
+ if (!(pkcolumn_name == rhs.pkcolumn_name))
+ return false;
+ if (!(fktable_db == rhs.fktable_db))
+ return false;
+ if (!(fktable_name == rhs.fktable_name))
+ return false;
+ if (!(fkcolumn_name == rhs.fkcolumn_name))
+ return false;
+ if (!(key_seq == rhs.key_seq))
+ return false;
+ if (!(update_rule == rhs.update_rule))
+ return false;
+ if (!(delete_rule == rhs.delete_rule))
+ return false;
+ if (!(fk_name == rhs.fk_name))
+ return false;
+ if (!(pk_name == rhs.pk_name))
+ return false;
+ if (!(enable_cstr == rhs.enable_cstr))
+ return false;
+ if (!(validate_cstr == rhs.validate_cstr))
+ return false;
+ if (!(rely_cstr == rhs.rely_cstr))
+ return false;
+ return true;
+ }
+ bool operator != (const SQLForeignKey &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const SQLForeignKey & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(SQLForeignKey &a, SQLForeignKey &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const SQLForeignKey& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _SQLUniqueConstraint__isset {
+ _SQLUniqueConstraint__isset() : table_db(false), table_name(false), column_name(false), key_seq(false), uk_name(false), enable_cstr(false), validate_cstr(false), rely_cstr(false) {}
+ bool table_db :1;
+ bool table_name :1;
+ bool column_name :1;
+ bool key_seq :1;
+ bool uk_name :1;
+ bool enable_cstr :1;
+ bool validate_cstr :1;
+ bool rely_cstr :1;
+ } _SQLUniqueConstraint__isset;
+
+ class SQLUniqueConstraint {
+ public:
+
+ SQLUniqueConstraint(const SQLUniqueConstraint&);
+ SQLUniqueConstraint& operator=(const SQLUniqueConstraint&);
+ SQLUniqueConstraint() : table_db(), table_name(), column_name(), key_seq(0), uk_name(), enable_cstr(0), validate_cstr(0), rely_cstr(0) {
+ }
+
+ virtual ~SQLUniqueConstraint() throw();
+ std::string table_db;
+ std::string table_name;
+ std::string column_name;
+ int32_t key_seq;
+ std::string uk_name;
+ bool enable_cstr;
+ bool validate_cstr;
+ bool rely_cstr;
+
+ _SQLUniqueConstraint__isset __isset;
+
+ void __set_table_db(const std::string& val);
+
+ void __set_table_name(const std::string& val);
+
+ void __set_column_name(const std::string& val);
+
+ void __set_key_seq(const int32_t val);
+
+ void __set_uk_name(const std::string& val);
+
+ void __set_enable_cstr(const bool val);
+
+ void __set_validate_cstr(const bool val);
+
+ void __set_rely_cstr(const bool val);
+
+ bool operator == (const SQLUniqueConstraint & rhs) const
+ {
+ if (!(table_db == rhs.table_db))
+ return false;
+ if (!(table_name == rhs.table_name))
+ return false;
+ if (!(column_name == rhs.column_name))
+ return false;
+ if (!(key_seq == rhs.key_seq))
+ return false;
+ if (!(uk_name == rhs.uk_name))
+ return false;
+ if (!(enable_cstr == rhs.enable_cstr))
+ return false;
+ if (!(validate_cstr == rhs.validate_cstr))
+ return false;
+ if (!(rely_cstr == rhs.rely_cstr))
+ return false;
+ return true;
+ }
+ bool operator != (const SQLUniqueConstraint &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const SQLUniqueConstraint & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(SQLUniqueConstraint &a, SQLUniqueConstraint &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const SQLUniqueConstraint& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _SQLNotNullConstraint__isset {
+ _SQLNotNullConstraint__isset() : table_db(false), table_name(false), column_name(false), nn_name(false), enable_cstr(false), validate_cstr(false), rely_cstr(false) {}
+ bool table_db :1;
+ bool table_name :1;
+ bool column_name :1;
+ bool nn_name :1;
+ bool enable_cstr :1;
+ bool validate_cstr :1;
+ bool rely_cstr :1;
+ } _SQLNotNullConstraint__isset;
+
+ class SQLNotNullConstraint {
+ public:
+
+ SQLNotNullConstraint(const SQLNotNullConstraint&);
+ SQLNotNullConstraint& operator=(const SQLNotNullConstraint&);
+ SQLNotNullConstraint() : table_db(), table_name(), column_name(), nn_name(), enable_cstr(0), validate_cstr(0), rely_cstr(0) {
+ }
+
+ virtual ~SQLNotNullConstraint() throw();
+ std::string table_db;
+ std::string table_name;
+ std::string column_name;
+ std::string nn_name;
+ bool enable_cstr;
+ bool validate_cstr;
+ bool rely_cstr;
+
+ _SQLNotNullConstraint__isset __isset;
+
+ void __set_table_db(const std::string& val);
+
+ void __set_table_name(const std::string& val);
+
+ void __set_column_name(const std::string& val);
+
+ void __set_nn_name(const std::string& val);
+
+ void __set_enable_cstr(const bool val);
+
+ void __set_validate_cstr(const bool val);
+
+ void __set_rely_cstr(const bool val);
+
+ bool operator == (const SQLNotNullConstraint & rhs) const
+ {
+ if (!(table_db == rhs.table_db))
+ return false;
+ if (!(table_name == rhs.table_name))
+ return false;
+ if (!(column_name == rhs.column_name))
+ return false;
+ if (!(nn_name == rhs.nn_name))
+ return false;
+ if (!(enable_cstr == rhs.enable_cstr))
+ return false;
+ if (!(validate_cstr == rhs.validate_cstr))
+ return false;
+ if (!(rely_cstr == rhs.rely_cstr))
+ return false;
+ return true;
+ }
+ bool operator != (const SQLNotNullConstraint &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const SQLNotNullConstraint & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(SQLNotNullConstraint &a, SQLNotNullConstraint &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const SQLNotNullConstraint& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _Type__isset {
+ _Type__isset() : name(false), type1(false), type2(false), fields(false) {}
+ bool name :1;
+ bool type1 :1;
+ bool type2 :1;
+ bool fields :1;
+ } _Type__isset;
+
+ class Type {
+ public:
+
+ Type(const Type&);
+ Type& operator=(const Type&);
+ Type() : name(), type1(), type2() {
+ }
+
+ virtual ~Type() throw();
+ std::string name;
+ std::string type1;
+ std::string type2;
+ std::vector<FieldSchema> fields;
+
+ _Type__isset __isset;
+
+ void __set_name(const std::string& val);
+
+ void __set_type1(const std::string& val);
+
+ void __set_type2(const std::string& val);
+
+ void __set_fields(const std::vector<FieldSchema> & val);
+
+ bool operator == (const Type & rhs) const
+ {
+ if (!(name == rhs.name))
+ return false;
+ if (__isset.type1 != rhs.__isset.type1)
+ return false;
+ else if (__isset.type1 && !(type1 == rhs.type1))
+ return false;
+ if (__isset.type2 != rhs.__isset.type2)
+ return false;
+ else if (__isset.type2 && !(type2 == rhs.type2))
+ return false;
+ if (__isset.fields != rhs.__isset.fields)
+ return false;
+ else if (__isset.fields && !(fields == rhs.fields))
+ return false;
+ return true;
+ }
+ bool operator != (const Type &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const Type & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(Type &a, Type &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const Type& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _HiveObjectRef__isset {
+ _HiveObjectRef__isset() : objectType(false), dbName(false), objectName(false), partValues(false), columnName(false) {}
+ bool objectType :1;
+ bool dbName :1;
+ bool objectName :1;
+ bool partValues :1;
+ bool columnName :1;
+ } _HiveObjectRef__isset;
+
+ class HiveObjectRef {
+ public:
+
+ HiveObjectRef(const HiveObjectRef&);
+ HiveObjectRef& operator=(const HiveObjectRef&);
+ HiveObjectRef() : objectType((HiveObjectType::type)0), dbName(), objectName(), columnName() {
+ }
+
+ virtual ~HiveObjectRef() throw();
+ HiveObjectType::type objectType;
+ std::string dbName;
+ std::string objectName;
+ std::vector<std::string> partValues;
+ std::string columnName;
+
+ _HiveObjectRef__isset __isset;
+
+ void __set_objectType(const HiveObjectType::type val);
+
+ void __set_dbName(const std::string& val);
+
+ void __set_objectName(const std::string& val);
+
+ void __set_partValues(const std::vector<std::string> & val);
+
+ void __set_columnName(const std::string& val);
+
+ bool operator == (const HiveObjectRef & rhs) const
+ {
+ if (!(objectType == rhs.objectType))
+ return false;
+ if (!(dbName == rhs.dbName))
+ return false;
+ if (!(objectName == rhs.objectName))
+ return false;
+ if (!(partValues == rhs.partValues))
+ return false;
+ if (!(columnName == rhs.columnName))
+ return false;
+ return true;
+ }
+ bool operator != (const HiveObjectRef &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const HiveObjectRef & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(HiveObjectRef &a, HiveObjectRef &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const HiveObjectRef& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _PrivilegeGrantInfo__isset {
+ _PrivilegeGrantInfo__isset() : privilege(false), createTime(false), grantor(false), grantorType(false), grantOption(false) {}
+ bool privilege :1;
+ bool createTime :1;
+ bool grantor :1;
+ bool grantorType :1;
+ bool grantOption :1;
+ } _PrivilegeGrantInfo__isset;
+
+ class PrivilegeGrantInfo {
+ public:
+
+ PrivilegeGrantInfo(const PrivilegeGrantInfo&);
+ PrivilegeGrantInfo& operator=(const PrivilegeGrantInfo&);
+ PrivilegeGrantInfo() : privilege(), createTime(0), grantor(), grantorType((PrincipalType::type)0), grantOption(0) {
+ }
+
+ virtual ~PrivilegeGrantInfo() throw();
+ std::string privilege;
+ int32_t createTime;
+ std::string grantor;
+ PrincipalType::type grantorType;
+ bool grantOption;
+
+ _PrivilegeGrantInfo__isset __isset;
+
+ void __set_privilege(const std::string& val);
+
+ void __set_createTime(const int32_t val);
+
+ void __set_grantor(const std::string& val);
+
+ void __set_grantorType(const PrincipalType::type val);
+
+ void __set_grantOption(const bool val);
+
+ bool operator == (const PrivilegeGrantInfo & rhs) const
+ {
+ if (!(privilege == rhs.privilege))
+ return false;
+ if (!(createTime == rhs.createTime))
+ return false;
+ if (!(grantor == rhs.grantor))
+ return false;
+ if (!(grantorType == rhs.grantorType))
+ return false;
+ if (!(grantOption == rhs.grantOption))
+ return false;
+ return true;
+ }
+ bool operator != (const PrivilegeGrantInfo &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const PrivilegeGrantInfo & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(PrivilegeGrantInfo &a, PrivilegeGrantInfo &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const PrivilegeGrantInfo& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _HiveObjectPrivilege__isset {
+ _HiveObjectPrivilege__isset() : hiveObject(false), principalName(false), principalType(false), grantInfo(false) {}
+ bool hiveObject :1;
+ bool principalName :1;
+ bool principalType :1;
+ bool grantInfo :1;
+ } _HiveObjectPrivilege__isset;
+
+ class HiveObjectPrivilege {
+ public:
+
+ HiveObjectPrivilege(const HiveObjectPrivilege&);
+ HiveObjectPrivilege& operator=(const HiveObjectPrivilege&);
+ HiveObjectPrivilege() : principalName(), principalType((PrincipalType::type)0) {
+ }
+
+ virtual ~HiveObjectPrivilege() throw();
+ HiveObjectRef hiveObject;
+ std::string principalName;
+ PrincipalType::type principalType;
+ PrivilegeGrantInfo grantInfo;
+
+ _HiveObjectPrivilege__isset __isset;
+
+ void __set_hiveObject(const HiveObjectRef& val);
+
+ void __set_principalName(const std::string& val);
+
+ void __set_principalType(const PrincipalType::type val);
+
+ void __set_grantInfo(const PrivilegeGrantInfo& val);
+
+ bool operator == (const HiveObjectPrivilege & rhs) const
+ {
+ if (!(hiveObject == rhs.hiveObject))
+ return false;
+ if (!(principalName == rhs.principalName))
+ return false;
+ if (!(principalType == rhs.principalType))
+ return false;
+ if (!(grantInfo == rhs.grantInfo))
+ return false;
+ return true;
+ }
+ bool operator != (const HiveObjectPrivilege &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const HiveObjectPrivilege & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(HiveObjectPrivilege &a, HiveObjectPrivilege &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const HiveObjectPrivilege& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _PrivilegeBag__isset {
+ _PrivilegeBag__isset() : privileges(false) {}
+ bool privileges :1;
+ } _PrivilegeBag__isset;
+
+ class PrivilegeBag {
+ public:
+
+ PrivilegeBag(const PrivilegeBag&);
+ PrivilegeBag& operator=(const PrivilegeBag&);
+ PrivilegeBag() {
+ }
+
+ virtual ~PrivilegeBag() throw();
+ std::vector<HiveObjectPrivilege> privileges;
+
+ _PrivilegeBag__isset __isset;
+
+ void __set_privileges(const std::vector<HiveObjectPrivilege> & val);
+
+ bool operator == (const PrivilegeBag & rhs) const
+ {
+ if (!(privileges == rhs.privileges))
+ return false;
+ return true;
+ }
+ bool operator != (const PrivilegeBag &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const PrivilegeBag & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(PrivilegeBag &a, PrivilegeBag &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const PrivilegeBag& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _PrincipalPrivilegeSet__isset {
+ _PrincipalPrivilegeSet__isset() : userPrivileges(false), groupPrivileges(false), rolePrivileges(false) {}
+ bool userPrivileges :1;
+ bool groupPrivileges :1;
+ bool rolePrivileges :1;
+ } _PrincipalPrivilegeSet__isset;
+
+ class PrincipalPrivilegeSet {
+ public:
+
+ PrincipalPrivilegeSet(const PrincipalPrivilegeSet&);
+ PrincipalPrivilegeSet& operator=(const PrincipalPrivilegeSet&);
+ PrincipalPrivilegeSet() {
+ }
+
+ virtual ~PrincipalPrivilegeSet() throw();
+ std::map<std::string, std::vector<PrivilegeGrantInfo> > userPrivileges;
+ std::map<std::string, std::vector<PrivilegeGrantInfo> > groupPrivileges;
+ std::map<std::string, std::vector<PrivilegeGrantInfo> > rolePrivileges;
+
+ _PrincipalPrivilegeSet__isset __isset;
+
+ void __set_userPrivileges(const std::map<std::string, std::vector<PrivilegeGrantInfo> > & val);
+
+ void __set_groupPrivileges(const std::map<std::string, std::vector<PrivilegeGrantInfo> > & val);
+
+ void __set_rolePrivileges(const std::map<std::string, std::vector<PrivilegeGrantInfo> > & val);
+
+ bool operator == (const PrincipalPrivilegeSet & rhs) const
+ {
+ if (!(userPrivileges == rhs.userPrivileges))
+ return false;
+ if (!(groupPrivileges == rhs.groupPrivileges))
+ return false;
+ if (!(rolePrivileges == rhs.rolePrivileges))
+ return false;
+ return true;
+ }
+ bool operator != (const PrincipalPrivilegeSet &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const PrincipalPrivilegeSet & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(PrincipalPrivilegeSet &a, PrincipalPrivilegeSet &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const PrincipalPrivilegeSet& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _GrantRevokePrivilegeRequest__isset {
+ _GrantRevokePrivilegeRequest__isset() : requestType(false), privileges(false), revokeGrantOption(false) {}
+ bool requestType :1;
+ bool privileges :1;
+ bool revokeGrantOption :1;
+ } _GrantRevokePrivilegeRequest__isset;
+
+ class GrantRevokePrivilegeRequest {
+ public:
+
+ GrantRevokePrivilegeRequest(const GrantRevokePrivilegeRequest&);
+ GrantRevokePrivilegeRequest& operator=(const GrantRevokePrivilegeRequest&);
+ GrantRevokePrivilegeRequest() : requestType((GrantRevokeType::type)0), revokeGrantOption(0) {
+ }
+
+ virtual ~GrantRevokePrivilegeRequest() throw();
+ GrantRevokeType::type requestType;
+ PrivilegeBag privileges;
+ bool revokeGrantOption;
+
+ _GrantRevokePrivilegeRequest__isset __isset;
+
+ void __set_requestType(const GrantRevokeType::type val);
+
+ void __set_privileges(const PrivilegeBag& val);
+
+ void __set_revokeGrantOption(const bool val);
+
+ bool operator == (const GrantRevokePrivilegeRequest & rhs) const
+ {
+ if (!(requestType == rhs.requestType))
+ return false;
+ if (!(privileges == rhs.privileges))
+ return false;
+ if (__isset.revokeGrantOption != rhs.__isset.revokeGrantOption)
+ return false;
+ else if (__isset.revokeGrantOption && !(revokeGrantOption == rhs.revokeGrantOption))
+ return false;
+ return true;
+ }
+ bool operator != (const GrantRevokePrivilegeRequest &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const GrantRevokePrivilegeRequest & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(GrantRevokePrivilegeRequest &a, GrantRevokePrivilegeRequest &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const GrantRevokePrivilegeRequest& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _GrantRevokePrivilegeResponse__isset {
+ _GrantRevokePrivilegeResponse__isset() : success(false) {}
+ bool success :1;
+ } _GrantRevokePrivilegeResponse__isset;
+
+ class GrantRevokePrivilegeResponse {
+ public:
+
+ GrantRevokePrivilegeResponse(const GrantRevokePrivilegeResponse&);
+ GrantRevokePrivilegeResponse& operator=(const GrantRevokePrivilegeResponse&);
+ GrantRevokePrivilegeResponse() : success(0) {
+ }
+
+ virtual ~GrantRevokePrivilegeResponse() throw();
+ bool success;
+
+ _GrantRevokePrivilegeResponse__isset __isset;
+
+ void __set_success(const bool val);
+
+ bool operator == (const GrantRevokePrivilegeResponse & rhs) const
+ {
+ if (__isset.success != rhs.__isset.success)
+ return false;
+ else if (__isset.success && !(success == rhs.success))
+ return false;
+ return true;
+ }
+ bool operator != (const GrantRevokePrivilegeResponse &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const GrantRevokePrivilegeResponse & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(GrantRevokePrivilegeResponse &a, GrantRevokePrivilegeResponse &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const GrantRevokePrivilegeResponse& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _Role__isset {
+ _Role__isset() : roleName(false), createTime(false), ownerName(false) {}
+ bool roleName :1;
+ bool createTime :1;
+ bool ownerName :1;
+ } _Role__isset;
+
+ class Role {
+ public:
+
+ Role(const Role&);
+ Role& operator=(const Role&);
+ Role() : roleName(), createTime(0), ownerName() {
+ }
+
+ virtual ~Role() throw();
+ std::string roleName;
+ int32_t createTime;
+ std::string ownerName;
+
+ _Role__isset __isset;
+
+ void __set_roleName(const std::string& val);
+
+ void __set_createTime(const int32_t val);
+
+ void __set_ownerName(const std::string& val);
+
+ bool operator == (const Role & rhs) const
+ {
+ if (!(roleName == rhs.roleName))
+ return false;
+ if (!(createTime == rhs.createTime))
+ return false;
+ if (!(ownerName == rhs.ownerName))
+ return false;
+ return true;
+ }
+ bool operator != (const Role &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const Role & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(Role &a, Role &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const Role& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _RolePrincipalGrant__isset {
+ _RolePrincipalGrant__isset() : roleName(false), principalName(false), principalType(false), grantOption(false), grantTime(false), grantorName(false), grantorPrincipalType(false) {}
+ bool roleName :1;
+ bool principalName :1;
+ bool principalType :1;
+ bool grantOption :1;
+ bool grantTime :1;
+ bool grantorName :1;
+ bool grantorPrincipalType :1;
+ } _RolePrincipalGrant__isset;
+
+ class RolePrincipalGrant {
+ public:
+
+ RolePrincipalGrant(const RolePrincipalGrant&);
+ RolePrincipalGrant& operator=(const RolePrincipalGrant&);
+ RolePrincipalGrant() : roleName(), principalName(), principalType((PrincipalType::type)0), grantOption(0), grantTime(0), grantorName(), grantorPrincipalType((PrincipalType::type)0) {
+ }
+
+ virtual ~RolePrincipalGrant() throw();
+ std::string roleName;
+ std::string principalName;
+ PrincipalType::type principalType;
+ bool grantOption;
+ int32_t grantTime;
+ std::string grantorName;
+ PrincipalType::type grantorPrincipalType;
+
+ _RolePrincipalGrant__isset __isset;
+
+ void __set_roleName(const std::string& val);
+
+ void __set_principalName(const std::string& val);
+
+ void __set_principalType(const PrincipalType::type val);
+
+ void __set_grantOption(const bool val);
+
+ void __set_grantTime(const int32_t val);
+
+ void __set_grantorName(const std::string& val);
+
+ void __set_grantorPrincipalType(const PrincipalType::type val);
+
+ bool operator == (const RolePrincipalGrant & rhs) const
+ {
+ if (!(roleName == rhs.roleName))
+ return false;
+ if (!(principalName == rhs.principalName))
+ return false;
+ if (!(principalType == rhs.principalType))
+ return false;
+ if (!(grantOption == rhs.grantOption))
+ return false;
+ if (!(grantTime == rhs.grantTime))
+ return false;
+ if (!(grantorName == rhs.grantorName))
+ return false;
+ if (!(grantorPrincipalType == rhs.grantorPrincipalType))
+ return false;
+ return true;
+ }
+ bool operator != (const RolePrincipalGrant &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const RolePrincipalGrant & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(RolePrincipalGrant &a, RolePrincipalGrant &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const RolePrincipalGrant& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+
+ class GetRoleGrantsForPrincipalRequest {
+ public:
+
+ GetRoleGrantsForPrincipalRequest(const GetRoleGrantsForPrincipalRequest&);
+ GetRoleGrantsForPrincipalRequest& operator=(const GetRoleGrantsForPrincipalRequest&);
+ GetRoleGrantsForPrincipalRequest() : principal_name(), principal_type((PrincipalType::type)0) {
+ }
+
+ virtual ~GetRoleGrantsForPrincipalRequest() throw();
+ std::string principal_name;
+ PrincipalType::type principal_type;
+
+ void __set_principal_name(const std::string& val);
+
+ void __set_principal_type(const PrincipalType::type val);
+
+ bool operator == (const GetRoleGrantsForPrincipalRequest & rhs) const
+ {
+ if (!(principal_name == rhs.principal_name))
+ return false;
+ if (!(principal_type == rhs.principal_type))
+ return false;
+ return true;
+ }
+ bool operator != (const GetRoleGrantsForPrincipalRequest &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const GetRoleGrantsForPrincipalRequest & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(GetRoleGrantsForPrincipalRequest &a, GetRoleGrantsForPrincipalRequest &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const GetRoleGrantsForPrincipalRequest& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+
+ class GetRoleGrantsForPrincipalResponse {
+ public:
+
+ GetRoleGrantsForPrincipalResponse(const GetRoleGrantsForPrincipalResponse&);
+ GetRoleGrantsForPrincipalResponse& operator=(const GetRoleGrantsForPrincipalResponse&);
+ GetRoleGrantsForPrincipalResponse() {
+ }
+
+ virtual ~GetRoleGrantsForPrincipalResponse() throw();
+ std::vector<RolePrincipalGrant> principalGrants;
+
+ void __set_principalGrants(const std::vector<RolePrincipalGrant> & val);
+
+ bool operator == (const GetRoleGrantsForPrincipalResponse & rhs) const
+ {
+ if (!(principalGrants == rhs.principalGrants))
+ return false;
+ return true;
+ }
+ bool operator != (const GetRoleGrantsForPrincipalResponse &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const GetRoleGrantsForPrincipalResponse & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(GetRoleGrantsForPrincipalResponse &a, GetRoleGrantsForPrincipalResponse &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const GetRoleGrantsForPrincipalResponse& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+
+ class GetPrincipalsInRoleRequest {
+ public:
+
+ GetPrincipalsInRoleRequest(const GetPrincipalsInRoleRequest&);
+ GetPrincipalsInRoleRequest& operator=(const GetPrincipalsInRoleRequest&);
+ GetPrincipalsInRoleRequest() : roleName() {
+ }
+
+ virtual ~GetPrincipalsInRoleRequest() throw();
+ std::string roleName;
+
+ void __set_roleName(const std::string& val);
+
+ bool operator == (const GetPrincipalsInRoleRequest & rhs) const
+ {
+ if (!(roleName == rhs.roleName))
+ return false;
+ return true;
+ }
+ bool operator != (const GetPrincipalsInRoleRequest &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const GetPrincipalsInRoleRequest & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(GetPrincipalsInRoleRequest &a, GetPrincipalsInRoleRequest &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const GetPrincipalsInRoleRequest& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+
+ class GetPrincipalsInRoleResponse {
+ public:
+
+ GetPrincipalsInRoleResponse(const GetPrincipalsInRoleResponse&);
+ GetPrincipalsInRoleResponse& operator=(const GetPrincipalsInRoleResponse&);
+ GetPrincipalsInRoleResponse() {
+ }
+
+ virtual ~GetPrincipalsInRoleResponse() throw();
+ std::vector<RolePrincipalGrant> principalGrants;
+
+ void __set_principalGrants(const std::vector<RolePrincipalGrant> & val);
+
+ bool operator == (const GetPrincipalsInRoleResponse & rhs) const
+ {
+ if (!(principalGrants == rhs.principalGrants))
+ return false;
+ return true;
+ }
+ bool operator != (const GetPrincipalsInRoleResponse &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const GetPrincipalsInRoleResponse & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(GetPrincipalsInRoleResponse &a, GetPrincipalsInRoleResponse &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const GetPrincipalsInRoleResponse& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _GrantRevokeRoleRequest__isset {
+ _GrantRevokeRoleRequest__isset() : requestType(false), roleName(false), principalName(false), principalType(false), grantor(false), grantorType(false), grantOption(false) {}
+ bool requestType :1;
+ bool roleName :1;
+ bool principalName :1;
+ bool principalType :1;
+ bool grantor :1;
+ bool grantorType :1;
+ bool grantOption :1;
+ } _GrantRevokeRoleRequest__isset;
+
+ class GrantRevokeRoleRequest {
+ public:
+
+ GrantRevokeRoleRequest(const GrantRevokeRoleRequest&);
+ GrantRevokeRoleRequest& operator=(const GrantRevokeRoleRequest&);
+ GrantRevokeRoleRequest() : requestType((GrantRevokeType::type)0), roleName(), principalName(), principalType((PrincipalType::type)0), grantor(), grantorType((PrincipalType::type)0), grantOption(0) {
+ }
+
+ virtual ~GrantRevokeRoleRequest() throw();
+ GrantRevokeType::type requestType;
+ std::string roleName;
+ std::string principalName;
+ PrincipalType::type principalType;
+ std::string grantor;
+ PrincipalType::type grantorType;
+ bool grantOption;
+
+ _GrantRevokeRoleRequest__isset __isset;
+
+ void __set_requestType(const GrantRevokeType::type val);
+
+ void __set_roleName(const std::string& val);
+
+ void __set_principalName(const std::string& val);
+
+ void __set_principalType(const PrincipalType::type val);
+
+ void __set_grantor(const std::string& val);
+
+ void __set_grantorType(const PrincipalType::type val);
+
+ void __set_grantOption(const bool val);
+
+ bool operator == (const GrantRevokeRoleRequest & rhs) const
+ {
+ if (!(requestType == rhs.requestType))
+ return false;
+ if (!(roleName == rhs.roleName))
+ return false;
+ if (!(principalName == rhs.principalName))
+ return false;
+ if (!(principalType == rhs.principalType))
+ return false;
+ if (__isset.grantor != rhs.__isset.grantor)
+ return false;
+ else if (__isset.grantor && !(grantor == rhs.grantor))
+ return false;
+ if (__isset.grantorType != rhs.__isset.grantorType)
+ return false;
+ else if (__isset.grantorType && !(grantorType == rhs.grantorType))
+ return false;
+ if (__isset.grantOption != rhs.__isset.grantOption)
+ return false;
+ else if (__isset.grantOption && !(grantOption == rhs.grantOption))
+ return false;
+ return true;
+ }
+ bool operator != (const GrantRevokeRoleRequest &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const GrantRevokeRoleRequest & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(GrantRevokeRoleRequest &a, GrantRevokeRoleRequest &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const GrantRevokeRoleRequest& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _GrantRevokeRoleResponse__isset {
+ _GrantRevokeRoleResponse__isset() : success(false) {}
+ bool success :1;
+ } _GrantRevokeRoleResponse__isset;
+
+ class GrantRevokeRoleResponse {
+ public:
+
+ GrantRevokeRoleResponse(const GrantRevokeRoleResponse&);
+ GrantRevokeRoleResponse& operator=(const GrantRevokeRoleResponse&);
+ GrantRevokeRoleResponse() : success(0) {
+ }
+
+ virtual ~GrantRevokeRoleResponse() throw();
+ bool success;
+
+ _GrantRevokeRoleResponse__isset __isset;
+
+ void __set_success(const bool val);
+
+ bool operator == (const GrantRevokeRoleResponse & rhs) const
+ {
+ if (__isset.success != rhs.__isset.success)
+ return false;
+ else if (__isset.success && !(success == rhs.success))
+ return false;
+ return true;
+ }
+ bool operator != (const GrantRevokeRoleResponse &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const GrantRevokeRoleResponse & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(GrantRevokeRoleResponse &a, GrantRevokeRoleResponse &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const GrantRevokeRoleResponse& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _Database__isset {
+ _Database__isset() : name(false), description(false), locationUri(false), parameters(false), privileges(false), ownerName(false), ownerType(false) {}
+ bool name :1;
+ bool description :1;
+ bool locationUri :1;
+ bool parameters :1;
+ bool privileges :1;
+ bool ownerName :1;
+ bool ownerType :1;
+ } _Database__isset;
+
+ class Database {
+ public:
+
+ Database(const Database&);
+ Database& operator=(const Database&);
+ Database() : name(), description(), locationUri(), ownerName(), ownerType((PrincipalType::type)0) {
+ }
+
+ virtual ~Database() throw();
+ std::string name;
+ std::string description;
+ std::string locationUri;
+ std::map<std::string, std::string> parameters;
+ PrincipalPrivilegeSet privileges;
+ std::string ownerName;
+ PrincipalType::type ownerType;
+
+ _Database__isset __isset;
+
+ void __set_name(const std::string& val);
+
+ void __set_description(const std::string& val);
+
+ void __set_locationUri(const std::string& val);
+
+ void __set_parameters(const std::map<std::string, std::string> & val);
+
+ void __set_privileges(const PrincipalPrivilegeSet& val);
+
+ void __set_ownerName(const std::string& val);
+
+ void __set_ownerType(const PrincipalType::type val);
+
+ bool operator == (const Database & rhs) const
+ {
+ if (!(name == rhs.name))
+ return false;
+ if (!(description == rhs.description))
+ return false;
+ if (!(locationUri == rhs.locationUri))
+ return false;
+ if (!(parameters == rhs.parameters))
+ return false;
+ if (__isset.privileges != rhs.__isset.privileges)
+ return false;
+ else if (__isset.privileges && !(privileges == rhs.privileges))
+ return false;
+ if (__isset.ownerName != rhs.__isset.ownerName)
+ return false;
+ else if (__isset.ownerName && !(ownerName == rhs.ownerName))
+ return false;
+ if (__isset.ownerType != rhs.__isset.ownerType)
+ return false;
+ else if (__isset.ownerType && !(ownerType == rhs.ownerType))
+ return false;
+ return true;
+ }
+ bool operator != (const Database &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const Database & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(Database &a, Database &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const Database& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _SerDeInfo__isset {
+ _SerDeInfo__isset() : name(false), serializationLib(false), parameters(false) {}
+ bool name :1;
+ bool serializationLib :1;
+ bool parameters :1;
+ } _SerDeInfo__isset;
+
+ class SerDeInfo {
+ public:
+
+ SerDeInfo(const SerDeInfo&);
+ SerDeInfo& operator=(const SerDeInfo&);
+ SerDeInfo() : name(), serializationLib() {
+ }
+
+ virtual ~SerDeInfo() throw();
+ std::string name;
+ std::string serializationLib;
+ std::map<std::string, std::string> parameters;
+
+ _SerDeInfo__isset __isset;
+
+ void __set_name(const std::string& val);
+
+ void __set_serializationLib(const std::string& val);
+
+ void __set_parameters(const std::map<std::string, std::string> & val);
+
+ bool operator == (const SerDeInfo & rhs) const
+ {
+ if (!(name == rhs.name))
+ return false;
+ if (!(serializationLib == rhs.serializationLib))
+ return false;
+ if (!(parameters == rhs.parameters))
+ return false;
+ return true;
+ }
+ bool operator != (const SerDeInfo &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const SerDeInfo & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(SerDeInfo &a, SerDeInfo &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const SerDeInfo& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _Order__isset {
+ _Order__isset() : col(false), order(false) {}
+ bool col :1;
+ bool order :1;
+ } _Order__isset;
+
+ class Order {
+ public:
+
+ Order(const Order&);
+ Order& operator=(const Order&);
+ Order() : col(), order(0) {
+ }
+
+ virtual ~Order() throw();
+ std::string col;
+ int32_t order;
+
+ _Order__isset __isset;
+
+ void __set_col(const std::string& val);
+
+ void __set_order(const int32_t val);
+
+ bool operator == (const Order & rhs) const
+ {
+ if (!(col == rhs.col))
+ return false;
+ if (!(order == rhs.order))
+ return false;
+ return true;
+ }
+ bool operator != (const Order &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const Order & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(Order &a, Order &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const Order& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _SkewedInfo__isset {
+ _SkewedInfo__isset() : skewedColNames(false), skewedColValues(false), skewedColValueLocationMaps(false) {}
+ bool skewedColNames :1;
+ bool skewedColValues :1;
+ bool skewedColValueLocationMaps :1;
+ } _SkewedInfo__isset;
+
+ class SkewedInfo {
+ public:
+
+ SkewedInfo(const SkewedInfo&);
+ SkewedInfo& operator=(const SkewedInfo&);
+ SkewedInfo() {
+ }
+
+ virtual ~SkewedInfo() throw();
+ std::vector<std::string> skewedColNames;
+ std::vector<std::vector<std::string> > skewedColValues;
+ std::map<std::vector<std::string> , std::string> skewedColValueLocationMaps;
+
+ _SkewedInfo__isset __isset;
+
+ void __set_skewedColNames(const std::vector<std::string> & val);
+
+ void __set_skewedColValues(const std::vector<std::vector<std::string> > & val);
+
+ void __set_skewedColValueLocationMaps(const std::map<std::vector<std::string> , std::string> & val);
+
+ bool operator == (const SkewedInfo & rhs) const
+ {
+ if (!(skewedColNames == rhs.skewedColNames))
+ return false;
+ if (!(skewedColValues == rhs.skewedColValues))
+ return false;
+ if (!(skewedColValueLocationMaps == rhs.skewedColValueLocationMaps))
+ return false;
+ return true;
+ }
+ bool operator != (const SkewedInfo &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const SkewedInfo & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(SkewedInfo &a, SkewedInfo &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const SkewedInfo& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _StorageDescriptor__isset {
+ _StorageDescriptor__isset() : cols(false), location(false), inputFormat(false), outputFormat(false), compressed(false), numBuckets(false), serdeInfo(false), bucketCols(false), sortCols(false), parameters(false), skewedInfo(false), storedAsSubDirectories(false) {}
+ bool cols :1;
+ bool location :1;
+ bool inputFormat :1;
+ bool outputFormat :1;
+ bool compressed :1;
+ bool numBuckets :1;
+ bool serdeInfo :1;
+ bool bucketCols :1;
+ bool sortCols :1;
+ bool parameters :1;
+ bool skewedInfo :1;
+ bool storedAsSubDirectories :1;
+ } _StorageDescriptor__isset;
+
+ class StorageDescriptor {
+ public:
+
+ StorageDescriptor(const StorageDescriptor&);
+ StorageDescriptor& operator=(const StorageDescriptor&);
+ StorageDescriptor() : location(), inputFormat(), outputFormat(), compressed(0), numBuckets(0), storedAsSubDirectories(0) {
+ }
+
+ virtual ~StorageDescriptor() throw();
+ std::vector<FieldSchema> cols;
+ std::string location;
+ std::string inputFormat;
+ std::string outputFormat;
+ bool compressed;
+ int32_t numBuckets;
+ SerDeInfo serdeInfo;
+ std::vector<std::string> bucketCols;
+ std::vector<Order> sortCols;
+ std::map<std::string, std::string> parameters;
+ SkewedInfo skewedInfo;
+ bool storedAsSubDirectories;
+
+ _StorageDescriptor__isset __isset;
+
+ void __set_cols(const std::vector<FieldSchema> & val);
+
+ void __set_location(const std::string& val);
+
+ void __set_inputFormat(const std::string& val);
+
+ void __set_outputFormat(const std::string& val);
+
+ void __set_compressed(const bool val);
+
+ void __set_numBuckets(const int32_t val);
+
+ void __set_serdeInfo(const SerDeInfo& val);
+
+ void __set_bucketCols(const std::vector<std::string> & val);
+
+ void __set_sortCols(const std::vector<Order> & val);
+
+ void __set_parameters(const std::map<std::string, std::string> & val);
+
+ void __set_skewedInfo(const SkewedInfo& val);
+
+ void __set_storedAsSubDirectories(const bool val);
+
+ bool operator == (const StorageDescriptor & rhs) const
+ {
+ if (!(cols == rhs.cols))
+ return false;
+ if (!(location == rhs.location))
+ return false;
+ if (!(inputFormat == rhs.inputFormat))
+ return false;
+ if (!(outputFormat == rhs.outputFormat))
+ return false;
+ if (!(compressed == rhs.compressed))
+ return false;
+ if (!(numBuckets == rhs.numBuckets))
+ return false;
+ if (!(serdeInfo == rhs.serdeInfo))
+ return false;
+ if (!(bucketCols == rhs.bucketCols))
+ return false;
+ if (!(sortCols == rhs.sortCols))
+ return false;
+ if (!(parameters == rhs.parameters))
+ return false;
+ if (__isset.skewedInfo != rhs.__isset.skewedInfo)
+ return false;
+ else if (__isset.skewedInfo && !(skewedInfo == rhs.skewedInfo))
+ return false;
+ if (__isset.storedAsSubDirectories != rhs.__isset.storedAsSubDirectories)
+ return false;
+ else if (__isset.storedAsSubDirectories && !(storedAsSubDirectories == rhs.storedAsSubDirectories))
+ return false;
+ return true;
+ }
+ bool operator != (const StorageDescriptor &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const StorageDescriptor & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(StorageDescriptor &a, StorageDescriptor &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const StorageDescriptor& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _Table__isset {
+ _Table__isset() : tableName(false), dbName(false), owner(false), createTime(false), lastAccessTime(false), retention(false), sd(false), partitionKeys(false), parameters(false), viewOriginalText(false), viewExpandedText(false), tableType(false), privileges(false), temporary(true), rewriteEnabled(false) {}
+ bool tableName :1;
+ bool dbName :1;
+ bool owner :1;
+ bool createTime :1;
+ bool lastAccessTime :1;
+ bool retention :1;
+ bool sd :1;
+ bool partitionKeys :1;
+ bool parameters :1;
+ bool viewOriginalText :1;
+ bool viewExpandedText :1;
+ bool tableType :1;
+ bool privileges :1;
+ bool temporary :1;
+ bool rewriteEnabled :1;
+ } _Table__isset;
+
+ class Table {
+ public:
+
+ Table(const Table&);
+ Table& operator=(const Table&);
+ Table() : tableName(), dbName(), owner(), createTime(0), lastAccessTime(0), retention(0), viewOriginalText(), viewExpandedText(), tableType(), temporary(false), rewriteEnabled(0) {
+ }
+
+ virtual ~Table() throw();
+ std::string tableName;
+ std::string dbName;
+ std::string owner;
+ int32_t createTime;
+ int32_t lastAccessTime;
+ int32_t retention;
+ StorageDescriptor sd;
+ std::vector<FieldSchema> partitionKeys;
+ std::map<std::string, std::string> parameters;
+ std::string viewOriginalText;
+ std::string viewExpandedText;
+ std::string tableType;
+ PrincipalPrivilegeSet privileges;
+ bool temporary;
+ bool rewriteEnabled;
+
+ _Table__isset __isset;
+
+ void __set_tableName(const std::string& val);
+
+ void __set_dbName(const std::string& val);
+
+ void __set_owner(const std::string& val);
+
+ void __set_createTime(const int32_t val);
+
+ void __set_lastAccessTime(const int32_t val);
+
+ void __set_retention(const int32_t val);
+
+ void __set_sd(const StorageDescriptor& val);
+
+ void __set_partitionKeys(const std::vector<FieldSchema> & val);
+
+ void __set_parameters(const std::map<std::string, std::string> & val);
+
+ void __set_viewOriginalText(const std::string& val);
+
+ void __set_viewExpandedText(const std::string& val);
+
+ void __set_tableType(const std::string& val);
+
+ void __set_privileges(const PrincipalPrivilegeSet& val);
+
+ void __set_temporary(const bool val);
+
+ void __set_rewriteEnabled(const bool val);
+
+ bool operator == (const Table & rhs) const
+ {
+ if (!(tableName == rhs.tableName))
+ return false;
+ if (!(dbName == rhs.dbName))
+ return false;
+ if (!(owner == rhs.owner))
+ return false;
+ if (!(createTime == rhs.createTime))
+ return false;
+ if (!(lastAccessTime == rhs.lastAccessTime))
+ return false;
+ if (!(retention == rhs.retention))
+ return false;
+ if (!(sd == rhs.sd))
+ return false;
+ if (!(partitionKeys == rhs.partitionKeys))
+ return false;
+ if (!(parameters == rhs.parameters))
+ return false;
+ if (!(viewOriginalText == rhs.viewOriginalText))
+ return false;
+ if (!(viewExpandedText == rhs.viewExpandedText))
+ return false;
+ if (!(tableType == rhs.tableType))
+ return false;
+ if (__isset.privileges != rhs.__isset.privileges)
+ return false;
+ else if (__isset.privileges && !(privileges == rhs.privileges))
+ return false;
+ if (__isset.temporary != rhs.__isset.temporary)
+ return false;
+ else if (__isset.temporary && !(temporary == rhs.temporary))
+ return false;
+ if (__isset.rewriteEnabled != rhs.__isset.rewriteEnabled)
+ return false;
+ else if (__isset.rewriteEnabled && !(rewriteEnabled == rhs.rewriteEnabled))
+ return false;
+ return true;
+ }
+ bool operator != (const Table &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const Table & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(Table &a, Table &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const Table& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _Partition__isset {
+ _Partition__isset() : values(false), dbName(false), tableName(false), createTime(false), lastAccessTime(false), sd(false), parameters(false), privileges(false) {}
+ bool values :1;
+ bool dbName :1;
+ bool tableName :1;
+ bool createTime :1;
+ bool lastAccessTime :1;
+ bool sd :1;
+ bool parameters :1;
+ bool privileges :1;
+ } _Partition__isset;
+
+ class Partition {
+ public:
+
+ Partition(const Partition&);
+ Partition& operator=(const Partition&);
+ Partition() : dbName(), tableName(), createTime(0), lastAccessTime(0) {
+ }
+
+ virtual ~Partition() throw();
+ std::vector<std::string> values;
+ std::string dbName;
+ std::string tableName;
+ int32_t createTime;
+ int32_t lastAccessTime;
+ StorageDescriptor sd;
+ std::map<std::string, std::string> parameters;
+ PrincipalPrivilegeSet privileges;
+
+ _Partition__isset __isset;
+
+ void __set_values(const std::vector<std::string> & val);
+
+ void __set_dbName(const std::string& val);
+
+ void __set_tableName(const std::string& val);
+
+ void __set_createTime(const int32_t val);
+
+ void __set_lastAccessTime(const int32_t val);
+
+ void __set_sd(const StorageDescriptor& val);
+
+ void __set_parameters(const std::map<std::string, std::string> & val);
+
+ void __set_privileges(const PrincipalPrivilegeSet& val);
+
+ bool operator == (const Partition & rhs) const
+ {
+ if (!(values == rhs.values))
+ return false;
+ if (!(dbName == rhs.dbName))
+ return false;
+ if (!(tableName == rhs.tableName))
+ return false;
+ if (!(createTime == rhs.createTime))
+ return false;
+ if (!(lastAccessTime == rhs.lastAccessTime))
+ return false;
+ if (!(sd == rhs.sd))
+ return false;
+ if (!(parameters == rhs.parameters))
+ return false;
+ if (__isset.privileges != rhs.__isset.privileges)
+ return false;
+ else if (__isset.privileges && !(privileges == rhs.privileges))
+ return false;
+ return true;
+ }
+ bool operator != (const Partition &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const Partition & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(Partition &a, Partition &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const Partition& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _PartitionWithoutSD__isset {
+ _PartitionWithoutSD__isset() : values(false), createTime(false), lastAccessTime(false), relativePath(false), parameters(false), privileges(false) {}
+ bool values :1;
+ bool createTime :1;
+ bool lastAccessTime :1;
+ bool relativePath :1;
+ bool parameters :1;
+ bool privileges :1;
+ } _PartitionWithoutSD__isset;
+
+ class PartitionWithoutSD {
+ public:
+
+ PartitionWithoutSD(const PartitionWithoutSD&);
+ PartitionWithoutSD& operator=(const PartitionWithoutSD&);
+ PartitionWithoutSD() : createTime(0), lastAccessTime(0), relativePath() {
+ }
+
+ virtual ~PartitionWithoutSD() throw();
+ std::vector<std::string> values;
+ int32_t createTime;
+ int32_t lastAccessTime;
+ std::string relativePath;
+ std::map<std::string, std::string> parameters;
+ PrincipalPrivilegeSet privileges;
+
+ _PartitionWithoutSD__isset __isset;
+
+ void __set_values(const std::vector<std::string> & val);
+
+ void __set_createTime(const int32_t val);
+
+ void __set_lastAccessTime(const int32_t val);
+
+ void __set_relativePath(const std::string& val);
+
+ void __set_parameters(const std::map<std::string, std::string> & val);
+
+ void __set_privileges(const PrincipalPrivilegeSet& val);
+
+ bool operator == (const PartitionWithoutSD & rhs) const
+ {
+ if (!(values == rhs.values))
+ return false;
+ if (!(createTime == rhs.createTime))
+ return false;
+ if (!(lastAccessTime == rhs.lastAccessTime))
+ return false;
+ if (!(relativePath == rhs.relativePath))
+ return false;
+ if (!(parameters == rhs.parameters))
+ return false;
+ if (__isset.privileges != rhs.__isset.privileges)
+ return false;
+ else if (__isset.privileges && !(privileges == rhs.privileges))
+ return false;
+ return true;
+ }
+ bool operator != (const PartitionWithoutSD &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const PartitionWithoutSD & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(PartitionWithoutSD &a, PartitionWithoutSD &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const PartitionWithoutSD& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _PartitionSpecWithSharedSD__isset {
+ _PartitionSpecWithSharedSD__isset() : partitions(false), sd(false) {}
+ bool partitions :1;
+ bool sd :1;
+ } _PartitionSpecWithSharedSD__isset;
+
+ class PartitionSpecWithSharedSD {
+ public:
+
+ PartitionSpecWithSharedSD(const PartitionSpecWithSharedSD&);
+ PartitionSpecWithSharedSD& operator=(const PartitionSpecWithSharedSD&);
+ PartitionSpecWithSharedSD() {
+ }
+
+ virtual ~PartitionSpecWithSharedSD() throw();
+ std::vector<PartitionWithoutSD> partitions;
+ StorageDescriptor sd;
+
+ _PartitionSpecWithSharedSD__isset __isset;
+
+ void __set_partitions(const std::vector<PartitionWithoutSD> & val);
+
+ void __set_sd(const StorageDescriptor& val);
+
+ bool operator == (const PartitionSpecWithSharedSD & rhs) const
+ {
+ if (!(partitions == rhs.partitions))
+ return false;
+ if (!(sd == rhs.sd))
+ return false;
+ return true;
+ }
+ bool operator != (const PartitionSpecWithSharedSD &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const PartitionSpecWithSharedSD & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(PartitionSpecWithSharedSD &a, PartitionSpecWithSharedSD &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const PartitionSpecWithSharedSD& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _PartitionListComposingSpec__isset {
+ _PartitionListComposingSpec__isset() : partitions(false) {}
+ bool partitions :1;
+ } _PartitionListComposingSpec__isset;
+
+ class PartitionListComposingSpec {
+ public:
+
+ PartitionListComposingSpec(const PartitionListComposingSpec&);
+ PartitionListComposingSpec& operator=(const PartitionListComposingSpec&);
+ PartitionListComposingSpec() {
+ }
+
+ virtual ~PartitionListComposingSpec() throw();
+ std::vector<Partition> partitions;
+
+ _PartitionListComposingSpec__isset __isset;
+
+ void __set_partitions(const std::vector<Partition> & val);
+
+ bool operator == (const PartitionListComposingSpec & rhs) const
+ {
+ if (!(partitions == rhs.partitions))
+ return false;
+ return true;
+ }
+ bool operator != (const PartitionListComposingSpec &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const PartitionListComposingSpec & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(PartitionListComposingSpec &a, PartitionListComposingSpec &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const PartitionListComposingSpec& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _PartitionSpec__isset {
+ _PartitionSpec__isset() : dbName(false), tableName(false), rootPath(false), sharedSDPartitionSpec(false), partitionList(false) {}
+ bool dbName :1;
+ bool tableName :1;
+ bool rootPath :1;
+ bool sharedSDPartitionSpec :1;
+ bool partitionList :1;
+ } _PartitionSpec__isset;
+
+ class PartitionSpec {
+ public:
+
+ PartitionSpec(const PartitionSpec&);
+ PartitionSpec& operator=(const PartitionSpec&);
+ PartitionSpec() : dbName(), tableName(), rootPath() {
+ }
+
+ virtual ~PartitionSpec() throw();
+ std::string dbName;
+ std::string tableName;
+ std::string rootPath;
+ PartitionSpecWithSharedSD sharedSDPartitionSpec;
+ PartitionListComposingSpec partitionList;
+
+ _PartitionSpec__isset __isset;
+
+ void __set_dbName(const std::string& val);
+
+ void __set_tableName(const std::string& val);
+
+ void __set_rootPath(const std::string& val);
+
+ void __set_sharedSDPartitionSpec(const PartitionSpecWithSharedSD& val);
+
+ void __set_partitionList(const PartitionListComposingSpec& val);
+
+ bool operator == (const PartitionSpec & rhs) const
+ {
+ if (!(dbName == rhs.dbName))
+ return false;
+ if (!(tableName == rhs.tableName))
+ return false;
+ if (!(rootPath == rhs.rootPath))
+ return false;
+ if (__isset.sharedSDPartitionSpec != rhs.__isset.sharedSDPartitionSpec)
+ return false;
+ else if (__isset.sharedSDPartitionSpec && !(sharedSDPartitionSpec == rhs.sharedSDPartitionSpec))
+ return false;
+ if (__isset.partitionList != rhs.__isset.partitionList)
+ return false;
+ else if (__isset.partitionList && !(partitionList == rhs.partitionList))
+ return false;
+ return true;
+ }
+ bool operator != (const PartitionSpec &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const PartitionSpec & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(PartitionSpec &a, PartitionSpec &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const PartitionSpec& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _Index__isset {
+ _Index__isset() : indexName(false), indexHandlerClass(false), dbName(false), origTableName(false), createTime(false), lastAccessTime(false), indexTableName(false), sd(false), parameters(false), deferredRebuild(false) {}
+ bool indexName :1;
+ bool indexHandlerClass :1;
+ bool dbName :1;
+ bool origTableName :1;
+ bool createTime :1;
+ bool lastAccessTime :1;
+ bool indexTableName :1;
+ bool sd :1;
+ bool parameters :1;
+ bool deferredRebuild :1;
+ } _Index__isset;
+
+ class Index {
+ public:
+
+ Index(const Index&);
+ Index& operator=(const Index&);
+ Index() : indexName(), indexHandlerClass(), dbName(), origTableName(), createTime(0), lastAccessTime(0), indexTableName(), deferredRebuild(0) {
+ }
+
+ virtual ~Index() throw();
+ std::string indexName;
+ std::string indexHandlerClass;
+ std::string dbName;
+ std::string origTableName;
+ int32_t createTime;
+ int32_t lastAccessTime;
+ std::string indexTableName;
+ StorageDescriptor sd;
+ std::map<std::string, std::string> parameters;
+ bool deferredRebuild;
+
+ _Index__isset __isset;
+
+ void __set_indexName(const std::string& val);
+
+ void __set_indexHandlerClass(const std::string& val);
+
+ void __set_dbName(const std::string& val);
+
+ void __set_origTableName(const std::string& val);
+
+ void __set_createTime(const int32_t val);
+
+ void __set_lastAccessTime(const int32_t val);
+
+ void __set_indexTableName(const std::string& val);
+
+ void __set_sd(const StorageDescriptor& val);
+
+ void __set_parameters(const std::map<std::string, std::string> & val);
+
+ void __set_deferredRebuild(const bool val);
+
+ bool operator == (const Index & rhs) const
+ {
+ if (!(indexName == rhs.indexName))
+ return false;
+ if (!(indexHandlerClass == rhs.indexHandlerClass))
+ return false;
+ if (!(dbName == rhs.dbName))
+ return false;
+ if (!(origTableName == rhs.origTableName))
+ return false;
+ if (!(createTime == rhs.createTime))
+ return false;
+ if (!(lastAccessTime == rhs.lastAccessTime))
+ return false;
+ if (!(indexTableName == rhs.indexTableName))
+ return false;
+ if (!(sd == rhs.sd))
+ return false;
+ if (!(parameters == rhs.parameters))
+ return false;
+ if (!(deferredRebuild == rhs.deferredRebuild))
+ return false;
+ return true;
+ }
+ bool operator != (const Index &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const Index & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(Index &a, Index &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const Index& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _BooleanColumnStatsData__isset {
+ _BooleanColumnStatsData__isset() : bitVectors(false) {}
+ bool bitVectors :1;
+ } _BooleanColumnStatsData__isset;
+
+ class BooleanColumnStatsData {
+ public:
+
+ BooleanColumnStatsData(const BooleanColumnStatsData&);
+ BooleanColumnStatsData& operator=(const BooleanColumnStatsData&);
+ BooleanColumnStatsData() : numTrues(0), numFalses(0), numNulls(0), bitVectors() {
+ }
+
+ virtual ~BooleanColumnStatsData() throw();
+ int64_t numTrues;
+ int64_t numFalses;
+ int64_t numNulls;
+ std::string bitVectors;
+
+ _BooleanColumnStatsData__isset __isset;
+
+ void __set_numTrues(const int64_t val);
+
+ void __set_numFalses(const int64_t val);
+
+ void __set_numNulls(const int64_t val);
+
+ void __set_bitVectors(const std::string& val);
+
+ bool operator == (const BooleanColumnStatsData & rhs) const
+ {
+ if (!(numTrues == rhs.numTrues))
+ return false;
+ if (!(numFalses == rhs.numFalses))
+ return false;
+ if (!(numNulls == rhs.numNulls))
+ return false;
+ if (__isset.bitVectors != rhs.__isset.bitVectors)
+ return false;
+ else if (__isset.bitVectors && !(bitVectors == rhs.bitVectors))
+ return false;
+ return true;
+ }
+ bool operator != (const BooleanColumnStatsData &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const BooleanColumnStatsData & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(BooleanColumnStatsData &a, BooleanColumnStatsData &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const BooleanColumnStatsData& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _DoubleColumnStatsData__isset {
+ _DoubleColumnStatsData__isset() : lowValue(false), highValue(false), bitVectors(false) {}
+ bool lowValue :1;
+ bool highValue :1;
+ bool bitVectors :1;
+ } _DoubleColumnStatsData__isset;
+
+ class DoubleColumnStatsData {
+ public:
+
+ DoubleColumnStatsData(const DoubleColumnStatsData&);
+ DoubleColumnStatsData& operator=(const DoubleColumnStatsData&);
+ DoubleColumnStatsData() : lowValue(0), highValue(0), numNulls(0), numDVs(0), bitVectors() {
+ }
+
+ virtual ~DoubleColumnStatsData() throw();
+ double lowValue;
+ double highValue;
+ int64_t numNulls;
+ int64_t numDVs;
+ std::string bitVectors;
+
+ _DoubleColumnStatsData__isset __isset;
+
+ void __set_lowValue(const double val);
+
+ void __set_highValue(const double val);
+
+ void __set_numNulls(const int64_t val);
+
+ void __set_numDVs(const int64_t val);
+
+ void __set_bitVectors(const std::string& val);
+
+ bool operator == (const DoubleColumnStatsData & rhs) const
+ {
+ if (__isset.lowValue != rhs.__isset.lowValue)
+ return false;
+ else if (__isset.lowValue && !(lowValue == rhs.lowValue))
+ return false;
+ if (__isset.highValue != rhs.__isset.highValue)
+ return false;
+ else if (__isset.highValue && !(highValue == rhs.highValue))
+ return false;
+ if (!(numNulls == rhs.numNulls))
+ return false;
+ if (!(numDVs == rhs.numDVs))
+ return false;
+ if (__isset.bitVectors != rhs.__isset.bitVectors)
+ return false;
+ else if (__isset.bitVectors && !(bitVectors == rhs.bitVectors))
+ return false;
+ return true;
+ }
+ bool operator != (const DoubleColumnStatsData &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const DoubleColumnStatsData & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(DoubleColumnStatsData &a, DoubleColumnStatsData &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const DoubleColumnStatsData& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _LongColumnStatsData__isset {
+ _LongColumnStatsData__isset() : lowValue(false), highValue(false), bitVectors(false) {}
+ bool lowValue :1;
+ bool highValue :1;
+ bool bitVectors :1;
+ } _LongColumnStatsData__isset;
+
+ class LongColumnStatsData {
+ public:
+
+ LongColumnStatsData(const LongColumnStatsData&);
+ LongColumnStatsData& operator=(const LongColumnStatsData&);
+ LongColumnStatsData() : lowValue(0), highValue(0), numNulls(0), numDVs(0), bitVectors() {
+ }
+
+ virtual ~LongColumnStatsData() throw();
+ int64_t lowValue;
+ int64_t highValue;
+ int64_t numNulls;
+ int64_t numDVs;
+ std::string bitVectors;
+
+ _LongColumnStatsData__isset __isset;
+
+ void __set_lowValue(const int64_t val);
+
+ void __set_highValue(const int64_t val);
+
+ void __set_numNulls(const int64_t val);
+
+ void __set_numDVs(const int64_t val);
+
+ void __set_bitVectors(const std::string& val);
+
+ bool operator == (const LongColumnStatsData & rhs) const
+ {
+ if (__isset.lowValue != rhs.__isset.lowValue)
+ return false;
+ else if (__isset.lowValue && !(lowValue == rhs.lowValue))
+ return false;
+ if (__isset.highValue != rhs.__isset.highValue)
+ return false;
+ else if (__isset.highValue && !(highValue == rhs.highValue))
+ return false;
+ if (!(numNulls == rhs.numNulls))
+ return false;
+ if (!(numDVs == rhs.numDVs))
+ return false;
+ if (__isset.bitVectors != rhs.__isset.bitVectors)
+ return false;
+ else if (__isset.bitVectors && !(bitVectors == rhs.bitVectors))
+ return false;
+ return true;
+ }
+ bool operator != (const LongColumnStatsData &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const LongColumnStatsData & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(LongColumnStatsData &a, LongColumnStatsData &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const LongColumnStatsData& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _StringColumnStatsData__isset {
+ _StringColumnStatsData__isset() : bitVectors(false) {}
+ bool bitVectors :1;
+ } _StringColumnStatsData__isset;
+
+ class StringColumnStatsData {
+ public:
+
+ StringColumnStatsData(const StringColumnStatsData&);
+ StringColumnStatsData& operator=(const StringColumnStatsData&);
+ StringColumnStatsData() : maxColLen(0), avgColLen(0), numNulls(0), numDVs(0), bitVectors() {
+ }
+
+ virtual ~StringColumnStatsData() throw();
+ int64_t maxColLen;
+ double avgColLen;
+ int64_t numNulls;
+ int64_t numDVs;
+ std::string bitVectors;
+
+ _StringColumnStatsData__isset __isset;
+
+ void __set_maxColLen(const int64_t val);
+
+ void __set_avgColLen(const double val);
+
+ void __set_numNulls(const int64_t val);
+
+ void __set_numDVs(const int64_t val);
+
+ void __set_bitVectors(const std::string& val);
+
+ bool operator == (const StringColumnStatsData & rhs) const
+ {
+ if (!(maxColLen == rhs.maxColLen))
+ return false;
+ if (!(avgColLen == rhs.avgColLen))
+ return false;
+ if (!(numNulls == rhs.numNulls))
+ return false;
+ if (!(numDVs == rhs.numDVs))
+ return false;
+ if (__isset.bitVectors != rhs.__isset.bitVectors)
+ return false;
+ else if (__isset.bitVectors && !(bitVectors == rhs.bitVectors))
+ return false;
+ return true;
+ }
+ bool operator != (const StringColumnStatsData &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const StringColumnStatsData & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(StringColumnStatsData &a, StringColumnStatsData &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const StringColumnStatsData& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+ typedef struct _BinaryColumnStatsData__isset {
+ _BinaryColumnStatsData__isset() : bitVectors(false) {}
+ bool bitVectors :1;
+ } _BinaryColumnStatsData__isset;
+
+ class BinaryColumnStatsData {
+ public:
+
+ BinaryColumnStatsData(const BinaryColumnStatsData&);
+ BinaryColumnStatsData& operator=(const BinaryColumnStatsData&);
+ BinaryColumnStatsData() : maxColLen(0), avgColLen(0), numNulls(0), bitVectors() {
+ }
+
+ virtual ~BinaryColumnStatsData() throw();
+ int64_t maxColLen;
+ double avgColLen;
+ int64_t numNulls;
+ std::string bitVectors;
+
+ _BinaryColumnStatsData__isset __isset;
+
+ void __set_maxColLen(const int64_t val);
+
+ void __set_avgColLen(const double val);
+
+ void __set_numNulls(const int64_t val);
+
+ void __set_bitVectors(const std::string& val);
+
+ bool operator == (const BinaryColumnStatsData & rhs) const
+ {
+ if (!(maxColLen == rhs.maxColLen))
+ return false;
+ if (!(avgColLen == rhs.avgColLen))
+ return false;
+ if (!(numNulls == rhs.numNulls))
+ return false;
+ if (__isset.bitVectors != rhs.__isset.bitVectors)
+ return false;
+ else if (__isset.bitVectors && !(bitVectors == rhs.bitVectors))
+ return false;
+ return true;
+ }
+ bool operator != (const BinaryColumnStatsData &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const BinaryColumnStatsData & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+ };
+
+ void swap(BinaryColumnStatsData &a, BinaryColumnStatsData &b);
+
+ inline std::ostream& operator<<(std::ostream& out, const BinaryColumnStatsData& obj)
+ {
+ obj.printTo(out);
+ return out;
+ }
+
+
+ class Decimal {
+ public:
+
+ Decimal(const Decimal&);
+ Decimal& operator=(const Decimal&);
+ Decimal() : unscaled(), scale(0) {
+ }
+
+ virtual ~Decimal() throw();
+ std::string unscaled;
+ int16_t scale;
+
+ void __set_unscaled(const std::string& val);
+
+ void __set_scale(const int16_t val);
+
+ bool operator == (const Decimal & rhs) const
+ {
+ if (!(unscaled == rhs.unscaled))
+ return false;
+ if (!(scale =
<TRUNCATED>
[49/50] [abbrv] hive git commit: HIVE-14671 : merge master into
hive-14535 (Wei Zheng)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --cc common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index ff8a7aa,e4b09a2..21c4cc9
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@@ -1864,14 -1855,9 +1857,16 @@@ public class HiveConf extends Configura
" of the lock manager is dumped to log file. This is for debugging. See also " +
"hive.lock.numretries and hive.lock.sleep.between.retries."),
- HIVE_TXN_OPERATIONAL_PROPERTIES("hive.txn.operational.properties", 0,
+ HIVE_TXN_OPERATIONAL_PROPERTIES("hive.txn.operational.properties", 1,
- "This is intended to be used as an internal property for future versions of ACID. (See\n" +
- "HIVE-14035 for details.)"),
+ "Sets the operational properties that control the appropriate behavior for various\n"
+ + "versions of the Hive ACID subsystem. Mostly it is intended to be used as an internal property\n"
+ + "for future versions of ACID. (See HIVE-14035 for details.)\n"
+ + "0: Turn on the legacy mode for ACID\n"
+ + "1: Enable split-update feature found in the newer version of Hive ACID subsystem\n"
+ + "2: Hash-based merge, which combines delta files using GRACE hash join based approach (not implemented)\n"
- + "3: Make the table 'quarter-acid' as it only supports insert. But it doesn't require ORC or bucketing."),
++ + "3: Make the table 'quarter-acid' as it only supports insert. But it doesn't require ORC or bucketing.\n"
++ + "This is intended to be used as an internal property for future versions of ACID. (See\n" +
++ "HIVE-14035 for details.)"),
HIVE_MAX_OPEN_TXNS("hive.max.open.txns", 100000, "Maximum number of open transactions. If \n" +
"current open transactions reach this limit, future open transaction requests will be \n" +
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --cc metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 913e333,5812a1b..ab5f7b7
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@@ -192,13 -194,14 +197,13 @@@ public class HiveMetaStore extends Thri
};
};
- /**
- * default port on which to start the Hive server
- */
public static final String ADMIN = "admin";
public static final String PUBLIC = "public";
+ /** MM write states. */
+ public static final char MM_WRITE_OPEN = 'o', MM_WRITE_COMMITTED = 'c', MM_WRITE_ABORTED = 'a';
private static HadoopThriftAuthBridge.Server saslServer;
- private static HiveDelegationTokenManager delegationTokenManager;
+ private static MetastoreDelegationTokenManager delegationTokenManager;
private static boolean useSasl;
public static final String NO_FILTER_STRING = "";
@@@ -437,41 -495,14 +497,15 @@@
}
}
- //Start Metrics for Embedded mode
+ //Start Metrics
if (hiveConf.getBoolVar(ConfVars.METASTORE_METRICS)) {
- try {
- MetricsFactory.init(hiveConf);
- } catch (Exception e) {
- // log exception, but ignore inability to start
- LOG.error("error in Metrics init: " + e.getClass().getName() + " "
- + e.getMessage(), e);
- }
- }
-
- Metrics metrics = MetricsFactory.getInstance();
- if (metrics != null && hiveConf.getBoolVar(ConfVars.METASTORE_INIT_METADATA_COUNT_ENABLED)) {
LOG.info("Begin calculating metadata count metrics.");
+ Metrics.initialize(hiveConf);
+ databaseCount = Metrics.getOrCreateGauge(MetricsConstants.TOTAL_DATABASES);
+ tableCount = Metrics.getOrCreateGauge(MetricsConstants.TOTAL_TABLES);
+ partCount = Metrics.getOrCreateGauge(MetricsConstants.TOTAL_PARTITIONS);
updateMetrics();
- LOG.info("Finished metadata count metrics: " + initDatabaseCount + " databases, " + initTableCount +
- " tables, " + initPartCount + " partitions.");
- metrics.addGauge(MetricsConstant.INIT_TOTAL_DATABASES, new MetricsVariable<Object>() {
- @Override
- public Object getValue() {
- return initDatabaseCount;
- }
- });
- metrics.addGauge(MetricsConstant.INIT_TOTAL_TABLES, new MetricsVariable<Object>() {
- @Override
- public Object getValue() {
- return initTableCount;
- }
- });
- metrics.addGauge(MetricsConstant.INIT_TOTAL_PARTITIONS, new MetricsVariable<Object>() {
- @Override
- public Object getValue() {
- return initPartCount;
- }
- });
++
}
preListeners = MetaStoreUtils.getMetaStoreListeners(MetaStorePreEventListener.class,
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------
diff --cc metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 8e89477,70451c4..37da2f8
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@@ -59,12 -60,11 +60,12 @@@ import org.apache.hadoop.hive.conf.Hive
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.conf.HiveConfUtil;
import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.hadoop.hive.metastore.TableType;
+ import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
+ import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
import org.apache.hadoop.hive.metastore.txn.TxnUtils;
- import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.shims.Utils;
- import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
import org.apache.thrift.TApplicationException;
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --cc metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index e7bbd62,bbe13fd..49005b9
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@@ -1981,59 -1967,156 +1970,212 @@@ public class MetaStoreUtils
return cols;
}
+ // TODO The following two utility methods can be moved to AcidUtils once no class in metastore is relying on them,
+ // right now ObjectStore.getAllMmTablesForCleanup is calling these method
+ /**
+ * Checks if a table is an ACID table that only supports INSERT, but not UPDATE/DELETE
+ * @param params table properties
+ * @return true if table is an INSERT_ONLY table, false otherwise
+ */
+ // TODO# also check that transactional is true
+ public static boolean isInsertOnlyTable(Map<String, String> params) {
+ return isInsertOnlyTable(params, false);
+ }
+
+ public static boolean isInsertOnlyTable(Map<String, String> params, boolean isCtas) {
+ String transactionalProp = params.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
+ return (transactionalProp != null && "insert_only".equalsIgnoreCase(transactionalProp));
+ }
+
+ public static boolean isInsertOnlyTable(Properties params) {
+ // TODO# redirect for now - fix before merge
+ HashMap<String, String> testMap = new HashMap<String, String>();
+ for (String n : params.stringPropertyNames()) {
+ testMap.put(n, params.getProperty(n));
+ }
+ return isInsertOnlyTable(testMap);
+ }
+
+ /** The method for altering table props; may set the table to MM, non-MM, or not affect MM. */
+ public static Boolean isToInsertOnlyTable(Map<String, String> props) {
+ // TODO# Setting these separately is a very hairy issue in certain combinations, since we
+ // cannot decide what type of table this becomes without taking both into account, and
+ // in many cases the conversion might be illegal.
+ // The only thing we allow is tx = true w/o tx-props, for backward compat.
+ String transactional = props.get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
+ String transactionalProp = props.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
+ if (transactional == null && transactionalProp == null) return null; // Not affected.
+ boolean isSetToTxn = "true".equalsIgnoreCase(transactional);
+ if (transactionalProp == null) {
+ if (isSetToTxn) return false; // Assume the full ACID table.
+ throw new RuntimeException("Cannot change '" + hive_metastoreConstants.TABLE_IS_TRANSACTIONAL
+ + "' without '" + hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES + "'");
+ }
+ if (!"insert_only".equalsIgnoreCase(transactionalProp)) return false; // Not MM.
+ if (!isSetToTxn) {
+ throw new RuntimeException("Cannot set '"
+ + hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES + "' to 'insert_only' without "
+ + "setting '" + hive_metastoreConstants.TABLE_IS_TRANSACTIONAL + "' to 'true'");
+ }
+ return true;
+ }
+
+ public static boolean isRemovedInsertOnlyTable(Set<String> removedSet) {
+ boolean hasTxn = removedSet.contains(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL),
+ hasProps = removedSet.contains(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
+ return hasTxn || hasProps;
+ }
++
+ // given a list of partStats, this function will give you an aggr stats
+ public static List<ColumnStatisticsObj> aggrPartitionStats(List<ColumnStatistics> partStats,
+ String dbName, String tableName, List<String> partNames, List<String> colNames,
+ boolean useDensityFunctionForNDVEstimation, double ndvTuner)
+ throws MetaException {
+ // 1. group by the stats by colNames
+ // map the colName to List<ColumnStatistics>
+ Map<String, List<ColumnStatistics>> map = new HashMap<>();
+ for (ColumnStatistics css : partStats) {
+ List<ColumnStatisticsObj> objs = css.getStatsObj();
+ for (ColumnStatisticsObj obj : objs) {
+ List<ColumnStatisticsObj> singleObj = new ArrayList<>();
+ singleObj.add(obj);
+ ColumnStatistics singleCS = new ColumnStatistics(css.getStatsDesc(), singleObj);
+ if (!map.containsKey(obj.getColName())) {
+ map.put(obj.getColName(), new ArrayList<ColumnStatistics>());
+ }
+ map.get(obj.getColName()).add(singleCS);
+ }
+ }
+ return aggrPartitionStats(map,dbName,tableName,partNames,colNames,useDensityFunctionForNDVEstimation, ndvTuner);
+ }
+
+ public static List<ColumnStatisticsObj> aggrPartitionStats(
+ Map<String, List<ColumnStatistics>> map, String dbName, String tableName,
+ final List<String> partNames, List<String> colNames,
+ final boolean useDensityFunctionForNDVEstimation,final double ndvTuner) throws MetaException {
+ List<ColumnStatisticsObj> colStats = new ArrayList<>();
+ // 2. Aggregate stats for each column in a separate thread
+ if (map.size()< 1) {
+ //stats are absent in RDBMS
+ LOG.debug("No stats data found for: dbName=" +dbName +" tblName=" + tableName +
+ " partNames= " + partNames + " colNames=" + colNames );
+ return colStats;
+ }
+ final ExecutorService pool = Executors.newFixedThreadPool(Math.min(map.size(), 16),
+ new ThreadFactoryBuilder().setDaemon(true).setNameFormat("aggr-col-stats-%d").build());
+ final List<Future<ColumnStatisticsObj>> futures = Lists.newLinkedList();
+
+ long start = System.currentTimeMillis();
+ for (final Entry<String, List<ColumnStatistics>> entry : map.entrySet()) {
+ futures.add(pool.submit(new Callable<ColumnStatisticsObj>() {
+ @Override
+ public ColumnStatisticsObj call() throws Exception {
+ List<ColumnStatistics> css = entry.getValue();
+ ColumnStatsAggregator aggregator = ColumnStatsAggregatorFactory.getColumnStatsAggregator(css
+ .iterator().next().getStatsObj().iterator().next().getStatsData().getSetField(),
+ useDensityFunctionForNDVEstimation, ndvTuner);
+ ColumnStatisticsObj statsObj = aggregator.aggregate(entry.getKey(), partNames, css);
+ return statsObj;
+ }}));
+ }
+ pool.shutdown();
+ for (Future<ColumnStatisticsObj> future : futures) {
+ try {
+ colStats.add(future.get());
+ } catch (InterruptedException | ExecutionException e) {
+ pool.shutdownNow();
+ LOG.debug(e.toString());
+ throw new MetaException(e.toString());
+ }
+ }
+ LOG.debug("Time for aggr col stats in seconds: {} Threads used: {}",
+ ((System.currentTimeMillis() - (double)start))/1000, Math.min(map.size(), 16));
+ return colStats;
+ }
+
+
+ /**
+ * Produce a hash for the storage descriptor
+ * @param sd storage descriptor to hash
+ * @param md message descriptor to use to generate the hash
+ * @return the hash as a byte array
+ */
+ public static byte[] hashStorageDescriptor(StorageDescriptor sd, MessageDigest md) {
+ // Note all maps and lists have to be absolutely sorted. Otherwise we'll produce different
+ // results for hashes based on the OS or JVM being used.
+ md.reset();
+ for (FieldSchema fs : sd.getCols()) {
+ md.update(fs.getName().getBytes(ENCODING));
+ md.update(fs.getType().getBytes(ENCODING));
+ if (fs.getComment() != null) md.update(fs.getComment().getBytes(ENCODING));
+ }
+ if (sd.getInputFormat() != null) {
+ md.update(sd.getInputFormat().getBytes(ENCODING));
+ }
+ if (sd.getOutputFormat() != null) {
+ md.update(sd.getOutputFormat().getBytes(ENCODING));
+ }
+ md.update(sd.isCompressed() ? "true".getBytes(ENCODING) : "false".getBytes(ENCODING));
+ md.update(Integer.toString(sd.getNumBuckets()).getBytes(ENCODING));
+ if (sd.getSerdeInfo() != null) {
+ SerDeInfo serde = sd.getSerdeInfo();
+ if (serde.getName() != null) {
+ md.update(serde.getName().getBytes(ENCODING));
+ }
+ if (serde.getSerializationLib() != null) {
+ md.update(serde.getSerializationLib().getBytes(ENCODING));
+ }
+ if (serde.getParameters() != null) {
+ SortedMap<String, String> params = new TreeMap<>(serde.getParameters());
+ for (Entry<String, String> param : params.entrySet()) {
+ md.update(param.getKey().getBytes(ENCODING));
+ md.update(param.getValue().getBytes(ENCODING));
+ }
+ }
+ }
+ if (sd.getBucketCols() != null) {
+ List<String> bucketCols = new ArrayList<>(sd.getBucketCols());
+ for (String bucket : bucketCols) md.update(bucket.getBytes(ENCODING));
+ }
+ if (sd.getSortCols() != null) {
+ SortedSet<Order> orders = new TreeSet<>(sd.getSortCols());
+ for (Order order : orders) {
+ md.update(order.getCol().getBytes(ENCODING));
+ md.update(Integer.toString(order.getOrder()).getBytes(ENCODING));
+ }
+ }
+ if (sd.getSkewedInfo() != null) {
+ SkewedInfo skewed = sd.getSkewedInfo();
+ if (skewed.getSkewedColNames() != null) {
+ SortedSet<String> colnames = new TreeSet<>(skewed.getSkewedColNames());
+ for (String colname : colnames) md.update(colname.getBytes(ENCODING));
+ }
+ if (skewed.getSkewedColValues() != null) {
+ SortedSet<String> sortedOuterList = new TreeSet<>();
+ for (List<String> innerList : skewed.getSkewedColValues()) {
+ SortedSet<String> sortedInnerList = new TreeSet<>(innerList);
+ sortedOuterList.add(StringUtils.join(sortedInnerList, "."));
+ }
+ for (String colval : sortedOuterList) md.update(colval.getBytes(ENCODING));
+ }
+ if (skewed.getSkewedColValueLocationMaps() != null) {
+ SortedMap<String, String> sortedMap = new TreeMap<>();
+ for (Entry<List<String>, String> smap : skewed.getSkewedColValueLocationMaps().entrySet()) {
+ SortedSet<String> sortedKey = new TreeSet<>(smap.getKey());
+ sortedMap.put(StringUtils.join(sortedKey, "."), smap.getValue());
+ }
+ for (Entry<String, String> e : sortedMap.entrySet()) {
+ md.update(e.getKey().getBytes(ENCODING));
+ md.update(e.getValue().getBytes(ENCODING));
+ }
+ }
+ md.update(sd.isStoredAsSubDirectories() ? "true".getBytes(ENCODING) : "false".getBytes(ENCODING));
+ }
+
+ return md.digest();
+ }
+
+ public static double decimalToDouble(Decimal decimal) {
+ return new BigDecimal(new BigInteger(decimal.getUnscaled()), decimal.getScale()).doubleValue();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --cc metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 88afe03,0db1bc0..903b2c7
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@@ -59,9 -58,11 +59,12 @@@ import javax.jdo.PersistenceManagerFact
import javax.jdo.Query;
import javax.jdo.Transaction;
import javax.jdo.datastore.DataStoreCache;
+import javax.jdo.datastore.JDOConnection;
import javax.jdo.identity.IntIdentity;
+ import javax.sql.DataSource;
+ import com.codahale.metrics.Counter;
+ import com.codahale.metrics.MetricRegistry;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configurable;
@@@ -415,10 -431,13 +434,14 @@@ public class ObjectStore implements Raw
pm = getPersistenceManager();
isInitialized = pm != null;
if (isInitialized) {
+ dbType = determineDatabaseProduct();
expressionProxy = createExpressionProxy(hiveConf);
if (HiveConf.getBoolVar(getConf(), ConfVars.METASTORE_TRY_DIRECT_SQL)) {
- directSql = new MetaStoreDirectSql(pm, hiveConf, dbType);
+ String schema = prop.getProperty("javax.jdo.mapping.Schema");
+ if (schema != null && schema.isEmpty()) {
+ schema = null;
+ }
+ directSql = new MetaStoreDirectSql(pm, hiveConf, schema);
}
}
LOG.debug("RawStore: " + this + ", with PersistenceManager: " + pm +
@@@ -2733,8 -2759,7 +2783,8 @@@
boolean isConfigEnabled = HiveConf.getBoolVar(getConf(), ConfVars.METASTORE_TRY_DIRECT_SQL)
&& (HiveConf.getBoolVar(getConf(), ConfVars.METASTORE_TRY_DIRECT_SQL_DDL) || !isInTxn);
if (isConfigEnabled && directSql == null) {
- directSql = new MetaStoreDirectSql(pm, getConf());
+ dbType = determineDatabaseProduct();
- directSql = new MetaStoreDirectSql(pm, getConf(), dbType);
++ directSql = new MetaStoreDirectSql(pm, getConf(), "");
}
if (!allowJdo && isConfigEnabled && !directSql.isCompatibleDatastore()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
----------------------------------------------------------------------
diff --cc metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
index ed05381,3a3d184..dabede4
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
@@@ -39,8 -39,6 +39,7 @@@ public final class TransactionalValidat
// These constants are also imported by org.apache.hadoop.hive.ql.io.AcidUtils.
public static final String DEFAULT_TRANSACTIONAL_PROPERTY = "default";
- public static final String LEGACY_TRANSACTIONAL_PROPERTY = "legacy";
+ public static final String INSERTONLY_TRANSACTIONAL_PROPERTY = "insert_only";
TransactionalValidationListener(Configuration conf) {
super(conf);
@@@ -104,13 -114,10 +115,13 @@@
//normalize prop name
parameters.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, transactionalValue);
}
- if ("true".equalsIgnoreCase(transactionalValue)) {
+ if ("true".equalsIgnoreCase(transactionalValue) && !"true".equalsIgnoreCase(oldTransactionalValue)) {
+ //only need to check conformance if alter table enabled aicd
if (!conformToAcid(newTable)) {
- throw new MetaException("The table must be stored using an ACID compliant format (such as ORC)");
+ // INSERT_ONLY tables don't have to conform to ACID requirement like ORC or bucketing
+ if (transactionalPropertiesValue == null || !"insert_only".equalsIgnoreCase(transactionalPropertiesValue)) {
- throw new MetaException("The table must be bucketed and stored using an ACID compliant" +
- " format (such as ORC)");
++ throw new MetaException("The table must be stored using an ACID compliant format (such as ORC)");
+ }
}
if (newTable.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) {
@@@ -204,13 -194,9 +204,12 @@@
return;
}
- if ("true".equalsIgnoreCase(transactionalValue)) {
+ if ("true".equalsIgnoreCase(transactional)) {
if (!conformToAcid(newTable)) {
- throw new MetaException("The table must be stored using an ACID compliant format (such as ORC)");
+ // INSERT_ONLY tables don't have to conform to ACID requirement like ORC or bucketing
+ if (transactionalProperties == null || !"insert_only".equalsIgnoreCase(transactionalProperties)) {
- throw new MetaException("The table must be bucketed and stored using an ACID compliant" +
- " format (such as ORC)");
++ throw new MetaException("The table must be stored using an ACID compliant format (such as ORC)");
+ }
}
if (newTable.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) {
@@@ -289,8 -273,6 +286,7 @@@
boolean isValid = false;
switch (transactionalProperties) {
case DEFAULT_TRANSACTIONAL_PROPERTY:
- case LEGACY_TRANSACTIONAL_PROPERTY:
+ case INSERTONLY_TRANSACTIONAL_PROPERTY:
isValid = true;
break;
default:
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
----------------------------------------------------------------------
diff --cc metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
index 906591d,93d1ba6..a97ba1a
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
@@@ -18,7 -18,11 +18,16 @@@
package org.apache.hadoop.hive.metastore.cache;
import java.nio.ByteBuffer;
- import java.util.*;
+ import java.util.ArrayList;
++import java.util.ArrayList;
++import java.util.HashMap;
+ import java.util.HashMap;
+ import java.util.LinkedList;
++import java.util.LinkedList;
+ import java.util.List;
++import java.util.List;
++import java.util.Map;
+ import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
----------------------------------------------------------------------
diff --cc metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
index 6d851a0,fb16cfc..0f7827b
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
@@@ -110,7 -114,14 +112,12 @@@ public class DummyRawStoreForJdoConnect
}
@Override
+ public boolean isActiveTransaction() {
+ return false;
+ }
+
+ @Override
public void rollbackTransaction() {
-
-
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
----------------------------------------------------------------------
diff --cc metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
index 9542990,08228af..7573fb2
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
@@@ -379,13 -362,14 +377,14 @@@ public class TestObjectStore
public void testDirectSqlErrorMetrics() throws Exception {
HiveConf conf = new HiveConf();
conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_METRICS_ENABLED, true);
- conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, MetricsReporting.JSON_FILE.name()
- + "," + MetricsReporting.JMX.name());
+ Metrics.initialize(conf);
- MetricsFactory.init(conf);
- CodahaleMetrics metrics = (CodahaleMetrics) MetricsFactory.getInstance();
+ // recall setup so that we get an object store with the metrics initalized
+ setUp();
+ Counter directSqlErrors =
+ Metrics.getRegistry().getCounters().get(MetricsConstants.DIRECTSQL_ERRORS);
- objectStore.new GetDbHelper("foo", null, true, true) {
+ objectStore.new GetDbHelper("foo", true, true) {
@Override
protected Database getSqlResult(ObjectStore.GetHelper<Database> ctx) throws MetaException {
return null;
@@@ -398,11 -382,9 +397,9 @@@
}
}.run(false);
- String json = metrics.dumpJson();
- MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER,
- MetricsConstant.DIRECTSQL_ERRORS, "");
+ Assert.assertEquals(0, directSqlErrors.getCount());
- objectStore.new GetDbHelper("foo", null, true, true) {
+ objectStore.new GetDbHelper("foo", true, true) {
@Override
protected Database getSqlResult(ObjectStore.GetHelper<Database> ctx) throws MetaException {
throw new RuntimeException();
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/Context.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 82a9fad,acc2390..b6eff94
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@@ -3846,11 -3867,42 +3884,22 @@@ public class DDLTask extends Task<DDLWo
throw new HiveException(ErrorMsg.REPLACE_CANNOT_DROP_COLUMNS, alterTbl.getOldName());
}
}
+
+ boolean partitioned = tbl.isPartitioned();
+ boolean droppingColumns = alterTbl.getNewCols().size() < sd.getCols().size();
+ if (ParquetHiveSerDe.isParquetTable(tbl) &&
+ isSchemaEvolutionEnabled(tbl) &&
+ !alterTbl.getIsCascade() &&
+ droppingColumns && partitioned) {
+ LOG.warn("Cannot drop columns from a partitioned parquet table without the CASCADE option");
+ throw new HiveException(ErrorMsg.REPLACE_CANNOT_DROP_COLUMNS,
+ alterTbl.getOldName());
+ }
sd.setCols(alterTbl.getNewCols());
} else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.ADDPROPS) {
- if (StatsSetupConst.USER.equals(environmentContext.getProperties()
- .get(StatsSetupConst.STATS_GENERATED))) {
- environmentContext.getProperties().remove(StatsSetupConst.DO_NOT_UPDATE_STATS);
- }
- if (part != null) {
- part.getTPartition().getParameters().putAll(alterTbl.getProps());
- } else {
- tbl.getTTable().getParameters().putAll(alterTbl.getProps());
- }
+ return alterTableAddProps(alterTbl, tbl, part, environmentContext);
} else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.DROPPROPS) {
- Iterator<String> keyItr = alterTbl.getProps().keySet().iterator();
- if (StatsSetupConst.USER.equals(environmentContext.getProperties()
- .get(StatsSetupConst.STATS_GENERATED))) {
- // drop a stats parameter, which triggers recompute stats update automatically
- environmentContext.getProperties().remove(StatsSetupConst.DO_NOT_UPDATE_STATS);
- }
- while (keyItr.hasNext()) {
- if (part != null) {
- part.getTPartition().getParameters().remove(keyItr.next());
- } else {
- tbl.getTTable().getParameters().remove(keyItr.next());
- }
- }
+ return alterTableDropProps(alterTbl, tbl, part, environmentContext);
} else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.ADDSERDEPROPS) {
StorageDescriptor sd = retrieveStorageDescriptor(tbl, part);
sd.getSerdeInfo().getParameters().putAll(alterTbl.getProps());
@@@ -4002,219 -4054,7 +4051,215 @@@
throw new HiveException(ErrorMsg.UNSUPPORTED_ALTER_TBL_OP, alterTbl.getOp().toString());
}
- return 0;
+ return null;
+ }
+
+ private List<Task<?>> alterTableDropProps(AlterTableDesc alterTbl, Table tbl,
+ Partition part, EnvironmentContext environmentContext) throws HiveException {
+ if (StatsSetupConst.USER.equals(environmentContext.getProperties()
+ .get(StatsSetupConst.STATS_GENERATED))) {
+ // drop a stats parameter, which triggers recompute stats update automatically
+ environmentContext.getProperties().remove(StatsSetupConst.DO_NOT_UPDATE_STATS);
+ }
+
+ List<Task<?>> result = null;
+ if (part == null) {
+ Set<String> removedSet = alterTbl.getProps().keySet();
+ boolean isFromMmTable = MetaStoreUtils.isInsertOnlyTable(tbl.getParameters()),
+ isRemoved = MetaStoreUtils.isRemovedInsertOnlyTable(removedSet);
+ if (isFromMmTable && isRemoved) {
+ result = generateRemoveMmTasks(tbl);
+ }
+ }
+ Iterator<String> keyItr = alterTbl.getProps().keySet().iterator();
+ while (keyItr.hasNext()) {
+ if (part != null) {
+ part.getTPartition().getParameters().remove(keyItr.next());
+ } else {
+ tbl.getTTable().getParameters().remove(keyItr.next());
+ }
+ }
+ return result;
+ }
+
+ private List<Task<?>> generateRemoveMmTasks(Table tbl) throws HiveException {
+ // To avoid confusion from nested MM directories when table is converted back and forth, we
+ // want to rename mm_ dirs to remove the prefix; however, given the unpredictable nested
+ // directory handling in Hive/MR, we will instead move all the files into the root directory.
+ // We will also delete any directories that are not committed.
+ // Note that this relies on locks. Note also that we only do the renames AFTER the metastore
+ // operation commits. Deleting uncommitted things is safe, but moving stuff before we convert
+ // could cause data loss.
+ List<Path> allMmDirs = new ArrayList<>();
+ if (tbl.isStoredAsSubDirectories()) {
+ // TODO: support this? we only bail because it's a PITA and hardly anyone seems to care.
+ throw new HiveException("Converting list bucketed tables stored as subdirectories "
+ + " to and from MM is not supported");
+ }
+ List<String> bucketCols = tbl.getBucketCols();
+ if (bucketCols != null && !bucketCols.isEmpty()
+ && HiveConf.getBoolVar(conf, ConfVars.HIVE_STRICT_CHECKS_BUCKETING)) {
+ throw new HiveException("Converting bucketed tables from MM is not supported by default; "
+ + "copying files from multiple MM directories may potentially break the buckets. You "
+ + "can set " + ConfVars.HIVE_STRICT_CHECKS_BUCKETING.varname
+ + " to false for this query if you want to force the conversion.");
+ }
+ Hive db = getHive();
+ String value = conf.get(ValidTxnList.VALID_TXNS_KEY);
+ ValidTxnList validTxnList = value == null ? new ValidReadTxnList() : new ValidReadTxnList(value);
+ if (tbl.getPartitionKeys().size() > 0) {
+ PartitionIterable parts = new PartitionIterable(db, tbl, null,
+ HiveConf.getIntVar(conf, ConfVars.METASTORE_BATCH_RETRIEVE_MAX));
+ Iterator<Partition> partIter = parts.iterator();
+ while (partIter.hasNext()) {
+ Partition part = partIter.next();
+ checkMmLb(part);
+ handleRemoveMm(part.getDataLocation(), validTxnList, allMmDirs);
+ }
+ } else {
+ checkMmLb(tbl);
+ handleRemoveMm(tbl.getDataLocation(), validTxnList, allMmDirs);
+ }
+ List<Path> targetPaths = new ArrayList<>(allMmDirs.size());
+ List<String> targetPrefix = new ArrayList<>(allMmDirs.size());
+ int prefixLen = JavaUtils.DELTA_PREFIX.length();
+ for (int i = 0; i < allMmDirs.size(); ++i) {
+ Path src = allMmDirs.get(i);
+ Path tgt = src.getParent();
+ String prefix = src.getName().substring(prefixLen + 1) + "_";
+ Utilities.LOG14535.info("Will move " + src + " to " + tgt + " (prefix " + prefix + ")");
+ targetPaths.add(tgt);
+ targetPrefix.add(prefix);
+ }
+ // Don't set inputs and outputs - the locks have already been taken so it's pointless.
+ MoveWork mw = new MoveWork(null, null, null, null, false);
+ mw.setMultiFilesDesc(new LoadMultiFilesDesc(
+ allMmDirs, targetPaths, targetPrefix, true, null, null));
+ return Lists.<Task<?>>newArrayList(TaskFactory.get(mw, conf));
+ }
+
+ private void checkMmLb(Table tbl) throws HiveException {
+ if (!tbl.isStoredAsSubDirectories()) return;
+ // TODO: support this?
+ throw new HiveException("Converting list bucketed tables stored as subdirectories "
+ + " to and from MM is not supported");
+ }
+
+ private void checkMmLb(Partition part) throws HiveException {
+ if (!part.isStoredAsSubDirectories()) return;
+ // TODO: support this?
+ throw new HiveException("Converting list bucketed tables stored as subdirectories "
+ + " to and from MM is not supported. Please create a table in the desired format.");
+ }
+
+ private void handleRemoveMm(
+ Path path, ValidTxnList validTxnList, List<Path> result) throws HiveException {
+ // Note: doesn't take LB into account; that is not presently supported here (throws above).
+ try {
+ FileSystem fs = path.getFileSystem(conf);
+ for (FileStatus file : fs.listStatus(path)) {
+ Path childPath = file.getPath();
+ if (!file.isDirectory()) {
+ ensureDelete(fs, childPath, "a non-directory file");
+ continue;
+ }
+ Long writeId = JavaUtils.extractTxnId(childPath);
+ if (writeId == null) {
+ ensureDelete(fs, childPath, "an unknown directory");
+ } else if (!validTxnList.isTxnValid(writeId)) {
+ // Assume no concurrent active writes - we rely on locks here. We could check and fail.
+ ensureDelete(fs, childPath, "an uncommitted directory");
+ } else {
+ result.add(childPath);
+ }
+ }
+ } catch (IOException ex) {
+ throw new HiveException(ex);
+ }
+ }
+
+ private static void ensureDelete(FileSystem fs, Path path, String what) throws IOException {
+ Utilities.LOG14535.info("Deleting " + what + " " + path);
+ try {
+ if (!fs.delete(path, true)) throw new IOException("delete returned false");
+ } catch (Exception ex) {
+ String error = "Couldn't delete " + path + "; cannot remove MM setting from the table";
+ LOG.error(error, ex);
+ throw (ex instanceof IOException) ? (IOException)ex : new IOException(ex);
+ }
+ }
+
+ private List<Task<?>> generateAddMmTasks(Table tbl) throws HiveException {
+ // We will move all the files in the table/partition directories into the first MM
+ // directory, then commit the first write ID.
+ List<Path> srcs = new ArrayList<>(), tgts = new ArrayList<>();
+ long mmWriteId = 0;
+ try {
+ HiveTxnManager txnManager = SessionState.get().getTxnMgr();
+ if (txnManager.isTxnOpen()) {
+ mmWriteId = txnManager.getCurrentTxnId();
+ } else {
+ mmWriteId = txnManager.openTxn(new Context(conf), conf.getUser());
+ txnManager.commitTxn();
+ }
+ } catch (Exception e) {
+ String errorMessage = "FAILED: Error in acquiring locks: " + e.getMessage();
+ console.printError(errorMessage, "\n"
+ + org.apache.hadoop.util.StringUtils.stringifyException(e));
+ }
+ int stmtId = 0;
+ String mmDir = AcidUtils.deltaSubdir(mmWriteId, mmWriteId, stmtId);
+ Hive db = getHive();
+ if (tbl.getPartitionKeys().size() > 0) {
+ PartitionIterable parts = new PartitionIterable(db, tbl, null,
+ HiveConf.getIntVar(conf, ConfVars.METASTORE_BATCH_RETRIEVE_MAX));
+ Iterator<Partition> partIter = parts.iterator();
+ while (partIter.hasNext()) {
+ Partition part = partIter.next();
+ checkMmLb(part);
+ Path src = part.getDataLocation(), tgt = new Path(src, mmDir);
+ srcs.add(src);
+ tgts.add(tgt);
+ Utilities.LOG14535.info("Will move " + src + " to " + tgt);
+ }
+ } else {
+ checkMmLb(tbl);
+ Path src = tbl.getDataLocation(), tgt = new Path(src, mmDir);
+ srcs.add(src);
+ tgts.add(tgt);
+ Utilities.LOG14535.info("Will move " + src + " to " + tgt);
+ }
+ // Don't set inputs and outputs - the locks have already been taken so it's pointless.
+ MoveWork mw = new MoveWork(null, null, null, null, false);
+ mw.setMultiFilesDesc(new LoadMultiFilesDesc(srcs, tgts, true, null, null));
+ ImportCommitWork icw = new ImportCommitWork(tbl.getDbName(), tbl.getTableName(), mmWriteId, stmtId);
+ Task<?> mv = TaskFactory.get(mw, conf), ic = TaskFactory.get(icw, conf);
+ mv.addDependentTask(ic);
+ return Lists.<Task<?>>newArrayList(mv);
+ }
+
+ private List<Task<?>> alterTableAddProps(AlterTableDesc alterTbl, Table tbl,
+ Partition part, EnvironmentContext environmentContext) throws HiveException {
+ if (StatsSetupConst.USER.equals(environmentContext.getProperties()
+ .get(StatsSetupConst.STATS_GENERATED))) {
+ environmentContext.getProperties().remove(StatsSetupConst.DO_NOT_UPDATE_STATS);
+ }
- if(alterTbl.getProps().containsKey(ParquetTableUtils.PARQUET_INT96_WRITE_ZONE_PROPERTY)) {
- NanoTimeUtils.validateTimeZone(
- alterTbl.getProps().get(ParquetTableUtils.PARQUET_INT96_WRITE_ZONE_PROPERTY));
- }
+ List<Task<?>> result = null;
+ if (part != null) {
+ part.getTPartition().getParameters().putAll(alterTbl.getProps());
+ } else {
+ boolean isFromMmTable = MetaStoreUtils.isInsertOnlyTable(tbl.getParameters());
+ Boolean isToMmTable = MetaStoreUtils.isToInsertOnlyTable(alterTbl.getProps());
+ if (isToMmTable != null) {
+ if (!isFromMmTable && isToMmTable) {
+ result = generateAddMmTasks(tbl);
+ } else if (isFromMmTable && !isToMmTable) {
+ result = generateRemoveMmTasks(tbl);
+ }
+ }
+ tbl.getTTable().getParameters().putAll(alterTbl.getProps());
+ }
+ return result;
}
private int dropConstraint(Hive db, AlterTableDesc alterTbl)
@@@ -4583,22 -4426,8 +4631,8 @@@
}
}
- // If HIVE_PARQUET_INT96_DEFAULT_UTC_WRITE_ZONE is set to True, then set new Parquet tables timezone
- // to UTC by default (only if the table property is not set)
- if (ParquetHiveSerDe.isParquetTable(tbl)) {
- SessionState ss = SessionState.get();
- String parquetTimezone = tbl.getProperty(ParquetTableUtils.PARQUET_INT96_WRITE_ZONE_PROPERTY);
- if (parquetTimezone == null || parquetTimezone.isEmpty()) {
- if (ss.getConf().getBoolVar(ConfVars.HIVE_PARQUET_INT96_DEFAULT_UTC_WRITE_ZONE)) {
- tbl.setProperty(ParquetTableUtils.PARQUET_INT96_WRITE_ZONE_PROPERTY, ParquetTableUtils.PARQUET_INT96_NO_ADJUSTMENT_ZONE);
- }
- } else {
- NanoTimeUtils.validateTimeZone(parquetTimezone);
- }
- }
-
// create the table
- if (crtTbl.getReplaceMode()){
+ if (crtTbl.getReplaceMode()) {
// replace-mode creates are really alters using CreateTableDesc.
try {
db.alterTable(tbl.getDbName()+"."+tbl.getTableName(),tbl,null);
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
index 0582f94,13750cd..d2d9946
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
@@@ -382,14 -368,7 +379,11 @@@ public class FetchOperator implements S
Class<? extends InputFormat> formatter = currDesc.getInputFileFormatClass();
Utilities.copyTableJobPropertiesToConf(currDesc.getTableDesc(), job);
- if (ParquetHiveSerDe.class.getName().equals(currDesc.getTableDesc().getSerdeClassName())) {
- ParquetTableUtils.setParquetTimeZoneIfAbsent(job, currDesc.getTableDesc().getProperties());
- }
InputFormat inputFormat = getInputFormatFromCache(formatter, job);
+ String inputs = processCurrPathForMmWriteIds(inputFormat);
+ Utilities.LOG14535.info("Setting fetch inputs to " + inputs);
+ if (inputs == null) return null;
+ job.set("mapred.input.dir", inputs);
InputSplit[] splits = inputFormat.getSplits(job, 1);
FetchInputFormatSplit[] inputSplits = new FetchInputFormatSplit[splits.length];
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index 7e1b8fa,bc265eb..3544884
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@@ -41,11 -28,11 +41,13 @@@ import org.apache.hadoop.hive.common.St
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConfUtil;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+ import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.CompilationOpContext;
import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.exec.Utilities.MissingBucketsContext;
import org.apache.hadoop.hive.ql.io.AcidUtils;
+import org.apache.hadoop.hive.ql.io.AcidUtils.Operation;
+ import org.apache.hadoop.hive.ql.io.BucketCodec;
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
import org.apache.hadoop.hive.ql.io.HiveKey;
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
@@@ -165,28 -149,11 +167,28 @@@ public class FileSinkOperator extends T
Path[] finalPaths;
RecordWriter[] outWriters;
RecordUpdater[] updaters;
- private Stat stat;
+ Stat stat;
+ int acidLastBucket = -1;
+ int acidFileOffset = -1;
+ private boolean isMmTable;
+ private Long txnId;
+ private int stmtId;
+
+ public FSPaths(Path specPath, boolean isMmTable) {
+ this.isMmTable = isMmTable;
+ if (!isMmTable) {
+ tmpPath = Utilities.toTempPath(specPath);
+ taskOutputTempPath = Utilities.toTaskTempPath(specPath);
+ } else {
+ tmpPath = specPath;
+ taskOutputTempPath = null; // Should not be used.
+ txnId = conf.getTransactionId();
+ stmtId = conf.getStatementId();
+ }
+ Utilities.LOG14535.info("new FSPaths for " + numFiles + " files, dynParts = " + bDynParts
+ + ": tmpPath " + tmpPath + ", task path " + taskOutputTempPath
+ + " (spec path " + specPath + ")"/*, new Exception()*/);
- public FSPaths(Path specPath) {
- tmpPath = Utilities.toTempPath(specPath);
- taskOutputTempPath = Utilities.toTaskTempPath(specPath);
outPaths = new Path[numFiles];
finalPaths = new Path[numFiles];
outWriters = new RecordWriter[numFiles];
@@@ -881,10 -758,10 +885,10 @@@
// for a given operator branch prediction should work quite nicely on it.
// RecordUpdateer expects to get the actual row, not a serialized version of it. Thus we
// pass the row rather than recordValue.
- if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID) {
+ if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID || conf.isMmTable()) {
- rowOutWriters[writerOffset].write(recordValue);
+ rowOutWriters[findWriterOffset(row)].write(recordValue);
} else if (conf.getWriteType() == AcidUtils.Operation.INSERT) {
- fpaths.updaters[writerOffset].insert(conf.getTransactionId(), row);
+ fpaths.updaters[findWriterOffset(row)].insert(conf.getTransactionId(), row);
} else {
// TODO I suspect we could skip much of the stuff above this in the function in the case
// of update and delete. But I don't understand all of the side effects of the above
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index e86ca3a,cde2805..34d0598
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@@ -403,232 -579,6 +436,231 @@@ public class MoveTask extends Task<Move
return (1);
}
}
+
+ private DataContainer handleStaticParts(Hive db, Table table, LoadTableDesc tbd,
+ TaskInformation ti) throws HiveException, IOException, InvalidOperationException {
+ List<String> partVals = MetaStoreUtils.getPvals(table.getPartCols(), tbd.getPartitionSpec());
+ db.validatePartitionNameCharacters(partVals);
+ Utilities.LOG14535.info("loadPartition called from " + tbd.getSourcePath()
+ + " into " + tbd.getTable().getTableName());
+ db.loadPartition(tbd.getSourcePath(), tbd.getTable().getTableName(),
+ tbd.getPartitionSpec(), tbd.getReplace(),
+ tbd.getInheritTableSpecs(), isSkewedStoredAsDirs(tbd), work.isSrcLocal(),
+ work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID &&
+ !tbd.isMmTable(),
+ hasFollowingStatsTask(), tbd.getTxnId(), tbd.getStmtId());
+ Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false);
+
+ // See the comment inside updatePartitionBucketSortColumns.
+ if (!tbd.isMmTable() && (ti.bucketCols != null || ti.sortCols != null)) {
+ updatePartitionBucketSortColumns(db, table, partn, ti.bucketCols,
+ ti.numBuckets, ti.sortCols);
+ }
+
+ DataContainer dc = new DataContainer(table.getTTable(), partn.getTPartition());
+ // add this partition to post-execution hook
+ if (work.getOutputs() != null) {
+ DDLTask.addIfAbsentByName(new WriteEntity(partn,
+ getWriteType(tbd, work.getLoadTableWork().getWriteType())), work.getOutputs());
+ }
+ return dc;
+ }
+
+ private DataContainer handleDynParts(Hive db, Table table, LoadTableDesc tbd,
+ TaskInformation ti, DynamicPartitionCtx dpCtx) throws HiveException,
+ IOException, InvalidOperationException {
+ DataContainer dc;
+ List<LinkedHashMap<String, String>> dps = Utilities.getFullDPSpecs(conf, dpCtx);
+
+ console.printInfo(System.getProperty("line.separator"));
+ long startTime = System.currentTimeMillis();
+ // load the list of DP partitions and return the list of partition specs
+ // TODO: In a follow-up to HIVE-1361, we should refactor loadDynamicPartitions
+ // to use Utilities.getFullDPSpecs() to get the list of full partSpecs.
+ // After that check the number of DPs created to not exceed the limit and
+ // iterate over it and call loadPartition() here.
+ // The reason we don't do inside HIVE-1361 is the latter is large and we
+ // want to isolate any potential issue it may introduce.
+ if (tbd.isMmTable() && !tbd.isCommitMmWrite()) {
+ throw new HiveException("Only single-partition LoadTableDesc can skip commiting write ID");
+ }
+ Map<Map<String, String>, Partition> dp =
+ db.loadDynamicPartitions(
+ tbd.getSourcePath(),
+ tbd.getTable().getTableName(),
+ tbd.getPartitionSpec(),
+ tbd.getReplace(),
+ dpCtx.getNumDPCols(),
+ (tbd.getLbCtx() == null) ? 0 : tbd.getLbCtx().calculateListBucketingLevel(),
+ work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID &&
+ !tbd.isMmTable(),
+ SessionState.get().getTxnMgr().getCurrentTxnId(), tbd.getStmtId(), hasFollowingStatsTask(),
+ work.getLoadTableWork().getWriteType());
+
+ // publish DP columns to its subscribers
+ if (dps != null && dps.size() > 0) {
+ pushFeed(FeedType.DYNAMIC_PARTITIONS, dp.values());
+ }
+
+ String loadTime = "\t Time taken to load dynamic partitions: " +
+ (System.currentTimeMillis() - startTime)/1000.0 + " seconds";
+ console.printInfo(loadTime);
+ LOG.info(loadTime);
+
+ if (dp.size() == 0 && conf.getBoolVar(HiveConf.ConfVars.HIVE_ERROR_ON_EMPTY_PARTITION)) {
+ throw new HiveException("This query creates no partitions." +
+ " To turn off this error, set hive.error.on.empty.partition=false.");
+ }
+
+ startTime = System.currentTimeMillis();
+ // for each partition spec, get the partition
+ // and put it to WriteEntity for post-exec hook
+ for(Map.Entry<Map<String, String>, Partition> entry : dp.entrySet()) {
+ Partition partn = entry.getValue();
+
+ // See the comment inside updatePartitionBucketSortColumns.
+ if (!tbd.isMmTable() && (ti.bucketCols != null || ti.sortCols != null)) {
+ updatePartitionBucketSortColumns(
+ db, table, partn, ti.bucketCols, ti.numBuckets, ti.sortCols);
+ }
+
+ WriteEntity enty = new WriteEntity(partn,
+ getWriteType(tbd, work.getLoadTableWork().getWriteType()));
+ if (work.getOutputs() != null) {
+ DDLTask.addIfAbsentByName(enty, work.getOutputs());
+ }
+ // Need to update the queryPlan's output as well so that post-exec hook get executed.
+ // This is only needed for dynamic partitioning since for SP the the WriteEntity is
+ // constructed at compile time and the queryPlan already contains that.
+ // For DP, WriteEntity creation is deferred at this stage so we need to update
+ // queryPlan here.
+ if (queryPlan.getOutputs() == null) {
+ queryPlan.setOutputs(new LinkedHashSet<WriteEntity>());
+ }
+ queryPlan.getOutputs().add(enty);
+
+ // update columnar lineage for each partition
+ dc = new DataContainer(table.getTTable(), partn.getTPartition());
+
+ // Don't set lineage on delete as we don't have all the columns
+ if (SessionState.get() != null &&
+ work.getLoadTableWork().getWriteType() != AcidUtils.Operation.DELETE &&
+ work.getLoadTableWork().getWriteType() != AcidUtils.Operation.UPDATE) {
+ SessionState.get().getLineageState().setLineage(tbd.getSourcePath(), dc,
+ table.getCols());
+ }
+ LOG.info("\tLoading partition " + entry.getKey());
+ }
+ console.printInfo("\t Time taken for adding to write entity : " +
+ (System.currentTimeMillis() - startTime)/1000.0 + " seconds");
+ dc = null; // reset data container to prevent it being added again.
+ return dc;
+ }
+
+ private void inferTaskInformation(TaskInformation ti) {
+ // Find the first ancestor of this MoveTask which is some form of map reduce task
+ // (Either standard, local, or a merge)
+ while (ti.task.getParentTasks() != null && ti.task.getParentTasks().size() == 1) {
+ ti.task = (Task)ti.task.getParentTasks().get(0);
+ // If it was a merge task or a local map reduce task, nothing can be inferred
+ if (ti.task instanceof MergeFileTask || ti.task instanceof MapredLocalTask) {
+ break;
+ }
+
+ // If it's a standard map reduce task, check what, if anything, it inferred about
+ // the directory this move task is moving
+ if (ti.task instanceof MapRedTask) {
+ MapredWork work = (MapredWork)ti.task.getWork();
+ MapWork mapWork = work.getMapWork();
+ ti.bucketCols = mapWork.getBucketedColsByDirectory().get(ti.path);
+ ti.sortCols = mapWork.getSortedColsByDirectory().get(ti.path);
+ if (work.getReduceWork() != null) {
+ ti.numBuckets = work.getReduceWork().getNumReduceTasks();
+ }
+
+ if (ti.bucketCols != null || ti.sortCols != null) {
+ // This must be a final map reduce task (the task containing the file sink
+ // operator that writes the final output)
+ assert work.isFinalMapRed();
+ }
+ break;
+ }
+
+ // If it's a move task, get the path the files were moved from, this is what any
+ // preceding map reduce task inferred information about, and moving does not invalidate
+ // those assumptions
+ // This can happen when a conditional merge is added before the final MoveTask, but the
+ // condition for merging is not met, see GenMRFileSink1.
+ if (ti.task instanceof MoveTask) {
+ MoveTask mt = (MoveTask)ti.task;
+ if (mt.getWork().getLoadFileWork() != null) {
+ ti.path = mt.getWork().getLoadFileWork().getSourcePath().toUri().toString();
+ }
+ }
+ }
+ }
+
+ private void checkFileFormats(Hive db, LoadTableDesc tbd, Table table)
+ throws HiveException {
+ if (work.getCheckFileFormat()) {
+ // Get all files from the src directory
+ FileStatus[] dirs;
+ ArrayList<FileStatus> files;
+ FileSystem srcFs; // source filesystem
+ try {
+ srcFs = tbd.getSourcePath().getFileSystem(conf);
+ dirs = srcFs.globStatus(tbd.getSourcePath());
+ files = new ArrayList<FileStatus>();
+ for (int i = 0; (dirs != null && i < dirs.length); i++) {
+ files.addAll(Arrays.asList(srcFs.listStatus(dirs[i].getPath(), FileUtils.HIDDEN_FILES_PATH_FILTER)));
+ // We only check one file, so exit the loop when we have at least
+ // one.
+ if (files.size() > 0) {
+ break;
+ }
+ }
+ } catch (IOException e) {
+ throw new HiveException(
+ "addFiles: filesystem error in check phase", e);
+ }
+
+ // handle file format check for table level
+ if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVECHECKFILEFORMAT)) {
+ boolean flag = true;
+ // work.checkFileFormat is set to true only for Load Task, so assumption here is
+ // dynamic partition context is null
+ if (tbd.getDPCtx() == null) {
+ if (tbd.getPartitionSpec() == null || tbd.getPartitionSpec().isEmpty()) {
+ // Check if the file format of the file matches that of the table.
+ flag = HiveFileFormatUtils.checkInputFormat(
+ srcFs, conf, tbd.getTable().getInputFileFormatClass(), files);
+ } else {
+ // Check if the file format of the file matches that of the partition
+ Partition oldPart = db.getPartition(table, tbd.getPartitionSpec(), false);
+ if (oldPart == null) {
+ // this means we have just created a table and are specifying partition in the
+ // load statement (without pre-creating the partition), in which case lets use
+ // table input format class. inheritTableSpecs defaults to true so when a new
+ // partition is created later it will automatically inherit input format
+ // from table object
+ flag = HiveFileFormatUtils.checkInputFormat(
+ srcFs, conf, tbd.getTable().getInputFileFormatClass(), files);
+ } else {
+ flag = HiveFileFormatUtils.checkInputFormat(
+ srcFs, conf, oldPart.getInputFormatClass(), files);
+ }
+ }
+ if (!flag) {
- throw new HiveException(
- "Wrong file format. Please check the file's format.");
++ throw new HiveException(ErrorMsg.WRONG_FILE_FORMAT);
+ }
+ } else {
+ LOG.warn("Skipping file format check as dpCtx is not null");
+ }
+ }
+ }
+ }
+
+
/**
* so to make sure we crate WriteEntity with the right WriteType. This is (at this point) only
* for consistency since LockManager (which is the only thing that pays attention to WriteType)
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index ea0a0fd,aca99f2..88f5a0d
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@@ -1445,14 -1447,21 +1471,21 @@@ public final class Utilities
// create empty buckets if necessary
if (emptyBuckets.size() > 0) {
perfLogger.PerfLogBegin("FileSinkOperator", "CreateEmptyBuckets");
- createEmptyBuckets(hconf, emptyBuckets, conf, reporter);
+ createEmptyBuckets(
+ hconf, emptyBuckets, conf.getCompressed(), conf.getTableInfo(), reporter);
+ filesKept.addAll(emptyBuckets);
perfLogger.PerfLogEnd("FileSinkOperator", "CreateEmptyBuckets");
}
-
// move to the file destination
- log.info("Moving tmp dir: " + tmpPath + " to: " + specPath);
+ Utilities.LOG14535.info("Moving tmp dir: " + tmpPath + " to: " + specPath);
perfLogger.PerfLogBegin("FileSinkOperator", "RenameOrMoveFiles");
- Utilities.renameOrMoveFiles(fs, tmpPath, specPath);
+ if (HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVE_EXEC_MOVE_FILES_FROM_SOURCE_DIR)) {
+ // HIVE-17113 - avoid copying files that may have been written to the temp dir by runaway tasks,
+ // by moving just the files we've tracked from removeTempOrDuplicateFiles().
+ Utilities.moveSpecifiedFiles(fs, tmpPath, specPath, filesKept);
+ } else {
+ Utilities.renameOrMoveFiles(fs, tmpPath, specPath);
+ }
perfLogger.PerfLogEnd("FileSinkOperator", "RenameOrMoveFiles");
}
} else {
@@@ -1534,27 -1552,7 +1578,27 @@@
* @return a list of path names corresponding to should-be-created empty buckets.
*/
public static List<Path> removeTempOrDuplicateFiles(FileSystem fs, FileStatus[] fileStats,
- DynamicPartitionCtx dpCtx, FileSinkDesc conf, Configuration hconf) throws IOException {
+ DynamicPartitionCtx dpCtx, FileSinkDesc conf, Configuration hconf, Set<Path> filesKept) throws IOException {
+ int dpLevels = dpCtx == null ? 0 : dpCtx.getNumDPCols(),
+ numBuckets = (conf != null && conf.getTable() != null)
+ ? conf.getTable().getNumBuckets() : 0;
+ return removeTempOrDuplicateFiles(fs, fileStats, dpLevels, numBuckets, hconf, null, 0, false);
+ }
+
+ private static boolean removeEmptyDpDirectory(FileSystem fs, Path path) throws IOException {
+ FileStatus[] items = fs.listStatus(path);
+ // remove empty directory since DP insert should not generate empty partitions.
+ // empty directories could be generated by crashed Task/ScriptOperator
+ if (items.length != 0) return false;
+ if (!fs.delete(path, true)) {
+ LOG.error("Cannot delete empty directory " + path);
+ throw new IOException("Cannot delete empty directory " + path);
+ }
+ return true;
+ }
+
+ public static List<Path> removeTempOrDuplicateFiles(FileSystem fs, FileStatus[] fileStats,
+ int dpLevels, int numBuckets, Configuration hconf, Long txnId, int stmtId, boolean isMmTable) throws IOException {
if (fileStats == null) {
return null;
}
@@@ -3145,21 -3150,14 +3235,21 @@@
LOG.info("Processing alias " + alias);
// The alias may not have any path
+ Collection<Map.Entry<Path, ArrayList<String>>> pathToAliases =
+ work.getPathToAliases().entrySet();
+ if (!skipDummy) {
+ // ConcurrentModification otherwise if adding dummy.
+ pathToAliases = new ArrayList<>(pathToAliases);
+ }
boolean isEmptyTable = true;
boolean hasLogged = false;
- // Note: this copies the list because createDummyFileForEmptyPartition may modify the map.
- for (Path file : new LinkedList<Path>(work.getPathToAliases().keySet())) {
+ Path path = null;
+ for (Map.Entry<Path, ArrayList<String>> e : pathToAliases) {
if (lDrvStat != null && lDrvStat.driverState == DriverState.INTERRUPT)
- throw new IOException("Operation is Canceled. ");
+ throw new IOException("Operation is Canceled.");
- List<String> aliases = work.getPathToAliases().get(file);
+ Path file = e.getKey();
+ List<String> aliases = e.getValue();
if (aliases.contains(alias)) {
if (file != null) {
isEmptyTable = false;
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
index 0000000,c944a13..2ae18cf
mode 000000,100644..100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
@@@ -1,0 -1,314 +1,316 @@@
+ /*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+ package org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.table;
+
+ import org.apache.hadoop.fs.Path;
+ import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+ import org.apache.hadoop.hive.metastore.Warehouse;
+ import org.apache.hadoop.hive.metastore.api.Database;
+ import org.apache.hadoop.hive.metastore.api.MetaException;
+ import org.apache.hadoop.hive.ql.exec.ReplCopyTask;
+ import org.apache.hadoop.hive.ql.exec.Task;
+ import org.apache.hadoop.hive.ql.exec.TaskFactory;
+ import org.apache.hadoop.hive.ql.exec.Utilities;
+ import org.apache.hadoop.hive.ql.exec.repl.ReplStateLogWork;
+ import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.TableEvent;
+ import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.ReplicationState;
+ import org.apache.hadoop.hive.ql.exec.repl.bootstrap.ReplLoadTask;
+ import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.TaskTracker;
+ import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.util.Context;
+ import org.apache.hadoop.hive.ql.metadata.HiveException;
+ import org.apache.hadoop.hive.ql.metadata.Partition;
+ import org.apache.hadoop.hive.ql.metadata.Table;
+ import org.apache.hadoop.hive.ql.parse.ImportSemanticAnalyzer;
+ import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
+ import org.apache.hadoop.hive.ql.parse.SemanticException;
+ import org.apache.hadoop.hive.ql.parse.repl.ReplLogger;
+ import org.apache.hadoop.hive.ql.plan.AddPartitionDesc;
+ import org.apache.hadoop.hive.ql.plan.DDLWork;
+ import org.apache.hadoop.hive.ql.plan.ImportTableDesc;
+ import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
+ import org.apache.hadoop.hive.ql.plan.MoveWork;
++import org.apache.hadoop.hive.ql.session.SessionState;
++import org.mortbay.jetty.servlet.AbstractSessionManager;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ import java.io.IOException;
+ import java.io.Serializable;
+ import java.util.*;
+
+ import static org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.ReplicationState.PartitionState;
+ import static org.apache.hadoop.hive.ql.parse.ImportSemanticAnalyzer.isPartitioned;
+ import static org.apache.hadoop.hive.ql.parse.ImportSemanticAnalyzer.partSpecToString;
+
+ public class LoadPartitions {
+ private static Logger LOG = LoggerFactory.getLogger(LoadPartitions.class);
+
+ private final Context context;
+ private final ReplLogger replLogger;
+ private final TableContext tableContext;
+ private final TableEvent event;
+ private final TaskTracker tracker;
+ private final AddPartitionDesc lastReplicatedPartition;
+
+ private final ImportTableDesc tableDesc;
+ private Table table;
+
+ public LoadPartitions(Context context, ReplLogger replLogger, TaskTracker tableTracker,
+ TableEvent event, String dbNameToLoadIn,
+ TableContext tableContext) throws HiveException, IOException {
+ this(context, replLogger, tableContext, tableTracker, event, dbNameToLoadIn, null);
+ }
+
+ public LoadPartitions(Context context, ReplLogger replLogger, TableContext tableContext,
+ TaskTracker limiter, TableEvent event, String dbNameToLoadIn,
+ AddPartitionDesc lastReplicatedPartition) throws HiveException, IOException {
+ this.tracker = new TaskTracker(limiter);
+ this.event = event;
+ this.context = context;
+ this.replLogger = replLogger;
+ this.lastReplicatedPartition = lastReplicatedPartition;
+ this.tableContext = tableContext;
+
+ this.tableDesc = tableContext.overrideProperties(event.tableDesc(dbNameToLoadIn));
+ this.table = ImportSemanticAnalyzer.tableIfExists(tableDesc, context.hiveDb);
+ }
+
+ private String location() throws MetaException, HiveException {
+ Database parentDb = context.hiveDb.getDatabase(tableDesc.getDatabaseName());
+ if (!tableContext.waitOnPrecursor()) {
+ return context.warehouse.getDefaultTablePath(parentDb, tableDesc.getTableName()).toString();
+ } else {
+ Path tablePath = new Path(
+ context.warehouse.getDefaultDatabasePath(tableDesc.getDatabaseName()),
+ MetaStoreUtils.encodeTableName(tableDesc.getTableName().toLowerCase())
+ );
+ return context.warehouse.getDnsPath(tablePath).toString();
+ }
+ }
+
+ private void createTableReplLogTask() throws SemanticException {
+ ReplStateLogWork replLogWork = new ReplStateLogWork(replLogger,
+ tableDesc.getTableName(), tableDesc.tableType());
+ Task<ReplStateLogWork> replLogTask = TaskFactory.get(replLogWork, context.hiveConf);
+
+ if (tracker.tasks().isEmpty()) {
+ tracker.addTask(replLogTask);
+ } else {
+ ReplLoadTask.dependency(tracker.tasks(), replLogTask);
+
+ List<Task<? extends Serializable>> visited = new ArrayList<>();
+ tracker.updateTaskCount(replLogTask, visited);
+ }
+ }
+
+ public TaskTracker tasks() throws SemanticException {
+ try {
+ /*
+ We are doing this both in load table and load partitions
+ */
+ if (tableDesc.getLocation() == null) {
+ tableDesc.setLocation(location());
+ }
+
+ if (table == null) {
+ //new table
+
+ table = new Table(tableDesc.getDatabaseName(), tableDesc.getTableName());
+ if (isPartitioned(tableDesc)) {
+ updateReplicationState(initialReplicationState());
+ if (!forNewTable().hasReplicationState()) {
+ // Add ReplStateLogTask only if no pending table load tasks left for next cycle
+ createTableReplLogTask();
+ }
+ return tracker;
+ }
+ } else {
+ // existing
+
+ if (table.isPartitioned()) {
+ List<AddPartitionDesc> partitionDescs = event.partitionDescriptions(tableDesc);
+ if (!event.replicationSpec().isMetadataOnly() && !partitionDescs.isEmpty()) {
+ updateReplicationState(initialReplicationState());
+ if (!forExistingTable(lastReplicatedPartition).hasReplicationState()) {
+ // Add ReplStateLogTask only if no pending table load tasks left for next cycle
+ createTableReplLogTask();
+ }
+ return tracker;
+ }
+ }
+ }
+ return tracker;
+ } catch (Exception e) {
+ throw new SemanticException(e);
+ }
+ }
+
+ private void updateReplicationState(ReplicationState replicationState) throws SemanticException {
+ if (!tracker.canAddMoreTasks()) {
+ tracker.setReplicationState(replicationState);
+ }
+ }
+
+ private ReplicationState initialReplicationState() throws SemanticException {
+ return new ReplicationState(
+ new PartitionState(tableDesc.getTableName(), lastReplicatedPartition)
+ );
+ }
+
+ private TaskTracker forNewTable() throws Exception {
+ Iterator<AddPartitionDesc> iterator = event.partitionDescriptions(tableDesc).iterator();
+ while (iterator.hasNext() && tracker.canAddMoreTasks()) {
+ AddPartitionDesc addPartitionDesc = iterator.next();
+ tracker.addTask(addSinglePartition(table, addPartitionDesc));
+ if (iterator.hasNext() && !tracker.canAddMoreTasks()) {
+ ReplicationState currentReplicationState =
+ new ReplicationState(new PartitionState(table.getTableName(), addPartitionDesc));
+ updateReplicationState(currentReplicationState);
+ }
+ }
+ return tracker;
+ }
+
+ /**
+ * returns the root task for adding a partition
+ */
+ private Task<? extends Serializable> addSinglePartition(Table table,
+ AddPartitionDesc addPartitionDesc) throws MetaException, IOException, HiveException {
+ AddPartitionDesc.OnePartitionDesc partSpec = addPartitionDesc.getPartition(0);
+ Path sourceWarehousePartitionLocation = new Path(partSpec.getLocation());
+ Path replicaWarehousePartitionLocation = locationOnReplicaWarehouse(table, partSpec);
+ partSpec.setLocation(replicaWarehousePartitionLocation.toString());
+ LOG.debug("adding dependent CopyWork/AddPart/MoveWork for partition "
+ + partSpecToString(partSpec.getPartSpec()) + " with source location: "
+ + partSpec.getLocation());
+ Path tmpPath = context.utils.getExternalTmpPath(replicaWarehousePartitionLocation);
+
+ Task<?> copyTask = ReplCopyTask.getLoadCopyTask(
+ event.replicationSpec(),
+ sourceWarehousePartitionLocation,
+ tmpPath,
+ context.hiveConf
+ );
+
+ Task<?> addPartTask = TaskFactory.get(
+ new DDLWork(new HashSet<>(), new HashSet<>(), addPartitionDesc),
+ context.hiveConf
+ );
+
+ Task<?> movePartitionTask = movePartitionTask(table, partSpec, tmpPath);
+
+ copyTask.addDependentTask(addPartTask);
+ addPartTask.addDependentTask(movePartitionTask);
+ return copyTask;
+ }
+
+ /**
+ * This will create the move of partition data from temp path to actual path
+ */
+ private Task<?> movePartitionTask(Table table, AddPartitionDesc.OnePartitionDesc partSpec,
+ Path tmpPath) {
+ LoadTableDesc loadTableWork = new LoadTableDesc(
+ tmpPath, Utilities.getTableDesc(table), partSpec.getPartSpec(),
- event.replicationSpec().isReplace()
++ event.replicationSpec().isReplace(), SessionState.get().getTxnMgr().getCurrentTxnId()
+ );
+ loadTableWork.setInheritTableSpecs(false);
+ MoveWork work = new MoveWork(new HashSet<>(), new HashSet<>(), loadTableWork, null, false);
+ return TaskFactory.get(work, context.hiveConf);
+ }
+
+ private Path locationOnReplicaWarehouse(Table table, AddPartitionDesc.OnePartitionDesc partSpec)
+ throws MetaException, HiveException, IOException {
+ String child = Warehouse.makePartPath(partSpec.getPartSpec());
+ if (tableDesc.getLocation() == null) {
+ if (table.getDataLocation() == null) {
+ Database parentDb = context.hiveDb.getDatabase(tableDesc.getDatabaseName());
+ return new Path(
+ context.warehouse.getDefaultTablePath(parentDb, tableDesc.getTableName()), child);
+ } else {
+ return new Path(table.getDataLocation().toString(), child);
+ }
+ } else {
+ return new Path(tableDesc.getLocation(), child);
+ }
+ }
+
+ private Task<? extends Serializable> alterSinglePartition(AddPartitionDesc desc,
+ ReplicationSpec replicationSpec, Partition ptn) {
+ desc.setReplaceMode(true);
+ if ((replicationSpec != null) && (replicationSpec.isInReplicationScope())) {
+ desc.setReplicationSpec(replicationSpec);
+ }
+ desc.getPartition(0).setLocation(ptn.getLocation()); // use existing location
+ return TaskFactory.get(
+ new DDLWork(new HashSet<>(), new HashSet<>(), desc),
+ context.hiveConf
+ );
+ }
+
+ private TaskTracker forExistingTable(AddPartitionDesc lastPartitionReplicated) throws Exception {
+ boolean encounteredTheLastReplicatedPartition = (lastPartitionReplicated == null);
+ ReplicationSpec replicationSpec = event.replicationSpec();
+ LOG.debug("table partitioned");
+
+ Iterator<AddPartitionDesc> iterator = event.partitionDescriptions(tableDesc).iterator();
+ while (iterator.hasNext()) {
+ /*
+ encounteredTheLastReplicatedPartition will be set, when we break creation of partition tasks
+ for a table, as we have reached the limit of number of tasks we should create for execution.
+ in this case on the next run we have to iterate over the partitions desc to reach the last replicated
+ partition so that we can start replicating partitions after that.
+ */
+ AddPartitionDesc addPartitionDesc = iterator.next();
+ if (encounteredTheLastReplicatedPartition && tracker.canAddMoreTasks()) {
+ Map<String, String> partSpec = addPartitionDesc.getPartition(0).getPartSpec();
+ Partition ptn;
+
+ if ((ptn = context.hiveDb.getPartition(table, partSpec, false)) == null) {
+ if (!replicationSpec.isMetadataOnly()) {
+ forNewTable();
+ }
+ } else {
+ // If replicating, then the partition already existing means we need to replace, maybe, if
+ // the destination ptn's repl.last.id is older than the replacement's.
+ if (replicationSpec.allowReplacementInto(ptn.getParameters())) {
+ if (replicationSpec.isMetadataOnly()) {
+ tracker.addTask(alterSinglePartition(addPartitionDesc, replicationSpec, ptn));
+ if (iterator.hasNext() && !tracker.canAddMoreTasks()) {
+ tracker.setReplicationState(
+ new ReplicationState(new PartitionState(table.getTableName(), addPartitionDesc)
+ )
+ );
+ }
+ } else {
+ forNewTable();
+ }
+ } else {
+ // ignore this ptn, do nothing, not an error.
+ }
+ }
+ } else {
+ Map<String, String> currentSpec = addPartitionDesc.getPartition(0).getPartSpec();
+ Map<String, String> lastReplicatedPartSpec =
+ lastPartitionReplicated.getPartition(0).getPartSpec();
+ encounteredTheLastReplicatedPartition = lastReplicatedPartSpec.equals(currentSpec);
+ }
+ }
+ return tracker;
+ }
+ }
+
[24/50] [abbrv] hive git commit: HIVE-17100 : Improve HS2 operation
logs for REPL commands (Sankar Hariappan, reviewed by Anishek Agarwal,
Thejas Nair)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/ReplState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/ReplState.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/ReplState.java
new file mode 100644
index 0000000..a003d37
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/ReplState.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.SerializationConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class ReplState {
+ @JsonIgnoreProperties
+ private static final Logger REPL_LOG = LoggerFactory.getLogger("ReplState");
+
+ @JsonIgnoreProperties
+ private static final ObjectMapper mapper = new ObjectMapper(); // Thread-safe.
+
+ static {
+ mapper.configure(SerializationConfig.Feature.AUTO_DETECT_GETTERS, false);
+ mapper.configure(SerializationConfig.Feature.AUTO_DETECT_IS_GETTERS, false);
+ mapper.configure(SerializationConfig.Feature.AUTO_DETECT_FIELDS, false);
+ }
+
+ public enum LogTag {
+ START,
+ TABLE_DUMP,
+ FUNCTION_DUMP,
+ EVENT_DUMP,
+ TABLE_LOAD,
+ FUNCTION_LOAD,
+ EVENT_LOAD,
+ END
+ }
+
+ public void log(LogTag tag) {
+ try {
+ REPL_LOG.info("REPL::{}: {}", tag.name(), mapper.writeValueAsString(this));
+ } catch (Exception exception) {
+ REPL_LOG.error("Could not serialize REPL log: {}", exception.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java
index f40c703..a48a17e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java
@@ -31,6 +31,7 @@ import com.google.common.collect.Collections2;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.util.Collection;
import java.util.List;
public class Utils {
@@ -64,14 +65,18 @@ public class Utils {
public static Iterable<? extends String> matchesTbl(Hive db, String dbName, String tblPattern)
throws HiveException {
if (tblPattern == null) {
- return Collections2.filter(db.getAllTables(dbName),
- tableName -> {
- assert tableName != null;
- return !tableName.toLowerCase().startsWith(
- SemanticAnalyzer.VALUES_TMP_TABLE_NAME_PREFIX.toLowerCase());
- });
+ return getAllTables(db, dbName);
} else {
return db.getTablesByPattern(dbName, tblPattern);
}
}
+
+ public static Collection<String> getAllTables(Hive db, String dbName) throws HiveException {
+ return Collections2.filter(db.getAllTables(dbName),
+ tableName -> {
+ assert tableName != null;
+ return !tableName.toLowerCase().startsWith(
+ SemanticAnalyzer.VALUES_TMP_TABLE_NAME_PREFIX.toLowerCase());
+ });
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/BootstrapDumpLogger.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/BootstrapDumpLogger.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/BootstrapDumpLogger.java
new file mode 100644
index 0000000..1b0ec59
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/BootstrapDumpLogger.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.log;
+
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.ql.parse.repl.dump.log.state.BootstrapDumpBegin;
+import org.apache.hadoop.hive.ql.parse.repl.dump.log.state.BootstrapDumpEnd;
+import org.apache.hadoop.hive.ql.parse.repl.dump.log.state.BootstrapDumpFunction;
+import org.apache.hadoop.hive.ql.parse.repl.dump.log.state.BootstrapDumpTable;
+import org.apache.hadoop.hive.ql.parse.repl.ReplLogger;
+import org.apache.hadoop.hive.ql.parse.repl.ReplState.LogTag;
+
+public class BootstrapDumpLogger extends ReplLogger {
+ private String dbName;
+ private String dumpDir;
+ private long estimatedNumTables;
+ private long estimatedNumFunctions;
+ private long tableSeqNo;
+ private long functionSeqNo;
+
+ public BootstrapDumpLogger(String dbName, String dumpDir,
+ int estimatedNumTables, int estimatedNumFunctions) {
+ this.dbName = dbName;
+ this.dumpDir = dumpDir;
+ this.estimatedNumTables = estimatedNumTables;
+ this.estimatedNumFunctions = estimatedNumFunctions;
+ this.tableSeqNo = 0;
+ this.functionSeqNo = 0;
+ }
+
+ @Override
+ public void startLog() {
+ (new BootstrapDumpBegin(dbName, estimatedNumTables, estimatedNumFunctions))
+ .log(LogTag.START);
+ }
+
+ @Override
+ public void tableLog(String tableName, TableType tableType) {
+ tableSeqNo++;
+ (new BootstrapDumpTable(dbName, tableName, tableType, tableSeqNo, estimatedNumTables))
+ .log(LogTag.TABLE_DUMP);
+ }
+
+ @Override
+ public void functionLog(String funcName) {
+ functionSeqNo++;
+ (new BootstrapDumpFunction(dbName, funcName, functionSeqNo, estimatedNumFunctions))
+ .log(LogTag.FUNCTION_DUMP);
+ }
+
+ @Override
+ public void endLog(String lastReplId) {
+ (new BootstrapDumpEnd(dbName, tableSeqNo, functionSeqNo, dumpDir, lastReplId))
+ .log(LogTag.END);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/IncrementalDumpLogger.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/IncrementalDumpLogger.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/IncrementalDumpLogger.java
new file mode 100644
index 0000000..03144df
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/IncrementalDumpLogger.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.log;
+
+import org.apache.hadoop.hive.ql.parse.repl.dump.log.state.IncrementalDumpBegin;
+import org.apache.hadoop.hive.ql.parse.repl.dump.log.state.IncrementalDumpEnd;
+import org.apache.hadoop.hive.ql.parse.repl.dump.log.state.IncrementalDumpEvent;
+import org.apache.hadoop.hive.ql.parse.repl.ReplLogger;
+import org.apache.hadoop.hive.ql.parse.repl.ReplState.LogTag;
+
+public class IncrementalDumpLogger extends ReplLogger {
+ private String dbName;
+ private String dumpDir;
+ private long estimatedNumEvents;
+ private long eventSeqNo;
+
+ public IncrementalDumpLogger(String dbName, String dumpDir, long estimatedNumEvents) {
+ this.dbName = dbName;
+ this.dumpDir = dumpDir;
+ this.estimatedNumEvents = estimatedNumEvents;
+ this.eventSeqNo = 0;
+ }
+
+ @Override
+ public void startLog() {
+ (new IncrementalDumpBegin(dbName, estimatedNumEvents)).log(LogTag.START);
+ }
+
+ @Override
+ public void eventLog(String eventId, String eventType) {
+ eventSeqNo++;
+ (new IncrementalDumpEvent(dbName, eventId, eventType, eventSeqNo, estimatedNumEvents))
+ .log(LogTag.EVENT_DUMP);
+ }
+
+ @Override
+ public void endLog(String lastReplId) {
+ (new IncrementalDumpEnd(dbName, eventSeqNo, dumpDir, lastReplId)).log(LogTag.END);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpBegin.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpBegin.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpBegin.java
new file mode 100644
index 0000000..3c4ebd3
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpBegin.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.log.state;
+
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class BootstrapDumpBegin extends ReplState {
+ @JsonProperty
+ private String dbName;
+
+ @JsonProperty
+ private DumpType dumpType;
+
+ @JsonProperty
+ private Long estimatedNumTables;
+
+ @JsonProperty
+ private Long estimatedNumFunctions;
+
+ @JsonProperty
+ private Long dumpStartTime;
+
+ public BootstrapDumpBegin(String dbName, long estimatedNumTables, long estimatedNumFunctions) {
+ this.dbName = dbName;
+ this.dumpType = DumpType.BOOTSTRAP;
+ this.estimatedNumTables = estimatedNumTables;
+ this.estimatedNumFunctions = estimatedNumFunctions;
+ this.dumpStartTime = System.currentTimeMillis() / 1000;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpEnd.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpEnd.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpEnd.java
new file mode 100644
index 0000000..a4dcfb5
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpEnd.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.log.state;
+
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class BootstrapDumpEnd extends ReplState {
+ @JsonProperty
+ private String dbName;
+
+ @JsonProperty
+ private DumpType dumpType;
+
+ @JsonProperty
+ private Long actualNumTables;
+
+ @JsonProperty
+ private Long actualNumFunctions;
+
+ @JsonProperty
+ private Long dumpEndTime;
+
+ @JsonProperty
+ private String dumpDir;
+
+ @JsonProperty
+ private String lastReplId;
+
+ public BootstrapDumpEnd(String dbName,
+ long actualNumTables,
+ long actualNumFunctions,
+ String dumpDir,
+ String lastReplId) {
+ this.dbName = dbName;
+ this.dumpType = DumpType.BOOTSTRAP;
+ this.actualNumTables = actualNumTables;
+ this.actualNumFunctions = actualNumFunctions;
+ this.dumpEndTime = System.currentTimeMillis() / 1000;
+ this.dumpDir = dumpDir;
+ this.lastReplId = lastReplId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpFunction.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpFunction.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpFunction.java
new file mode 100644
index 0000000..a414daa
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpFunction.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.log.state;
+
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class BootstrapDumpFunction extends ReplState {
+ @JsonProperty
+ private String dbName;
+
+ @JsonProperty
+ private String functionName;
+
+ @JsonProperty
+ private String functionsDumpProgress;
+
+ @JsonProperty
+ private Long dumpTime;
+
+ public BootstrapDumpFunction(String dbName, String funcName,
+ long functionSeqNo, long estimatedNumFunctions) {
+ this.dbName = dbName;
+ this.functionName = funcName;
+ this.functionsDumpProgress = new String(new StringBuilder()
+ .append(functionSeqNo).append("/").append(estimatedNumFunctions));
+ this.dumpTime = System.currentTimeMillis() / 1000;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpTable.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpTable.java
new file mode 100644
index 0000000..4bf4905
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpTable.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.log.state;
+
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class BootstrapDumpTable extends ReplState {
+ @JsonProperty
+ private String dbName;
+
+ @JsonProperty
+ private String tableName;
+
+ @JsonProperty
+ private String tableType;
+
+ @JsonProperty
+ private String tablesDumpProgress;
+
+ @JsonProperty
+ private Long dumpTime;
+
+ public BootstrapDumpTable(String dbName,
+ String tableName,
+ TableType tableType,
+ long tableSeqNo,
+ long estimatedNumTables) {
+ this.dbName = dbName;
+ this.tableName = tableName;
+ this.tableType = tableType.name();
+ this.tablesDumpProgress = new String(new StringBuilder()
+ .append(tableSeqNo).append("/").append(estimatedNumTables));
+ this.dumpTime = System.currentTimeMillis() / 1000;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpBegin.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpBegin.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpBegin.java
new file mode 100644
index 0000000..166d359
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpBegin.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.log.state;
+
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class IncrementalDumpBegin extends ReplState {
+ @JsonProperty
+ private String dbName;
+
+ @JsonProperty
+ private DumpType dumpType;
+
+ @JsonProperty
+ private Long estimatedNumEvents;
+
+ @JsonProperty
+ private Long dumpStartTime;
+
+ public IncrementalDumpBegin(String dbName, long estimatedNumEvents) {
+ this.dbName = dbName;
+ this.dumpType = DumpType.INCREMENTAL;
+ this.estimatedNumEvents = estimatedNumEvents;
+ this.dumpStartTime = System.currentTimeMillis() / 1000;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpEnd.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpEnd.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpEnd.java
new file mode 100644
index 0000000..a7cec21
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpEnd.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.log.state;
+
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class IncrementalDumpEnd extends ReplState {
+ @JsonProperty
+ private String dbName;
+
+ @JsonProperty
+ private DumpType dumpType;
+
+ @JsonProperty
+ private Long actualNumEvents;
+
+ @JsonProperty
+ private Long dumpEndTime;
+
+ @JsonProperty
+ private String dumpDir;
+
+ @JsonProperty
+ private String lastReplId;
+
+ public IncrementalDumpEnd(String dbName,
+ long actualNumEvents,
+ String dumpDir,
+ String lastReplId) {
+ this.dbName = dbName;
+ this.dumpType = DumpType.INCREMENTAL;
+ this.actualNumEvents = actualNumEvents;
+ this.dumpEndTime = System.currentTimeMillis() / 1000;
+ this.dumpDir = dumpDir;
+ this.lastReplId = lastReplId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpEvent.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpEvent.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpEvent.java
new file mode 100644
index 0000000..ebeb468
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpEvent.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.log.state;
+
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class IncrementalDumpEvent extends ReplState {
+ @JsonProperty
+ private String dbName;
+
+ @JsonProperty
+ private String eventId;
+
+ @JsonProperty
+ private String eventType;
+
+ @JsonProperty
+ private String eventsDumpProgress;
+
+ @JsonProperty
+ private Long dumpTime;
+
+ public IncrementalDumpEvent(String dbName,
+ String eventId,
+ String eventType,
+ long eventSeqNo,
+ long estimatedNumEvents) {
+ this.dbName = dbName;
+ this.eventId = eventId;
+ this.eventType = eventType;
+ this.eventsDumpProgress = new String(new StringBuilder()
+ .append(eventSeqNo).append("/").append(estimatedNumEvents));
+ this.dumpTime = System.currentTimeMillis() / 1000;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/BootstrapLoadLogger.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/BootstrapLoadLogger.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/BootstrapLoadLogger.java
new file mode 100644
index 0000000..739bd12
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/BootstrapLoadLogger.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load.log;
+
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.ql.parse.repl.load.log.state.*;
+import org.apache.hadoop.hive.ql.parse.repl.ReplLogger;
+import org.apache.hadoop.hive.ql.parse.repl.ReplState.LogTag;
+
+public class BootstrapLoadLogger extends ReplLogger {
+ private String dbName;
+ private String dumpDir;
+ private long numTables;
+ private long numFunctions;
+ private long tableSeqNo;
+ private long functionSeqNo;
+
+ public BootstrapLoadLogger(String dbName, String dumpDir, long numTables, long numFunctions) {
+ this.dbName = dbName;
+ this.dumpDir = dumpDir;
+ this.numTables = numTables;
+ this.numFunctions = numFunctions;
+ this.tableSeqNo = 0;
+ this.functionSeqNo = 0;
+ }
+
+ @Override
+ public void startLog() {
+ (new BootstrapLoadBegin(dbName, dumpDir, numTables, numFunctions)).log(LogTag.START);
+ }
+
+ @Override
+ public void tableLog(String tableName, TableType tableType) {
+ tableSeqNo++;
+ (new BootstrapLoadTable(dbName, tableName, tableType, tableSeqNo, numTables))
+ .log(LogTag.TABLE_LOAD);
+ }
+
+ @Override
+ public void functionLog(String funcName) {
+ functionSeqNo++;
+ (new BootstrapLoadFunction(dbName, funcName, functionSeqNo, numFunctions))
+ .log(LogTag.FUNCTION_LOAD);
+ }
+
+ @Override
+ public void endLog(String lastReplId) {
+ (new BootstrapLoadEnd(dbName, numTables, numFunctions, dumpDir, lastReplId))
+ .log(LogTag.END);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/IncrementalLoadLogger.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/IncrementalLoadLogger.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/IncrementalLoadLogger.java
new file mode 100644
index 0000000..d2114f0
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/IncrementalLoadLogger.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load.log;
+
+import org.apache.hadoop.hive.ql.parse.repl.load.log.state.IncrementalLoadBegin;
+import org.apache.hadoop.hive.ql.parse.repl.load.log.state.IncrementalLoadEnd;
+import org.apache.hadoop.hive.ql.parse.repl.load.log.state.IncrementalLoadEvent;
+import org.apache.hadoop.hive.ql.parse.repl.ReplLogger;
+import org.apache.hadoop.hive.ql.parse.repl.ReplState.LogTag;
+
+public class IncrementalLoadLogger extends ReplLogger {
+ private String dbName;
+ private String dumpDir;
+ private long numEvents;
+ private long eventSeqNo;
+
+ public IncrementalLoadLogger(String dbName, String dumpDir, int numEvents) {
+ this.dbName = dbName;
+ this.dumpDir = dumpDir;
+ this.numEvents = numEvents;
+ this.eventSeqNo = 0;
+ }
+
+ @Override
+ public void startLog() {
+ (new IncrementalLoadBegin(dbName, dumpDir, numEvents)).log(LogTag.START);
+ }
+
+ @Override
+ public void eventLog(String eventId, String eventType) {
+ eventSeqNo++;
+ (new IncrementalLoadEvent(dbName, eventId, eventType, eventSeqNo, numEvents))
+ .log(LogTag.EVENT_LOAD);
+ }
+
+ @Override
+ public void endLog(String lastReplId) {
+ (new IncrementalLoadEnd(dbName, numEvents, dumpDir, lastReplId)).log(LogTag.END);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadBegin.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadBegin.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadBegin.java
new file mode 100644
index 0000000..5574454
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadBegin.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load.log.state;
+
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class BootstrapLoadBegin extends ReplState {
+ @JsonProperty
+ private String dbName;
+
+ @JsonProperty
+ private String dumpDir;
+
+ @JsonProperty
+ private DumpType loadType;
+
+ @JsonProperty
+ private Long numTables;
+
+ @JsonProperty
+ private Long numFunctions;
+
+ @JsonProperty
+ private Long loadStartTime;
+
+ public BootstrapLoadBegin(String dbName, String dumpDir, long numTables, long numFunctions) {
+ this.dbName = dbName;
+ this.dumpDir = dumpDir;
+ this.loadType = DumpType.BOOTSTRAP;
+ this.numTables = numTables;
+ this.numFunctions = numFunctions;
+ this.loadStartTime = System.currentTimeMillis() / 1000;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadEnd.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadEnd.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadEnd.java
new file mode 100644
index 0000000..3db8144
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadEnd.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load.log.state;
+
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class BootstrapLoadEnd extends ReplState {
+ @JsonProperty
+ private String dbName;
+
+ @JsonProperty
+ private DumpType loadType;
+
+ @JsonProperty
+ private Long numTables;
+
+ @JsonProperty
+ private Long numFunctions;
+
+ @JsonProperty
+ private Long loadEndTime;
+
+ @JsonProperty
+ private String dumpDir;
+
+ @JsonProperty
+ private String lastReplId;
+
+ public BootstrapLoadEnd(String dbName,
+ long numTables,
+ long numFunctions,
+ String dumpDir,
+ String lastReplId) {
+ this.dbName = dbName;
+ this.loadType = DumpType.BOOTSTRAP;
+ this.numTables = numTables;
+ this.numFunctions = numFunctions;
+ this.loadEndTime = System.currentTimeMillis() / 1000;
+ this.dumpDir = dumpDir;
+ this.lastReplId = lastReplId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadFunction.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadFunction.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadFunction.java
new file mode 100644
index 0000000..e2434ce
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadFunction.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load.log.state;
+
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class BootstrapLoadFunction extends ReplState {
+ @JsonProperty
+ private String dbName;
+
+ @JsonProperty
+ private String functionName;
+
+ @JsonProperty
+ private String functionsLoadProgress;
+
+ @JsonProperty
+ private Long loadTime;
+
+ public BootstrapLoadFunction(String dbName, String funcName,
+ long functionSeqNo, long numFunctions) {
+ this.dbName = dbName;
+ this.functionName = funcName;
+ this.functionsLoadProgress = new String(new StringBuilder()
+ .append(functionSeqNo).append("/").append(numFunctions));
+ this.loadTime = System.currentTimeMillis() / 1000;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadTable.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadTable.java
new file mode 100644
index 0000000..63ff467
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadTable.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load.log.state;
+
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class BootstrapLoadTable extends ReplState {
+ @JsonProperty
+ private String dbName;
+
+ @JsonProperty
+ private String tableName;
+
+ @JsonProperty
+ private String tableType;
+
+ @JsonProperty
+ private String tablesLoadProgress;
+
+ @JsonProperty
+ private Long loadTime;
+
+ public BootstrapLoadTable(String dbName,
+ String tableName,
+ TableType tableType,
+ long tableSeqNo,
+ long numTables) {
+ this.dbName = dbName;
+ this.tableName = tableName;
+ this.tableType = tableType.name();
+ this.tablesLoadProgress = new String(new StringBuilder()
+ .append(tableSeqNo).append("/").append(numTables));
+ this.loadTime = System.currentTimeMillis() / 1000;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/IncrementalLoadBegin.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/IncrementalLoadBegin.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/IncrementalLoadBegin.java
new file mode 100644
index 0000000..93f1d7b
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/IncrementalLoadBegin.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load.log.state;
+
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class IncrementalLoadBegin extends ReplState {
+ @JsonProperty
+ private String dbName;
+
+ @JsonProperty
+ private String dumpDir;
+
+ @JsonProperty
+ private DumpType loadType;
+
+ @JsonProperty
+ private Long numEvents;
+
+ @JsonProperty
+ private Long loadStartTime;
+
+ public IncrementalLoadBegin(String dbName, String dumpDir, long numEvents) {
+ this.dbName = dbName;
+ this.dumpDir = dumpDir;
+ this.loadType = DumpType.INCREMENTAL;
+ this.numEvents = numEvents;
+ this.loadStartTime = System.currentTimeMillis() / 1000;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/IncrementalLoadEnd.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/IncrementalLoadEnd.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/IncrementalLoadEnd.java
new file mode 100644
index 0000000..f633c72
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/IncrementalLoadEnd.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load.log.state;
+
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class IncrementalLoadEnd extends ReplState {
+ @JsonProperty
+ private String dbName;
+
+ @JsonProperty
+ private DumpType loadType;
+
+ @JsonProperty
+ private Long numEvents;
+
+ @JsonProperty
+ private Long loadEndTime;
+
+ @JsonProperty
+ private String dumpDir;
+
+ @JsonProperty
+ private String lastReplId;
+
+ public IncrementalLoadEnd(String dbName,
+ long numEvents,
+ String dumpDir,
+ String lastReplId) {
+ this.dbName = dbName;
+ this.loadType = DumpType.INCREMENTAL;
+ this.numEvents = numEvents;
+ this.loadEndTime = System.currentTimeMillis() / 1000;
+ this.dumpDir = dumpDir;
+ this.lastReplId = lastReplId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/IncrementalLoadEvent.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/IncrementalLoadEvent.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/IncrementalLoadEvent.java
new file mode 100644
index 0000000..224aeb2
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/IncrementalLoadEvent.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load.log.state;
+
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class IncrementalLoadEvent extends ReplState {
+ @JsonProperty
+ private String dbName;
+
+ @JsonProperty
+ private String eventId;
+
+ @JsonProperty
+ private String eventType;
+
+ @JsonProperty
+ private String eventsLoadProgress;
+
+ @JsonProperty
+ private Long loadTime;
+
+ public IncrementalLoadEvent(String dbName,
+ String eventId,
+ String eventType,
+ long eventSeqNo,
+ long numEvents) {
+ this.dbName = dbName;
+ this.eventId = eventId;
+ this.eventType = eventType;
+ this.eventsLoadProgress = new String(new StringBuilder()
+ .append(eventSeqNo).append("/").append(numEvents));
+ this.loadTime = System.currentTimeMillis() / 1000;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java
index 3f176aa..caf6f3f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java
@@ -49,12 +49,20 @@ import java.util.List;
import static org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.toReadEntity;
public class CreateFunctionHandler extends AbstractMessageHandler {
+ private String functionName;
+
+ public String getFunctionName() {
+ return functionName;
+ }
+
@Override
public List<Task<? extends Serializable>> handle(Context context)
throws SemanticException {
try {
FunctionDescBuilder builder = new FunctionDescBuilder(context);
CreateFunctionDesc descToLoad = builder.build();
+ this.functionName = builder.metadata.function.getFunctionName();
+
context.log.debug("Loading function desc : {}", descToLoad.toString());
Task<FunctionWork> createTask = TaskFactory.get(
new FunctionWork(descToLoad), context.hiveConf
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/MessageHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/MessageHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/MessageHandler.java
index 8daff6d..3ccd639 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/MessageHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/MessageHandler.java
@@ -45,9 +45,10 @@ public interface MessageHandler {
UpdatedMetaDataTracker getUpdatedMetadata();
class Context {
- final String dbName, tableName, location;
- final Task<? extends Serializable> precursor;
- DumpMetaData dmd;
+ public String dbName;
+ public final String tableName, location;
+ public final Task<? extends Serializable> precursor;
+ public DumpMetaData dmd;
final HiveConf hiveConf;
final Hive db;
final org.apache.hadoop.hive.ql.Context nestedContext;
@@ -83,7 +84,7 @@ public interface MessageHandler {
return StringUtils.isEmpty(tableName);
}
- boolean isDbNameEmpty() {
+ public boolean isDbNameEmpty() {
return StringUtils.isEmpty(dbName);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java
index a4fb378..1770046 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.ql.exec.Task;
@@ -34,11 +35,8 @@ import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
-import org.apache.hadoop.hive.ql.parse.EximUtil;
import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
-import org.apache.hadoop.hive.ql.plan.CreateViewDesc;
/**
* ImportTableDesc.
@@ -56,7 +54,7 @@ public class ImportTableDesc {
this.dbName = dbName;
this.table = table;
- switch (getTableType()) {
+ switch (getDescType()) {
case TABLE:
this.createTblDesc = new CreateTableDesc(dbName,
table.getTableName(),
@@ -122,7 +120,7 @@ public class ImportTableDesc {
}
}
- public TYPE getTableType() {
+ public TYPE getDescType() {
if (table.isView() || table.isMaterializedView()) {
return TYPE.VIEW;
}
@@ -143,7 +141,7 @@ public class ImportTableDesc {
}
public void setReplicationSpec(ReplicationSpec replSpec) {
- switch (getTableType()) {
+ switch (getDescType()) {
case TABLE:
createTblDesc.setReplicationSpec(replSpec);
break;
@@ -154,20 +152,20 @@ public class ImportTableDesc {
}
public void setExternal(boolean isExternal) {
- if (TYPE.TABLE.equals(getTableType())) {
+ if (TYPE.TABLE.equals(getDescType())) {
createTblDesc.setExternal(isExternal);
}
}
public boolean isExternal() {
- if (TYPE.TABLE.equals(getTableType())) {
+ if (TYPE.TABLE.equals(getDescType())) {
return createTblDesc.isExternal();
}
return false;
}
public void setLocation(String location) {
- switch (getTableType()) {
+ switch (getDescType()) {
case TABLE:
createTblDesc.setLocation(location);
break;
@@ -178,7 +176,7 @@ public class ImportTableDesc {
}
public String getLocation() {
- switch (getTableType()) {
+ switch (getDescType()) {
case TABLE:
return createTblDesc.getLocation();
case VIEW:
@@ -188,7 +186,7 @@ public class ImportTableDesc {
}
public void setTableName(String tableName) throws SemanticException {
- switch (getTableType()) {
+ switch (getDescType()) {
case TABLE:
createTblDesc.setTableName(tableName);
break;
@@ -201,7 +199,7 @@ public class ImportTableDesc {
}
public String getTableName() throws SemanticException {
- switch (getTableType()) {
+ switch (getDescType()) {
case TABLE:
return createTblDesc.getTableName();
case VIEW:
@@ -213,7 +211,7 @@ public class ImportTableDesc {
}
public List<FieldSchema> getPartCols() {
- switch (getTableType()) {
+ switch (getDescType()) {
case TABLE:
return createTblDesc.getPartCols();
case VIEW:
@@ -223,7 +221,7 @@ public class ImportTableDesc {
}
public List<FieldSchema> getCols() {
- switch (getTableType()) {
+ switch (getDescType()) {
case TABLE:
return createTblDesc.getCols();
case VIEW:
@@ -233,7 +231,7 @@ public class ImportTableDesc {
}
public Map<String, String> getTblProps() {
- switch (getTableType()) {
+ switch (getDescType()) {
case TABLE:
return createTblDesc.getTblProps();
case VIEW:
@@ -243,7 +241,7 @@ public class ImportTableDesc {
}
public String getInputFormat() {
- switch (getTableType()) {
+ switch (getDescType()) {
case TABLE:
return createTblDesc.getInputFormat();
case VIEW:
@@ -253,7 +251,7 @@ public class ImportTableDesc {
}
public String getOutputFormat() {
- switch (getTableType()) {
+ switch (getDescType()) {
case TABLE:
return createTblDesc.getOutputFormat();
case VIEW:
@@ -263,7 +261,7 @@ public class ImportTableDesc {
}
public String getSerName() {
- switch (getTableType()) {
+ switch (getDescType()) {
case TABLE:
return createTblDesc.getSerName();
case VIEW:
@@ -273,7 +271,7 @@ public class ImportTableDesc {
}
public Map<String, String> getSerdeProps() {
- switch (getTableType()) {
+ switch (getDescType()) {
case TABLE:
return createTblDesc.getSerdeProps();
case VIEW:
@@ -283,14 +281,14 @@ public class ImportTableDesc {
}
public List<String> getBucketCols() {
- if (TYPE.TABLE.equals(getTableType())) {
+ if (TYPE.TABLE.equals(getDescType())) {
return createTblDesc.getBucketCols();
}
return null;
}
public List<Order> getSortCols() {
- if (TYPE.TABLE.equals(getTableType())) {
+ if (TYPE.TABLE.equals(getDescType())) {
return createTblDesc.getSortCols();
}
return null;
@@ -300,7 +298,7 @@ public class ImportTableDesc {
* @param replaceMode Determine if this CreateTable should behave like a replace-into alter instead
*/
public void setReplaceMode(boolean replaceMode) {
- switch (getTableType()) {
+ switch (getDescType()) {
case TABLE:
createTblDesc.setReplaceMode(replaceMode);
break;
@@ -315,7 +313,7 @@ public class ImportTableDesc {
public Task<? extends Serializable> getCreateTableTask(HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs,
HiveConf conf) {
- switch (getTableType()) {
+ switch (getDescType()) {
case TABLE:
return TaskFactory.get(new DDLWork(inputs, outputs, createTblDesc), conf);
case VIEW:
@@ -332,4 +330,13 @@ public class ImportTableDesc {
public boolean isMaterializedView() {
return table.isMaterializedView();
}
+
+ public TableType tableType() {
+ if (isView()) {
+ return TableType.VIRTUAL_VIEW;
+ } else if (isMaterializedView()) {
+ return TableType.MATERIALIZED_VIEW;
+ }
+ return TableType.MANAGED_TABLE;
+ }
}
[07/50] [abbrv] hive git commit: HIVE-17205 - add functional support
for unbucketed tables (Eugene Koifman, reviewed by Wei Zheng)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
index d61b24b..37aaeb6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
@@ -50,6 +50,8 @@ public class OrcSplit extends FileSplit implements ColumnarSplit, LlapAwareSplit
private boolean hasFooter;
private boolean isOriginal;
private boolean hasBase;
+ //partition root
+ private Path rootDir;
private final List<AcidInputFormat.DeltaMetaData> deltas = new ArrayList<>();
private long projColsUncompressedSize;
private transient Object fileKey;
@@ -70,7 +72,7 @@ public class OrcSplit extends FileSplit implements ColumnarSplit, LlapAwareSplit
public OrcSplit(Path path, Object fileId, long offset, long length, String[] hosts,
OrcTail orcTail, boolean isOriginal, boolean hasBase,
- List<AcidInputFormat.DeltaMetaData> deltas, long projectedDataSize, long fileLen) {
+ List<AcidInputFormat.DeltaMetaData> deltas, long projectedDataSize, long fileLen, Path rootDir) {
super(path, offset, length, hosts);
// For HDFS, we could avoid serializing file ID and just replace the path with inode-based
// path. However, that breaks bunch of stuff because Hive later looks up things by split path.
@@ -79,6 +81,7 @@ public class OrcSplit extends FileSplit implements ColumnarSplit, LlapAwareSplit
hasFooter = this.orcTail != null;
this.isOriginal = isOriginal;
this.hasBase = hasBase;
+ this.rootDir = rootDir;
this.deltas.addAll(deltas);
this.projColsUncompressedSize = projectedDataSize <= 0 ? length : projectedDataSize;
// setting file length to Long.MAX_VALUE will let orc reader read file length from file system
@@ -129,6 +132,7 @@ public class OrcSplit extends FileSplit implements ColumnarSplit, LlapAwareSplit
((Writable)fileKey).write(out);
}
out.writeLong(fileLen);
+ out.writeUTF(rootDir.toString());
}
@Override
@@ -168,6 +172,7 @@ public class OrcSplit extends FileSplit implements ColumnarSplit, LlapAwareSplit
this.fileKey = fileId;
}
fileLen = in.readLong();
+ rootDir = new Path(in.readUTF());
}
public OrcTail getOrcTail() {
@@ -186,6 +191,9 @@ public class OrcSplit extends FileSplit implements ColumnarSplit, LlapAwareSplit
return hasBase;
}
+ public Path getRootDir() {
+ return rootDir;
+ }
public List<AcidInputFormat.DeltaMetaData> getDeltas() {
return deltas;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
index 8f80710..138e56e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.hive.common.ValidReadTxnList;
import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector;
@@ -357,7 +358,7 @@ public class VectorizedOrcAcidRowBatchReader
int bucket = AcidUtils.parseBaseOrDeltaBucketFilename(orcSplit.getPath(), conf).getBucketId();
String txnString = conf.get(ValidTxnList.VALID_TXNS_KEY);
this.validTxnList = (txnString == null) ? new ValidReadTxnList() : new ValidReadTxnList(txnString);
- OrcRawRecordMerger.Options mergerOptions = new OrcRawRecordMerger.Options().isCompacting(false);
+ OrcRawRecordMerger.Options mergerOptions = new OrcRawRecordMerger.Options().isDeleteReader(true);
assert !orcSplit.isOriginal() : "If this now supports Original splits, set up mergeOptions properly";
this.deleteRecords = new OrcRawRecordMerger(conf, true, null, false, bucket,
validTxnList, readerOptions, deleteDeltas,
@@ -530,6 +531,9 @@ public class VectorizedOrcAcidRowBatchReader
* For every call to next(), it returns the next smallest record id in the file if available.
* Internally, the next() buffers a row batch and maintains an index pointer, reading the
* next batch when the previous batch is exhausted.
+ *
+ * For unbucketed tables this will currently return all delete events. Once we trust that
+ * the N in bucketN for "base" spit is reliable, all delete events not matching N can be skipped.
*/
static class DeleteReaderValue {
private VectorizedRowBatch batch;
@@ -538,9 +542,10 @@ public class VectorizedOrcAcidRowBatchReader
private final int bucketForSplit; // The bucket value should be same for all the records.
private final ValidTxnList validTxnList;
private boolean isBucketPropertyRepeating;
+ private final boolean isBucketedTable;
public DeleteReaderValue(Reader deleteDeltaReader, Reader.Options readerOptions, int bucket,
- ValidTxnList validTxnList) throws IOException {
+ ValidTxnList validTxnList, boolean isBucketedTable) throws IOException {
this.recordReader = deleteDeltaReader.rowsOptions(readerOptions);
this.bucketForSplit = bucket;
this.batch = deleteDeltaReader.getSchema().createRowBatch();
@@ -549,6 +554,7 @@ public class VectorizedOrcAcidRowBatchReader
}
this.indexPtrInBatch = 0;
this.validTxnList = validTxnList;
+ this.isBucketedTable = isBucketedTable;
checkBucketId();//check 1st batch
}
@@ -615,6 +621,13 @@ public class VectorizedOrcAcidRowBatchReader
* either the split computation got messed up or we found some corrupted records.
*/
private void checkBucketId(int bucketPropertyFromRecord) throws IOException {
+ if(!isBucketedTable) {
+ /**
+ * in this case a file inside a delete_delta_x_y/bucketN may contain any value for
+ * bucketId in {@link RecordIdentifier#getBucketProperty()}
+ */
+ return;
+ }
int bucketIdFromRecord = BucketCodec.determineVersion(bucketPropertyFromRecord)
.decodeWriterId(bucketPropertyFromRecord);
if(bucketIdFromRecord != bucketForSplit) {
@@ -686,14 +699,16 @@ public class VectorizedOrcAcidRowBatchReader
this.rowIds = null;
this.compressedOtids = null;
int maxEventsInMemory = HiveConf.getIntVar(conf, ConfVars.HIVE_TRANSACTIONAL_NUM_EVENTS_IN_MEMORY);
+ final boolean isBucketedTable = conf.getInt(hive_metastoreConstants.BUCKET_COUNT, 0) > 0;
try {
final Path[] deleteDeltaDirs = getDeleteDeltaDirsFromSplit(orcSplit);
if (deleteDeltaDirs.length > 0) {
int totalDeleteEventCount = 0;
for (Path deleteDeltaDir : deleteDeltaDirs) {
- Path deleteDeltaFile = AcidUtils.createBucketFile(deleteDeltaDir, bucket);
- FileSystem fs = deleteDeltaFile.getFileSystem(conf);
+ FileSystem fs = deleteDeltaDir.getFileSystem(conf);
+ for(Path deleteDeltaFile : OrcRawRecordMerger.getDeltaFiles(deleteDeltaDir, bucket, conf,
+ new OrcRawRecordMerger.Options().isCompacting(false), isBucketedTable)) {
// NOTE: Calling last flush length below is more for future-proofing when we have
// streaming deletes. But currently we don't support streaming deletes, and this can
// be removed if this becomes a performance issue.
@@ -721,7 +736,7 @@ public class VectorizedOrcAcidRowBatchReader
throw new DeleteEventsOverflowMemoryException();
}
DeleteReaderValue deleteReaderValue = new DeleteReaderValue(deleteDeltaReader,
- readerOptions, bucket, validTxnList);
+ readerOptions, bucket, validTxnList, isBucketedTable);
DeleteRecordKey deleteRecordKey = new DeleteRecordKey();
if (deleteReaderValue.next(deleteRecordKey)) {
sortMerger.put(deleteRecordKey, deleteReaderValue);
@@ -730,6 +745,7 @@ public class VectorizedOrcAcidRowBatchReader
}
}
}
+ }
if (totalDeleteEventCount > 0) {
// Initialize the rowId array when we have some delete events.
rowIds = new long[totalDeleteEventCount];
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
index 76aa39f..4b11a4a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
@@ -206,6 +206,7 @@ public class SortedDynPartitionOptimizer extends Transform {
if(!VirtualColumn.ROWID.getTypeInfo().equals(ci.getType())) {
throw new IllegalStateException("expected 1st column to be ROW__ID but got wrong type: " + ci.toString());
}
+ //HIVE-17328: not sure this is correct... I don't think is gets wrapped in UDFToInteger....
bucketColumns.add(new ExprNodeColumnDesc(ci));
} else {
if (!destTable.getSortCols().isEmpty()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index bc6e0d5..e8acabe 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -7377,9 +7377,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
// Check constraints on acid tables. This includes
- // * no insert overwrites
- // * no use of vectorization
- // * turns off reduce deduplication optimization, as that sometimes breaks acid
// * Check that the table is bucketed
// * Check that the table is not sorted
// This method assumes you have already decided that this is an Acid write. Don't call it if
@@ -7397,9 +7394,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
*/
conf.set(AcidUtils.CONF_ACID_KEY, "true");
- if (table.getNumBuckets() < 1) {
- throw new SemanticException(ErrorMsg.ACID_OP_ON_NONACID_TABLE, table.getTableName());
- }
if (table.getSortCols() != null && table.getSortCols().size() > 0) {
throw new SemanticException(ErrorMsg.ACID_NO_SORTED_BUCKETS, table.getTableName());
}
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
index 5e2146e..04ef7fc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hive.metastore.api.CompactionType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
import org.apache.hadoop.hive.metastore.txn.TxnStore;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
@@ -93,7 +94,7 @@ public class CompactorMR {
static final private String IS_MAJOR = "hive.compactor.is.major";
static final private String IS_COMPRESSED = "hive.compactor.is.compressed";
static final private String TABLE_PROPS = "hive.compactor.table.props";
- static final private String NUM_BUCKETS = "hive.compactor.num.buckets";
+ static final private String NUM_BUCKETS = hive_metastoreConstants.BUCKET_COUNT;
static final private String BASE_DIR = "hive.compactor.base.dir";
static final private String DELTA_DIRS = "hive.compactor.delta.dirs";
static final private String DIRS_TO_SEARCH = "hive.compactor.dirs.to.search";
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
index bff9884..0f129fc 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
@@ -69,68 +69,17 @@ import java.util.concurrent.TimeUnit;
* Tests here are for multi-statement transactions (WIP) and those that don't need to
* run with Acid 2.0 (see subclasses of TestTxnCommands2)
*/
-public class TestTxnCommands {
+public class TestTxnCommands extends TestTxnCommandsBase {
static final private Logger LOG = LoggerFactory.getLogger(TestTxnCommands.class);
private static final String TEST_DATA_DIR = new File(System.getProperty("java.io.tmpdir") +
File.separator + TestTxnCommands.class.getCanonicalName()
+ "-" + System.currentTimeMillis()
).getPath().replaceAll("\\\\", "/");
- private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
- //bucket count for test tables; set it to 1 for easier debugging
- private static int BUCKET_COUNT = 2;
- @Rule
- public TestName testName = new TestName();
- private HiveConf hiveConf;
- private Driver d;
- private static enum Table {
- ACIDTBL("acidTbl"),
- ACIDTBLPART("acidTblPart"),
- ACIDTBL2("acidTbl2"),
- NONACIDORCTBL("nonAcidOrcTbl"),
- NONACIDORCTBL2("nonAcidOrcTbl2");
-
- private final String name;
- @Override
- public String toString() {
- return name;
- }
- Table(String name) {
- this.name = name;
- }
+ @Override
+ String getTestDataDir() {
+ return TEST_DATA_DIR;
}
- @Before
- public void setUp() throws Exception {
- tearDown();
- hiveConf = new HiveConf(this.getClass());
- hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
- hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
- hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, TEST_WAREHOUSE_DIR);
- hiveConf.setVar(HiveConf.ConfVars.HIVEMAPREDMODE, "nonstrict");
- hiveConf.setVar(HiveConf.ConfVars.HIVEINPUTFORMAT, HiveInputFormat.class.getName());
- hiveConf
- .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
- "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory");
- hiveConf.setBoolVar(HiveConf.ConfVars.MERGE_CARDINALITY_VIOLATION_CHECK, true);
- TxnDbUtil.setConfValues(hiveConf);
- TxnDbUtil.prepDb();
- File f = new File(TEST_WAREHOUSE_DIR);
- if (f.exists()) {
- FileUtil.fullyDelete(f);
- }
- if (!(new File(TEST_WAREHOUSE_DIR).mkdirs())) {
- throw new RuntimeException("Could not create " + TEST_WAREHOUSE_DIR);
- }
- SessionState.start(new SessionState(hiveConf));
- d = new Driver(hiveConf);
- d.setMaxRows(10000);
- dropTables();
- runStatementOnDriver("create table " + Table.ACIDTBL + "(a int, b int) clustered by (a) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES ('transactional'='true')");
- runStatementOnDriver("create table " + Table.ACIDTBLPART + "(a int, b int) partitioned by (p string) clustered by (a) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES ('transactional'='true')");
- runStatementOnDriver("create table " + Table.NONACIDORCTBL + "(a int, b int) clustered by (a) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES ('transactional'='false')");
- runStatementOnDriver("create table " + Table.NONACIDORCTBL2 + "(a int, b int) clustered by (a) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES ('transactional'='false')");
- runStatementOnDriver("create temporary table " + Table.ACIDTBL2 + "(a int, b int, c int) clustered by (c) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES ('transactional'='true')");
- }
private void dropTables() throws Exception {
for(Table t : Table.values()) {
runStatementOnDriver("drop table if exists " + t);
@@ -150,7 +99,7 @@ public class TestTxnCommands {
FileUtils.deleteDirectory(new File(TEST_DATA_DIR));
}
}
- @Test
+ @Test//todo: what is this for?
public void testInsertOverwrite() throws Exception {
runStatementOnDriver("insert overwrite table " + Table.NONACIDORCTBL + " select a,b from " + Table.NONACIDORCTBL2);
runStatementOnDriver("create table " + Table.NONACIDORCTBL2 + "3(a int, b int) clustered by (a) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES ('transactional'='false')");
@@ -172,7 +121,7 @@ public class TestTxnCommands {
if(true) {
return;
}
- Path bucket = AcidUtils.createBucketFile(new Path(new Path(TEST_WAREHOUSE_DIR, table.toString().toLowerCase()), AcidUtils.deltaSubdir(txnId, txnId, stmtId)), bucketNum);
+ Path bucket = AcidUtils.createBucketFile(new Path(new Path(getWarehouseDir(), table.toString().toLowerCase()), AcidUtils.deltaSubdir(txnId, txnId, stmtId)), bucketNum);
FileOutputStream delta = new FileOutputStream(testName.getMethodName() + "_" + bucket.getParent().getName() + "_" + bucket.getName());
// try {
// FileDump.printJsonData(hiveConf, bucket.toString(), delta);
@@ -446,7 +395,7 @@ public class TestTxnCommands {
}
}
Assert.assertNotNull(txnInfo);
- Assert.assertEquals(12, txnInfo.getId());
+ Assert.assertEquals(14, txnInfo.getId());
Assert.assertEquals(TxnState.OPEN, txnInfo.getState());
String s =TxnDbUtil.queryToString("select TXN_STARTED, TXN_LAST_HEARTBEAT from TXNS where TXN_ID = " + txnInfo.getId(), false);
String[] vals = s.split("\\s+");
@@ -490,33 +439,6 @@ public class TestTxnCommands {
}
}
- /**
- * takes raw data and turns it into a string as if from Driver.getResults()
- * sorts rows in dictionary order
- */
- private List<String> stringifyValues(int[][] rowsIn) {
- return TestTxnCommands2.stringifyValues(rowsIn);
- }
- private String makeValuesClause(int[][] rows) {
- return TestTxnCommands2.makeValuesClause(rows);
- }
-
- private List<String> runStatementOnDriver(String stmt) throws Exception {
- CommandProcessorResponse cpr = d.run(stmt);
- if(cpr.getResponseCode() != 0) {
- throw new RuntimeException(stmt + " failed: " + cpr);
- }
- List<String> rs = new ArrayList<String>();
- d.getResults(rs);
- return rs;
- }
- private CommandProcessorResponse runStatementOnDriverNegative(String stmt) throws Exception {
- CommandProcessorResponse cpr = d.run(stmt);
- if(cpr.getResponseCode() != 0) {
- return cpr;
- }
- throw new RuntimeException("Didn't get expected failure!");
- }
@Test
public void exchangePartition() throws Exception {
@@ -872,8 +794,8 @@ public class TestTxnCommands {
Assert.assertTrue(rs.get(1), rs.get(1).endsWith("nonacidorctbl/000001_0"));
Assert.assertTrue(rs.get(2), rs.get(2).startsWith("{\"transactionid\":0,\"bucketid\":536936448,\"rowid\":1}\t1\t5"));
Assert.assertTrue(rs.get(2), rs.get(2).endsWith("nonacidorctbl/000001_0_copy_1"));
- Assert.assertTrue(rs.get(3), rs.get(3).startsWith("{\"transactionid\":14,\"bucketid\":536936448,\"rowid\":0}\t1\t17"));
- Assert.assertTrue(rs.get(3), rs.get(3).endsWith("nonacidorctbl/delta_0000014_0000014_0000/bucket_00001"));
+ Assert.assertTrue(rs.get(3), rs.get(3).startsWith("{\"transactionid\":16,\"bucketid\":536936448,\"rowid\":0}\t1\t17"));
+ Assert.assertTrue(rs.get(3), rs.get(3).endsWith("nonacidorctbl/delta_0000016_0000016_0000/bucket_00001"));
//run Compaction
runStatementOnDriver("alter table "+ TestTxnCommands2.Table.NONACIDORCTBL +" compact 'major'");
TestTxnCommands2.runWorker(hiveConf);
@@ -884,13 +806,13 @@ public class TestTxnCommands {
}
Assert.assertEquals("", 4, rs.size());
Assert.assertTrue(rs.get(0), rs.get(0).startsWith("{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}\t0\t12"));
- Assert.assertTrue(rs.get(0), rs.get(0).endsWith("nonacidorctbl/base_0000014/bucket_00000"));
+ Assert.assertTrue(rs.get(0), rs.get(0).endsWith("nonacidorctbl/base_0000016/bucket_00000"));
Assert.assertTrue(rs.get(1), rs.get(1).startsWith("{\"transactionid\":0,\"bucketid\":536936448,\"rowid\":0}\t1\t2"));
- Assert.assertTrue(rs.get(1), rs.get(1).endsWith("nonacidorctbl/base_0000014/bucket_00001"));
+ Assert.assertTrue(rs.get(1), rs.get(1).endsWith("nonacidorctbl/base_0000016/bucket_00001"));
Assert.assertTrue(rs.get(2), rs.get(2).startsWith("{\"transactionid\":0,\"bucketid\":536936448,\"rowid\":1}\t1\t5"));
- Assert.assertTrue(rs.get(2), rs.get(2).endsWith("nonacidorctbl/base_0000014/bucket_00001"));
- Assert.assertTrue(rs.get(3), rs.get(3).startsWith("{\"transactionid\":14,\"bucketid\":536936448,\"rowid\":0}\t1\t17"));
- Assert.assertTrue(rs.get(3), rs.get(3).endsWith("nonacidorctbl/base_0000014/bucket_00001"));
+ Assert.assertTrue(rs.get(2), rs.get(2).endsWith("nonacidorctbl/base_0000016/bucket_00001"));
+ Assert.assertTrue(rs.get(3), rs.get(3).startsWith("{\"transactionid\":16,\"bucketid\":536936448,\"rowid\":0}\t1\t17"));
+ Assert.assertTrue(rs.get(3), rs.get(3).endsWith("nonacidorctbl/base_0000016/bucket_00001"));
//make sure they are the same before and after compaction
}
@@ -940,4 +862,4 @@ public class TestTxnCommands {
int[][] expected = {{0, -1},{0, -1}, {1, -1}, {1, -1}, {2, -1}, {2, -1}, {3, -1}, {3, -1}};
Assert.assertEquals(stringifyValues(expected), r);
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
index 0e0fca3..21b4a2c 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
@@ -304,8 +304,8 @@ public class TestTxnCommands2 {
// 1. Insert five rows to Non-ACID table.
runStatementOnDriver("insert into " + Table.NONACIDORCTBL + "(a,b) values(1,2),(3,4),(5,6),(7,8),(9,10)");
- // 2. Convert NONACIDORCTBL to ACID table.
- runStatementOnDriver("alter table " + Table.NONACIDORCTBL + " SET TBLPROPERTIES ('transactional'='true')");
+ // 2. Convert NONACIDORCTBL to ACID table. //todo: remove trans_prop after HIVE-17089
+ runStatementOnDriver("alter table " + Table.NONACIDORCTBL + " SET TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default')");
runStatementOnDriver("update " + Table.NONACIDORCTBL + " set b = b*2 where b in (4,10)");
runStatementOnDriver("delete from " + Table.NONACIDORCTBL + " where a = 7");
@@ -331,6 +331,7 @@ public class TestTxnCommands2 {
/**
* see HIVE-16177
* See also {@link TestTxnCommands#testNonAcidToAcidConversion01()}
+ * {@link TestTxnNoBuckets#testCTAS()}
*/
@Test
public void testNonAcidToAcidConversion02() throws Exception {
@@ -341,8 +342,8 @@ public class TestTxnCommands2 {
//create 1 row in a file 000001_0_copy2 (and empty 000000_0_copy2?)
runStatementOnDriver("insert into " + Table.NONACIDORCTBL + "(a,b) values(1,6)");
- //convert the table to Acid
- runStatementOnDriver("alter table " + Table.NONACIDORCTBL + " SET TBLPROPERTIES ('transactional'='true')");
+ //convert the table to Acid //todo: remove trans_prop after HIVE-17089
+ runStatementOnDriver("alter table " + Table.NONACIDORCTBL + " SET TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default')");
List<String> rs1 = runStatementOnDriver("describe "+ Table.NONACIDORCTBL);
//create a some of delta directories
runStatementOnDriver("insert into " + Table.NONACIDORCTBL + "(a,b) values(0,15),(1,16)");
@@ -361,6 +362,7 @@ public class TestTxnCommands2 {
* All ROW__IDs are unique on read after conversion to acid
* ROW__IDs are exactly the same before and after compaction
* Also check the file name (only) after compaction for completeness
+ * Note: order of rows in a file ends up being the reverse of order in values clause (why?!)
*/
String[][] expected = {
{"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}\t0\t13", "bucket_00000"},
@@ -2022,7 +2024,7 @@ public class TestTxnCommands2 {
}
static String makeValuesClause(int[][] rows) {
assert rows.length > 0;
- StringBuilder sb = new StringBuilder("values");
+ StringBuilder sb = new StringBuilder(" values");
for(int[] row : rows) {
assert row.length > 0;
if(row.length > 1) {
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsBase.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsBase.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsBase.java
new file mode 100644
index 0000000..d6e709d
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsBase.java
@@ -0,0 +1,162 @@
+package org.apache.hadoop.hive.ql;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
+import org.apache.hadoop.hive.ql.io.HiveInputFormat;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.rules.TestName;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public abstract class TestTxnCommandsBase {
+ //bucket count for test tables; set it to 1 for easier debugging
+ final static int BUCKET_COUNT = 2;
+ @Rule
+ public TestName testName = new TestName();
+ HiveConf hiveConf;
+ Driver d;
+ enum Table {
+ ACIDTBL("acidTbl"),
+ ACIDTBLPART("acidTblPart"),
+ ACIDTBL2("acidTbl2"),
+ NONACIDORCTBL("nonAcidOrcTbl"),
+ NONACIDORCTBL2("nonAcidOrcTbl2"),
+ NONACIDNONBUCKET("nonAcidNonBucket");
+
+ final String name;
+ @Override
+ public String toString() {
+ return name;
+ }
+ Table(String name) {
+ this.name = name;
+ }
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ setUpInternal();
+ }
+ void setUpInternal() throws Exception {
+ tearDown();
+ hiveConf = new HiveConf(this.getClass());
+ hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
+ hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
+ hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, getWarehouseDir());
+ hiveConf.setVar(HiveConf.ConfVars.HIVEMAPREDMODE, "nonstrict");
+ hiveConf.setVar(HiveConf.ConfVars.HIVEINPUTFORMAT, HiveInputFormat.class.getName());
+ hiveConf
+ .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
+ "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory");
+ hiveConf.setBoolVar(HiveConf.ConfVars.MERGE_CARDINALITY_VIOLATION_CHECK, true);
+ TxnDbUtil.setConfValues(hiveConf);
+ TxnDbUtil.prepDb();
+ File f = new File(getWarehouseDir());
+ if (f.exists()) {
+ FileUtil.fullyDelete(f);
+ }
+ if (!(new File(getWarehouseDir()).mkdirs())) {
+ throw new RuntimeException("Could not create " + getWarehouseDir());
+ }
+ SessionState.start(new SessionState(hiveConf));
+ d = new Driver(hiveConf);
+ d.setMaxRows(10000);
+ dropTables();
+ runStatementOnDriver("create table " + Table.ACIDTBL + "(a int, b int) clustered by (a) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES ('transactional'='true')");
+ runStatementOnDriver("create table " + Table.ACIDTBLPART + "(a int, b int) partitioned by (p string) clustered by (a) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES ('transactional'='true')");
+ runStatementOnDriver("create table " + Table.NONACIDORCTBL + "(a int, b int) clustered by (a) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES ('transactional'='false')");
+ runStatementOnDriver("create table " + Table.NONACIDORCTBL2 + "(a int, b int) clustered by (a) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES ('transactional'='false')");
+ runStatementOnDriver("create temporary table " + Table.ACIDTBL2 + "(a int, b int, c int) clustered by (c) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES ('transactional'='true')");
+ runStatementOnDriver("create table " + Table.NONACIDNONBUCKET + "(a int, b int) stored as orc");
+ }
+ private void dropTables() throws Exception {
+ for(TestTxnCommandsBase.Table t : TestTxnCommandsBase.Table.values()) {
+ runStatementOnDriver("drop table if exists " + t);
+ }
+ }
+ @After
+ public void tearDown() throws Exception {
+ try {
+ if (d != null) {
+ dropTables();
+ d.destroy();
+ d.close();
+ d = null;
+ }
+ } finally {
+ TxnDbUtil.cleanDb();
+ FileUtils.deleteDirectory(new File(getTestDataDir()));
+ }
+ }
+ String getWarehouseDir() {
+ return getTestDataDir() + "/warehouse";
+ }
+ abstract String getTestDataDir();
+ /**
+ * takes raw data and turns it into a string as if from Driver.getResults()
+ * sorts rows in dictionary order
+ */
+ List<String> stringifyValues(int[][] rowsIn) {
+ return TestTxnCommands2.stringifyValues(rowsIn);
+ }
+ String makeValuesClause(int[][] rows) {
+ return TestTxnCommands2.makeValuesClause(rows);
+ }
+
+ List<String> runStatementOnDriver(String stmt) throws Exception {
+ CommandProcessorResponse cpr = d.run(stmt);
+ if(cpr.getResponseCode() != 0) {
+ throw new RuntimeException(stmt + " failed: " + cpr);
+ }
+ List<String> rs = new ArrayList<String>();
+ d.getResults(rs);
+ return rs;
+ }
+ CommandProcessorResponse runStatementOnDriverNegative(String stmt) throws Exception {
+ CommandProcessorResponse cpr = d.run(stmt);
+ if(cpr.getResponseCode() != 0) {
+ return cpr;
+ }
+ throw new RuntimeException("Didn't get expected failure!");
+ }
+ /**
+ * Will assert that actual files match expected.
+ * @param expectedFiles - suffixes of expected Paths. Must be the same length
+ * @param rootPath - table or patition root where to start looking for actual files, recursively
+ */
+ void assertExpectedFileSet(Set<String> expectedFiles, String rootPath) throws Exception {
+ int suffixLength = 0;
+ for(String s : expectedFiles) {
+ if(suffixLength > 0) {
+ assert suffixLength == s.length() : "all entries must be the same length. current: " + s;
+ }
+ suffixLength = s.length();
+ }
+ FileSystem fs = FileSystem.get(hiveConf);
+ Set<String> actualFiles = new HashSet<>();
+ RemoteIterator<LocatedFileStatus> remoteIterator = fs.listFiles(new Path(rootPath), true);
+ while (remoteIterator.hasNext()) {
+ LocatedFileStatus lfs = remoteIterator.next();
+ if(!lfs.isDirectory() && org.apache.hadoop.hive.common.FileUtils.HIDDEN_FILES_PATH_FILTER.accept(lfs.getPath())) {
+ String p = lfs.getPath().toString();
+ actualFiles.add(p.substring(p.length() - suffixLength, p.length()));
+ }
+ }
+ Assert.assertEquals("Unexpected file list", expectedFiles, actualFiles);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
new file mode 100644
index 0000000..7aca6b2
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
@@ -0,0 +1,297 @@
+package org.apache.hadoop.hive.ql;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.io.BucketCodec;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public class TestTxnNoBuckets extends TestTxnCommandsBase {
+ static final private Logger LOG = LoggerFactory.getLogger(TestTxnNoBuckets.class);
+ private static final String TEST_DATA_DIR = new File(System.getProperty("java.io.tmpdir") +
+ File.separator + TestTxnNoBuckets.class.getCanonicalName()
+ + "-" + System.currentTimeMillis()
+ ).getPath().replaceAll("\\\\", "/");
+ @Override
+ String getTestDataDir() {
+ return TEST_DATA_DIR;
+ }
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ setUpInternal();
+ hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, true);
+ }
+
+ /**
+ * Tests that Acid can work with un-bucketed tables.
+ */
+ @Test
+ public void testNoBuckets() throws Exception {
+ int[][] sourceVals1 = {{0,0,0},{3,3,3}};
+ int[][] sourceVals2 = {{1,1,1},{2,2,2}};
+ runStatementOnDriver("drop table if exists tmp");
+ runStatementOnDriver("create table tmp (c1 integer, c2 integer, c3 integer) stored as orc");
+ runStatementOnDriver("insert into tmp " + makeValuesClause(sourceVals1));
+ runStatementOnDriver("insert into tmp " + makeValuesClause(sourceVals2));
+ runStatementOnDriver("drop table if exists nobuckets");
+ runStatementOnDriver("create table nobuckets (c1 integer, c2 integer, c3 integer) stored " +
+ "as orc tblproperties('transactional'='true', 'transactional_properties'='default')");
+ String stmt = "insert into nobuckets select * from tmp";
+ runStatementOnDriver(stmt);
+ List<String> rs = runStatementOnDriver(
+ "select ROW__ID, c1, c2, c3, INPUT__FILE__NAME from nobuckets order by ROW__ID");
+ Assert.assertEquals("", 4, rs.size());
+ LOG.warn("after insert");
+ for(String s : rs) {
+ LOG.warn(s);
+ }
+ /**the insert creates 2 output files (presumably because there are 2 input files)
+ * The number in the file name is writerId. This is the number encoded in ROW__ID.bucketId -
+ * see {@link org.apache.hadoop.hive.ql.io.BucketCodec}*/
+ Assert.assertTrue(rs.get(0), rs.get(0).startsWith("{\"transactionid\":19,\"bucketid\":536870912,\"rowid\":0}\t0\t0\t0\t"));
+ Assert.assertTrue(rs.get(0), rs.get(0).endsWith("nobuckets/delta_0000019_0000019_0000/bucket_00000"));
+ Assert.assertTrue(rs.get(1), rs.get(1).startsWith("{\"transactionid\":19,\"bucketid\":536870912,\"rowid\":1}\t3\t3\t3\t"));
+ Assert.assertTrue(rs.get(1), rs.get(1).endsWith("nobuckets/delta_0000019_0000019_0000/bucket_00000"));
+ Assert.assertTrue(rs.get(2), rs.get(2).startsWith("{\"transactionid\":19,\"bucketid\":536936448,\"rowid\":0}\t1\t1\t1\t"));
+ Assert.assertTrue(rs.get(2), rs.get(2).endsWith("nobuckets/delta_0000019_0000019_0000/bucket_00001"));
+ Assert.assertTrue(rs.get(3), rs.get(3).startsWith("{\"transactionid\":19,\"bucketid\":536936448,\"rowid\":1}\t2\t2\t2\t"));
+ Assert.assertTrue(rs.get(3), rs.get(3).endsWith("nobuckets/delta_0000019_0000019_0000/bucket_00001"));
+ /*todo: WTF?
+ RS for update seems to spray randomly... is that OK? maybe as long as all resultant files have different names... will they?
+ Assuming we name them based on taskId, we should create bucketX and bucketY.
+ we delete events can be written to bucketX file it could be useful for filter delete for a split by file name since the insert
+ events seem to be written to a proper bucketX file. In fact this may reduce the number of changes elsewhere like compactor... maybe
+ But this limits the parallelism - what is worse, you don't know what the parallelism should be until you have a list of all the
+ input files since bucket count is no longer a metadata property. Also, with late Update split, the file name has already been determined
+ from taskId so the Insert part won't end up matching the bucketX property necessarily.
+ With early Update split, the Insert can still be an insert - i.e. go to appropriate bucketX. But deletes will still go wherever (random shuffle)
+ unless you know all the bucketX files to be read - may not be worth the trouble.
+ * 2nd: something in FS fails. ArrayIndexOutOfBoundsException: 1 at FileSinkOperator.process(FileSinkOperator.java:779)*/
+ runStatementOnDriver("update nobuckets set c3 = 17 where c3 in(0,1)");
+ rs = runStatementOnDriver("select ROW__ID, c1, c2, c3, INPUT__FILE__NAME from nobuckets order by INPUT__FILE__NAME, ROW__ID");
+ LOG.warn("after update");
+ for(String s : rs) {
+ LOG.warn(s);
+ }
+ Assert.assertTrue(rs.get(0), rs.get(0).startsWith("{\"transactionid\":19,\"bucketid\":536870912,\"rowid\":1}\t3\t3\t3\t"));
+ Assert.assertTrue(rs.get(0), rs.get(0).endsWith("nobuckets/delta_0000019_0000019_0000/bucket_00000"));
+ Assert.assertTrue(rs.get(1), rs.get(1).startsWith("{\"transactionid\":19,\"bucketid\":536936448,\"rowid\":1}\t2\t2\t2\t"));
+ Assert.assertTrue(rs.get(1), rs.get(1).endsWith("nobuckets/delta_0000019_0000019_0000/bucket_00001"));
+ //so update has 1 writer which creates bucket0 where both new rows land
+ Assert.assertTrue(rs.get(2), rs.get(2).startsWith("{\"transactionid\":21,\"bucketid\":536870912,\"rowid\":0}\t0\t0\t17\t"));
+ Assert.assertTrue(rs.get(2), rs.get(2).endsWith("nobuckets/delta_0000021_0000021_0000/bucket_00000"));
+ Assert.assertTrue(rs.get(3), rs.get(3).startsWith("{\"transactionid\":21,\"bucketid\":536870912,\"rowid\":1}\t1\t1\t17\t"));
+ Assert.assertTrue(rs.get(3), rs.get(3).endsWith("nobuckets/delta_0000021_0000021_0000/bucket_00000"));
+
+ Set<String> expectedFiles = new HashSet<>();
+ //both delete events land in a single bucket0. Each has a different ROW__ID.bucketId value (even writerId in it is different)
+ expectedFiles.add("ts/delete_delta_0000021_0000021_0000/bucket_00000");
+ expectedFiles.add("nobuckets/delta_0000019_0000019_0000/bucket_00000");
+ expectedFiles.add("nobuckets/delta_0000019_0000019_0000/bucket_00001");
+ expectedFiles.add("nobuckets/delta_0000021_0000021_0000/bucket_00000");
+ //check that we get the right files on disk
+ assertExpectedFileSet(expectedFiles, getWarehouseDir() + "/nobuckets");
+ //todo: it would be nice to check the contents of the files... could use orc.FileDump - it has
+ // methods to print to a supplied stream but those are package private
+
+ runStatementOnDriver("alter table nobuckets compact 'major'");
+ TestTxnCommands2.runWorker(hiveConf);
+ rs = runStatementOnDriver("select ROW__ID, c1, c2, c3, INPUT__FILE__NAME from nobuckets order by INPUT__FILE__NAME, ROW__ID");
+ LOG.warn("after major compact");
+ for(String s : rs) {
+ LOG.warn(s);
+ }
+ /*
+├── base_0000021
+│  ├── bucket_00000
+│  └── bucket_00001
+├── delete_delta_0000021_0000021_0000
+│  └── bucket_00000
+├── delta_0000019_0000019_0000
+│  ├── bucket_00000
+│  └── bucket_00001
+└── delta_0000021_0000021_0000
+ └── bucket_00000
+ */
+ Assert.assertTrue(rs.get(0), rs.get(0).startsWith("{\"transactionid\":19,\"bucketid\":536870912,\"rowid\":1}\t3\t3\t3\t"));
+ Assert.assertTrue(rs.get(0), rs.get(0).endsWith("nobuckets/base_0000021/bucket_00000"));
+ Assert.assertTrue(rs.get(1), rs.get(1).startsWith("{\"transactionid\":21,\"bucketid\":536870912,\"rowid\":0}\t0\t0\t17\t"));
+ Assert.assertTrue(rs.get(1), rs.get(1).endsWith("nobuckets/base_0000021/bucket_00000"));
+ Assert.assertTrue(rs.get(2), rs.get(2).startsWith("{\"transactionid\":21,\"bucketid\":536870912,\"rowid\":1}\t1\t1\t17\t"));
+ Assert.assertTrue(rs.get(2), rs.get(2).endsWith("nobuckets/base_0000021/bucket_00000"));
+ Assert.assertTrue(rs.get(3), rs.get(3).startsWith("{\"transactionid\":19,\"bucketid\":536936448,\"rowid\":1}\t2\t2\t2\t"));
+ Assert.assertTrue(rs.get(3), rs.get(3).endsWith("nobuckets/base_0000021/bucket_00001"));
+
+ expectedFiles.clear();
+ expectedFiles.add("delete_delta_0000021_0000021_0000/bucket_00000");
+ expectedFiles.add("uckets/delta_0000019_0000019_0000/bucket_00000");
+ expectedFiles.add("uckets/delta_0000019_0000019_0000/bucket_00001");
+ expectedFiles.add("uckets/delta_0000021_0000021_0000/bucket_00000");
+ expectedFiles.add("/warehouse/nobuckets/base_0000021/bucket_00000");
+ expectedFiles.add("/warehouse/nobuckets/base_0000021/bucket_00001");
+ assertExpectedFileSet(expectedFiles, getWarehouseDir() + "/nobuckets");
+
+ TestTxnCommands2.runCleaner(hiveConf);
+ rs = runStatementOnDriver("select c1, c2, c3 from nobuckets order by c1, c2, c3");
+ int[][] result = {{0,0,17},{1,1,17},{2,2,2},{3,3,3}};
+ Assert.assertEquals("Unexpected result after clean", stringifyValues(result), rs);
+
+ expectedFiles.clear();
+ expectedFiles.add("nobuckets/base_0000021/bucket_00000");
+ expectedFiles.add("nobuckets/base_0000021/bucket_00001");
+ assertExpectedFileSet(expectedFiles, getWarehouseDir() + "/nobuckets");
+ }
+
+ /**
+ * all of these pass but don't do exactly the right thing
+ * files land as if it's not an acid table "warehouse/myctas4/000000_0"
+ * even though in {@link org.apache.hadoop.hive.metastore.TransactionalValidationListener} fires
+ * and sees it as transactional table
+ * look for QB.isCTAS() and CreateTableDesc() in SemanticAnalyzer
+ *
+ * On read, these files are treated like non acid to acid conversion
+ *
+ * see HIVE-15899
+ * See CTAS tests in TestAcidOnTez
+ */
+ @Test
+ public void testCTAS() throws Exception {
+ int[][] values = {{1,2},{3,4}};
+ runStatementOnDriver("insert into " + Table.NONACIDORCTBL + makeValuesClause(values));
+ runStatementOnDriver("create table myctas stored as ORC TBLPROPERTIES ('transactional" +
+ "'='true', 'transactional_properties'='default') as select a, b from " + Table.NONACIDORCTBL);
+ List<String> rs = runStatementOnDriver("select * from myctas order by a, b");
+ Assert.assertEquals(stringifyValues(values), rs);
+
+ runStatementOnDriver("insert into " + Table.ACIDTBL + makeValuesClause(values));
+ runStatementOnDriver("create table myctas2 stored as ORC TBLPROPERTIES ('transactional" +
+ "'='true', 'transactional_properties'='default') as select a, b from " + Table.ACIDTBL);
+ rs = runStatementOnDriver("select * from myctas2 order by a, b");
+ Assert.assertEquals(stringifyValues(values), rs);
+
+ runStatementOnDriver("create table myctas3 stored as ORC TBLPROPERTIES ('transactional" +
+ "'='true', 'transactional_properties'='default') as select a, b from " + Table.NONACIDORCTBL +
+ " union all select a, b from " + Table.ACIDTBL);
+ rs = runStatementOnDriver("select * from myctas3 order by a, b");
+ Assert.assertEquals(stringifyValues(new int[][] {{1,2},{1,2},{3,4},{3,4}}), rs);
+
+ runStatementOnDriver("create table myctas4 stored as ORC TBLPROPERTIES ('transactional" +
+ "'='true', 'transactional_properties'='default') as select a, b from " + Table.NONACIDORCTBL +
+ " union distinct select a, b from " + Table.ACIDTBL);
+ rs = runStatementOnDriver("select * from myctas4 order by a, b");
+ Assert.assertEquals(stringifyValues(values), rs);
+ }
+ /**
+ * see HIVE-16177
+ * See also {@link TestTxnCommands2#testNonAcidToAcidConversion02()} todo need test with > 1 bucket file
+ */
+ @Test
+ public void testToAcidConversion02() throws Exception {
+ //create 2 rows in a file 00000_0
+ runStatementOnDriver("insert into " + Table.NONACIDNONBUCKET + "(a,b) values(1,2),(1,3)");
+ //create 4 rows in a file 000000_0_copy_1
+ runStatementOnDriver("insert into " + Table.NONACIDNONBUCKET + "(a,b) values(0,12),(0,13),(1,4),(1,5)");
+ //create 1 row in a file 000000_0_copy_2
+ runStatementOnDriver("insert into " + Table.NONACIDNONBUCKET + "(a,b) values(1,6)");
+
+ //convert the table to Acid //todo: remove trans_prop after HIVE-17089
+ runStatementOnDriver("alter table " + Table.NONACIDNONBUCKET + " SET TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default')");
+ List<String> rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by ROW__ID");
+ LOG.warn("before acid ops (after convert)");
+ for(String s : rs) {
+ LOG.warn(s);
+ }
+ //create a some of delta directories
+ runStatementOnDriver("insert into " + Table.NONACIDNONBUCKET + "(a,b) values(0,15),(1,16)");
+ runStatementOnDriver("update " + Table.NONACIDNONBUCKET + " set b = 120 where a = 0 and b = 12");
+ runStatementOnDriver("insert into " + Table.NONACIDNONBUCKET + "(a,b) values(0,17)");
+ runStatementOnDriver("delete from " + Table.NONACIDNONBUCKET + " where a = 1 and b = 3");
+
+ rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by a,b");
+ LOG.warn("before compact");
+ for(String s : rs) {
+ LOG.warn(s);
+ }
+ Assert.assertEquals(0, BucketCodec.determineVersion(536870912).decodeWriterId(536870912));
+ /*
+ * All ROW__IDs are unique on read after conversion to acid
+ * ROW__IDs are exactly the same before and after compaction
+ * Also check the file name (only) after compaction for completeness
+ */
+ String[][] expected = {
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t0\t13", "bucket_00000", "000000_0_copy_1"},
+ {"{\"transactionid\":18,\"bucketid\":536870912,\"rowid\":0}\t0\t15", "bucket_00000", "bucket_00000"},
+ {"{\"transactionid\":20,\"bucketid\":536870912,\"rowid\":0}\t0\t17", "bucket_00000", "bucket_00000"},
+ {"{\"transactionid\":19,\"bucketid\":536870912,\"rowid\":0}\t0\t120", "bucket_00000", "bucket_00000"},
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}\t1\t2", "bucket_00000", "000000_0"},
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}\t1\t4", "bucket_00000", "000000_0_copy_1"},
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":5}\t1\t5", "bucket_00000", "000000_0_copy_1"},
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":6}\t1\t6", "bucket_00000", "000000_0_copy_2"},
+ {"{\"transactionid\":18,\"bucketid\":536870912,\"rowid\":1}\t1\t16", "bucket_00000", "bucket_00000"}
+ };
+ Assert.assertEquals("Unexpected row count before compaction", expected.length, rs.size());
+ for(int i = 0; i < expected.length; i++) {
+ Assert.assertTrue("Actual line " + i + " bc: " + rs.get(i), rs.get(i).startsWith(expected[i][0]));
+ Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), rs.get(i).endsWith(expected[i][2]));
+ }
+ //run Compaction
+ runStatementOnDriver("alter table "+ Table.NONACIDNONBUCKET +" compact 'major'");
+ TestTxnCommands2.runWorker(hiveConf);
+ /*
+ nonacidnonbucket/
+ ├── 000000_0
+ ├── 000000_0_copy_1
+ ├── 000000_0_copy_2
+ ├── base_0000021
+ │  └── bucket_00000
+ ├── delete_delta_0000019_0000019_0000
+ │  └── bucket_00000
+ ├── delete_delta_0000021_0000021_0000
+ │  └── bucket_00000
+ ├── delta_0000018_0000018_0000
+ │  └── bucket_00000
+ ├── delta_0000019_0000019_0000
+ │  └── bucket_00000
+ └── delta_0000020_0000020_0000
+ └── bucket_00000
+
+ 6 directories, 9 files
+ */
+ rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by a,b");
+ LOG.warn("after compact");
+ for(String s : rs) {
+ LOG.warn(s);
+ }
+ Assert.assertEquals("Unexpected row count after compaction", expected.length, rs.size());
+ for(int i = 0; i < expected.length; i++) {
+ Assert.assertTrue("Actual line " + i + " ac: " + rs.get(i), rs.get(i).startsWith(expected[i][0]));
+ Assert.assertTrue("Actual line(file) " + i + " ac: " + rs.get(i), rs.get(i).endsWith(expected[i][1]));
+ }
+ //make sure they are the same before and after compaction
+ }
+ /**
+ * Currently CTAS doesn't support bucketed tables. Correspondingly Acid only supports CTAS for
+ * unbucketed tables. This test is here to make sure that if CTAS is made to support unbucketed
+ * tables, that it raises a red flag for Acid.
+ */
+ @Test
+ public void testCtasBucketed() throws Exception {
+ runStatementOnDriver("insert into " + Table.NONACIDNONBUCKET + "(a,b) values(1,2),(1,3)");
+ CommandProcessorResponse cpr = runStatementOnDriverNegative("create table myctas " +
+ "clustered by (a) into 2 buckets stored as ORC TBLPROPERTIES ('transactional'='true') as " +
+ "select a, b from " + Table.NONACIDORCTBL);
+ int j = ErrorMsg.CTAS_PARCOL_COEXISTENCE.getErrorCode();//this code doesn't propagate
+// Assert.assertEquals("Wrong msg", ErrorMsg.CTAS_PARCOL_COEXISTENCE.getErrorCode(), cpr.getErrorCode());
+ Assert.assertTrue(cpr.getErrorMessage().contains("CREATE-TABLE-AS-SELECT does not support"));
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
index f73d058..4c30732 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
@@ -3420,6 +3420,7 @@ public class TestInputOutputFormat {
assertTrue(split.toString().contains("hasFooter=false"));
assertTrue(split.toString().contains("hasBase=true"));
assertTrue(split.toString().contains("deltas=0"));
+ assertTrue(split.toString().contains("isOriginal=true"));
if (split instanceof OrcSplit) {
assertFalse("No footer serialize test for non-vector reader, hasFooter is not expected in" +
" orc splits.", ((OrcSplit) split).hasFooter());
@@ -3435,11 +3436,13 @@ public class TestInputOutputFormat {
}
// call-1: open to read footer - split 1 => mock:/mocktable5/0_0
// call-2: open to read data - split 1 => mock:/mocktable5/0_0
- // call-3: open to read footer - split 2 => mock:/mocktable5/0_1
- // call-4: open to read data - split 2 => mock:/mocktable5/0_1
- // call-5: AcidUtils.getAcidState - getLen() mock:/mocktable5/0_0
- // call-6: AcidUtils.getAcidState - getLen() mock:/mocktable5/0_1
- assertEquals(6, readOpsDelta);
+ // call-3: getAcidState - split 1 => mock:/mocktable5 (to compute offset for original read)
+ // call-4: open to read footer - split 2 => mock:/mocktable5/0_1
+ // call-5: open to read data - split 2 => mock:/mocktable5/0_1
+ // call-6: getAcidState - split 2 => mock:/mocktable5 (to compute offset for original read)
+ // call-7: open to read footer - split 2 => mock:/mocktable5/0_0 (to get row count)
+ // call-8: file status - split 2 => mock:/mocktable5/0_0
+ assertEquals(8, readOpsDelta);
// revert back to local fs
conf.set("fs.defaultFS", "file:///");
@@ -3498,6 +3501,7 @@ public class TestInputOutputFormat {
assertTrue(split.toString().contains("hasFooter=true"));
assertTrue(split.toString().contains("hasBase=true"));
assertTrue(split.toString().contains("deltas=0"));
+ assertTrue(split.toString().contains("isOriginal=true"));
if (split instanceof OrcSplit) {
assertTrue("Footer serialize test for ACID reader, hasFooter is expected in" +
" orc splits.", ((OrcSplit) split).hasFooter());
@@ -3512,10 +3516,12 @@ public class TestInputOutputFormat {
}
}
// call-1: open to read data - split 1 => mock:/mocktable6/0_0
- // call-2: open to read data - split 2 => mock:/mocktable6/0_1
- // call-3: AcidUtils.getAcidState - getLen() mock:/mocktable6/0_0
- // call-4: AcidUtils.getAcidState - getLen() mock:/mocktable6/0_1
- assertEquals(4, readOpsDelta);
+ // call-2: AcidUtils.getAcidState - split 1 => ls mock:/mocktable6
+ // call-3: open to read data - split 2 => mock:/mocktable6/0_1
+ // call-4: AcidUtils.getAcidState - split 2 => ls mock:/mocktable6
+ // call-5: read footer - split 2 => mock:/mocktable6/0_0 (to get offset since it's original file)
+ // call-6: file stat - split 2 => mock:/mocktable6/0_0
+ assertEquals(6, readOpsDelta);
// revert back to local fs
conf.set("fs.defaultFS", "file:///");
@@ -3883,7 +3889,7 @@ public class TestInputOutputFormat {
conf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, "0,2");
OrcSplit split = new OrcSplit(testFilePath, null, 0, fileLength,
new String[0], null, false, true,
- new ArrayList<AcidInputFormat.DeltaMetaData>(), fileLength, fileLength);
+ new ArrayList<AcidInputFormat.DeltaMetaData>(), fileLength, fileLength, workDir);
OrcInputFormat inputFormat = new OrcInputFormat();
AcidInputFormat.RowReader<OrcStruct> reader = inputFormat.getReader(split,
new AcidInputFormat.Options(conf));
@@ -3911,7 +3917,7 @@ public class TestInputOutputFormat {
conf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, "0,2,3");
split = new OrcSplit(testFilePath, null, 0, fileLength,
new String[0], null, false, true,
- new ArrayList<AcidInputFormat.DeltaMetaData>(), fileLength, fileLength);
+ new ArrayList<AcidInputFormat.DeltaMetaData>(), fileLength, fileLength, workDir);
inputFormat = new OrcInputFormat();
reader = inputFormat.getReader(split, new AcidInputFormat.Options(conf));
record = 0;
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
index 82cf108..9628a40 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
@@ -826,12 +826,9 @@ public class TestOrcRawRecordMerger {
merger.close();
//now run as if it's a minor Compaction so we don't collapse events
- //it's not exactly like minor compaction since MC would not have a baseReader
//here there is only 1 "split" since we only have data for 1 bucket
- baseReader = OrcFile.createReader(basePath,
- OrcFile.readerOptions(conf));
merger =
- new OrcRawRecordMerger(conf, false, baseReader, false, BUCKET,
+ new OrcRawRecordMerger(conf, false, null, false, BUCKET,
createMaximalTxnList(), new Reader.Options(),
AcidUtils.getPaths(directory.getCurrentDirectories()), new OrcRawRecordMerger.Options().isCompacting(true));
assertEquals(null, merger.getMinKey());
@@ -844,40 +841,86 @@ public class TestOrcRawRecordMerger {
assertNull(OrcRecordUpdater.getRow(event));
assertEquals(true, merger.next(id, event));
+ assertEquals(OrcRecordUpdater.DELETE_OPERATION,
+ OrcRecordUpdater.getOperation(event));
+ assertEquals(new ReaderKey(0, BUCKET_PROPERTY, 2, 200), id);
+ assertNull(OrcRecordUpdater.getRow(event));
+
+ assertEquals(true, merger.next(id, event));
+ assertEquals(OrcRecordUpdater.DELETE_OPERATION,
+ OrcRecordUpdater.getOperation(event));
+ assertEquals(new ReaderKey(0, BUCKET_PROPERTY, 3, 200), id);
+ assertNull(OrcRecordUpdater.getRow(event));
+
+ assertEquals(true, merger.next(id, event));
+ assertEquals(OrcRecordUpdater.DELETE_OPERATION,
+ OrcRecordUpdater.getOperation(event));
+ assertEquals(new ReaderKey(0, BUCKET_PROPERTY, 7, 200), id);
+ assertNull(OrcRecordUpdater.getRow(event));
+
+ assertEquals(true, merger.next(id, event));
+ assertEquals(OrcRecordUpdater.DELETE_OPERATION,
+ OrcRecordUpdater.getOperation(event));
+ assertEquals(new ReaderKey(0, BUCKET_PROPERTY, 8, 200), id);
+ assertNull(OrcRecordUpdater.getRow(event));
+
+ //data from delta_200_200
+ assertEquals(true, merger.next(id, event));
assertEquals(OrcRecordUpdater.INSERT_OPERATION,
OrcRecordUpdater.getOperation(event));
- assertEquals(new ReaderKey(0, BUCKET_PROPERTY, 0, 0), id);
- assertEquals("first", getValue(event));
+ assertEquals(new ReaderKey(200, BUCKET_PROPERTY, 0, 200), id);
+ assertEquals("update 1", getValue(event));
assertEquals(true, merger.next(id, event));
assertEquals(OrcRecordUpdater.INSERT_OPERATION,
OrcRecordUpdater.getOperation(event));
- assertEquals(new ReaderKey(0, BUCKET_PROPERTY, 1, 0), id);
- assertEquals("second", getValue(event));
+ assertEquals(new ReaderKey(200, BUCKET_PROPERTY, 1, 200), id);
+ assertEquals("update 2", getValue(event));
+
+ assertEquals(true, merger.next(id, event));
+ assertEquals(OrcRecordUpdater.INSERT_OPERATION,
+ OrcRecordUpdater.getOperation(event));
+ assertEquals(new ReaderKey(200, BUCKET_PROPERTY, 2, 200), id);
+ assertEquals("update 3", getValue(event));
+
+ assertEquals(false, merger.next(id, event));
+ merger.close();
+
+ //now run as if it's a major Compaction so we collapse events
+ //here there is only 1 "split" since we only have data for 1 bucket
+ baseReader = OrcFile.createReader(basePath,
+ OrcFile.readerOptions(conf));
+ merger =
+ new OrcRawRecordMerger(conf, true, baseReader, false, BUCKET,
+ createMaximalTxnList(), new Reader.Options(),
+ AcidUtils.getPaths(directory.getCurrentDirectories()), new OrcRawRecordMerger.Options()
+ .isCompacting(true).isMajorCompaction(true));
+ assertEquals(null, merger.getMinKey());
+ assertEquals(null, merger.getMaxKey());
assertEquals(true, merger.next(id, event));
assertEquals(OrcRecordUpdater.DELETE_OPERATION,
OrcRecordUpdater.getOperation(event));
- assertEquals(new ReaderKey(0, BUCKET_PROPERTY, 2, 200), id);
+ assertEquals(new ReaderKey(0, BUCKET_PROPERTY, 0, 200), id);
assertNull(OrcRecordUpdater.getRow(event));
assertEquals(true, merger.next(id, event));
assertEquals(OrcRecordUpdater.INSERT_OPERATION,
OrcRecordUpdater.getOperation(event));
- assertEquals(new ReaderKey(0, BUCKET_PROPERTY, 2, 0), id);
- assertEquals("third", getValue(event));
+ assertEquals(new ReaderKey(0, BUCKET_PROPERTY, 1, 0), id);
+ assertEquals("second", getValue(event));
assertEquals(true, merger.next(id, event));
assertEquals(OrcRecordUpdater.DELETE_OPERATION,
OrcRecordUpdater.getOperation(event));
- assertEquals(new ReaderKey(0, BUCKET_PROPERTY, 3, 200), id);
+ assertEquals(new ReaderKey(0, BUCKET_PROPERTY, 2, 200), id);
assertNull(OrcRecordUpdater.getRow(event));
assertEquals(true, merger.next(id, event));
- assertEquals(OrcRecordUpdater.INSERT_OPERATION,
+ assertEquals(OrcRecordUpdater.DELETE_OPERATION,
OrcRecordUpdater.getOperation(event));
- assertEquals(new ReaderKey(0, BUCKET_PROPERTY, 3, 0), id);
- assertEquals("fourth", getValue(event));
+ assertEquals(new ReaderKey(0, BUCKET_PROPERTY, 3, 200), id);
+ assertNull(OrcRecordUpdater.getRow(event));
assertEquals(true, merger.next(id, event));
assertEquals(OrcRecordUpdater.INSERT_OPERATION,
@@ -904,12 +947,6 @@ public class TestOrcRawRecordMerger {
assertNull(OrcRecordUpdater.getRow(event));
assertEquals(true, merger.next(id, event));
- assertEquals(OrcRecordUpdater.INSERT_OPERATION,
- OrcRecordUpdater.getOperation(event));
- assertEquals(new ReaderKey(0, BUCKET_PROPERTY, 7, 0), id);
- assertEquals("eighth", getValue(event));
-
- assertEquals(true, merger.next(id, event));
assertEquals(OrcRecordUpdater.DELETE_OPERATION,
OrcRecordUpdater.getOperation(event));
assertEquals(new ReaderKey(0, BUCKET_PROPERTY, 8, 200), id);
@@ -918,12 +955,6 @@ public class TestOrcRawRecordMerger {
assertEquals(true, merger.next(id, event));
assertEquals(OrcRecordUpdater.INSERT_OPERATION,
OrcRecordUpdater.getOperation(event));
- assertEquals(new ReaderKey(0, BUCKET_PROPERTY, 8, 0), id);
- assertEquals("ninth", getValue(event));
-
- assertEquals(true, merger.next(id, event));
- assertEquals(OrcRecordUpdater.INSERT_OPERATION,
- OrcRecordUpdater.getOperation(event));
assertEquals(new ReaderKey(0, BUCKET_PROPERTY, 9, 0), id);
assertEquals("tenth", getValue(event));
@@ -949,7 +980,6 @@ public class TestOrcRawRecordMerger {
assertEquals(false, merger.next(id, event));
merger.close();
-
// try ignoring the 200 transaction and make sure it works still
ValidTxnList txns = new ValidReadTxnList("2000:200:200");
//again 1st split is for base/
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/ql/src/test/queries/clientnegative/create_not_acid.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/create_not_acid.q b/ql/src/test/queries/clientnegative/create_not_acid.q
index 8d6c9ac..8ed13fb 100644
--- a/ql/src/test/queries/clientnegative/create_not_acid.q
+++ b/ql/src/test/queries/clientnegative/create_not_acid.q
@@ -2,5 +2,5 @@ set hive.support.concurrency=true;
set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
-create table acid_notbucketed(a int, b varchar(128)) stored as orc TBLPROPERTIES ('transactional'='true');
+create table acid_notbucketed(a int, b varchar(128)) TBLPROPERTIES ('transactional'='true');
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/ql/src/test/queries/clientpositive/acid_no_buckets.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/acid_no_buckets.q b/ql/src/test/queries/clientpositive/acid_no_buckets.q
new file mode 100644
index 0000000..c2f713e
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/acid_no_buckets.q
@@ -0,0 +1,210 @@
+--this has 4 groups of tests
+--Acid tables w/o bucketing
+--the tests with bucketing (make sure we get the same results)
+--same tests with and w/o vectorization
+
+set hive.mapred.mode=nonstrict;
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+set hive.exec.dynamic.partition.mode=nonstrict;
+set hive.vectorized.execution.enabled=false;
+set hive.explain.user=false;
+set hive.merge.cardinality.check=true;
+
+drop table if exists srcpart_acid;
+CREATE TABLE srcpart_acid (key STRING, value STRING) PARTITIONED BY (ds STRING, hr STRING) stored as ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default');
+insert into srcpart_acid PARTITION (ds, hr) select * from srcpart;
+
+--2 rows for 413, 1 row for 43, 2 for 213, 1 for 44 in kv1.txt (in each partition)
+select ds, hr, key, value from srcpart_acid where cast(key as integer) in(413,43) and hr='11' order by ds, hr, cast(key as integer);
+
+analyze table srcpart_acid PARTITION(ds, hr) compute statistics;
+analyze table srcpart_acid PARTITION(ds, hr) compute statistics for columns;
+explain update srcpart_acid set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11';
+update srcpart_acid set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11';
+select ds, hr, key, value from srcpart_acid where value like '%updated' order by ds, hr, cast(key as integer);
+
+insert into srcpart_acid PARTITION (ds='2008-04-08', hr=='11') values ('1001','val1001'),('1002','val1002'),('1003','val1003');
+select ds, hr, key, value from srcpart_acid where cast(key as integer) > 1000 order by ds, hr, cast(key as integer);
+
+analyze table srcpart_acid PARTITION(ds, hr) compute statistics;
+analyze table srcpart_acid PARTITION(ds, hr) compute statistics for columns;
+explain delete from srcpart_acid where key in( '1001', '213', '43');
+--delete some rows from initial load, some that were updated and some that were inserted
+delete from srcpart_acid where key in( '1001', '213', '43');
+
+--make sure we deleted everything that should've been deleted
+select count(*) from srcpart_acid where key in( '1001', '213', '43');
+--make sure nothing extra was deleted (2000 + 3 (insert) - 4 - 1 - 8 = 1990)
+select count(*) from srcpart_acid;
+
+--todo: should really have a way to run compactor here....
+
+--update should match 1 rows in 1 partition
+--delete should drop everything from 1 partition
+--insert should do nothing
+merge into srcpart_acid t using (select distinct ds, hr, key, value from srcpart_acid) s
+on s.ds=t.ds and s.hr=t.hr and s.key=t.key and s.value=t.value
+when matched and s.ds='2008-04-08' and s.hr=='11' and s.key='44' then update set value=concat(s.value,'updated by merge')
+when matched and s.ds='2008-04-08' and s.hr=='12' then delete
+when not matched then insert values('this','should','not','be there');
+
+--check results
+--should be 0
+select count(*) from srcpart_acid where ds='2008-04-08' and hr=='12';
+--should be 1 rows
+select ds, hr, key, value from srcpart_acid where value like '%updated by merge';
+--should be 0
+select count(*) from srcpart_acid where ds = 'this' and hr = 'should' and key = 'not' and value = 'be there';
+drop table if exists srcpart_acid;
+
+
+drop table if exists srcpart_acidb;
+CREATE TABLE srcpart_acidb (key STRING, value STRING) PARTITIONED BY (ds STRING, hr STRING) CLUSTERED BY(key) INTO 2 BUCKETS stored as ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default');
+insert into srcpart_acidb PARTITION (ds, hr) select * from srcpart;
+
+--2 rows for 413, 1 row for 43, 2 for 213, 2 for 12 in kv1.txt (in each partition)
+select ds, hr, key, value from srcpart_acidb where cast(key as integer) in(413,43) and hr='11' order by ds, hr, cast(key as integer);
+
+analyze table srcpart_acidb PARTITION(ds, hr) compute statistics;
+analyze table srcpart_acidb PARTITION(ds, hr) compute statistics for columns;
+explain update srcpart_acidb set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11';
+update srcpart_acidb set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11';
+select ds, hr, key, value from srcpart_acidb where value like '%updated' order by ds, hr, cast(key as integer);
+
+insert into srcpart_acidb PARTITION (ds='2008-04-08', hr=='11') values ('1001','val1001'),('1002','val1002'),('1003','val1003');
+select ds, hr, key, value from srcpart_acidb where cast(key as integer) > 1000 order by ds, hr, cast(key as integer);
+
+analyze table srcpart_acidb PARTITION(ds, hr) compute statistics;
+analyze table srcpart_acidb PARTITION(ds, hr) compute statistics for columns;
+explain delete from srcpart_acidb where key in( '1001', '213', '43');
+--delete some rows from initial load, some that were updated and some that were inserted
+delete from srcpart_acidb where key in( '1001', '213', '43');
+
+--make sure we deleted everything that should've been deleted
+select count(*) from srcpart_acidb where key in( '1001', '213', '43');
+--make sure nothing extra was deleted (2000 + 3 (insert) - 4 - 1 - 8 = 1990)
+select count(*) from srcpart_acidb;
+
+
+--todo: should really have a way to run compactor here....
+
+--update should match 1 rows in 1 partition
+--delete should drop everything from 1 partition
+--insert should do nothing
+merge into srcpart_acidb t using (select distinct ds, hr, key, value from srcpart_acidb) s
+on s.ds=t.ds and s.hr=t.hr and s.key=t.key and s.value=t.value
+when matched and s.ds='2008-04-08' and s.hr=='11' and s.key='44' then update set value=concat(s.value,'updated by merge')
+when matched and s.ds='2008-04-08' and s.hr=='12' then delete
+when not matched then insert values('this','should','not','be there');
+
+--check results
+--should be 0
+select count(*) from srcpart_acidb where ds='2008-04-08' and hr=='12';
+--should be 1 rows
+select ds, hr, key, value from srcpart_acidb where value like '%updated by merge';
+--should be 0
+select count(*) from srcpart_acidb where ds = 'this' and hr = 'should' and key = 'not' and value = 'be there';
+drop table if exists srcpart_acidb;
+
+
+
+--now same thing but vectorized
+set hive.vectorized.execution.enabled=true;
+
+drop table if exists srcpart_acidv;
+CREATE TABLE srcpart_acidv (key STRING, value STRING) PARTITIONED BY (ds STRING, hr STRING) stored as ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default');
+insert into srcpart_acidv PARTITION (ds, hr) select * from srcpart;
+
+--2 rows for 413, 21 row for 43, 2 for 213, 2 for 12 in kv1.txt (in each partition)
+select ds, hr, key, value from srcpart_acidv where cast(key as integer) in(413,43) and hr='11' order by ds, hr, cast(key as integer);
+
+analyze table srcpart_acidv PARTITION(ds, hr) compute statistics;
+analyze table srcpart_acidv PARTITION(ds, hr) compute statistics for columns;
+explain update srcpart_acidv set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11';
+update srcpart_acidv set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11';
+select ds, hr, key, value from srcpart_acidv where value like '%updated' order by ds, hr, cast(key as integer);
+
+insert into srcpart_acidv PARTITION (ds='2008-04-08', hr=='11') values ('1001','val1001'),('1002','val1002'),('1003','val1003');
+select ds, hr, key, value from srcpart_acidv where cast(key as integer) > 1000 order by ds, hr, cast(key as integer);
+
+analyze table srcpart_acidv PARTITION(ds, hr) compute statistics;
+analyze table srcpart_acidv PARTITION(ds, hr) compute statistics for columns;
+explain delete from srcpart_acidv where key in( '1001', '213', '43');
+--delete some rows from initial load, some that were updated and some that were inserted
+delete from srcpart_acidv where key in( '1001', '213', '43');
+
+--make sure we deleted everything that should've been deleted
+select count(*) from srcpart_acidv where key in( '1001', '213', '43');
+--make sure nothing extra was deleted (2000 + 3 - 4 - 1 - 8 = 1990)
+select count(*) from srcpart_acidv;
+
+--todo: should really have a way to run compactor here....
+
+--update should match 1 rows in 1 partition
+--delete should drop everything from 1 partition
+--insert should do nothing
+merge into srcpart_acidv t using (select distinct ds, hr, key, value from srcpart_acidv) s
+on s.ds=t.ds and s.hr=t.hr and s.key=t.key and s.value=t.value
+when matched and s.ds='2008-04-08' and s.hr=='11' and s.key='44' then update set value=concat(s.value,'updated by merge')
+when matched and s.ds='2008-04-08' and s.hr=='12' then delete
+when not matched then insert values('this','should','not','be there');
+
+--check results
+--should be 0
+select count(*) from srcpart_acidv where ds='2008-04-08' and hr=='12';
+--should be 1 rows
+select ds, hr, key, value from srcpart_acidv where value like '%updated by merge';
+--should be 0
+select count(*) from srcpart_acidv where ds = 'this' and hr = 'should' and key = 'not' and value = 'be there';
+drop table if exists srcpart_acidv;
+
+
+
+drop table if exists srcpart_acidvb;
+CREATE TABLE srcpart_acidvb (key STRING, value STRING) PARTITIONED BY (ds STRING, hr STRING) CLUSTERED BY(key) INTO 2 BUCKETS stored as ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default');
+insert into srcpart_acidvb PARTITION (ds, hr) select * from srcpart;
+
+--2 rows for 413, 1 row for 43, 2 for 213, 2 for 12 in kv1.txt (in each partition)
+select ds, hr, key, value from srcpart_acidvb where cast(key as integer) in(413,43) and hr='11' order by ds, hr, cast(key as integer);
+
+analyze table srcpart_acidvb PARTITION(ds, hr) compute statistics;
+analyze table srcpart_acidvb PARTITION(ds, hr) compute statistics for columns;
+explain update srcpart_acidvb set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11';
+update srcpart_acidvb set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11';
+select ds, hr, key, value from srcpart_acidvb where value like '%updated' order by ds, hr, cast(key as integer);
+
+insert into srcpart_acidvb PARTITION (ds='2008-04-08', hr=='11') values ('1001','val1001'),('1002','val1002'),('1003','val1003');
+select ds, hr, key, value from srcpart_acidvb where cast(key as integer) > 1000 order by ds, hr, cast(key as integer);
+
+analyze table srcpart_acidvb PARTITION(ds, hr) compute statistics;
+analyze table srcpart_acidvb PARTITION(ds, hr) compute statistics for columns;
+explain delete from srcpart_acidvb where key in( '1001', '213', '43');
+--delete some rows from initial load, some that were updated and some that were inserted
+delete from srcpart_acidvb where key in( '1001', '213', '43');
+
+--make sure we deleted everything that should've been deleted
+select count(*) from srcpart_acidvb where key in( '1001', '213', '43');
+--make sure nothing extra was deleted (2000 + 3 (insert) - 4 - 1 - 8 = 1990)
+select count(*) from srcpart_acidvb;
+
+
+--todo: should really have a way to run compactor here....
+
+--update should match 1 rows in 1 partition
+--delete should drop everything from 1 partition
+--insert should do nothing
+merge into srcpart_acidvb t using (select distinct ds, hr, key, value from srcpart_acidvb) s
+on s.ds=t.ds and s.hr=t.hr and s.key=t.key and s.value=t.value
+when matched and s.ds='2008-04-08' and s.hr=='11' and s.key='44' then update set value=concat(s.value,'updated by merge')
+when matched and s.ds='2008-04-08' and s.hr=='12' then delete
+when not matched then insert values('this','should','not','be there');
+
+--check results
+--should be 0
+select count(*) from srcpart_acidvb where ds='2008-04-08' and hr=='12';
+--should be 1 rows
+select ds, hr, key, value from srcpart_acidvb where value like '%updated by merge';
+--should be 0
+select count(*) from srcpart_acidvb where ds = 'this' and hr = 'should' and key = 'not' and value = 'be there';
+drop table if exists srcpart_acidvb;
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/ql/src/test/results/clientnegative/create_not_acid.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/create_not_acid.q.out b/ql/src/test/results/clientnegative/create_not_acid.q.out
index bb8f6c9..4e775e5 100644
--- a/ql/src/test/results/clientnegative/create_not_acid.q.out
+++ b/ql/src/test/results/clientnegative/create_not_acid.q.out
@@ -1,5 +1,5 @@
-PREHOOK: query: create table acid_notbucketed(a int, b varchar(128)) stored as orc TBLPROPERTIES ('transactional'='true')
+PREHOOK: query: create table acid_notbucketed(a int, b varchar(128)) TBLPROPERTIES ('transactional'='true')
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@acid_notbucketed
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:The table must be bucketed and stored using an ACID compliant format (such as ORC))
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:The table must be stored using an ACID compliant format (such as ORC))
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/ql/src/test/results/clientnegative/delete_non_acid_table.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/delete_non_acid_table.q.out b/ql/src/test/results/clientnegative/delete_non_acid_table.q.out
index a9b884a..19fd5fb 100644
--- a/ql/src/test/results/clientnegative/delete_non_acid_table.q.out
+++ b/ql/src/test/results/clientnegative/delete_non_acid_table.q.out
@@ -34,4 +34,4 @@ POSTHOOK: Input: default@not_an_acid_table2
-1070883071 0ruyd6Y50JpdGRf6HqD
-1070551679 iUR3Q
-1069736047 k17Am8uPHWk02cEf1jet
-FAILED: SemanticException [Error 10297]: Attempt to do update or delete on table default.not_an_acid_table2 that does not use an AcidOutputFormat or is not bucketed
+FAILED: SemanticException [Error 10297]: Attempt to do update or delete on table default.not_an_acid_table2 that is not transactional
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/ql/src/test/results/clientnegative/update_non_acid_table.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/update_non_acid_table.q.out b/ql/src/test/results/clientnegative/update_non_acid_table.q.out
index 381b0db..02946fc 100644
--- a/ql/src/test/results/clientnegative/update_non_acid_table.q.out
+++ b/ql/src/test/results/clientnegative/update_non_acid_table.q.out
@@ -34,4 +34,4 @@ POSTHOOK: Input: default@not_an_acid_table
-1070883071 0ruyd6Y50JpdGRf6HqD
-1070551679 iUR3Q
-1069736047 k17Am8uPHWk02cEf1jet
-FAILED: SemanticException [Error 10297]: Attempt to do update or delete on table default.not_an_acid_table that does not use an AcidOutputFormat or is not bucketed
+FAILED: SemanticException [Error 10297]: Attempt to do update or delete on table default.not_an_acid_table that is not transactional
[30/50] [abbrv] hive git commit: HIVE-17006 : LLAP: Parquet caching
v1 (Sergey Shelukhin, reviewed by Gunther Hagleitner)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/ql/src/test/queries/clientpositive/parquet_types.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/parquet_types.q b/ql/src/test/queries/clientpositive/parquet_types.q
index db37d2e..1a8844b 100644
--- a/ql/src/test/queries/clientpositive/parquet_types.q
+++ b/ql/src/test/queries/clientpositive/parquet_types.q
@@ -1,4 +1,5 @@
set hive.mapred.mode=nonstrict;
+set hive.llap.cache.allow.synthetic.fileid=true;
DROP TABLE parquet_types_staging;
DROP TABLE parquet_types;
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/ql/src/test/queries/clientpositive/parquet_types_vectorization.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/parquet_types_vectorization.q b/ql/src/test/queries/clientpositive/parquet_types_vectorization.q
index bb0e5b2..1f353ae 100644
--- a/ql/src/test/queries/clientpositive/parquet_types_vectorization.q
+++ b/ql/src/test/queries/clientpositive/parquet_types_vectorization.q
@@ -7,6 +7,7 @@ set hive.vectorized.execution.reduce.enabled=true;
set hive.vectorized.use.row.serde.deserialize=true;
set hive.vectorized.use.vector.serde.deserialize=true;
set hive.vectorized.execution.reduce.groupby.enabled = true;
+set hive.llap.cache.allow.synthetic.fileid=true;
CREATE TABLE parquet_types_staging (
cint int,
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/ql/src/test/queries/clientpositive/vectorized_parquet.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vectorized_parquet.q b/ql/src/test/queries/clientpositive/vectorized_parquet.q
index db02ec0..1efc60e 100644
--- a/ql/src/test/queries/clientpositive/vectorized_parquet.q
+++ b/ql/src/test/queries/clientpositive/vectorized_parquet.q
@@ -1,6 +1,7 @@
set hive.explain.user=false;
set hive.exec.submitviachild=false;
set hive.exec.submit.local.task.via.child=false;
+set hive.llap.cache.allow.synthetic.fileid=true;
create table if not exists alltypes_parquet (
cint int,
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/ql/src/test/queries/clientpositive/vectorized_parquet_types.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vectorized_parquet_types.q b/ql/src/test/queries/clientpositive/vectorized_parquet_types.q
index 7467cb3..63f811b 100644
--- a/ql/src/test/queries/clientpositive/vectorized_parquet_types.q
+++ b/ql/src/test/queries/clientpositive/vectorized_parquet_types.q
@@ -1,5 +1,6 @@
set hive.mapred.mode=nonstrict;
set hive.explain.user=false;
+set hive.llap.cache.allow.synthetic.fileid=true;
DROP TABLE parquet_types_staging;
DROP TABLE parquet_types;
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out b/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out
index aecbcfd..01458fe 100644
--- a/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out
+++ b/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out
@@ -140,7 +140,7 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint)
Execution mode: llap
- LLAP IO: no inputs
+ LLAP IO: all inputs (cache only)
Reducer 2
Execution mode: llap
Reduce Operator Tree:
@@ -198,7 +198,7 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint)
Execution mode: llap
- LLAP IO: no inputs
+ LLAP IO: all inputs (cache only)
Reducer 2
Execution mode: llap
Reduce Operator Tree:
@@ -540,7 +540,7 @@ STAGE PLANS:
sort order: ++
Statistics: Num rows: 29 Data size: 319 Basic stats: COMPLETE Column stats: NONE
Execution mode: llap
- LLAP IO: no inputs
+ LLAP IO: all inputs (cache only)
Reducer 2
Execution mode: llap
Reduce Operator Tree:
@@ -606,7 +606,7 @@ STAGE PLANS:
sort order: ++
Statistics: Num rows: 29 Data size: 319 Basic stats: COMPLETE Column stats: NONE
Execution mode: llap
- LLAP IO: no inputs
+ LLAP IO: all inputs (cache only)
Reducer 2
Execution mode: llap
Reduce Operator Tree:
@@ -739,7 +739,7 @@ STAGE PLANS:
TopN Hash Memory Usage: 0.1
value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double)
Execution mode: llap
- LLAP IO: no inputs
+ LLAP IO: all inputs (cache only)
Reducer 2
Execution mode: llap
Reduce Operator Tree:
@@ -818,7 +818,7 @@ STAGE PLANS:
TopN Hash Memory Usage: 0.1
value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double)
Execution mode: llap
- LLAP IO: no inputs
+ LLAP IO: all inputs (cache only)
Reducer 2
Execution mode: llap
Reduce Operator Tree:
@@ -1008,7 +1008,7 @@ STAGE PLANS:
TopN Hash Memory Usage: 0.1
value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double)
Execution mode: llap
- LLAP IO: no inputs
+ LLAP IO: all inputs (cache only)
Reducer 2
Execution mode: llap
Reduce Operator Tree:
@@ -1108,7 +1108,7 @@ STAGE PLANS:
TopN Hash Memory Usage: 0.1
value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double)
Execution mode: llap
- LLAP IO: no inputs
+ LLAP IO: all inputs (cache only)
Reducer 2
Execution mode: llap
Reduce Operator Tree:
@@ -1220,7 +1220,7 @@ STAGE PLANS:
TopN Hash Memory Usage: 0.1
value expressions: _col1 (type: int), _col2 (type: bigint)
Execution mode: llap
- LLAP IO: no inputs
+ LLAP IO: all inputs (cache only)
Reducer 2
Execution mode: llap
Reduce Operator Tree:
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/ql/src/test/results/clientpositive/llap/parquet_types_vectorization.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/parquet_types_vectorization.q.out b/ql/src/test/results/clientpositive/llap/parquet_types_vectorization.q.out
new file mode 100644
index 0000000..66bfb81
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/parquet_types_vectorization.q.out
@@ -0,0 +1,849 @@
+PREHOOK: query: DROP TABLE parquet_types_staging
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE parquet_types_staging
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE parquet_types
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE parquet_types
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE parquet_types_staging (
+ cint int,
+ ctinyint tinyint,
+ csmallint smallint,
+ cfloat float,
+ cdouble double,
+ cstring1 string,
+ t timestamp,
+ cchar char(5),
+ cvarchar varchar(10),
+ cbinary string,
+ m1 map<string, varchar(3)>,
+ l1 array<int>,
+ st1 struct<c1:int, c2:char(1)>,
+ d date
+) ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '|'
+COLLECTION ITEMS TERMINATED BY ','
+MAP KEYS TERMINATED BY ':'
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@parquet_types_staging
+POSTHOOK: query: CREATE TABLE parquet_types_staging (
+ cint int,
+ ctinyint tinyint,
+ csmallint smallint,
+ cfloat float,
+ cdouble double,
+ cstring1 string,
+ t timestamp,
+ cchar char(5),
+ cvarchar varchar(10),
+ cbinary string,
+ m1 map<string, varchar(3)>,
+ l1 array<int>,
+ st1 struct<c1:int, c2:char(1)>,
+ d date
+) ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '|'
+COLLECTION ITEMS TERMINATED BY ','
+MAP KEYS TERMINATED BY ':'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@parquet_types_staging
+PREHOOK: query: CREATE TABLE parquet_types (
+ cint int,
+ ctinyint tinyint,
+ csmallint smallint,
+ cfloat float,
+ cdouble double,
+ cstring1 string,
+ t timestamp,
+ cchar char(5),
+ cvarchar varchar(10),
+ cbinary binary,
+ m1 map<string, varchar(3)>,
+ l1 array<int>,
+ st1 struct<c1:int, c2:char(1)>,
+ d date
+) STORED AS PARQUET
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@parquet_types
+POSTHOOK: query: CREATE TABLE parquet_types (
+ cint int,
+ ctinyint tinyint,
+ csmallint smallint,
+ cfloat float,
+ cdouble double,
+ cstring1 string,
+ t timestamp,
+ cchar char(5),
+ cvarchar varchar(10),
+ cbinary binary,
+ m1 map<string, varchar(3)>,
+ l1 array<int>,
+ st1 struct<c1:int, c2:char(1)>,
+ d date
+) STORED AS PARQUET
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@parquet_types
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/parquet_types.txt' OVERWRITE INTO TABLE parquet_types_staging
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@parquet_types_staging
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/parquet_types.txt' OVERWRITE INTO TABLE parquet_types_staging
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@parquet_types_staging
+PREHOOK: query: SELECT * FROM parquet_types_staging
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_types_staging
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_types_staging
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_types_staging
+#### A masked pattern was here ####
+100 1 1 1.0 0.0 abc 2011-01-01 01:01:01.111111111 a a B4F3CAFDBEDD {"k1":"v1"} [101,200] {"c1":10,"c2":"a"} 2011-01-01
+101 2 2 1.1 0.3 def 2012-02-02 02:02:02.222222222 ab ab 68692CCAC0BDE7 {"k2":"v2"} [102,200] {"c1":10,"c2":"d"} 2012-02-02
+102 3 3 1.2 0.6 ghi 2013-03-03 03:03:03.333333333 abc abc B4F3CAFDBEDD {"k3":"v3"} [103,200] {"c1":10,"c2":"g"} 2013-03-03
+103 1 4 1.3 0.9 jkl 2014-04-04 04:04:04.444444444 abcd abcd 68692CCAC0BDE7 {"k4":"v4"} [104,200] {"c1":10,"c2":"j"} 2014-04-04
+104 2 5 1.4 1.2 mno 2015-05-05 05:05:05.555555555 abcde abcde B4F3CAFDBEDD {"k5":"v5"} [105,200] {"c1":10,"c2":"m"} 2015-05-05
+105 3 1 1.0 1.5 pqr 2016-06-06 06:06:06.666666666 abcde abcdef 68692CCAC0BDE7 {"k6":"v6"} [106,200] {"c1":10,"c2":"p"} 2016-06-06
+106 1 2 1.1 1.8 stu 2017-07-07 07:07:07.777777777 abcde abcdefg B4F3CAFDBEDD {"k7":"v7"} [107,200] {"c1":10,"c2":"s"} 2017-07-07
+107 2 3 1.2 2.1 vwx 2018-08-08 08:08:08.888888888 bcdef abcdefgh 68692CCAC0BDE7 {"k8":"v8"} [108,200] {"c1":10,"c2":"v"} 2018-08-08
+108 3 4 1.3 2.4 yza 2019-09-09 09:09:09.999999999 cdefg B4F3CAFDBE 68656C6C6F {"k9":"v9"} [109,200] {"c1":10,"c2":"y"} 2019-09-09
+109 1 5 1.4 2.7 bcd 2020-10-10 10:10:10.101010101 klmno abcdedef 68692CCAC0BDE7 {"k10":"v10"} [110,200] {"c1":10,"c2":"b"} 2020-10-10
+110 2 1 1.0 3.0 efg 2021-11-11 11:11:11.111111111 pqrst abcdede B4F3CAFDBEDD {"k11":"v11"} [111,200] {"c1":10,"c2":"e"} 2021-11-11
+111 3 2 1.1 3.3 hij 2022-12-12 12:12:12.121212121 nopqr abcded 68692CCAC0BDE7 {"k12":"v12"} [112,200] {"c1":10,"c2":"h"} 2022-12-12
+112 1 3 1.2 3.6 klm 2023-01-02 13:13:13.131313131 opqrs abcdd B4F3CAFDBEDD {"k13":"v13"} [113,200] {"c1":10,"c2":"k"} 2023-01-02
+113 2 4 1.3 3.9 nop 2024-02-02 14:14:14.141414141 pqrst abc 68692CCAC0BDE7 {"k14":"v14"} [114,200] {"c1":10,"c2":"n"} 2024-02-02
+114 3 5 1.4 4.2 qrs 2025-03-03 15:15:15.151515151 qrstu b B4F3CAFDBEDD {"k15":"v15"} [115,200] {"c1":10,"c2":"q"} 2025-03-03
+115 1 1 1.0 4.5 qrs 2026-04-04 16:16:16.161616161 rstuv abcded 68692CCAC0BDE7 {"k16":"v16"} [116,200] {"c1":10,"c2":"q"} 2026-04-04
+116 2 2 1.1 4.8 wxy 2027-05-05 17:17:17.171717171 stuvw abcded B4F3CAFDBEDD {"k17":"v17"} [117,200] {"c1":10,"c2":"w"} 2027-05-05
+117 3 3 1.2 5.1 zab 2028-06-06 18:18:18.181818181 tuvwx abcded 68692CCAC0BDE7 {"k18":"v18"} [118,200] {"c1":10,"c2":"z"} 2028-06-06
+118 1 4 1.3 5.4 cde 2029-07-07 19:19:19.191919191 uvwzy abcdede B4F3CAFDBEDD {"k19":"v19"} [119,200] {"c1":10,"c2":"c"} 2029-07-07
+119 2 5 1.4 5.7 fgh 2030-08-08 20:20:20.202020202 vwxyz abcdede 68692CCAC0BDE7 {"k20":"v20"} [120,200] {"c1":10,"c2":"f"} 2030-08-08
+120 3 1 1.0 6.0 ijk 2031-09-09 21:21:21.212121212 wxyza abcde B4F3CAFDBEDD {"k21":"v21"} [121,200] {"c1":10,"c2":"i"} 2031-09-09
+121 1 2 1.1 6.3 lmn 2032-10-10 22:22:22.222222222 bcdef abcde {"k22":"v22"} [122,200] {"c1":10,"c2":"l"} 2032-10-10
+PREHOOK: query: INSERT OVERWRITE TABLE parquet_types
+SELECT cint, ctinyint, csmallint, cfloat, cdouble, cstring1, t, cchar, cvarchar,
+unhex(cbinary), m1, l1, st1, d FROM parquet_types_staging
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_types_staging
+PREHOOK: Output: default@parquet_types
+POSTHOOK: query: INSERT OVERWRITE TABLE parquet_types
+SELECT cint, ctinyint, csmallint, cfloat, cdouble, cstring1, t, cchar, cvarchar,
+unhex(cbinary), m1, l1, st1, d FROM parquet_types_staging
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_types_staging
+POSTHOOK: Output: default@parquet_types
+POSTHOOK: Lineage: parquet_types.cbinary EXPRESSION [(parquet_types_staging)parquet_types_staging.FieldSchema(name:cbinary, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_types.cchar SIMPLE [(parquet_types_staging)parquet_types_staging.FieldSchema(name:cchar, type:char(5), comment:null), ]
+POSTHOOK: Lineage: parquet_types.cdouble SIMPLE [(parquet_types_staging)parquet_types_staging.FieldSchema(name:cdouble, type:double, comment:null), ]
+POSTHOOK: Lineage: parquet_types.cfloat SIMPLE [(parquet_types_staging)parquet_types_staging.FieldSchema(name:cfloat, type:float, comment:null), ]
+POSTHOOK: Lineage: parquet_types.cint SIMPLE [(parquet_types_staging)parquet_types_staging.FieldSchema(name:cint, type:int, comment:null), ]
+POSTHOOK: Lineage: parquet_types.csmallint SIMPLE [(parquet_types_staging)parquet_types_staging.FieldSchema(name:csmallint, type:smallint, comment:null), ]
+POSTHOOK: Lineage: parquet_types.cstring1 SIMPLE [(parquet_types_staging)parquet_types_staging.FieldSchema(name:cstring1, type:string, comment:null), ]
+POSTHOOK: Lineage: parquet_types.ctinyint SIMPLE [(parquet_types_staging)parquet_types_staging.FieldSchema(name:ctinyint, type:tinyint, comment:null), ]
+POSTHOOK: Lineage: parquet_types.cvarchar SIMPLE [(parquet_types_staging)parquet_types_staging.FieldSchema(name:cvarchar, type:varchar(10), comment:null), ]
+POSTHOOK: Lineage: parquet_types.d SIMPLE [(parquet_types_staging)parquet_types_staging.FieldSchema(name:d, type:date, comment:null), ]
+POSTHOOK: Lineage: parquet_types.l1 SIMPLE [(parquet_types_staging)parquet_types_staging.FieldSchema(name:l1, type:array<int>, comment:null), ]
+POSTHOOK: Lineage: parquet_types.m1 SIMPLE [(parquet_types_staging)parquet_types_staging.FieldSchema(name:m1, type:map<string,varchar(3)>, comment:null), ]
+POSTHOOK: Lineage: parquet_types.st1 SIMPLE [(parquet_types_staging)parquet_types_staging.FieldSchema(name:st1, type:struct<c1:int,c2:char(1)>, comment:null), ]
+POSTHOOK: Lineage: parquet_types.t SIMPLE [(parquet_types_staging)parquet_types_staging.FieldSchema(name:t, type:timestamp, comment:null), ]
+PREHOOK: query: EXPLAIN SELECT ctinyint,
+ MAX(cint),
+ MIN(csmallint),
+ COUNT(cstring1),
+ ROUND(AVG(cfloat), 5),
+ ROUND(STDDEV_POP(cdouble),5)
+FROM parquet_types
+GROUP BY ctinyint
+ORDER BY ctinyint
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT ctinyint,
+ MAX(cint),
+ MIN(csmallint),
+ COUNT(cstring1),
+ ROUND(AVG(cfloat), 5),
+ ROUND(STDDEV_POP(cdouble),5)
+FROM parquet_types
+GROUP BY ctinyint
+ORDER BY ctinyint
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: parquet_types
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cint (type: int), ctinyint (type: tinyint), csmallint (type: smallint), cfloat (type: float), cdouble (type: double), cstring1 (type: string)
+ outputColumnNames: cint, ctinyint, csmallint, cfloat, cdouble, cstring1
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: max(cint), min(csmallint), count(cstring1), avg(cfloat), stddev_pop(cdouble)
+ keys: ctinyint (type: tinyint)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: tinyint)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: tinyint)
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), _col4 (type: struct<count:bigint,sum:double,input:float>), _col5 (type: struct<count:bigint,sum:double,variance:double>)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs (cache only)
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: max(VALUE._col0), min(VALUE._col1), count(VALUE._col2), avg(VALUE._col3), stddev_pop(VALUE._col4)
+ keys: KEY._col0 (type: tinyint)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: tinyint), _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), round(_col4, 5) (type: double), round(_col5, 5) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: tinyint)
+ sort order: +
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), _col4 (type: double), _col5 (type: double)
+ Reducer 3
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: tinyint), VALUE._col0 (type: int), VALUE._col1 (type: smallint), VALUE._col2 (type: bigint), VALUE._col3 (type: double), VALUE._col4 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT ctinyint,
+ MAX(cint),
+ MIN(csmallint),
+ COUNT(cstring1),
+ ROUND(AVG(cfloat), 5),
+ ROUND(STDDEV_POP(cdouble),5)
+FROM parquet_types
+GROUP BY ctinyint
+ORDER BY ctinyint
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_types
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT ctinyint,
+ MAX(cint),
+ MIN(csmallint),
+ COUNT(cstring1),
+ ROUND(AVG(cfloat), 5),
+ ROUND(STDDEV_POP(cdouble),5)
+FROM parquet_types
+GROUP BY ctinyint
+ORDER BY ctinyint
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_types
+#### A masked pattern was here ####
+1 121 1 8 1.175 2.06216
+2 119 1 7 1.21429 1.8
+3 120 1 7 1.17143 1.8
+PREHOOK: query: EXPLAIN SELECT cfloat, count(*) FROM parquet_types GROUP BY cfloat ORDER BY cfloat
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT cfloat, count(*) FROM parquet_types GROUP BY cfloat ORDER BY cfloat
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: parquet_types
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cfloat (type: float)
+ outputColumnNames: cfloat
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ keys: cfloat (type: float)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: float)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: float)
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs (cache only)
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: float)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: float)
+ sort order: +
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Reducer 3
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: float), VALUE._col0 (type: bigint)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT cfloat, count(*) FROM parquet_types GROUP BY cfloat ORDER BY cfloat
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_types
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT cfloat, count(*) FROM parquet_types GROUP BY cfloat ORDER BY cfloat
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_types
+#### A masked pattern was here ####
+1.0 5
+1.1 5
+1.2 4
+1.3 4
+1.4 4
+PREHOOK: query: EXPLAIN SELECT cchar, count(*) FROM parquet_types GROUP BY cchar ORDER BY cchar
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT cchar, count(*) FROM parquet_types GROUP BY cchar ORDER BY cchar
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: parquet_types
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cchar (type: char(5))
+ outputColumnNames: cchar
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ keys: cchar (type: char(5))
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: char(5))
+ sort order: +
+ Map-reduce partition columns: _col0 (type: char(5))
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs (cache only)
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: char(5))
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: char(5))
+ sort order: +
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Reducer 3
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: char(5)), VALUE._col0 (type: bigint)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT cchar, count(*) FROM parquet_types GROUP BY cchar ORDER BY cchar
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_types
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT cchar, count(*) FROM parquet_types GROUP BY cchar ORDER BY cchar
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_types
+#### A masked pattern was here ####
+a 1
+ab 1
+abc 1
+abcd 1
+abcde 3
+bcdef 2
+cdefg 1
+klmno 1
+nopqr 1
+opqrs 1
+pqrst 2
+qrstu 1
+rstuv 1
+stuvw 1
+tuvwx 1
+uvwzy 1
+vwxyz 1
+wxyza 1
+PREHOOK: query: EXPLAIN SELECT cvarchar, count(*) FROM parquet_types GROUP BY cvarchar ORDER BY cvarchar
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT cvarchar, count(*) FROM parquet_types GROUP BY cvarchar ORDER BY cvarchar
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: parquet_types
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cvarchar (type: varchar(10))
+ outputColumnNames: cvarchar
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ keys: cvarchar (type: varchar(10))
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: varchar(10))
+ sort order: +
+ Map-reduce partition columns: _col0 (type: varchar(10))
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs (cache only)
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: varchar(10))
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: varchar(10))
+ sort order: +
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Reducer 3
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: varchar(10)), VALUE._col0 (type: bigint)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT cvarchar, count(*) FROM parquet_types GROUP BY cvarchar ORDER BY cvarchar
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_types
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT cvarchar, count(*) FROM parquet_types GROUP BY cvarchar ORDER BY cvarchar
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_types
+#### A masked pattern was here ####
+B4F3CAFDBE 1
+a 1
+ab 1
+abc 2
+abcd 1
+abcdd 1
+abcde 3
+abcded 4
+abcdede 3
+abcdedef 1
+abcdef 1
+abcdefg 1
+abcdefgh 1
+b 1
+PREHOOK: query: EXPLAIN SELECT cstring1, count(*) FROM parquet_types GROUP BY cstring1 ORDER BY cstring1
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT cstring1, count(*) FROM parquet_types GROUP BY cstring1 ORDER BY cstring1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: parquet_types
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cstring1 (type: string)
+ outputColumnNames: cstring1
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ keys: cstring1 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs (cache only)
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Reducer 3
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT cstring1, count(*) FROM parquet_types GROUP BY cstring1 ORDER BY cstring1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_types
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT cstring1, count(*) FROM parquet_types GROUP BY cstring1 ORDER BY cstring1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_types
+#### A masked pattern was here ####
+abc 1
+bcd 1
+cde 1
+def 1
+efg 1
+fgh 1
+ghi 1
+hij 1
+ijk 1
+jkl 1
+klm 1
+lmn 1
+mno 1
+nop 1
+pqr 1
+qrs 2
+stu 1
+vwx 1
+wxy 1
+yza 1
+zab 1
+PREHOOK: query: EXPLAIN SELECT t, count(*) FROM parquet_types GROUP BY t ORDER BY t
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT t, count(*) FROM parquet_types GROUP BY t ORDER BY t
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: parquet_types
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: t (type: timestamp)
+ outputColumnNames: t
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ keys: t (type: timestamp)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: timestamp)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: timestamp)
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs (cache only)
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: timestamp)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: timestamp)
+ sort order: +
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Reducer 3
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: timestamp), VALUE._col0 (type: bigint)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT t, count(*) FROM parquet_types GROUP BY t ORDER BY t
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_types
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT t, count(*) FROM parquet_types GROUP BY t ORDER BY t
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_types
+#### A masked pattern was here ####
+2011-01-01 01:01:01.111111111 1
+2012-02-02 02:02:02.222222222 1
+2013-03-03 03:03:03.333333333 1
+2014-04-04 04:04:04.444444444 1
+2015-05-05 05:05:05.555555555 1
+2016-06-06 06:06:06.666666666 1
+2017-07-07 07:07:07.777777777 1
+2018-08-08 08:08:08.888888888 1
+2019-09-09 09:09:09.999999999 1
+2020-10-10 10:10:10.101010101 1
+2021-11-11 11:11:11.111111111 1
+2022-12-12 12:12:12.121212121 1
+2023-01-02 13:13:13.131313131 1
+2024-02-02 14:14:14.141414141 1
+2025-03-03 15:15:15.151515151 1
+2026-04-04 16:16:16.161616161 1
+2027-05-05 17:17:17.171717171 1
+2028-06-06 18:18:18.181818181 1
+2029-07-07 19:19:19.191919191 1
+2030-08-08 20:20:20.202020202 1
+2031-09-09 21:21:21.212121212 1
+2032-10-10 22:22:22.222222222 1
+PREHOOK: query: EXPLAIN SELECT hex(cbinary), count(*) FROM parquet_types GROUP BY cbinary
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT hex(cbinary), count(*) FROM parquet_types GROUP BY cbinary
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: parquet_types
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cbinary (type: binary)
+ outputColumnNames: cbinary
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ keys: cbinary (type: binary)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: binary)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: binary)
+ Statistics: Num rows: 22 Data size: 308 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs (cache only)
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: binary)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: hex(_col0) (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 11 Data size: 154 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT hex(cbinary), count(*) FROM parquet_types GROUP BY cbinary
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_types
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT hex(cbinary), count(*) FROM parquet_types GROUP BY cbinary
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_types
+#### A masked pattern was here ####
+68692CCAC0BDE7 10
+ 1
+68656C6C6F 1
+B4F3CAFDBEDD 10
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out b/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out
index c5f7128..dd62df6 100644
--- a/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out
@@ -2734,7 +2734,7 @@ STAGE PLANS:
TopN Hash Memory Usage: 0.1
value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: timestamp), _col4 (type: float)
Execution mode: vectorized, llap
- LLAP IO: no inputs
+ LLAP IO: all inputs (cache only)
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
@@ -2915,7 +2915,7 @@ STAGE PLANS:
Statistics: Num rows: 137 Data size: 822 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
Execution mode: vectorized, llap
- LLAP IO: no inputs
+ LLAP IO: all inputs (cache only)
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
@@ -3411,7 +3411,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Execution mode: vectorized, llap
- LLAP IO: no inputs
+ LLAP IO: all inputs (cache only)
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
@@ -3646,7 +3646,7 @@ STAGE PLANS:
TopN Hash Memory Usage: 0.1
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp), _col3 (type: float)
Execution mode: vectorized, llap
- LLAP IO: no inputs
+ LLAP IO: all inputs (cache only)
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
@@ -3851,7 +3851,7 @@ STAGE PLANS:
Statistics: Num rows: 12 Data size: 768 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint)
Execution mode: vectorized, llap
- LLAP IO: no inputs
+ LLAP IO: all inputs (cache only)
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
@@ -4371,7 +4371,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Execution mode: vectorized, llap
- LLAP IO: no inputs
+ LLAP IO: all inputs (cache only)
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
@@ -4606,7 +4606,7 @@ STAGE PLANS:
TopN Hash Memory Usage: 0.1
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: date), _col3 (type: float)
Execution mode: vectorized, llap
- LLAP IO: no inputs
+ LLAP IO: all inputs (cache only)
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
@@ -4811,7 +4811,7 @@ STAGE PLANS:
Statistics: Num rows: 12 Data size: 576 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint)
Execution mode: vectorized, llap
- LLAP IO: no inputs
+ LLAP IO: all inputs (cache only)
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/ql/src/test/results/clientpositive/llap/vectorized_parquet.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vectorized_parquet.q.out b/ql/src/test/results/clientpositive/llap/vectorized_parquet.q.out
index 8a84d3d..781c8a1 100644
--- a/ql/src/test/results/clientpositive/llap/vectorized_parquet.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorized_parquet.q.out
@@ -159,7 +159,7 @@ STAGE PLANS:
Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), _col4 (type: struct<count:bigint,sum:double,input:float>), _col5 (type: struct<count:bigint,sum:double,variance:double>)
Execution mode: vectorized, llap
- LLAP IO: no inputs
+ LLAP IO: all inputs (cache only)
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/ql/src/test/results/clientpositive/llap/vectorized_parquet_types.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vectorized_parquet_types.q.out b/ql/src/test/results/clientpositive/llap/vectorized_parquet_types.q.out
index cdf6b3d..0303e0e 100644
--- a/ql/src/test/results/clientpositive/llap/vectorized_parquet_types.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorized_parquet_types.q.out
@@ -312,7 +312,7 @@ STAGE PLANS:
Statistics: Num rows: 22 Data size: 242 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), _col4 (type: struct<count:bigint,sum:double,input:float>), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: decimal(4,2))
Execution mode: vectorized, llap
- LLAP IO: no inputs
+ LLAP IO: all inputs (cache only)
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/storage-api/src/java/org/apache/hadoop/hive/common/io/FileMetadataCache.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/FileMetadataCache.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/FileMetadataCache.java
new file mode 100644
index 0000000..403c3ad
--- /dev/null
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/io/FileMetadataCache.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common.io;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.hadoop.hive.common.io.encoded.MemoryBufferOrBuffers;
+
+public interface FileMetadataCache {
+ /**
+ * @return Metadata for a given file (ORC or Parquet footer).
+ * The caller must decref this buffer when done.
+ */
+ MemoryBufferOrBuffers getFileMetadata(Object fileKey);
+
+ // TODO: add BB put method(s) when merging with ORC off-heap metadata cache
+
+ /**
+ * Puts the metadata for a given file (e.g. a footer buffer into cache).
+ * @param fileKey The file key.
+ * @param length The footer length.
+ * @param is The stream to read the footer from.
+ * @return The buffer or buffers representing the cached footer.
+ * The caller must decref this buffer when done.
+ */
+ MemoryBufferOrBuffers putFileMetadata(
+ Object fileKey, int length, InputStream is) throws IOException;
+
+ /**
+ * Releases the buffer returned from getFileMetadata or putFileMetadata method.
+ * @param buffer The buffer to release.
+ */
+ void decRefBuffer(MemoryBufferOrBuffers buffer);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBufferOrBuffers.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBufferOrBuffers.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBufferOrBuffers.java
new file mode 100644
index 0000000..c04310a
--- /dev/null
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBufferOrBuffers.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common.io.encoded;
+
+public interface MemoryBufferOrBuffers {
+ MemoryBuffer getSingleBuffer();
+ MemoryBuffer[] getMultipleBuffers();
+}
\ No newline at end of file
[38/50] [abbrv] hive git commit: HIVE-14671 : merge master into
hive-14535 (Wei Zheng)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java
----------------------------------------------------------------------
diff --cc standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java
index 0000000,6cc7157..3759348
mode 000000,100644..100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java
@@@ -1,0 -1,256 +1,256 @@@
+ /**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.hadoop.hive.metastore.model;
+
+ import java.util.List;
+ import java.util.Map;
+
+ public class MTable {
+
+ private String tableName;
+ private MDatabase database;
+ private MStorageDescriptor sd;
+ private String owner;
+ private int createTime;
+ private int lastAccessTime;
+ private int retention;
+ private List<MFieldSchema> partitionKeys;
+ private Map<String, String> parameters;
+ private String viewOriginalText;
+ private String viewExpandedText;
+ private boolean rewriteEnabled;
+ private String tableType;
+
+ public MTable() {}
+
+ /**
+ * @param tableName
+ * @param database
+ * @param sd
+ * @param owner
+ * @param createTime
+ * @param lastAccessTime
+ * @param retention
+ * @param partitionKeys
+ * @param parameters
+ * @param viewOriginalText
+ * @param viewExpandedText
+ * @param tableType
+ */
+ public MTable(String tableName, MDatabase database, MStorageDescriptor sd, String owner,
+ int createTime, int lastAccessTime, int retention, List<MFieldSchema> partitionKeys,
- Map<String, String> parameters, String viewOriginalText, String viewExpandedText,
- boolean rewriteEnabled, String tableType) {
++ Map<String, String> parameters,
++ String viewOriginalText, String viewExpandedText, boolean rewriteEnabled, String tableType) {
+ this.tableName = tableName;
+ this.database = database;
+ this.sd = sd;
+ this.owner = owner;
+ this.createTime = createTime;
+ this.setLastAccessTime(lastAccessTime);
+ this.retention = retention;
+ this.partitionKeys = partitionKeys;
+ this.parameters = parameters;
+ this.viewOriginalText = viewOriginalText;
+ this.viewExpandedText = viewExpandedText;
+ this.rewriteEnabled = rewriteEnabled;
+ this.tableType = tableType;
+ }
+
+ /**
+ * @return the tableName
+ */
+ public String getTableName() {
+ return tableName;
+ }
+
+ /**
+ * @param tableName the tableName to set
+ */
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ /**
+ * @return the sd
+ */
+ public MStorageDescriptor getSd() {
+ return sd;
+ }
+
+ /**
+ * @param sd the sd to set
+ */
+ public void setSd(MStorageDescriptor sd) {
+ this.sd = sd;
+ }
+
+ /**
+ * @return the partKeys
+ */
+ public List<MFieldSchema> getPartitionKeys() {
+ return partitionKeys;
+ }
+
+ /**
+ * @param partKeys the partKeys to set
+ */
+ public void setPartitionKeys(List<MFieldSchema> partKeys) {
+ this.partitionKeys = partKeys;
+ }
+
+ /**
+ * @return the parameters
+ */
+ public Map<String, String> getParameters() {
+ return parameters;
+ }
+
+ /**
+ * @param parameters the parameters to set
+ */
+ public void setParameters(Map<String, String> parameters) {
+ this.parameters = parameters;
+ }
+
+ /**
+ * @return the original view text, or null if this table is not a view
+ */
+ public String getViewOriginalText() {
+ return viewOriginalText;
+ }
+
+ /**
+ * @param viewOriginalText the original view text to set
+ */
+ public void setViewOriginalText(String viewOriginalText) {
+ this.viewOriginalText = viewOriginalText;
+ }
+
+ /**
+ * @return the expanded view text, or null if this table is not a view
+ */
+ public String getViewExpandedText() {
+ return viewExpandedText;
+ }
+
+ /**
+ * @param viewExpandedText the expanded view text to set
+ */
+ public void setViewExpandedText(String viewExpandedText) {
+ this.viewExpandedText = viewExpandedText;
+ }
+
+ /**
+ * @return whether the view can be used for rewriting queries
+ */
+ public boolean isRewriteEnabled() {
+ return rewriteEnabled;
+ }
+
+ /**
+ * @param rewriteEnabled whether the view can be used for rewriting queries
+ */
+ public void setRewriteEnabled(boolean rewriteEnabled) {
+ this.rewriteEnabled = rewriteEnabled;
+ }
+
+ /**
+ * @return the owner
+ */
+ public String getOwner() {
+ return owner;
+ }
+
+ /**
+ * @param owner the owner to set
+ */
+ public void setOwner(String owner) {
+ this.owner = owner;
+ }
+
+ /**
+ * @return the createTime
+ */
+ public int getCreateTime() {
+ return createTime;
+ }
+
+ /**
+ * @param createTime the createTime to set
+ */
+ public void setCreateTime(int createTime) {
+ this.createTime = createTime;
+ }
+
+ /**
+ * @return the database
+ */
+ public MDatabase getDatabase() {
+ return database;
+ }
+
+ /**
+ * @param database the database to set
+ */
+ public void setDatabase(MDatabase database) {
+ this.database = database;
+ }
+
+ /**
+ * @return the retention
+ */
+ public int getRetention() {
+ return retention;
+ }
+
+ /**
+ * @param retention the retention to set
+ */
+ public void setRetention(int retention) {
+ this.retention = retention;
+ }
+
+ /**
+ * @param lastAccessTime the lastAccessTime to set
+ */
+ public void setLastAccessTime(int lastAccessTime) {
+ this.lastAccessTime = lastAccessTime;
+ }
+
+ /**
+ * @return the lastAccessTime
+ */
+ public int getLastAccessTime() {
+ return lastAccessTime;
+ }
+
+ /**
+ * @param tableType the tableType to set
+ */
+ public void setTableType(String tableType) {
+ this.tableType = tableType;
+ }
+
+ /**
+ * @return the tableType
+ */
+ public String getTableType() {
+ return tableType;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/standalone-metastore/src/main/resources/package.jdo
----------------------------------------------------------------------
diff --cc standalone-metastore/src/main/resources/package.jdo
index 0000000,570fd44..bbb8f2d
mode 000000,100644..100644
--- a/standalone-metastore/src/main/resources/package.jdo
+++ b/standalone-metastore/src/main/resources/package.jdo
@@@ -1,0 -1,1088 +1,1087 @@@
+ <?xml version="1.0"?>
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <!DOCTYPE jdo PUBLIC "-//Sun Microsystems, Inc.//DTD Java Data Objects Metadata 2.0//EN"
+ "http://java.sun.com/dtd/jdo_2_0.dtd">
+ <!--
+ Size Limitations:
+
+ Indexed VARCHAR: 767 bytes (MySQL running on InnoDB Engine http://bugs.mysql.com/bug.php?id=13315)
+ Non-indexed VARCHAR: 4000 bytes (max length on Oracle 9i/10g/11g)
+
+ -->
+ <jdo>
+ <package name="org.apache.hadoop.hive.metastore.model">
+ <class name="MDatabase" identity-type="datastore" table="DBS" detachable="true">
+ <datastore-identity>
+ <column name="DB_ID"/>
+ </datastore-identity>
+ <field name="name">
+ <column name="NAME" length="128" jdbc-type="VARCHAR"/>
+ <index name="UniqueDatabase" unique="true"/>
+ </field>
+ <field name="description">
+ <column name="DESC" length="4000" jdbc-type="VARCHAR" allows-null="true"/>
+ </field>
+ <field name="locationUri">
+ <column name="DB_LOCATION_URI" length="4000" jdbc-type="VARCHAR" allows-null="false"/>
+ </field>
+ <field name="parameters" table="DATABASE_PARAMS">
+ <map key-type="java.lang.String" value-type="java.lang.String"/>
+ <join>
+ <column name="DB_ID"/>
+ </join>
+ <key>
+ <column name="PARAM_KEY" length="180" jdbc-type="VARCHAR"/>
+ </key>
+ <value>
+ <column name="PARAM_VALUE" length="4000" jdbc-type="VARCHAR"/>
+ </value>
+ </field>
- <field name="ownerName">
++ <field name="ownerName">
+ <column name="OWNER_NAME" length="128" jdbc-type="VARCHAR" allows-null="true"/>
+ </field>
+ <field name="ownerType">
+ <column name="OWNER_TYPE" length="10" jdbc-type="VARCHAR" allows-null="true"/>
+ </field>
+ </class>
+
+ <class name="MFieldSchema" embedded-only="true" table="TYPE_FIELDS" detachable="true">
+ <field name="name">
+ <column name="FNAME" length="767" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="type" >
+ <column name="FTYPE" length="32672" jdbc-type="VARCHAR" allows-null="false"/>
+ </field>
+ <field name="comment" >
+ <column name="FCOMMENT" length="4000" jdbc-type="VARCHAR" allows-null="true"/>
+ </field>
+ </class>
+
+ <class name="MType" table="TYPES" detachable="true">
+ <field name="name" >
+ <column name="TYPE_NAME" length="128" jdbc-type="VARCHAR"/>
+ <index name="UniqueType" unique="true"/>
+ </field>
+ <field name="type1">
+ <column name="TYPE1" length="767" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="type2">
+ <column name="TYPE2" length="767" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="fields" table="TYPE_FIELDS" >
+ <collection element-type="MFieldSchema"/>
+ <join>
+ <primary-key name="TYPE_FIELDS_PK">
+ <column name="TYPE_NAME"/>
+ <column name="FIELD_NAME"/>
+ </primary-key>
+ <column name="TYPE_NAME"/>
+ </join>
+ <element>
+ <embedded>
+ <field name="name">
+ <column name="FIELD_NAME" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="type">
+ <column name="FIELD_TYPE" length="767" jdbc-type="VARCHAR" allows-null="false"/>
+ </field>
+ <field name="comment" >
+ <column name="COMMENT" length="256" jdbc-type="VARCHAR" allows-null="true"/>
+ </field>
+ </embedded>
+ </element>
+ </field>
+ </class>
+
+ <class name="MTable" table="TBLS" identity-type="datastore" detachable="true">
+ <datastore-identity>
+ <column name="TBL_ID"/>
+ </datastore-identity>
+ <index name="UniqueTable" unique="true">
+ <column name="TBL_NAME"/>
+ <column name="DB_ID"/>
+ </index>
+ <field name="tableName">
+ <column name="TBL_NAME" length="256" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="database">
+ <column name="DB_ID"/>
+ </field>
+ <field name="partitionKeys" table="PARTITION_KEYS" >
+ <collection element-type="MFieldSchema"/>
+ <join>
+ <primary-key name="PARTITION_KEY_PK">
+ <column name="TBL_ID"/>
+ <column name="PKEY_NAME"/>
+ </primary-key>
+ <column name="TBL_ID"/>
+ </join>
+ <element>
+ <embedded>
+ <field name="name">
+ <column name="PKEY_NAME" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="type">
+ <column name="PKEY_TYPE" length="767" jdbc-type="VARCHAR" allows-null="false"/>
+ </field>
+ <field name="comment" >
+ <column name="PKEY_COMMENT" length="4000" jdbc-type="VARCHAR" allows-null="true"/>
+ </field>
+ </embedded>
+ </element>
+ </field>
+ <field name="sd" dependent="true">
+ <column name="SD_ID"/>
+ </field>
+ <field name="owner">
+ <column name="OWNER" length="767" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="createTime">
+ <column name="CREATE_TIME" jdbc-type="integer"/>
+ </field>
+ <field name="lastAccessTime">
+ <column name="LAST_ACCESS_TIME" jdbc-type="integer"/>
+ </field>
+ <field name="retention">
+ <column name="RETENTION" jdbc-type="integer"/>
+ </field>
+ <field name="parameters" table="TABLE_PARAMS">
+ <map key-type="java.lang.String" value-type="java.lang.String"/>
+ <join>
+ <column name="TBL_ID"/>
+ </join>
+ <key>
+ <column name="PARAM_KEY" length="256" jdbc-type="VARCHAR"/>
+ </key>
+ <value>
+ <column name="PARAM_VALUE" length="32672" jdbc-type="VARCHAR"/>
+ </value>
+ </field>
+ <field name="viewOriginalText" default-fetch-group="false">
+ <column name="VIEW_ORIGINAL_TEXT" jdbc-type="LONGVARCHAR"/>
+ </field>
+ <field name="viewExpandedText" default-fetch-group="false">
+ <column name="VIEW_EXPANDED_TEXT" jdbc-type="LONGVARCHAR"/>
+ </field>
+ <field name="rewriteEnabled">
+ <column name="IS_REWRITE_ENABLED"/>
+ </field>
+ <field name="tableType">
+ <column name="TBL_TYPE" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ </class>
+
+ <class name="MConstraint" identity-type="application" table="KEY_CONSTRAINTS" detachable="true" objectid-class="MConstraint$PK">
+ <field name="constraintName" primary-key="true">
+ <column name="CONSTRAINT_NAME"/>
+ </field>
+ <field name="position" primary-key="true">
+ <column name="POSITION"/>
+ </field>
+ <field name="childColumn">
+ <column name="CHILD_CD_ID"/>
+ </field>
+ <field name="childIntegerIndex">
+ <column name="CHILD_INTEGER_IDX"/>
+ </field>
+ <field name="childTable">
+ <column name="CHILD_TBL_ID"/>
+ </field>
+ <field name="parentColumn">
+ <column name="PARENT_CD_ID"/>
+ </field>
+ <field name="parentIntegerIndex">
- <column name="PARENT_INTEGER_IDX"/>
++ <column name="PARENT_INTEGER_IDX"/>
+ </field>
+ <field name="parentTable">
+ <column name="PARENT_TBL_ID"/>
+ </field>
+ <field name="constraintType">
+ <column name="CONSTRAINT_TYPE"/>
+ </field>
+ <field name="deleteRule">
- <column name="DELETE_RULE"/>
++ <column name="DELETE_RULE"/>
+ </field>
+ <field name="updateRule">
+ <column name="UPDATE_RULE"/>
+ </field>
+ <field name="enableValidateRely">
+ <column name="ENABLE_VALIDATE_RELY"/>
+ </field>
+ </class>
+
+ <class name="MSerDeInfo" identity-type="datastore" table="SERDES" detachable="true">
+ <datastore-identity>
+ <column name="SERDE_ID"/>
+ </datastore-identity>
+ <field name="name">
+ <column name="NAME" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="serializationLib">
+ <column name="SLIB" length="4000" jdbc-type="VARCHAR" allows-null="true"/>
+ </field>
+ <field name="parameters" table="SERDE_PARAMS">
+ <map key-type="java.lang.String" value-type="java.lang.String"/>
+ <join>
+ <column name="SERDE_ID"/>
+ </join>
+ <key>
+ <column name="PARAM_KEY" length="256" jdbc-type="VARCHAR"/>
+ </key>
+ <value>
+ <column name="PARAM_VALUE" length="32672" jdbc-type="VARCHAR"/>
+ </value>
+ </field>
+ </class>
+
+ <class name="MOrder" embedded-only="true" table="SORT_ORDER" detachable="true">
+ <field name="col">
+ <column name="COL_NAME" length="767" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="order">
+ <column name="ORDER" jdbc-type="INTEGER" allows-null="false"/>
+ </field>
+ </class>
+
+ <class name="MColumnDescriptor" identity-type="datastore" table="CDS" detachable="true">
+ <datastore-identity>
+ <column name="CD_ID"/>
+ </datastore-identity>
+ <field name="cols" table="COLUMNS_V2" >
+ <collection element-type="MFieldSchema"/>
+ <join>
+ <primary-key name="COLUMNS_PK">
+ <column name="COLUMN_NAME"/>
+ </primary-key>
+ <column name="CD_ID"/>
+ </join>
+ <element>
+ <embedded>
+ <field name="name">
+ <column name="COLUMN_NAME" length="767" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="type">
+ <column name="TYPE_NAME" length="32672" jdbc-type="VARCHAR" allows-null="false"/>
+ </field>
+ <field name="comment">
+ <column name="COMMENT" length="256" jdbc-type="VARCHAR" allows-null="true"/>
+ </field>
+ </embedded>
+ </element>
+ </field>
- </class>
++ </class>
+
+ <class name="MStringList" identity-type="datastore" table="Skewed_STRING_LIST" detachable="true">
+ <datastore-identity>
+ <column name="STRING_LIST_ID"/>
+ </datastore-identity>
+ <field name="internalList" table="Skewed_STRING_LIST_VALUES">
+ <collection element-type="java.lang.String"/>
+ <join>
+ <column name="STRING_LIST_ID"/>
+ </join>
+ <element column="STRING_LIST_VALUE"/>
+ </field>
+ </class>
+
+ <class name="MStorageDescriptor" identity-type="datastore" table="SDS" detachable="true">
+ <datastore-identity>
+ <column name="SD_ID"/>
+ </datastore-identity>
+ <field name="cd">
- <column name="CD_ID"/>
++ <column name="CD_ID"/>
+ </field>
+ <field name="location">
+ <column name="LOCATION" length="4000" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="inputFormat">
+ <column name="INPUT_FORMAT" length="4000" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="outputFormat">
+ <column name="OUTPUT_FORMAT" length="4000" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="isCompressed">
+ <column name="IS_COMPRESSED"/>
+ </field>
+ <field name="isStoredAsSubDirectories">
+ <column name="IS_STOREDASSUBDIRECTORIES"/>
+ </field>
+ <field name="numBuckets">
+ <column name="NUM_BUCKETS" jdbc-type="integer"/>
+ </field>
+ <field name="serDeInfo" dependent="true">
+ <column name="SERDE_ID"/>
+ </field>
+ <field name="bucketCols" table="BUCKETING_COLS">
+ <collection element-type="java.lang.String"/>
+ <join>
+ <column name="SD_ID"/>
+ </join>
+ <element column="BUCKET_COL_NAME"/>
+ </field>
+ <field name="sortCols" table="SORT_COLS">
+ <collection element-type="MOrder"/>
+ <join>
+ <column name="SD_ID"/>
+ </join>
+ <element>
+ <embedded>
+ <field name="col">
+ <column name="COLUMN_NAME" length="767" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="order">
+ <column name="ORDER" jdbc-type="INTEGER" allows-null="false"/>
+ </field>
+ </embedded>
+ </element>
+ </field>
+ <field name="parameters" table="SD_PARAMS">
+ <map key-type="java.lang.String" value-type="java.lang.String"/>
+ <join>
+ <column name="SD_ID"/>
+ </join>
+ <key>
+ <column name="PARAM_KEY" length="256" jdbc-type="VARCHAR"/>
+ </key>
+ <value>
+ <column name="PARAM_VALUE" length="32672" jdbc-type="VARCHAR"/>
+ </value>
+ </field>
+ <field name="skewedColNames" table="SKEWED_COL_NAMES">
+ <collection element-type="java.lang.String"/>
+ <join>
+ <column name="SD_ID"/>
+ </join>
+ <element column="SKEWED_COL_NAME"/>
+ </field>
+ <field name="skewedColValues" table="SKEWED_VALUES">
+ <collection element-type="MStringList"/>
+ <join/>
+ </field>
+ <field name="skewedColValueLocationMaps" table="SKEWED_COL_VALUE_LOC_MAP">
+ <map key-type="MStringList" value-type="java.lang.String"/>
+ <join>
+ <column name="SD_ID"/>
+ </join>
+ <value>
+ <column name="location" length="4000" jdbc-type="VARCHAR"/>
+ </value>
+ </field>
+ </class>
+
+ <class name="MPartition" table="PARTITIONS" identity-type="datastore" detachable="true">
+ <index name="UniquePartition" unique="true">
+ <column name="PART_NAME"/>
+ <column name="TBL_ID"/>
+ </index>
+ <datastore-identity>
+ <column name="PART_ID"/>
+ </datastore-identity>
+ <field name="partitionName">
+ <column name="PART_NAME" length="767" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="table">
+ <column name="TBL_ID"/>
+ </field>
+ <field name="createTime">
+ <column name="CREATE_TIME" jdbc-type="integer"/>
+ </field>
+ <field name="lastAccessTime">
+ <column name="LAST_ACCESS_TIME" jdbc-type="integer"/>
+ </field>
+ <field name="values" table="PARTITION_KEY_VALS">
+ <collection element-type="java.lang.String"/>
+ <join>
+ <column name="PART_ID"/>
+ </join>
+ <element column="PART_KEY_VAL"/>
+ </field>
+ <field name="sd" dependent="true">
+ <column name="SD_ID"/>
+ </field>
+ <field name="parameters" table="PARTITION_PARAMS">
+ <map key-type="java.lang.String" value-type="java.lang.String"/>
+ <join>
+ <column name="PART_ID"/>
+ </join>
+ <key>
+ <column name="PARAM_KEY" length="256" jdbc-type="VARCHAR"/>
+ </key>
+ <value>
+ <column name="PARAM_VALUE" length="4000" jdbc-type="VARCHAR"/>
+ </value>
+ </field>
+ </class>
+
+ <class name="MIndex" table="IDXS" identity-type="datastore" detachable="true">
+ <index name="UniqueINDEX" unique="true">
+ <column name="INDEX_NAME"/>
+ <column name="ORIG_TBL_ID"/>
+ </index>
+
+ <datastore-identity>
+ <column name="INDEX_ID"/>
+ </datastore-identity>
+ <field name="indexName">
+ <column name="INDEX_NAME" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="origTable">
+ <column name="ORIG_TBL_ID"/>
+ </field>
+ <field name="indexTable">
+ <column name="INDEX_TBL_ID"/>
+ </field>
+ <field name="indexHandlerClass">
+ <column name="INDEX_HANDLER_CLASS" length="4000" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="deferredRebuild">
+ <column name="DEFERRED_REBUILD"/>
+ </field>
+ <field name="createTime">
+ <column name="CREATE_TIME" jdbc-type="integer"/>
+ </field>
+ <field name="lastAccessTime">
+ <column name="LAST_ACCESS_TIME" jdbc-type="integer"/>
+ </field>
+ <field name="sd" dependent="true">
+ <column name="SD_ID"/>
+ </field>
+ <field name="parameters" table="INDEX_PARAMS">
+ <map key-type="java.lang.String" value-type="java.lang.String"/>
+ <join>
+ <column name="INDEX_ID"/>
+ </join>
+ <key>
+ <column name="PARAM_KEY" length="256" jdbc-type="VARCHAR"/>
+ </key>
+ <value>
+ <column name="PARAM_VALUE" length="4000" jdbc-type="VARCHAR"/>
+ </value>
+ </field>
+ </class>
+
+ <class name="MRole" table="ROLES" identity-type="datastore" detachable="true">
+
+ <index name="RoleEntityINDEX" unique="true">
+ <column name="ROLE_NAME"/>
+ </index>
+
+ <datastore-identity>
+ <column name="ROLE_ID"/>
+ </datastore-identity>
+
+ <field name="roleName">
+ <column name="ROLE_NAME" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="createTime">
+ <column name="CREATE_TIME" jdbc-type="integer"/>
+ </field>
+ <field name="ownerName">
+ <column name="OWNER_NAME" length="128" jdbc-type="VARCHAR"/>
+ </field>
+
+ </class>
+
+ <class name="MRoleMap" table="ROLE_MAP" identity-type="datastore" detachable="true">
+ <index name="UserRoleMapINDEX" unique="true">
+ <column name="PRINCIPAL_NAME"/>
+ <column name="ROLE_ID"/>
+ <column name="GRANTOR"/>
+ <column name="GRANTOR_TYPE"/>
+ </index>
+
+ <datastore-identity>
+ <column name="ROLE_GRANT_ID"/>
+ </datastore-identity>
+
+ <field name="role">
+ <column name="ROLE_ID" />
+ </field>
+ <field name="principalName">
+ <column name="PRINCIPAL_NAME" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="principalType">
+ <column name="PRINCIPAL_TYPE" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="addTime">
+ <column name="ADD_TIME" jdbc-type="integer"/>
+ </field>
+ <field name="grantor">
+ <column name="GRANTOR" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="grantorType">
+ <column name="GRANTOR_TYPE" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="grantOption">
+ <column name="GRANT_OPTION" jdbc-type="SMALLINT"/>
+ </field>
+ </class>
+
+ <class name="MGlobalPrivilege" table="GLOBAL_PRIVS" identity-type="datastore" detachable="true">
+
+ <index name="GlobalPrivilegeIndex" unique="true">
+ <column name="PRINCIPAL_NAME"/>
+ <column name="PRINCIPAL_TYPE"/>
+ <column name="USER_PRIV"/>
+ <column name="GRANTOR"/>
+ <column name="GRANTOR_TYPE"/>
+ </index>
+
+ <datastore-identity>
+ <column name="USER_GRANT_ID"/>
+ </datastore-identity>
+
+ <field name="principalName">
+ <column name="PRINCIPAL_NAME" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="principalType">
+ <column name="PRINCIPAL_TYPE" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="privilege">
+ <column name="USER_PRIV" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="createTime">
+ <column name="CREATE_TIME" jdbc-type="integer"/>
+ </field>
+ <field name="grantor">
+ <column name="GRANTOR" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="grantorType">
+ <column name="GRANTOR_TYPE" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="grantOption">
+ <column name="GRANT_OPTION" jdbc-type="SMALLINT"/>
+ </field>
+ </class>
+
+ <class name="MDBPrivilege" table="DB_PRIVS" identity-type="datastore" detachable="true">
+
+ <index name="DBPrivilegeIndex" unique="true">
+ <column name="DB_ID"/>
+ <column name="PRINCIPAL_NAME"/>
+ <column name="PRINCIPAL_TYPE"/>
+ <column name="DB_PRIV"/>
+ <column name="GRANTOR"/>
+ <column name="GRANTOR_TYPE"/>
+ </index>
+
+ <datastore-identity>
+ <column name="DB_GRANT_ID"/>
+ </datastore-identity>
+
+ <field name="principalName">
+ <column name="PRINCIPAL_NAME" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="principalType">
+ <column name="PRINCIPAL_TYPE" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="database">
+ <column name="DB_ID" />
+ </field>
+ <field name="privilege">
+ <column name="DB_PRIV" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="createTime">
+ <column name="CREATE_TIME" jdbc-type="integer"/>
+ </field>
+ <field name="grantor">
+ <column name="GRANTOR" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="grantorType">
+ <column name="GRANTOR_TYPE" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="grantOption">
+ <column name="GRANT_OPTION" jdbc-type="SMALLINT"/>
+ </field>
+ </class>
+
+ <class name="MTablePrivilege" table="TBL_PRIVS" identity-type="datastore" detachable="true">
+
+ <index name="TablePrivilegeIndex" unique="false">
+ <column name="TBL_ID"/>
+ <column name="PRINCIPAL_NAME"/>
+ <column name="PRINCIPAL_TYPE"/>
+ <column name="TBL_PRIV"/>
+ <column name="GRANTOR"/>
+ <column name="GRANTOR_TYPE"/>
+ </index>
+
+ <datastore-identity>
+ <column name="TBL_GRANT_ID"/>
+ </datastore-identity>
+
+ <field name="principalName">
+ <column name="PRINCIPAL_NAME" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="principalType">
+ <column name="PRINCIPAL_TYPE" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="table">
+ <column name="TBL_ID"/>
+ </field>
+ <field name="privilege">
+ <column name="TBL_PRIV" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="createTime">
+ <column name="CREATE_TIME" jdbc-type="integer"/>
+ </field>
+ <field name="grantor">
+ <column name="GRANTOR" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="grantorType">
+ <column name="GRANTOR_TYPE" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="grantOption">
+ <column name="GRANT_OPTION" jdbc-type="SMALLINT"/>
+ </field>
+ </class>
+
+ <class name="MPartitionPrivilege" table="PART_PRIVS" identity-type="datastore" detachable="true">
+
+ <index name="PartPrivilegeIndex" unique="false">
+ <column name="PART_ID"/>
+ <column name="PRINCIPAL_NAME"/>
+ <column name="PRINCIPAL_TYPE"/>
+ <column name="PART_PRIV"/>
+ <column name="GRANTOR"/>
+ <column name="GRANTOR_TYPE"/>
+ </index>
+
+ <datastore-identity>
+ <column name="PART_GRANT_ID"/>
+ </datastore-identity>
+
+ <field name="principalName">
+ <column name="PRINCIPAL_NAME" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="principalType">
+ <column name="PRINCIPAL_TYPE" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="partition">
+ <column name="PART_ID" />
+ </field>
+ <field name="privilege">
+ <column name="PART_PRIV" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="createTime">
+ <column name="CREATE_TIME" jdbc-type="integer"/>
+ </field>
+ <field name="grantor">
+ <column name="GRANTOR" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="grantorType">
+ <column name="GRANTOR_TYPE" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="grantOption">
+ <column name="GRANT_OPTION" jdbc-type="SMALLINT"/>
+ </field>
+ </class>
+
+ <class name="MTableColumnPrivilege" table="TBL_COL_PRIVS" identity-type="datastore" detachable="true">
+
+ <index name="TableColumnPrivilegeIndex" unique="false">
+ <column name="TBL_ID"/>
+ <column name="COLUMN_NAME"/>
+ <column name="PRINCIPAL_NAME"/>
+ <column name="PRINCIPAL_TYPE"/>
+ <column name="TBL_COL_PRIV"/>
+ <column name="GRANTOR"/>
+ <column name="GRANTOR_TYPE"/>
+ </index>
+
+ <datastore-identity>
+ <column name="TBL_COLUMN_GRANT_ID"/>
+ </datastore-identity>
+
+ <field name="principalName">
+ <column name="PRINCIPAL_NAME" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="principalType">
+ <column name="PRINCIPAL_TYPE" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="table">
+ <column name="TBL_ID" />
+ </field>
+ <field name="columnName">
+ <column name="COLUMN_NAME" length="767" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="privilege">
+ <column name="TBL_COL_PRIV" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="createTime">
+ <column name="CREATE_TIME" jdbc-type="integer"/>
+ </field>
+ <field name="grantor">
+ <column name="GRANTOR" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="grantorType">
+ <column name="GRANTOR_TYPE" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="grantOption">
+ <column name="GRANT_OPTION" jdbc-type="SMALLINT"/>
+ </field>
+ </class>
+
+ <class name="MPartitionColumnPrivilege" table="PART_COL_PRIVS" identity-type="datastore" detachable="true">
+
+ <index name="PartitionColumnPrivilegeIndex" unique="false">
+ <column name="PART_ID"/>
+ <column name="COLUMN_NAME"/>
+ <column name="PRINCIPAL_NAME"/>
+ <column name="PRINCIPAL_TYPE"/>
+ <column name="PART_COL_PRIV"/>
+ <column name="GRANTOR"/>
+ <column name="GRANTOR_TYPE"/>
+ </index>
+
+ <datastore-identity>
+ <column name="PART_COLUMN_GRANT_ID"/>
+ </datastore-identity>
+
+ <field name="principalName">
+ <column name="PRINCIPAL_NAME" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="principalType">
+ <column name="PRINCIPAL_TYPE" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="partition">
+ <column name="PART_ID" />
+ </field>
+ <field name="columnName">
+ <column name="COLUMN_NAME" length="767" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="privilege">
+ <column name="PART_COL_PRIV" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="createTime">
+ <column name="CREATE_TIME" jdbc-type="integer"/>
+ </field>
+ <field name="grantor">
+ <column name="GRANTOR" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="grantorType">
+ <column name="GRANTOR_TYPE" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="grantOption">
+ <column name="GRANT_OPTION" jdbc-type="SMALLINT"/>
+ </field>
+ </class>
+
+ <class name="MPartitionEvent" table="PARTITION_EVENTS" identity-type="datastore" detachable="true">
+
+ <index name="PartitionEventIndex" unique="false">
+ <column name="PARTITION_NAME"/>
+ </index>
+
+ <datastore-identity>
+ <column name="PART_NAME_ID"/>
+ </datastore-identity>
+
+ <field name="dbName">
+ <column name="DB_NAME" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="tblName">
+ <column name="TBL_NAME" length="256" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="partName">
+ <column name="PARTITION_NAME" length="767" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="eventType">
+ <column name="EVENT_TYPE" jdbc-type="integer"/>
+ </field>
+ <field name="eventTime">
+ <column name="EVENT_TIME" jdbc-type="BIGINT"/>
+ </field>
+
+ </class>
+
+ <class name="MMasterKey" table="MASTER_KEYS" identity-type="application" detachable="true">
+
+ <field name="keyId" primary-key="true" value-strategy="native">
+ <column name="KEY_ID" jdbc-type="integer" />
+ </field>
+
+ <field name="masterKey">
+ <column name="MASTER_KEY" length="767" jdbc-type="VARCHAR" />
+ </field>
+
+ </class>
+
+ <class name="MDelegationToken" table="DELEGATION_TOKENS" identity-type="application" detachable="true">
+
+ <field name="tokenIdentifier" primary-key="true">
+ <column name="TOKEN_IDENT" length="767" jdbc-type="VARCHAR" />
+ </field>
+
+ <field name="tokenStr">
+ <column name="TOKEN" length="767" jdbc-type="VARCHAR" />
+ </field>
+
+ </class>
+
+ <class name="MTableColumnStatistics" table="TAB_COL_STATS" identity-type="datastore" detachable="true">
+ <datastore-identity>
+ <column name="CS_ID"/>
+ </datastore-identity>
+
+ <field name ="dbName">
+ <column name="DB_NAME" length="128" jdbc-type="VARCHAR" allows-null="false"/>
+ </field>
+ <field name="tableName">
+ <column name="TABLE_NAME" length="256" jdbc-type="VARCHAR" allows-null="false"/>
+ </field>
+ <field name="table">
+ <column name="TBL_ID"/>
+ </field>
+ <field name="colName">
+ <column name="COLUMN_NAME" length="767" jdbc-type="VARCHAR" allows-null="false"/>
+ </field>
+ <field name="colType">
+ <column name="COLUMN_TYPE" length="128" jdbc-type="VARCHAR" allows-null="false"/>
+ </field>
+ <field name="longLowValue">
+ <column name="LONG_LOW_VALUE" jdbc-type="BIGINT" allows-null="true"/>
+ </field>
+ <field name="longHighValue">
+ <column name="LONG_HIGH_VALUE" jdbc-type="BIGINT" allows-null="true"/>
+ </field>
+ <field name="doubleLowValue">
+ <column name="DOUBLE_LOW_VALUE" jdbc-type="DOUBLE" allows-null="true"/>
+ </field>
+ <field name="doubleHighValue">
+ <column name="DOUBLE_HIGH_VALUE" jdbc-type="DOUBLE" allows-null="true"/>
+ </field>
+ <field name="decimalLowValue">
+ <column name="BIG_DECIMAL_LOW_VALUE" jdbc-type="VARCHAR" allows-null="true"/>
+ </field>
+ <field name="decimalHighValue">
+ <column name="BIG_DECIMAL_HIGH_VALUE" jdbc-type="VARCHAR" allows-null="true"/>
+ </field>
+ <field name="numNulls">
+ <column name="NUM_NULLS" jdbc-type="BIGINT" allows-null="false"/>
+ </field>
+ <field name="numDVs">
+ <column name="NUM_DISTINCTS" jdbc-type="BIGINT" allows-null="true"/>
+ </field>
+ <field name="bitVector">
+ <column name="BIT_VECTOR" jdbc-type="BLOB" allows-null="true"/>
+ </field>
+ <field name="avgColLen">
+ <column name="AVG_COL_LEN" jdbc-type="DOUBLE" allows-null="true"/>
+ </field>
+ <field name="maxColLen">
+ <column name="MAX_COL_LEN" jdbc-type="BIGINT" allows-null="true"/>
+ </field>
+ <field name="numTrues">
+ <column name="NUM_TRUES" jdbc-type="BIGINT" allows-null="true"/>
+ </field>
+ <field name="numFalses">
+ <column name="NUM_FALSES" jdbc-type="BIGINT" allows-null="true"/>
+ </field>
+ <field name="lastAnalyzed">
+ <column name="LAST_ANALYZED" jdbc-type="BIGINT" allows-null="false"/>
+ </field>
+ </class>
+
+ <class name="MPartitionColumnStatistics" table="PART_COL_STATS" identity-type="datastore" detachable="true">
+ <datastore-identity>
+ <column name="CS_ID"/>
+ </datastore-identity>
+
+ <field name ="dbName">
+ <column name="DB_NAME" length="128" jdbc-type="VARCHAR" allows-null="false"/>
+ </field>
+ <field name="tableName">
+ <column name="TABLE_NAME" length="256" jdbc-type="VARCHAR" allows-null="false"/>
+ </field>
+ <field name="partitionName">
+ <column name="PARTITION_NAME" length="767" jdbc-type="VARCHAR" allows-null="false"/>
+ </field>
+ <field name="partition">
+ <column name="PART_ID"/>
+ </field>
+ <field name="colName">
+ <column name="COLUMN_NAME" length="767" jdbc-type="VARCHAR" allows-null="false"/>
+ </field>
+ <field name="colType">
+ <column name="COLUMN_TYPE" length="128" jdbc-type="VARCHAR" allows-null="false"/>
+ </field>
+ <field name="longLowValue">
+ <column name="LONG_LOW_VALUE" jdbc-type="BIGINT" allows-null="true"/>
+ </field>
+ <field name="longHighValue">
+ <column name="LONG_HIGH_VALUE" jdbc-type="BIGINT" allows-null="true"/>
+ </field>
+ <field name="doubleLowValue">
+ <column name="DOUBLE_LOW_VALUE" jdbc-type="DOUBLE" allows-null="true"/>
+ </field>
+ <field name="doubleHighValue">
+ <column name="DOUBLE_HIGH_VALUE" jdbc-type="DOUBLE" allows-null="true"/>
+ </field>
+ <field name="decimalLowValue">
+ <column name="BIG_DECIMAL_LOW_VALUE" jdbc-type="VARCHAR" allows-null="true"/>
+ </field>
+ <field name="decimalHighValue">
+ <column name="BIG_DECIMAL_HIGH_VALUE" jdbc-type="VARCHAR" allows-null="true"/>
+ </field>
+ <field name="numNulls">
+ <column name="NUM_NULLS" jdbc-type="BIGINT" allows-null="false"/>
+ </field>
+ <field name="numDVs">
+ <column name="NUM_DISTINCTS" jdbc-type="BIGINT" allows-null="true"/>
+ </field>
+ <field name="bitVector">
+ <column name="BIT_VECTOR" jdbc-type="BLOB" allows-null="true"/>
+ </field>
+ <field name="avgColLen">
+ <column name="AVG_COL_LEN" jdbc-type="DOUBLE" allows-null="true"/>
+ </field>
+ <field name="maxColLen">
+ <column name="MAX_COL_LEN" jdbc-type="BIGINT" allows-null="true"/>
+ </field>
+ <field name="numTrues">
+ <column name="NUM_TRUES" jdbc-type="BIGINT" allows-null="true"/>
+ </field>
+ <field name="numFalses">
+ <column name="NUM_FALSES" jdbc-type="BIGINT" allows-null="true"/>
+ </field>
+ <field name="lastAnalyzed">
+ <column name="LAST_ANALYZED" jdbc-type="BIGINT" allows-null="false"/>
+ </field>
+ </class>
+ <class name="MVersionTable" table="VERSION" identity-type="datastore" detachable="true">
+ <datastore-identity>
+ <column name="VER_ID"/>
+ </datastore-identity>
+ <field name ="schemaVersion">
+ <column name="SCHEMA_VERSION" length="127" jdbc-type="VARCHAR" allows-null="false"/>
+ </field>
+ <field name ="versionComment">
+ <column name="VERSION_COMMENT" length="255" jdbc-type="VARCHAR" allows-null="false"/>
+ </field>
+ </class>
+
+ <class name="MMetastoreDBProperties" table="METASTORE_DB_PROPERTIES" identity-type="application" detachable="true">
+ <field name ="propertyKey" primary-key="true">
+ <column name="PROPERTY_KEY" length="255" jdbc-type="VARCHAR" allows-null="false"/>
+ </field>
+ <field name ="propertyValue">
+ <column name="PROPERTY_VALUE" length="1000" jdbc-type="VARCHAR" allows-null="false"/>
+ </field>
+ <field name ="description">
+ <column name="DESCRIPTION" length="1000" jdbc-type="VARCHAR" allows-null="false"/>
+ </field>
+ </class>
+
+ <class name="MResourceUri" embedded-only="true" table="RESOURCE_URI" detachable="true">
+ <field name="resourceType">
+ <column name="RESOURCE_TYPE" jdbc-type="INTEGER"/>
+ </field>
+ <field name="uri">
+ <column name="RESOURCE_URI" length="4000" jdbc-type="VARCHAR"/>
+ </field>
+ </class>
+
+ <class name="MFunction" table="FUNCS" identity-type="datastore" detachable="true">
+ <datastore-identity>
+ <column name="FUNC_ID"/>
+ </datastore-identity>
+ <index name="UniqueFunction" unique="true">
+ <column name="FUNC_NAME"/>
+ <column name="DB_ID"/>
+ </index>
+ <field name="functionName">
+ <column name="FUNC_NAME" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="database">
+ <column name="DB_ID"/>
+ </field>
+ <field name="functionType">
+ <column name="FUNC_TYPE" jdbc-type="integer"/>
+ </field>
+ <field name="className">
+ <column name="CLASS_NAME" length="4000" jdbc-type="VARCHAR"/>
+ </field>
- <field name="ownerName">
++ <field name="ownerName">
+ <column name="OWNER_NAME" length="128" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="ownerType">
+ <column name="OWNER_TYPE" length="10" jdbc-type="VARCHAR"/>
+ </field>
+ <field name="createTime">
+ <column name="CREATE_TIME" jdbc-type="integer"/>
+ </field>
+ <field name="resourceUris" table="FUNC_RU">
+ <collection element-type="MResourceUri"/>
+ <join>
+ <column name="FUNC_ID"/>
+ </join>
+ <element>
+ <embedded>
+ <field name="resourceType">
+ <column name="RESOURCE_TYPE" jdbc-type="INTEGER"/>
+ </field>
+ <field name="uri">
+ <column name="RESOURCE_URI" length="4000" jdbc-type="VARCHAR"/>
+ </field>
+ </embedded>
+ </element>
+ </field>
+ </class>
+
+ <class name="MNotificationLog" table="NOTIFICATION_LOG" identity-type="datastore" detachable="true">
+ <datastore-identity>
+ <column name="NL_ID"/>
+ </datastore-identity>
+ <field name="eventId">
+ <column name="EVENT_ID" jdbc-type="BIGINT" allows-null="false"/>
+ </field>
+ <field name="eventTime">
+ <column name="EVENT_TIME" jdbc-type="INTEGER" allows-null="false"/>
+ </field>
+ <field name="eventType">
+ <column name="EVENT_TYPE" length="32" jdbc-type="VARCHAR" allows-null="false"/>
+ </field>
+ <field name="dbName">
+ <column name="DB_NAME" length="128" jdbc-type="VARCHAR" allows-null="true"/>
+ </field>
+ <field name="tableName">
+ <column name="TBL_NAME" length="256" jdbc-type="VARCHAR" allows-null="true"/>
+ </field>
+ <field name="message">
+ <column name="MESSAGE" jdbc-type="LONGVARCHAR"/>
+ </field>
+ <field name="messageFormat">
+ <column name="MESSAGE_FORMAT" length="16" jdbc-type="VARCHAR" allows-null="true"/>
+ </field>
+ </class>
+
+ <!-- I tried to use a sequence here but derby didn't handle it well. -->
+ <class name="MNotificationNextId" table="NOTIFICATION_SEQUENCE" identity-type="datastore" detachable="true">
+ <datastore-identity>
+ <column name="NNI_ID"/>
+ </datastore-identity>
+ <field name="nextEventId">
+ <column name="NEXT_EVENT_ID" jdbc-type="BIGINT" allows-null="false"/>
+ </field>
+ </class>
+
-
+ </package>
+ </jdo>
+
[20/50] [abbrv] hive git commit: HIVE-17100 : Improve HS2 operation
logs for REPL commands (Sankar Hariappan, reviewed by Anishek Agarwal,
Thejas Nair)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py
index 2ecf555..4b45c3a 100644
--- a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py
+++ b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py
@@ -1224,6 +1224,13 @@ class Iface(fb303.FacebookService.Iface):
def get_current_notificationEventId(self):
pass
+ def get_notification_events_count(self, rqst):
+ """
+ Parameters:
+ - rqst
+ """
+ pass
+
def fire_listener_event(self, rqst):
"""
Parameters:
@@ -6774,6 +6781,37 @@ class Client(fb303.FacebookService.Client, Iface):
return result.success
raise TApplicationException(TApplicationException.MISSING_RESULT, "get_current_notificationEventId failed: unknown result")
+ def get_notification_events_count(self, rqst):
+ """
+ Parameters:
+ - rqst
+ """
+ self.send_get_notification_events_count(rqst)
+ return self.recv_get_notification_events_count()
+
+ def send_get_notification_events_count(self, rqst):
+ self._oprot.writeMessageBegin('get_notification_events_count', TMessageType.CALL, self._seqid)
+ args = get_notification_events_count_args()
+ args.rqst = rqst
+ args.write(self._oprot)
+ self._oprot.writeMessageEnd()
+ self._oprot.trans.flush()
+
+ def recv_get_notification_events_count(self):
+ iprot = self._iprot
+ (fname, mtype, rseqid) = iprot.readMessageBegin()
+ if mtype == TMessageType.EXCEPTION:
+ x = TApplicationException()
+ x.read(iprot)
+ iprot.readMessageEnd()
+ raise x
+ result = get_notification_events_count_result()
+ result.read(iprot)
+ iprot.readMessageEnd()
+ if result.success is not None:
+ return result.success
+ raise TApplicationException(TApplicationException.MISSING_RESULT, "get_notification_events_count failed: unknown result")
+
def fire_listener_event(self, rqst):
"""
Parameters:
@@ -7200,6 +7238,7 @@ class Processor(fb303.FacebookService.Processor, Iface, TProcessor):
self._processMap["add_dynamic_partitions"] = Processor.process_add_dynamic_partitions
self._processMap["get_next_notification"] = Processor.process_get_next_notification
self._processMap["get_current_notificationEventId"] = Processor.process_get_current_notificationEventId
+ self._processMap["get_notification_events_count"] = Processor.process_get_notification_events_count
self._processMap["fire_listener_event"] = Processor.process_fire_listener_event
self._processMap["flushCache"] = Processor.process_flushCache
self._processMap["cm_recycle"] = Processor.process_cm_recycle
@@ -10946,6 +10985,25 @@ class Processor(fb303.FacebookService.Processor, Iface, TProcessor):
oprot.writeMessageEnd()
oprot.trans.flush()
+ def process_get_notification_events_count(self, seqid, iprot, oprot):
+ args = get_notification_events_count_args()
+ args.read(iprot)
+ iprot.readMessageEnd()
+ result = get_notification_events_count_result()
+ try:
+ result.success = self._handler.get_notification_events_count(args.rqst)
+ msg_type = TMessageType.REPLY
+ except (TTransport.TTransportException, KeyboardInterrupt, SystemExit):
+ raise
+ except Exception as ex:
+ msg_type = TMessageType.EXCEPTION
+ logging.exception(ex)
+ result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
+ oprot.writeMessageBegin("get_notification_events_count", msg_type, seqid)
+ result.write(oprot)
+ oprot.writeMessageEnd()
+ oprot.trans.flush()
+
def process_fire_listener_event(self, seqid, iprot, oprot):
args = fire_listener_event_args()
args.read(iprot)
@@ -36964,6 +37022,133 @@ class get_current_notificationEventId_result:
def __ne__(self, other):
return not (self == other)
+class get_notification_events_count_args:
+ """
+ Attributes:
+ - rqst
+ """
+
+ thrift_spec = None
+ def __init__(self, rqst=None,):
+ self.rqst = rqst
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == -1:
+ if ftype == TType.STRUCT:
+ self.rqst = NotificationEventsCountRequest()
+ self.rqst.read(iprot)
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('get_notification_events_count_args')
+ if self.rqst is not None:
+ oprot.writeFieldBegin('rqst', TType.STRUCT, -1)
+ self.rqst.write(oprot)
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.rqst)
+ return value
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
+class get_notification_events_count_result:
+ """
+ Attributes:
+ - success
+ """
+
+ thrift_spec = (
+ (0, TType.STRUCT, 'success', (NotificationEventsCountResponse, NotificationEventsCountResponse.thrift_spec), None, ), # 0
+ )
+
+ def __init__(self, success=None,):
+ self.success = success
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 0:
+ if ftype == TType.STRUCT:
+ self.success = NotificationEventsCountResponse()
+ self.success.read(iprot)
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('get_notification_events_count_result')
+ if self.success is not None:
+ oprot.writeFieldBegin('success', TType.STRUCT, 0)
+ self.success.write(oprot)
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.success)
+ return value
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
class fire_listener_event_args:
"""
Attributes:
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
index 53671e3..7570895 100644
--- a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
+++ b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
@@ -11935,6 +11935,155 @@ class CurrentNotificationEventId:
def __ne__(self, other):
return not (self == other)
+class NotificationEventsCountRequest:
+ """
+ Attributes:
+ - fromEventId
+ - dbName
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.I64, 'fromEventId', None, None, ), # 1
+ (2, TType.STRING, 'dbName', None, None, ), # 2
+ )
+
+ def __init__(self, fromEventId=None, dbName=None,):
+ self.fromEventId = fromEventId
+ self.dbName = dbName
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.I64:
+ self.fromEventId = iprot.readI64()
+ else:
+ iprot.skip(ftype)
+ elif fid == 2:
+ if ftype == TType.STRING:
+ self.dbName = iprot.readString()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('NotificationEventsCountRequest')
+ if self.fromEventId is not None:
+ oprot.writeFieldBegin('fromEventId', TType.I64, 1)
+ oprot.writeI64(self.fromEventId)
+ oprot.writeFieldEnd()
+ if self.dbName is not None:
+ oprot.writeFieldBegin('dbName', TType.STRING, 2)
+ oprot.writeString(self.dbName)
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ if self.fromEventId is None:
+ raise TProtocol.TProtocolException(message='Required field fromEventId is unset!')
+ if self.dbName is None:
+ raise TProtocol.TProtocolException(message='Required field dbName is unset!')
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.fromEventId)
+ value = (value * 31) ^ hash(self.dbName)
+ return value
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
+class NotificationEventsCountResponse:
+ """
+ Attributes:
+ - eventsCount
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.I64, 'eventsCount', None, None, ), # 1
+ )
+
+ def __init__(self, eventsCount=None,):
+ self.eventsCount = eventsCount
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.I64:
+ self.eventsCount = iprot.readI64()
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('NotificationEventsCountResponse')
+ if self.eventsCount is not None:
+ oprot.writeFieldBegin('eventsCount', TType.I64, 1)
+ oprot.writeI64(self.eventsCount)
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ if self.eventsCount is None:
+ raise TProtocol.TProtocolException(message='Required field eventsCount is unset!')
+ return
+
+
+ def __hash__(self):
+ value = 17
+ value = (value * 31) ^ hash(self.eventsCount)
+ return value
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
+
class InsertEventRequestData:
"""
Attributes:
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb b/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
index c67f3b0..ea73b34 100644
--- a/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
+++ b/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
@@ -2670,6 +2670,43 @@ class CurrentNotificationEventId
::Thrift::Struct.generate_accessors self
end
+class NotificationEventsCountRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ FROMEVENTID = 1
+ DBNAME = 2
+
+ FIELDS = {
+ FROMEVENTID => {:type => ::Thrift::Types::I64, :name => 'fromEventId'},
+ DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field fromEventId is unset!') unless @fromEventId
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbName is unset!') unless @dbName
+ end
+
+ ::Thrift::Struct.generate_accessors self
+end
+
+class NotificationEventsCountResponse
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ EVENTSCOUNT = 1
+
+ FIELDS = {
+ EVENTSCOUNT => {:type => ::Thrift::Types::I64, :name => 'eventsCount'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field eventsCount is unset!') unless @eventsCount
+ end
+
+ ::Thrift::Struct.generate_accessors self
+end
+
class InsertEventRequestData
include ::Thrift::Struct, ::Thrift::Struct_Union
REPLACE = 1
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/standalone-metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb b/standalone-metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
index cfe08f6..54b3dfb 100644
--- a/standalone-metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
+++ b/standalone-metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
@@ -2524,6 +2524,21 @@ module ThriftHiveMetastore
raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'get_current_notificationEventId failed: unknown result')
end
+ def get_notification_events_count(rqst)
+ send_get_notification_events_count(rqst)
+ return recv_get_notification_events_count()
+ end
+
+ def send_get_notification_events_count(rqst)
+ send_message('get_notification_events_count', Get_notification_events_count_args, :rqst => rqst)
+ end
+
+ def recv_get_notification_events_count()
+ result = receive_message(Get_notification_events_count_result)
+ return result.success unless result.success.nil?
+ raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'get_notification_events_count failed: unknown result')
+ end
+
def fire_listener_event(rqst)
send_fire_listener_event(rqst)
return recv_fire_listener_event()
@@ -4558,6 +4573,13 @@ module ThriftHiveMetastore
write_result(result, oprot, 'get_current_notificationEventId', seqid)
end
+ def process_get_notification_events_count(seqid, iprot, oprot)
+ args = read_args(iprot, Get_notification_events_count_args)
+ result = Get_notification_events_count_result.new()
+ result.success = @handler.get_notification_events_count(args.rqst)
+ write_result(result, oprot, 'get_notification_events_count', seqid)
+ end
+
def process_fire_listener_event(seqid, iprot, oprot)
args = read_args(iprot, Fire_listener_event_args)
result = Fire_listener_event_result.new()
@@ -10325,6 +10347,38 @@ module ThriftHiveMetastore
::Thrift::Struct.generate_accessors self
end
+ class Get_notification_events_count_args
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ RQST = -1
+
+ FIELDS = {
+ RQST => {:type => ::Thrift::Types::STRUCT, :name => 'rqst', :class => ::NotificationEventsCountRequest}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class Get_notification_events_count_result
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ SUCCESS = 0
+
+ FIELDS = {
+ SUCCESS => {:type => ::Thrift::Types::STRUCT, :name => 'success', :class => ::NotificationEventsCountResponse}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
class Fire_listener_event_args
include ::Thrift::Struct, ::Thrift::Struct_Union
RQST = 1
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/standalone-metastore/src/main/thrift/hive_metastore.thrift
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/thrift/hive_metastore.thrift b/standalone-metastore/src/main/thrift/hive_metastore.thrift
index 9b0a846..7268d53 100644
--- a/standalone-metastore/src/main/thrift/hive_metastore.thrift
+++ b/standalone-metastore/src/main/thrift/hive_metastore.thrift
@@ -859,6 +859,15 @@ struct CurrentNotificationEventId {
1: required i64 eventId,
}
+struct NotificationEventsCountRequest {
+ 1: required i64 fromEventId,
+ 2: required string dbName,
+}
+
+struct NotificationEventsCountResponse {
+ 1: required i64 eventsCount,
+}
+
struct InsertEventRequestData {
1: optional bool replace,
2: required list<string> filesAdded,
@@ -1546,6 +1555,7 @@ service ThriftHiveMetastore extends fb303.FacebookService
// Notification logging calls
NotificationEventResponse get_next_notification(1:NotificationEventRequest rqst)
CurrentNotificationEventId get_current_notificationEventId()
+ NotificationEventsCountResponse get_notification_events_count(NotificationEventsCountRequest rqst)
FireEventResponse fire_listener_event(1:FireEventRequest rqst)
void flushCache()
[13/50] [abbrv] hive git commit: HIVE-17375: stddev_samp,
var_samp standard compliance (Zoltan Haindrich, reviewed by Ashutosh Chauhan)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/dd04a92f/ql/src/test/results/clientpositive/vectorization_14.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorization_14.q.out b/ql/src/test/results/clientpositive/vectorization_14.q.out
index c6bd7cf..536a2a1 100644
--- a/ql/src/test/results/clientpositive/vectorization_14.q.out
+++ b/ql/src/test/results/clientpositive/vectorization_14.q.out
@@ -290,625 +290,625 @@ ORDER BY cstring1, cfloat, cdouble, ctimestamp1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-1969-12-31 15:59:55.491 50.0 NULL NULL -200.0 -226.28 226.28 0.0 -1314.0 50.0 -50.0 -50.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 250.0 0.0 0.0 0.0 -250.0
-1969-12-31 15:59:55.508 31.0 NULL NULL -200.0 -226.28 226.28 0.0 -814.68 31.0 -31.0 -31.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 231.0 0.0 0.0 0.0 -231.0
-1969-12-31 15:59:55.747 -3.0 NULL NULL -200.0 -226.28 226.28 0.0 78.840004 -3.0 3.0 3.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 197.0 0.0 0.0 0.0 -197.0
-1969-12-31 15:59:55.796 -43.0 NULL NULL -200.0 -226.28 226.28 0.0 1130.04 -43.0 43.0 43.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 157.0 0.0 0.0 0.0 -157.0
-1969-12-31 15:59:55.799 19.0 NULL NULL -200.0 -226.28 226.28 0.0 -499.32 19.0 -19.0 -19.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 219.0 0.0 0.0 0.0 -219.0
-1969-12-31 15:59:55.982 4.0 NULL NULL -200.0 -226.28 226.28 0.0 -105.12 4.0 -4.0 -4.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 204.0 0.0 0.0 0.0 -204.0
-1969-12-31 15:59:56.099 19.0 NULL NULL -200.0 -226.28 226.28 0.0 -499.32 19.0 -19.0 -19.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 219.0 0.0 0.0 0.0 -219.0
-1969-12-31 15:59:56.131 26.0 NULL NULL -200.0 -226.28 226.28 0.0 -683.28 26.0 -26.0 -26.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 226.0 0.0 0.0 0.0 -226.0
-1969-12-31 15:59:56.14 61.0 NULL NULL -200.0 -226.28 226.28 0.0 -1603.0801 61.0 -61.0 -61.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 261.0 0.0 0.0 0.0 -261.0
-1969-12-31 15:59:56.159 -49.0 NULL NULL -200.0 -226.28 226.28 0.0 1287.7201 -49.0 49.0 49.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 151.0 0.0 0.0 0.0 -151.0
-1969-12-31 15:59:56.174 -36.0 NULL NULL -200.0 -226.28 226.28 0.0 946.08 -36.0 36.0 36.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 164.0 0.0 0.0 0.0 -164.0
-1969-12-31 15:59:56.197 -42.0 NULL NULL -200.0 -226.28 226.28 0.0 1103.76 -42.0 42.0 42.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 158.0 0.0 0.0 0.0 -158.0
-1969-12-31 15:59:56.218 -4.0 NULL NULL -200.0 -226.28 226.28 0.0 105.12 -4.0 4.0 4.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 196.0 0.0 0.0 0.0 -196.0
-1969-12-31 15:59:56.276 -60.0 NULL NULL -200.0 -226.28 226.28 0.0 1576.8 -60.0 60.0 60.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 140.0 0.0 0.0 0.0 -140.0
-1969-12-31 15:59:56.319 -5.0 NULL NULL -200.0 -226.28 226.28 0.0 131.40001 -5.0 5.0 5.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 195.0 0.0 0.0 0.0 -195.0
-1969-12-31 15:59:56.345 54.0 NULL NULL -200.0 -226.28 226.28 0.0 -1419.12 54.0 -54.0 -54.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 254.0 0.0 0.0 0.0 -254.0
-1969-12-31 15:59:56.414 -23.0 NULL NULL -200.0 -226.28 226.28 0.0 604.44 -23.0 23.0 23.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 177.0 0.0 0.0 0.0 -177.0
-1969-12-31 15:59:56.436 -33.0 NULL NULL -200.0 -226.28 226.28 0.0 867.24005 -33.0 33.0 33.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 167.0 0.0 0.0 0.0 -167.0
-1969-12-31 15:59:56.477 -47.0 NULL NULL -200.0 -226.28 226.28 0.0 1235.16 -47.0 47.0 47.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 153.0 0.0 0.0 0.0 -153.0
-1969-12-31 15:59:56.691 41.0 NULL NULL -200.0 -226.28 226.28 0.0 -1077.48 41.0 -41.0 -41.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 241.0 0.0 0.0 0.0 -241.0
-1969-12-31 15:59:56.769 -38.0 NULL NULL -200.0 -226.28 226.28 0.0 998.64 -38.0 38.0 38.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 162.0 0.0 0.0 0.0 -162.0
-1969-12-31 15:59:56.776 55.0 NULL NULL -200.0 -226.28 226.28 0.0 -1445.4 55.0 -55.0 -55.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 255.0 0.0 0.0 0.0 -255.0
-1969-12-31 15:59:56.795 28.0 NULL NULL -200.0 -226.28 226.28 0.0 -735.84 28.0 -28.0 -28.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 228.0 0.0 0.0 0.0 -228.0
-1969-12-31 15:59:56.929 61.0 NULL NULL -200.0 -226.28 226.28 0.0 -1603.0801 61.0 -61.0 -61.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 261.0 0.0 0.0 0.0 -261.0
-1969-12-31 15:59:56.969 -57.0 NULL NULL -200.0 -226.28 226.28 0.0 1497.9601 -57.0 57.0 57.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 143.0 0.0 0.0 0.0 -143.0
-1969-12-31 15:59:57.027 -39.0 NULL NULL -200.0 -226.28 226.28 0.0 1024.92 -39.0 39.0 39.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 161.0 0.0 0.0 0.0 -161.0
-1969-12-31 15:59:57.048 9.0 NULL NULL -200.0 -226.28 226.28 0.0 -236.52 9.0 -9.0 -9.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 209.0 0.0 0.0 0.0 -209.0
-1969-12-31 15:59:57.063 8.0 NULL NULL -200.0 -226.28 226.28 0.0 -210.24 8.0 -8.0 -8.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 208.0 0.0 0.0 0.0 -208.0
-1969-12-31 15:59:57.118 37.0 NULL NULL -200.0 -226.28 226.28 0.0 -972.36005 37.0 -37.0 -37.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 237.0 0.0 0.0 0.0 -237.0
-1969-12-31 15:59:57.21 -42.0 NULL NULL -200.0 -226.28 226.28 0.0 1103.76 -42.0 42.0 42.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 158.0 0.0 0.0 0.0 -158.0
-1969-12-31 15:59:57.245 16.0 NULL NULL -200.0 -226.28 226.28 0.0 -420.48 16.0 -16.0 -16.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 216.0 0.0 0.0 0.0 -216.0
-1969-12-31 15:59:57.256 -48.0 NULL NULL -200.0 -226.28 226.28 0.0 1261.4401 -48.0 48.0 48.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 152.0 0.0 0.0 0.0 -152.0
-1969-12-31 15:59:57.269 -4.0 NULL NULL -200.0 -226.28 226.28 0.0 105.12 -4.0 4.0 4.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 196.0 0.0 0.0 0.0 -196.0
-1969-12-31 15:59:57.273 -6.0 NULL NULL -200.0 -226.28 226.28 0.0 157.68001 -6.0 6.0 6.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 194.0 0.0 0.0 0.0 -194.0
-1969-12-31 15:59:57.349 -56.0 NULL NULL -200.0 -226.28 226.28 0.0 1471.68 -56.0 56.0 56.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 144.0 0.0 0.0 0.0 -144.0
-1969-12-31 15:59:57.369 -54.0 NULL NULL -200.0 -226.28 226.28 0.0 1419.12 -54.0 54.0 54.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 146.0 0.0 0.0 0.0 -146.0
-1969-12-31 15:59:57.434 54.0 NULL NULL -200.0 -226.28 226.28 0.0 -1419.12 54.0 -54.0 -54.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 254.0 0.0 0.0 0.0 -254.0
-1969-12-31 15:59:57.528 -60.0 NULL NULL -200.0 -226.28 226.28 0.0 1576.8 -60.0 60.0 60.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 140.0 0.0 0.0 0.0 -140.0
-1969-12-31 15:59:57.543 32.0 NULL NULL -200.0 -226.28 226.28 0.0 -840.96 32.0 -32.0 -32.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 232.0 0.0 0.0 0.0 -232.0
-1969-12-31 15:59:57.56 56.0 NULL NULL -200.0 -226.28 226.28 0.0 -1471.68 56.0 -56.0 -56.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 256.0 0.0 0.0 0.0 -256.0
-1969-12-31 15:59:57.568 6.0 NULL NULL -200.0 -226.28 226.28 0.0 -157.68001 6.0 -6.0 -6.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 206.0 0.0 0.0 0.0 -206.0
-1969-12-31 15:59:57.693 -63.0 NULL NULL -200.0 -226.28 226.28 0.0 1655.64 -63.0 63.0 63.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 137.0 0.0 0.0 0.0 -137.0
-1969-12-31 15:59:57.747 -60.0 NULL NULL -200.0 -226.28 226.28 0.0 1576.8 -60.0 60.0 60.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 140.0 0.0 0.0 0.0 -140.0
-1969-12-31 15:59:57.794 -5.0 NULL NULL -200.0 -226.28 226.28 0.0 131.40001 -5.0 5.0 5.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 195.0 0.0 0.0 0.0 -195.0
-1969-12-31 15:59:57.828 -34.0 NULL NULL -200.0 -226.28 226.28 0.0 893.52 -34.0 34.0 34.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 166.0 0.0 0.0 0.0 -166.0
-1969-12-31 15:59:57.847 50.0 NULL NULL -200.0 -226.28 226.28 0.0 -1314.0 50.0 -50.0 -50.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 250.0 0.0 0.0 0.0 -250.0
-1969-12-31 15:59:57.882 -29.0 NULL NULL -200.0 -226.28 226.28 0.0 762.12 -29.0 29.0 29.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 171.0 0.0 0.0 0.0 -171.0
-1969-12-31 15:59:57.942 -40.0 NULL NULL -200.0 -226.28 226.28 0.0 1051.2001 -40.0 40.0 40.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 160.0 0.0 0.0 0.0 -160.0
-1969-12-31 15:59:57.957 -6.0 NULL NULL -200.0 -226.28 226.28 0.0 157.68001 -6.0 6.0 6.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 194.0 0.0 0.0 0.0 -194.0
-1969-12-31 15:59:57.965 -9.0 NULL NULL -200.0 -226.28 226.28 0.0 236.52 -9.0 9.0 9.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 191.0 0.0 0.0 0.0 -191.0
-1969-12-31 15:59:58.046 -62.0 NULL NULL -200.0 -226.28 226.28 0.0 1629.36 -62.0 62.0 62.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 138.0 0.0 0.0 0.0 -138.0
-1969-12-31 15:59:58.112 -54.0 NULL NULL -200.0 -226.28 226.28 0.0 1419.12 -54.0 54.0 54.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 146.0 0.0 0.0 0.0 -146.0
-1969-12-31 15:59:58.129 37.0 NULL NULL -200.0 -226.28 226.28 0.0 -972.36005 37.0 -37.0 -37.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 237.0 0.0 0.0 0.0 -237.0
-1969-12-31 15:59:58.158 -53.0 NULL NULL -200.0 -226.28 226.28 0.0 1392.8401 -53.0 53.0 53.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 147.0 0.0 0.0 0.0 -147.0
-1969-12-31 15:59:58.173 -17.0 NULL NULL -200.0 -226.28 226.28 0.0 446.76 -17.0 17.0 17.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 183.0 0.0 0.0 0.0 -183.0
-1969-12-31 15:59:58.214 22.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -578.16003 22.0 -22.0 -22.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7218.0 0.0 0.0 0.0 -7218.0
-1969-12-31 15:59:58.245 -35.0 NULL NULL -200.0 -226.28 226.28 0.0 919.80005 -35.0 35.0 35.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 165.0 0.0 0.0 0.0 -165.0
-1969-12-31 15:59:58.265 -8.0 NULL NULL -200.0 -226.28 226.28 0.0 210.24 -8.0 8.0 8.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 192.0 0.0 0.0 0.0 -192.0
-1969-12-31 15:59:58.272 7.0 NULL NULL -200.0 -226.28 226.28 0.0 -183.96 7.0 -7.0 -7.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 207.0 0.0 0.0 0.0 -207.0
-1969-12-31 15:59:58.298 32.0 NULL NULL -200.0 -226.28 226.28 0.0 -840.96 32.0 -32.0 -32.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 232.0 0.0 0.0 0.0 -232.0
-1969-12-31 15:59:58.309 52.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1366.56 52.0 -52.0 -52.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7248.0 0.0 0.0 0.0 -7248.0
-1969-12-31 15:59:58.455 57.0 NULL NULL -200.0 -226.28 226.28 0.0 -1497.9601 57.0 -57.0 -57.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 257.0 0.0 0.0 0.0 -257.0
-1969-12-31 15:59:58.463 -7.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 183.96 -7.0 7.0 7.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7189.0 0.0 0.0 0.0 -7189.0
-1969-12-31 15:59:58.512 -20.0 NULL NULL -200.0 -226.28 226.28 0.0 525.60004 -20.0 20.0 20.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 180.0 0.0 0.0 0.0 -180.0
-1969-12-31 15:59:58.544 -40.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1051.2001 -40.0 40.0 40.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7156.0 0.0 0.0 0.0 -7156.0
-1969-12-31 15:59:58.561 -17.0 NULL NULL -200.0 -226.28 226.28 0.0 446.76 -17.0 17.0 17.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 183.0 0.0 0.0 0.0 -183.0
-1969-12-31 15:59:58.594 -23.0 NULL NULL -200.0 -226.28 226.28 0.0 604.44 -23.0 23.0 23.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 177.0 0.0 0.0 0.0 -177.0
-1969-12-31 15:59:58.615 25.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -657.0 25.0 -25.0 -25.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7221.0 0.0 0.0 0.0 -7221.0
-1969-12-31 15:59:58.625 -6.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 157.68001 -6.0 6.0 6.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7190.0 0.0 0.0 0.0 -7190.0
-1969-12-31 15:59:58.65 43.0 NULL NULL -200.0 -226.28 226.28 0.0 -1130.04 43.0 -43.0 -43.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 243.0 0.0 0.0 0.0 -243.0
-1969-12-31 15:59:58.788 24.0 NULL NULL -200.0 -226.28 226.28 0.0 -630.72003 24.0 -24.0 -24.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 224.0 0.0 0.0 0.0 -224.0
-1969-12-31 15:59:58.825 -9.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 236.52 -9.0 9.0 9.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7187.0 0.0 0.0 0.0 -7187.0
-1969-12-31 15:59:58.863 -3.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 78.840004 -3.0 3.0 3.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7193.0 0.0 0.0 0.0 -7193.0
-1969-12-31 15:59:58.893 -10.0 NULL NULL -200.0 -226.28 226.28 0.0 262.80002 -10.0 10.0 10.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 190.0 0.0 0.0 0.0 -190.0
-1969-12-31 15:59:58.93 -22.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 578.16003 -22.0 22.0 22.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7174.0 0.0 0.0 0.0 -7174.0
-1969-12-31 15:59:58.93 2.0 NULL NULL -200.0 -226.28 226.28 0.0 -52.56 2.0 -2.0 -2.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 202.0 0.0 0.0 0.0 -202.0
-1969-12-31 15:59:58.98 -33.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 867.24005 -33.0 33.0 33.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7163.0 0.0 0.0 0.0 -7163.0
-1969-12-31 15:59:58.989 -10.0 NULL NULL -200.0 -226.28 226.28 0.0 262.80002 -10.0 10.0 10.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 190.0 0.0 0.0 0.0 -190.0
-1969-12-31 16:00:00.019 4.0 NULL NULL -200.0 -226.28 226.28 0.0 -105.12 4.0 -4.0 -4.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 204.0 0.0 0.0 0.0 -204.0
-1969-12-31 16:00:00.022 50.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1314.0 50.0 -50.0 -50.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7246.0 0.0 0.0 0.0 -7246.0
-1969-12-31 16:00:00.025 13.0 NULL NULL -200.0 -226.28 226.28 0.0 -341.64 13.0 -13.0 -13.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 213.0 0.0 0.0 0.0 -213.0
-1969-12-31 16:00:00.026 -43.0 NULL NULL -200.0 -226.28 226.28 0.0 1130.04 -43.0 43.0 43.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 157.0 0.0 0.0 0.0 -157.0
-1969-12-31 16:00:00.038 12.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -315.36002 12.0 -12.0 -12.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7208.0 0.0 0.0 0.0 -7208.0
-1969-12-31 16:00:00.073 22.0 NULL NULL -200.0 -226.28 226.28 0.0 -578.16003 22.0 -22.0 -22.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 222.0 0.0 0.0 0.0 -222.0
-1969-12-31 16:00:00.074 -47.0 NULL NULL -200.0 -226.28 226.28 0.0 1235.16 -47.0 47.0 47.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 153.0 0.0 0.0 0.0 -153.0
-1969-12-31 16:00:00.074 3.0 NULL NULL -200.0 -226.28 226.28 0.0 -78.840004 3.0 -3.0 -3.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 203.0 0.0 0.0 0.0 -203.0
-1969-12-31 16:00:00.11 42.0 NULL NULL -200.0 -226.28 226.28 0.0 -1103.76 42.0 -42.0 -42.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 242.0 0.0 0.0 0.0 -242.0
-1969-12-31 16:00:00.147 51.0 NULL NULL -200.0 -226.28 226.28 0.0 -1340.28 51.0 -51.0 -51.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 251.0 0.0 0.0 0.0 -251.0
-1969-12-31 16:00:00.148 49.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1287.7201 49.0 -49.0 -49.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7245.0 0.0 0.0 0.0 -7245.0
-1969-12-31 16:00:00.156 -51.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1340.28 -51.0 51.0 51.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7145.0 0.0 0.0 0.0 -7145.0
-1969-12-31 16:00:00.157 -28.0 NULL NULL -200.0 -226.28 226.28 0.0 735.84 -28.0 28.0 28.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 172.0 0.0 0.0 0.0 -172.0
-1969-12-31 16:00:00.199 -64.0 NULL NULL -200.0 -226.28 226.28 0.0 1681.92 -64.0 64.0 64.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 136.0 0.0 0.0 0.0 -136.0
-1969-12-31 16:00:00.229 -22.0 NULL NULL -200.0 -226.28 226.28 0.0 578.16003 -22.0 22.0 22.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 178.0 0.0 0.0 0.0 -178.0
-1969-12-31 16:00:00.247 -44.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1156.3201 -44.0 44.0 44.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7152.0 0.0 0.0 0.0 -7152.0
-1969-12-31 16:00:00.289 -44.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1156.3201 -44.0 44.0 44.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7152.0 0.0 0.0 0.0 -7152.0
-1969-12-31 16:00:00.29 -64.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1681.92 -64.0 64.0 64.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7132.0 0.0 0.0 0.0 -7132.0
-1969-12-31 16:00:00.306 22.0 NULL NULL -200.0 -226.28 226.28 0.0 -578.16003 22.0 -22.0 -22.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 222.0 0.0 0.0 0.0 -222.0
-1969-12-31 16:00:00.308 4.0 NULL NULL -200.0 -226.28 226.28 0.0 -105.12 4.0 -4.0 -4.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 204.0 0.0 0.0 0.0 -204.0
-1969-12-31 16:00:00.363 57.0 NULL NULL -200.0 -226.28 226.28 0.0 -1497.9601 57.0 -57.0 -57.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 257.0 0.0 0.0 0.0 -257.0
-1969-12-31 16:00:00.381 -2.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 52.56 -2.0 2.0 2.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7194.0 0.0 0.0 0.0 -7194.0
-1969-12-31 16:00:00.382 46.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1208.88 46.0 -46.0 -46.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7242.0 0.0 0.0 0.0 -7242.0
-1969-12-31 16:00:00.39 -39.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1024.92 -39.0 39.0 39.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7157.0 0.0 0.0 0.0 -7157.0
-1969-12-31 16:00:00.434 -30.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 788.4 -30.0 30.0 30.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7166.0 0.0 0.0 0.0 -7166.0
-1969-12-31 16:00:00.45 2.0 NULL NULL -200.0 -226.28 226.28 0.0 -52.56 2.0 -2.0 -2.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 202.0 0.0 0.0 0.0 -202.0
-1969-12-31 16:00:00.51 -20.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 525.60004 -20.0 20.0 20.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7176.0 0.0 0.0 0.0 -7176.0
-1969-12-31 16:00:00.515 15.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -394.2 15.0 -15.0 -15.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7211.0 0.0 0.0 0.0 -7211.0
-1969-12-31 16:00:00.519 1.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -26.28 1.0 -1.0 -1.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7197.0 0.0 0.0 0.0 -7197.0
-1969-12-31 16:00:00.52 25.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -657.0 25.0 -25.0 -25.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7221.0 0.0 0.0 0.0 -7221.0
-1969-12-31 16:00:00.526 12.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -315.36002 12.0 -12.0 -12.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7208.0 0.0 0.0 0.0 -7208.0
-1969-12-31 16:00:00.539 -37.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 972.36005 -37.0 37.0 37.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7159.0 0.0 0.0 0.0 -7159.0
-1969-12-31 16:00:00.543 45.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1182.6 45.0 -45.0 -45.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7241.0 0.0 0.0 0.0 -7241.0
-1969-12-31 16:00:00.546 35.0 NULL NULL -200.0 -226.28 226.28 0.0 -919.80005 35.0 -35.0 -35.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 235.0 0.0 0.0 0.0 -235.0
-1969-12-31 16:00:00.547 -55.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1445.4 -55.0 55.0 55.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7141.0 0.0 0.0 0.0 -7141.0
-1969-12-31 16:00:00.551 59.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1550.52 59.0 -59.0 -59.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7255.0 0.0 0.0 0.0 -7255.0
-1969-12-31 16:00:00.553 -10.0 NULL NULL -200.0 -226.28 226.28 0.0 262.80002 -10.0 10.0 10.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 190.0 0.0 0.0 0.0 -190.0
-1969-12-31 16:00:00.557 53.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1392.8401 53.0 -53.0 -53.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7249.0 0.0 0.0 0.0 -7249.0
-1969-12-31 16:00:00.563 4.0 NULL NULL -200.0 -226.28 226.28 0.0 -105.12 4.0 -4.0 -4.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 204.0 0.0 0.0 0.0 -204.0
-1969-12-31 16:00:00.564 -18.0 NULL NULL -200.0 -226.28 226.28 0.0 473.04 -18.0 18.0 18.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 182.0 0.0 0.0 0.0 -182.0
-1969-12-31 16:00:00.574 -2.0 NULL NULL -200.0 -226.28 226.28 0.0 52.56 -2.0 2.0 2.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 198.0 0.0 0.0 0.0 -198.0
-1969-12-31 16:00:00.611 -33.0 NULL NULL -200.0 -226.28 226.28 0.0 867.24005 -33.0 33.0 33.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 167.0 0.0 0.0 0.0 -167.0
-1969-12-31 16:00:00.612 -49.0 NULL NULL -200.0 -226.28 226.28 0.0 1287.7201 -49.0 49.0 49.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 151.0 0.0 0.0 0.0 -151.0
-1969-12-31 16:00:00.613 8.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -210.24 8.0 -8.0 -8.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7204.0 0.0 0.0 0.0 -7204.0
-1969-12-31 16:00:00.621 22.0 NULL NULL -200.0 -226.28 226.28 0.0 -578.16003 22.0 -22.0 -22.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 222.0 0.0 0.0 0.0 -222.0
-1969-12-31 16:00:00.664 8.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -210.24 8.0 -8.0 -8.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7204.0 0.0 0.0 0.0 -7204.0
-1969-12-31 16:00:00.692 -27.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 709.56 -27.0 27.0 27.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7169.0 0.0 0.0 0.0 -7169.0
-1969-12-31 16:00:00.738 59.0 NULL NULL -200.0 -226.28 226.28 0.0 -1550.52 59.0 -59.0 -59.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 259.0 0.0 0.0 0.0 -259.0
-1969-12-31 16:00:00.754 49.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1287.7201 49.0 -49.0 -49.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7245.0 0.0 0.0 0.0 -7245.0
-1969-12-31 16:00:00.761 79.553 NULL NULL -7196.0 -7222.28 7222.28 0.0 -2090.6528 79.553 -79.553 -79.553 709.8063882063881 0.0 1 -709.8063882063881 NULL 7275.553001403809 0.0 0.0 0.0 -7275.553001403809
-1969-12-31 16:00:00.767 22.0 NULL NULL -200.0 -226.28 226.28 0.0 -578.16003 22.0 -22.0 -22.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 222.0 0.0 0.0 0.0 -222.0
-1969-12-31 16:00:00.8 -35.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 919.80005 -35.0 35.0 35.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7161.0 0.0 0.0 0.0 -7161.0
-1969-12-31 16:00:00.82 34.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -893.52 34.0 -34.0 -34.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7230.0 0.0 0.0 0.0 -7230.0
-1969-12-31 16:00:00.835 -44.0 NULL NULL -200.0 -226.28 226.28 0.0 1156.3201 -44.0 44.0 44.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 156.0 0.0 0.0 0.0 -156.0
-1969-12-31 16:00:00.865 36.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -946.08 36.0 -36.0 -36.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7232.0 0.0 0.0 0.0 -7232.0
-1969-12-31 16:00:00.885 -23.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 604.44 -23.0 23.0 23.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7173.0 0.0 0.0 0.0 -7173.0
-1969-12-31 16:00:00.9 35.0 NULL NULL -200.0 -226.28 226.28 0.0 -919.80005 35.0 -35.0 -35.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 235.0 0.0 0.0 0.0 -235.0
-1969-12-31 16:00:00.909 56.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1471.68 56.0 -56.0 -56.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7252.0 0.0 0.0 0.0 -7252.0
-1969-12-31 16:00:00.911 12.0 NULL NULL -200.0 -226.28 226.28 0.0 -315.36002 12.0 -12.0 -12.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 212.0 0.0 0.0 0.0 -212.0
-1969-12-31 16:00:00.916 -10.0 NULL NULL -200.0 -226.28 226.28 0.0 262.80002 -10.0 10.0 10.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 190.0 0.0 0.0 0.0 -190.0
-1969-12-31 16:00:00.951 15.0 NULL NULL -200.0 -226.28 226.28 0.0 -394.2 15.0 -15.0 -15.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 215.0 0.0 0.0 0.0 -215.0
-1969-12-31 16:00:00.958 -39.0 NULL NULL -200.0 -226.28 226.28 0.0 1024.92 -39.0 39.0 39.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 161.0 0.0 0.0 0.0 -161.0
-1969-12-31 16:00:00.992 42.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1103.76 42.0 -42.0 -42.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7238.0 0.0 0.0 0.0 -7238.0
-1969-12-31 16:00:01.088 -16.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 420.48 -16.0 16.0 16.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7180.0 0.0 0.0 0.0 -7180.0
-1969-12-31 16:00:01.128 49.0 NULL NULL -200.0 -226.28 226.28 0.0 -1287.7201 49.0 -49.0 -49.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 249.0 0.0 0.0 0.0 -249.0
-1969-12-31 16:00:01.138 -55.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1445.4 -55.0 55.0 55.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7141.0 0.0 0.0 0.0 -7141.0
-1969-12-31 16:00:01.22 -62.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1629.36 -62.0 62.0 62.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7134.0 0.0 0.0 0.0 -7134.0
-1969-12-31 16:00:01.232 34.0 NULL NULL -200.0 -226.28 226.28 0.0 -893.52 34.0 -34.0 -34.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 234.0 0.0 0.0 0.0 -234.0
-1969-12-31 16:00:01.235 17.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -446.76 17.0 -17.0 -17.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7213.0 0.0 0.0 0.0 -7213.0
-1969-12-31 16:00:01.282 -38.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 998.64 -38.0 38.0 38.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7158.0 0.0 0.0 0.0 -7158.0
-1969-12-31 16:00:01.356 40.0 NULL NULL -200.0 -226.28 226.28 0.0 -1051.2001 40.0 -40.0 -40.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 240.0 0.0 0.0 0.0 -240.0
-1969-12-31 16:00:01.388 46.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1208.88 46.0 -46.0 -46.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7242.0 0.0 0.0 0.0 -7242.0
-1969-12-31 16:00:01.389 26.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -683.28 26.0 -26.0 -26.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7222.0 0.0 0.0 0.0 -7222.0
-1969-12-31 16:00:01.424 41.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1077.48 41.0 -41.0 -41.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7237.0 0.0 0.0 0.0 -7237.0
-1969-12-31 16:00:01.462 -11.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 289.08002 -11.0 11.0 11.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7185.0 0.0 0.0 0.0 -7185.0
-1969-12-31 16:00:01.489 2.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -52.56 2.0 -2.0 -2.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7198.0 0.0 0.0 0.0 -7198.0
-1969-12-31 16:00:01.496 59.0 NULL NULL -200.0 -226.28 226.28 0.0 -1550.52 59.0 -59.0 -59.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 259.0 0.0 0.0 0.0 -259.0
-1969-12-31 16:00:01.505 61.0 NULL NULL -200.0 -226.28 226.28 0.0 -1603.0801 61.0 -61.0 -61.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 261.0 0.0 0.0 0.0 -261.0
-1969-12-31 16:00:01.515 -62.0 NULL NULL -200.0 -226.28 226.28 0.0 1629.36 -62.0 62.0 62.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 138.0 0.0 0.0 0.0 -138.0
-1969-12-31 16:00:01.562 -34.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 893.52 -34.0 34.0 34.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7162.0 0.0 0.0 0.0 -7162.0
-1969-12-31 16:00:01.592 -45.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1182.6 -45.0 45.0 45.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7151.0 0.0 0.0 0.0 -7151.0
-1969-12-31 16:00:01.627 -28.0 NULL NULL -200.0 -226.28 226.28 0.0 735.84 -28.0 28.0 28.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 172.0 0.0 0.0 0.0 -172.0
-1969-12-31 16:00:01.673 -47.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1235.16 -47.0 47.0 47.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7149.0 0.0 0.0 0.0 -7149.0
-1969-12-31 16:00:01.694 47.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1235.16 47.0 -47.0 -47.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7243.0 0.0 0.0 0.0 -7243.0
-1969-12-31 16:00:01.723 -3.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 78.840004 -3.0 3.0 3.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7193.0 0.0 0.0 0.0 -7193.0
-1969-12-31 16:00:01.734 -62.0 NULL NULL -200.0 -226.28 226.28 0.0 1629.36 -62.0 62.0 62.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 138.0 0.0 0.0 0.0 -138.0
-1969-12-31 16:00:01.781 -19.0 NULL NULL -200.0 -226.28 226.28 0.0 499.32 -19.0 19.0 19.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 181.0 0.0 0.0 0.0 -181.0
-1969-12-31 16:00:01.792 35.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -919.80005 35.0 -35.0 -35.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7231.0 0.0 0.0 0.0 -7231.0
-1969-12-31 16:00:01.811 57.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1497.9601 57.0 -57.0 -57.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7253.0 0.0 0.0 0.0 -7253.0
-1969-12-31 16:00:01.841 54.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1419.12 54.0 -54.0 -54.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7250.0 0.0 0.0 0.0 -7250.0
-1969-12-31 16:00:01.849 31.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -814.68 31.0 -31.0 -31.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7227.0 0.0 0.0 0.0 -7227.0
-1969-12-31 16:00:01.873 14.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -367.92 14.0 -14.0 -14.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7210.0 0.0 0.0 0.0 -7210.0
-1969-12-31 16:00:01.901 -43.0 NULL NULL -200.0 -226.28 226.28 0.0 1130.04 -43.0 43.0 43.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 157.0 0.0 0.0 0.0 -157.0
-1969-12-31 16:00:01.951 41.0 NULL NULL -200.0 -226.28 226.28 0.0 -1077.48 41.0 -41.0 -41.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 241.0 0.0 0.0 0.0 -241.0
-1969-12-31 16:00:02 47.0 NULL NULL -200.0 -226.28 226.28 0.0 -1235.16 47.0 -47.0 -47.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 247.0 0.0 0.0 0.0 -247.0
-1969-12-31 16:00:02.014 -39.0 NULL NULL -200.0 -226.28 226.28 0.0 1024.92 -39.0 39.0 39.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 161.0 0.0 0.0 0.0 -161.0
-1969-12-31 16:00:02.021 -5.0 NULL NULL -200.0 -226.28 226.28 0.0 131.40001 -5.0 5.0 5.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 195.0 0.0 0.0 0.0 -195.0
-1969-12-31 16:00:02.171 -40.0 NULL NULL -200.0 -226.28 226.28 0.0 1051.2001 -40.0 40.0 40.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 160.0 0.0 0.0 0.0 -160.0
-1969-12-31 16:00:02.208 16.0 NULL NULL -200.0 -226.28 226.28 0.0 -420.48 16.0 -16.0 -16.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 216.0 0.0 0.0 0.0 -216.0
-1969-12-31 16:00:02.234 -30.0 NULL NULL -200.0 -226.28 226.28 0.0 788.4 -30.0 30.0 30.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 170.0 0.0 0.0 0.0 -170.0
-1969-12-31 16:00:02.269 52.0 NULL NULL -200.0 -226.28 226.28 0.0 -1366.56 52.0 -52.0 -52.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 252.0 0.0 0.0 0.0 -252.0
-1969-12-31 16:00:02.325 -49.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1287.7201 -49.0 49.0 49.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7147.0 0.0 0.0 0.0 -7147.0
-1969-12-31 16:00:02.344 28.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -735.84 28.0 -28.0 -28.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7224.0 0.0 0.0 0.0 -7224.0
-1969-12-31 16:00:02.363 31.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -814.68 31.0 -31.0 -31.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7227.0 0.0 0.0 0.0 -7227.0
-1969-12-31 16:00:02.38 62.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1629.36 62.0 -62.0 -62.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7258.0 0.0 0.0 0.0 -7258.0
-1969-12-31 16:00:02.434 -50.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1314.0 -50.0 50.0 50.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7146.0 0.0 0.0 0.0 -7146.0
-1969-12-31 16:00:02.445 -26.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 683.28 -26.0 26.0 26.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7170.0 0.0 0.0 0.0 -7170.0
-1969-12-31 16:00:02.492 -13.0 NULL NULL -200.0 -226.28 226.28 0.0 341.64 -13.0 13.0 13.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 187.0 0.0 0.0 0.0 -187.0
-1969-12-31 16:00:02.508 -47.0 NULL NULL -200.0 -226.28 226.28 0.0 1235.16 -47.0 47.0 47.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 153.0 0.0 0.0 0.0 -153.0
-1969-12-31 16:00:02.58 -28.0 NULL NULL -200.0 -226.28 226.28 0.0 735.84 -28.0 28.0 28.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 172.0 0.0 0.0 0.0 -172.0
-1969-12-31 16:00:02.582 16.0 NULL NULL -200.0 -226.28 226.28 0.0 -420.48 16.0 -16.0 -16.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 216.0 0.0 0.0 0.0 -216.0
-1969-12-31 16:00:02.613 -13.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 341.64 -13.0 13.0 13.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7183.0 0.0 0.0 0.0 -7183.0
-1969-12-31 16:00:02.621 -52.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1366.56 -52.0 52.0 52.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7144.0 0.0 0.0 0.0 -7144.0
-1969-12-31 16:00:02.657 49.0 NULL NULL -200.0 -226.28 226.28 0.0 -1287.7201 49.0 -49.0 -49.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 249.0 0.0 0.0 0.0 -249.0
-1969-12-31 16:00:02.659 18.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -473.04 18.0 -18.0 -18.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7214.0 0.0 0.0 0.0 -7214.0
-1969-12-31 16:00:02.67 -32.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 840.96 -32.0 32.0 32.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7164.0 0.0 0.0 0.0 -7164.0
-1969-12-31 16:00:02.698 -61.0 NULL NULL -200.0 -226.28 226.28 0.0 1603.0801 -61.0 61.0 61.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 139.0 0.0 0.0 0.0 -139.0
-1969-12-31 16:00:02.707 -57.0 NULL NULL -200.0 -226.28 226.28 0.0 1497.9601 -57.0 57.0 57.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 143.0 0.0 0.0 0.0 -143.0
-1969-12-31 16:00:02.71 0.0 NULL NULL -200.0 -226.28 226.28 0.0 -0.0 0.0 -0.0 -0.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 200.0 0.0 0.0 0.0 -200.0
-1969-12-31 16:00:02.722 25.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -657.0 25.0 -25.0 -25.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7221.0 0.0 0.0 0.0 -7221.0
-1969-12-31 16:00:02.723 -51.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1340.28 -51.0 51.0 51.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7145.0 0.0 0.0 0.0 -7145.0
-1969-12-31 16:00:02.752 2.0 NULL NULL -200.0 -226.28 226.28 0.0 -52.56 2.0 -2.0 -2.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 202.0 0.0 0.0 0.0 -202.0
-1969-12-31 16:00:02.777 29.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -762.12 29.0 -29.0 -29.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7225.0 0.0 0.0 0.0 -7225.0
-1969-12-31 16:00:02.795 28.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -735.84 28.0 -28.0 -28.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7224.0 0.0 0.0 0.0 -7224.0
-1969-12-31 16:00:02.804 39.0 NULL NULL -200.0 -226.28 226.28 0.0 -1024.92 39.0 -39.0 -39.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 239.0 0.0 0.0 0.0 -239.0
-1969-12-31 16:00:02.814 -45.0 NULL NULL -200.0 -226.28 226.28 0.0 1182.6 -45.0 45.0 45.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 155.0 0.0 0.0 0.0 -155.0
-1969-12-31 16:00:02.91 -20.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 525.60004 -20.0 20.0 20.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7176.0 0.0 0.0 0.0 -7176.0
-1969-12-31 16:00:02.925 -19.0 NULL NULL -200.0 -226.28 226.28 0.0 499.32 -19.0 19.0 19.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 181.0 0.0 0.0 0.0 -181.0
-1969-12-31 16:00:02.966 53.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1392.8401 53.0 -53.0 -53.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7249.0 0.0 0.0 0.0 -7249.0
-1969-12-31 16:00:02.969 -41.0 NULL NULL -200.0 -226.28 226.28 0.0 1077.48 -41.0 41.0 41.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 159.0 0.0 0.0 0.0 -159.0
-1969-12-31 16:00:02.974 -17.0 NULL NULL -200.0 -226.28 226.28 0.0 446.76 -17.0 17.0 17.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 183.0 0.0 0.0 0.0 -183.0
-1969-12-31 16:00:03.002 -44.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1156.3201 -44.0 44.0 44.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7152.0 0.0 0.0 0.0 -7152.0
-1969-12-31 16:00:03.066 57.0 NULL NULL -200.0 -226.28 226.28 0.0 -1497.9601 57.0 -57.0 -57.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 257.0 0.0 0.0 0.0 -257.0
-1969-12-31 16:00:03.09 -50.0 NULL NULL -200.0 -226.28 226.28 0.0 1314.0 -50.0 50.0 50.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 150.0 0.0 0.0 0.0 -150.0
-1969-12-31 16:00:03.116 -29.0 NULL NULL -200.0 -226.28 226.28 0.0 762.12 -29.0 29.0 29.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 171.0 0.0 0.0 0.0 -171.0
-1969-12-31 16:00:03.261 -15.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 394.2 -15.0 15.0 15.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7181.0 0.0 0.0 0.0 -7181.0
-1969-12-31 16:00:03.31 -21.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 551.88 -21.0 21.0 21.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7175.0 0.0 0.0 0.0 -7175.0
-1969-12-31 16:00:03.341 -43.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1130.04 -43.0 43.0 43.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7153.0 0.0 0.0 0.0 -7153.0
-1969-12-31 16:00:03.357 -44.0 NULL NULL -200.0 -226.28 226.28 0.0 1156.3201 -44.0 44.0 44.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 156.0 0.0 0.0 0.0 -156.0
-1969-12-31 16:00:03.381 -19.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 499.32 -19.0 19.0 19.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7177.0 0.0 0.0 0.0 -7177.0
-1969-12-31 16:00:03.395 -13.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 341.64 -13.0 13.0 13.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7183.0 0.0 0.0 0.0 -7183.0
-1969-12-31 16:00:03.4 21.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -551.88 21.0 -21.0 -21.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7217.0 0.0 0.0 0.0 -7217.0
-1969-12-31 16:00:03.506 -29.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 762.12 -29.0 29.0 29.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7167.0 0.0 0.0 0.0 -7167.0
-1969-12-31 16:00:03.52 -5.0 NULL NULL -200.0 -226.28 226.28 0.0 131.40001 -5.0 5.0 5.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 195.0 0.0 0.0 0.0 -195.0
-1969-12-31 16:00:03.571 32.0 NULL NULL -200.0 -226.28 226.28 0.0 -840.96 32.0 -32.0 -32.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 232.0 0.0 0.0 0.0 -232.0
-1969-12-31 16:00:03.63 44.0 NULL NULL -200.0 -226.28 226.28 0.0 -1156.3201 44.0 -44.0 -44.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 244.0 0.0 0.0 0.0 -244.0
-1969-12-31 16:00:03.741 -40.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1051.2001 -40.0 40.0 40.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7156.0 0.0 0.0 0.0 -7156.0
-1969-12-31 16:00:03.794 13.0 NULL NULL -200.0 -226.28 226.28 0.0 -341.64 13.0 -13.0 -13.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 213.0 0.0 0.0 0.0 -213.0
-1969-12-31 16:00:03.809 30.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -788.4 30.0 -30.0 -30.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7226.0 0.0 0.0 0.0 -7226.0
-1969-12-31 16:00:03.818 32.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -840.96 32.0 -32.0 -32.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7228.0 0.0 0.0 0.0 -7228.0
-1969-12-31 16:00:03.855 -29.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 762.12 -29.0 29.0 29.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7167.0 0.0 0.0 0.0 -7167.0
-1969-12-31 16:00:03.944 -64.0 NULL NULL -200.0 -226.28 226.28 0.0 1681.92 -64.0 64.0 64.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 136.0 0.0 0.0 0.0 -136.0
-1969-12-31 16:00:03.963 -52.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1366.56 -52.0 52.0 52.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7144.0 0.0 0.0 0.0 -7144.0
-1969-12-31 16:00:04.024 52.0 NULL NULL -200.0 -226.28 226.28 0.0 -1366.56 52.0 -52.0 -52.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 252.0 0.0 0.0 0.0 -252.0
-1969-12-31 16:00:04.058 5.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -131.40001 5.0 -5.0 -5.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7201.0 0.0 0.0 0.0 -7201.0
-1969-12-31 16:00:04.12 41.0 NULL NULL -200.0 -226.28 226.28 0.0 -1077.48 41.0 -41.0 -41.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 241.0 0.0 0.0 0.0 -241.0
-1969-12-31 16:00:04.136 -6.0 NULL NULL -200.0 -226.28 226.28 0.0 157.68001 -6.0 6.0 6.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 194.0 0.0 0.0 0.0 -194.0
-1969-12-31 16:00:04.16 -59.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1550.52 -59.0 59.0 59.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7137.0 0.0 0.0 0.0 -7137.0
-1969-12-31 16:00:04.199 30.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -788.4 30.0 -30.0 -30.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7226.0 0.0 0.0 0.0 -7226.0
-1969-12-31 16:00:04.228 50.0 NULL NULL -200.0 -226.28 226.28 0.0 -1314.0 50.0 -50.0 -50.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 250.0 0.0 0.0 0.0 -250.0
-1969-12-31 16:00:04.236 -30.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 788.4 -30.0 30.0 30.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7166.0 0.0 0.0 0.0 -7166.0
-1969-12-31 16:00:04.36 34.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -893.52 34.0 -34.0 -34.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7230.0 0.0 0.0 0.0 -7230.0
-1969-12-31 16:00:04.396 33.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -867.24005 33.0 -33.0 -33.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7229.0 0.0 0.0 0.0 -7229.0
-1969-12-31 16:00:04.431 44.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1156.3201 44.0 -44.0 -44.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7240.0 0.0 0.0 0.0 -7240.0
-1969-12-31 16:00:04.442 -44.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1156.3201 -44.0 44.0 44.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7152.0 0.0 0.0 0.0 -7152.0
-1969-12-31 16:00:04.443 -8.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 210.24 -8.0 8.0 8.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7188.0 0.0 0.0 0.0 -7188.0
-1969-12-31 16:00:04.513 17.0 NULL NULL -200.0 -226.28 226.28 0.0 -446.76 17.0 -17.0 -17.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 217.0 0.0 0.0 0.0 -217.0
-1969-12-31 16:00:04.572 45.0 NULL NULL -200.0 -226.28 226.28 0.0 -1182.6 45.0 -45.0 -45.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 245.0 0.0 0.0 0.0 -245.0
-1969-12-31 16:00:04.574 -14.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 367.92 -14.0 14.0 14.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7182.0 0.0 0.0 0.0 -7182.0
-1969-12-31 16:00:04.625 -18.0 NULL NULL -200.0 -226.28 226.28 0.0 473.04 -18.0 18.0 18.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 182.0 0.0 0.0 0.0 -182.0
-1969-12-31 16:00:04.682 -58.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1524.24 -58.0 58.0 58.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7138.0 0.0 0.0 0.0 -7138.0
-1969-12-31 16:00:04.747 -28.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 735.84 -28.0 28.0 28.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7168.0 0.0 0.0 0.0 -7168.0
-1969-12-31 16:00:04.756 -34.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 893.52 -34.0 34.0 34.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7162.0 0.0 0.0 0.0 -7162.0
-1969-12-31 16:00:04.827 -41.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1077.48 -41.0 41.0 41.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7155.0 0.0 0.0 0.0 -7155.0
-1969-12-31 16:00:04.836 -1.0 NULL NULL -200.0 -226.28 226.28 0.0 26.28 -1.0 1.0 1.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 199.0 0.0 0.0 0.0 -199.0
-1969-12-31 16:00:04.868 -49.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1287.7201 -49.0 49.0 49.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7147.0 0.0 0.0 0.0 -7147.0
-1969-12-31 16:00:04.916 1.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -26.28 1.0 -1.0 -1.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7197.0 0.0 0.0 0.0 -7197.0
-1969-12-31 16:00:04.928 32.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -840.96 32.0 -32.0 -32.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7228.0 0.0 0.0 0.0 -7228.0
-1969-12-31 16:00:04.967 62.0 NULL NULL -200.0 -226.28 226.28 0.0 -1629.36 62.0 -62.0 -62.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 262.0 0.0 0.0 0.0 -262.0
-1969-12-31 16:00:04.994 -1.0 NULL NULL -200.0 -226.28 226.28 0.0 26.28 -1.0 1.0 1.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 199.0 0.0 0.0 0.0 -199.0
-1969-12-31 16:00:05.028 13.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -341.64 13.0 -13.0 -13.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7209.0 0.0 0.0 0.0 -7209.0
-1969-12-31 16:00:05.051 50.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1314.0 50.0 -50.0 -50.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7246.0 0.0 0.0 0.0 -7246.0
-1969-12-31 16:00:05.066 30.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -788.4 30.0 -30.0 -30.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7226.0 0.0 0.0 0.0 -7226.0
-1969-12-31 16:00:05.092 42.0 NULL NULL -200.0 -226.28 226.28 0.0 -1103.76 42.0 -42.0 -42.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 242.0 0.0 0.0 0.0 -242.0
-1969-12-31 16:00:05.105 -19.0 NULL NULL -200.0 -226.28 226.28 0.0 499.32 -19.0 19.0 19.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 181.0 0.0 0.0 0.0 -181.0
-1969-12-31 16:00:05.113 -15.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 394.2 -15.0 15.0 15.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7181.0 0.0 0.0 0.0 -7181.0
-1969-12-31 16:00:05.13 59.0 NULL NULL -200.0 -226.28 226.28 0.0 -1550.52 59.0 -59.0 -59.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 259.0 0.0 0.0 0.0 -259.0
-1969-12-31 16:00:05.178 -32.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 840.96 -32.0 32.0 32.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7164.0 0.0 0.0 0.0 -7164.0
-1969-12-31 16:00:05.218 -22.0 NULL NULL -200.0 -226.28 226.28 0.0 578.16003 -22.0 22.0 22.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 178.0 0.0 0.0 0.0 -178.0
-1969-12-31 16:00:05.219 26.0 NULL NULL -200.0 -226.28 226.28 0.0 -683.28 26.0 -26.0 -26.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 226.0 0.0 0.0 0.0 -226.0
-1969-12-31 16:00:05.226 46.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1208.88 46.0 -46.0 -46.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7242.0 0.0 0.0 0.0 -7242.0
-1969-12-31 16:00:05.241 -18.0 NULL NULL -200.0 -226.28 226.28 0.0 473.04 -18.0 18.0 18.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 182.0 0.0 0.0 0.0 -182.0
-1969-12-31 16:00:05.29 38.0 NULL NULL -200.0 -226.28 226.28 0.0 -998.64 38.0 -38.0 -38.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 238.0 0.0 0.0 0.0 -238.0
-1969-12-31 16:00:05.356 37.0 NULL NULL -200.0 -226.28 226.28 0.0 -972.36005 37.0 -37.0 -37.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 237.0 0.0 0.0 0.0 -237.0
-1969-12-31 16:00:05.368 13.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -341.64 13.0 -13.0 -13.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7209.0 0.0 0.0 0.0 -7209.0
-1969-12-31 16:00:05.369 49.0 NULL NULL -200.0 -226.28 226.28 0.0 -1287.7201 49.0 -49.0 -49.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 249.0 0.0 0.0 0.0 -249.0
-1969-12-31 16:00:05.377 -52.0 NULL NULL -200.0 -226.28 226.28 0.0 1366.56 -52.0 52.0 52.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 148.0 0.0 0.0 0.0 -148.0
-1969-12-31 16:00:05.383 10.0 NULL NULL -200.0 -226.28 226.28 0.0 -262.80002 10.0 -10.0 -10.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 210.0 0.0 0.0 0.0 -210.0
-1969-12-31 16:00:05.43 54.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1419.12 54.0 -54.0 -54.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7250.0 0.0 0.0 0.0 -7250.0
-1969-12-31 16:00:05.451 28.0 NULL NULL -200.0 -226.28 226.28 0.0 -735.84 28.0 -28.0 -28.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 228.0 0.0 0.0 0.0 -228.0
-1969-12-31 16:00:05.495 -6.0 NULL NULL -200.0 -226.28 226.28 0.0 157.68001 -6.0 6.0 6.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 194.0 0.0 0.0 0.0 -194.0
-1969-12-31 16:00:05.5 -57.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1497.9601 -57.0 57.0 57.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7139.0 0.0 0.0 0.0 -7139.0
-1969-12-31 16:00:05.63 15.0 NULL NULL -200.0 -226.28 226.28 0.0 -394.2 15.0 -15.0 -15.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 215.0 0.0 0.0 0.0 -215.0
-1969-12-31 16:00:05.68 -45.0 NULL NULL -200.0 -226.28 226.28 0.0 1182.6 -45.0 45.0 45.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 155.0 0.0 0.0 0.0 -155.0
-1969-12-31 16:00:05.688 -54.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1419.12 -54.0 54.0 54.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7142.0 0.0 0.0 0.0 -7142.0
-1969-12-31 16:00:05.722 20.0 NULL NULL -200.0 -226.28 226.28 0.0 -525.60004 20.0 -20.0 -20.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 220.0 0.0 0.0 0.0 -220.0
-1969-12-31 16:00:05.731 22.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -578.16003 22.0 -22.0 -22.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7218.0 0.0 0.0 0.0 -7218.0
-1969-12-31 16:00:05.784 15.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -394.2 15.0 -15.0 -15.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7211.0 0.0 0.0 0.0 -7211.0
-1969-12-31 16:00:05.79 29.0 NULL NULL -200.0 -226.28 226.28 0.0 -762.12 29.0 -29.0 -29.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 229.0 0.0 0.0 0.0 -229.0
-1969-12-31 16:00:05.793 -55.0 NULL NULL -200.0 -226.28 226.28 0.0 1445.4 -55.0 55.0 55.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 145.0 0.0 0.0 0.0 -145.0
-1969-12-31 16:00:05.804 18.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -473.04 18.0 -18.0 -18.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7214.0 0.0 0.0 0.0 -7214.0
-1969-12-31 16:00:05.814 -49.0 NULL NULL -200.0 -226.28 226.28 0.0 1287.7201 -49.0 49.0 49.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 151.0 0.0 0.0 0.0 -151.0
-1969-12-31 16:00:05.865 16.0 NULL NULL -200.0 -226.28 226.28 0.0 -420.48 16.0 -16.0 -16.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 216.0 0.0 0.0 0.0 -216.0
-1969-12-31 16:00:05.892 31.0 NULL NULL -200.0 -226.28 226.28 0.0 -814.68 31.0 -31.0 -31.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 231.0 0.0 0.0 0.0 -231.0
-1969-12-31 16:00:05.927 -59.0 NULL NULL -200.0 -226.28 226.28 0.0 1550.52 -59.0 59.0 59.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 141.0 0.0 0.0 0.0 -141.0
-1969-12-31 16:00:05.944 29.0 NULL NULL -200.0 -226.28 226.28 0.0 -762.12 29.0 -29.0 -29.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 229.0 0.0 0.0 0.0 -229.0
-1969-12-31 16:00:05.978 -48.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1261.4401 -48.0 48.0 48.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7148.0 0.0 0.0 0.0 -7148.0
-1969-12-31 16:00:06.018 7.0 NULL NULL -200.0 -226.28 226.28 0.0 -183.96 7.0 -7.0 -7.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 207.0 0.0 0.0 0.0 -207.0
-1969-12-31 16:00:06.061 6.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -157.68001 6.0 -6.0 -6.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7202.0 0.0 0.0 0.0 -7202.0
-1969-12-31 16:00:06.132 3.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -78.840004 3.0 -3.0 -3.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7199.0 0.0 0.0 0.0 -7199.0
-1969-12-31 16:00:06.149 39.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1024.92 39.0 -39.0 -39.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7235.0 0.0 0.0 0.0 -7235.0
-1969-12-31 16:00:06.3 14.0 NULL NULL -200.0 -226.28 226.28 0.0 -367.92 14.0 -14.0 -14.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 214.0 0.0 0.0 0.0 -214.0
-1969-12-31 16:00:06.315 30.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -788.4 30.0 -30.0 -30.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7226.0 0.0 0.0 0.0 -7226.0
-1969-12-31 16:00:06.346 40.0 NULL NULL -200.0 -226.28 226.28 0.0 -1051.2001 40.0 -40.0 -40.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 240.0 0.0 0.0 0.0 -240.0
-1969-12-31 16:00:06.371 50.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1314.0 50.0 -50.0 -50.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7246.0 0.0 0.0 0.0 -7246.0
-1969-12-31 16:00:06.4 -6.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 157.68001 -6.0 6.0 6.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7190.0 0.0 0.0 0.0 -7190.0
-1969-12-31 16:00:06.404 20.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -525.60004 20.0 -20.0 -20.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7216.0 0.0 0.0 0.0 -7216.0
-1969-12-31 16:00:06.405 -34.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 893.52 -34.0 34.0 34.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7162.0 0.0 0.0 0.0 -7162.0
-1969-12-31 16:00:06.481 -16.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 420.48 -16.0 16.0 16.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7180.0 0.0 0.0 0.0 -7180.0
-1969-12-31 16:00:06.484 -54.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1419.12 -54.0 54.0 54.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7142.0 0.0 0.0 0.0 -7142.0
-1969-12-31 16:00:06.498 -2.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 52.56 -2.0 2.0 2.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7194.0 0.0 0.0 0.0 -7194.0
-1969-12-31 16:00:06.506 -25.0 NULL NULL -200.0 -226.28 226.28 0.0 657.0 -25.0 25.0 25.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 175.0 0.0 0.0 0.0 -175.0
-1969-12-31 16:00:06.51 36.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -946.08 36.0 -36.0 -36.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7232.0 0.0 0.0 0.0 -7232.0
-1969-12-31 16:00:06.511 27.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -709.56 27.0 -27.0 -27.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7223.0 0.0 0.0 0.0 -7223.0
-1969-12-31 16:00:06.523 -23.0 NULL NULL -200.0 -226.28 226.28 0.0 604.44 -23.0 23.0 23.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 177.0 0.0 0.0 0.0 -177.0
-1969-12-31 16:00:06.568 -24.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 630.72003 -24.0 24.0 24.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7172.0 0.0 0.0 0.0 -7172.0
-1969-12-31 16:00:06.578 43.0 NULL NULL -200.0 -226.28 226.28 0.0 -1130.04 43.0 -43.0 -43.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 243.0 0.0 0.0 0.0 -243.0
-1969-12-31 16:00:06.603 11.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -289.08002 11.0 -11.0 -11.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7207.0 0.0 0.0 0.0 -7207.0
-1969-12-31 16:00:06.624 3.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -78.840004 3.0 -3.0 -3.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7199.0 0.0 0.0 0.0 -7199.0
-1969-12-31 16:00:06.661 -36.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 946.08 -36.0 36.0 36.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7160.0 0.0 0.0 0.0 -7160.0
-1969-12-31 16:00:06.664 -57.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1497.9601 -57.0 57.0 57.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7139.0 0.0 0.0 0.0 -7139.0
-1969-12-31 16:00:06.688 22.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -578.16003 22.0 -22.0 -22.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7218.0 0.0 0.0 0.0 -7218.0
-1969-12-31 16:00:06.731 14.0 NULL NULL -200.0 -226.28 226.28 0.0 -367.92 14.0 -14.0 -14.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 214.0 0.0 0.0 0.0 -214.0
-1969-12-31 16:00:06.749 13.0 NULL NULL -200.0 -226.28 226.28 0.0 -341.64 13.0 -13.0 -13.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 213.0 0.0 0.0 0.0 -213.0
-1969-12-31 16:00:06.811 60.0 NULL NULL -200.0 -226.28 226.28 0.0 -1576.8 60.0 -60.0 -60.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 260.0 0.0 0.0 0.0 -260.0
-1969-12-31 16:00:06.848 -61.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1603.0801 -61.0 61.0 61.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7135.0 0.0 0.0 0.0 -7135.0
-1969-12-31 16:00:06.852 -63.0 NULL NULL -200.0 -226.28 226.28 0.0 1655.64 -63.0 63.0 63.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 137.0 0.0 0.0 0.0 -137.0
-1969-12-31 16:00:06.906 51.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1340.28 51.0 -51.0 -51.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7247.0 0.0 0.0 0.0 -7247.0
-1969-12-31 16:00:06.935 -53.0 NULL NULL -200.0 -226.28 226.28 0.0 1392.8401 -53.0 53.0 53.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 147.0 0.0 0.0 0.0 -147.0
-1969-12-31 16:00:07.022 -25.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 657.0 -25.0 25.0 25.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7171.0 0.0 0.0 0.0 -7171.0
-1969-12-31 16:00:07.046 -48.0 NULL NULL -200.0 -226.28 226.28 0.0 1261.4401 -48.0 48.0 48.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 152.0 0.0 0.0 0.0 -152.0
-1969-12-31 16:00:07.115 48.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1261.4401 48.0 -48.0 -48.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7244.0 0.0 0.0 0.0 -7244.0
-1969-12-31 16:00:07.163 4.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -105.12 4.0 -4.0 -4.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7200.0 0.0 0.0 0.0 -7200.0
-1969-12-31 16:00:07.175 41.0 NULL NULL -200.0 -226.28 226.28 0.0 -1077.48 41.0 -41.0 -41.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 241.0 0.0 0.0 0.0 -241.0
-1969-12-31 16:00:07.179 42.0 NULL NULL -200.0 -226.28 226.28 0.0 -1103.76 42.0 -42.0 -42.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 242.0 0.0 0.0 0.0 -242.0
-1969-12-31 16:00:07.204 -41.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1077.48 -41.0 41.0 41.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7155.0 0.0 0.0 0.0 -7155.0
-1969-12-31 16:00:07.212 -56.0 NULL NULL -200.0 -226.28 226.28 0.0 1471.68 -56.0 56.0 56.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 144.0 0.0 0.0 0.0 -144.0
-1969-12-31 16:00:07.243 17.0 NULL NULL -200.0 -226.28 226.28 0.0 -446.76 17.0 -17.0 -17.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 217.0 0.0 0.0 0.0 -217.0
-1969-12-31 16:00:07.257 48.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1261.4401 48.0 -48.0 -48.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7244.0 0.0 0.0 0.0 -7244.0
-1969-12-31 16:00:07.331 25.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -657.0 25.0 -25.0 -25.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7221.0 0.0 0.0 0.0 -7221.0
-1969-12-31 16:00:07.361 57.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1497.9601 57.0 -57.0 -57.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7253.0 0.0 0.0 0.0 -7253.0
-1969-12-31 16:00:07.365 10.0 NULL NULL -200.0 -226.28 226.28 0.0 -262.80002 10.0 -10.0 -10.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 210.0 0.0 0.0 0.0 -210.0
-1969-12-31 16:00:07.423 -6.0 NULL NULL -200.0 -226.28 226.28 0.0 157.68001 -6.0 6.0 6.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 194.0 0.0 0.0 0.0 -194.0
-1969-12-31 16:00:07.461 -41.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1077.48 -41.0 41.0 41.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7155.0 0.0 0.0 0.0 -7155.0
-1969-12-31 16:00:07.497 45.0 NULL NULL -200.0 -226.28 226.28 0.0 -1182.6 45.0 -45.0 -45.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 245.0 0.0 0.0 0.0 -245.0
-1969-12-31 16:00:07.504 36.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -946.08 36.0 -36.0 -36.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7232.0 0.0 0.0 0.0 -7232.0
-1969-12-31 16:00:07.541 39.0 NULL NULL -200.0 -226.28 226.28 0.0 -1024.92 39.0 -39.0 -39.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 239.0 0.0 0.0 0.0 -239.0
-1969-12-31 16:00:07.548 17.0 NULL NULL -200.0 -226.28 226.28 0.0 -446.76 17.0 -17.0 -17.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 217.0 0.0 0.0 0.0 -217.0
-1969-12-31 16:00:07.6 -14.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 367.92 -14.0 14.0 14.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7182.0 0.0 0.0 0.0 -7182.0
-1969-12-31 16:00:07.607 15.0 NULL NULL -200.0 -226.28 226.28 0.0 -394.2 15.0 -15.0 -15.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 215.0 0.0 0.0 0.0 -215.0
-1969-12-31 16:00:07.613 35.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -919.80005 35.0 -35.0 -35.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7231.0 0.0 0.0 0.0 -7231.0
-1969-12-31 16:00:07.642 -24.0 NULL NULL -200.0 -226.28 226.28 0.0 630.72003 -24.0 24.0 24.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 176.0 0.0 0.0 0.0 -176.0
-1969-12-31 16:00:07.651 -9.0 NULL NULL -200.0 -226.28 226.28 0.0 236.52 -9.0 9.0 9.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 191.0 0.0 0.0 0.0 -191.0
-1969-12-31 16:00:07.675 32.0 NULL NULL -200.0 -226.28 226.28 0.0 -840.96 32.0 -32.0 -32.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 232.0 0.0 0.0 0.0 -232.0
-1969-12-31 16:00:07.678 16.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -420.48 16.0 -16.0 -16.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7212.0 0.0 0.0 0.0 -7212.0
-1969-12-31 16:00:07.711 -2.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 52.56 -2.0 2.0 2.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7194.0 0.0 0.0 0.0 -7194.0
-1969-12-31 16:00:07.712 -4.0 NULL NULL -200.0 -226.28 226.28 0.0 105.12 -4.0 4.0 4.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 196.0 0.0 0.0 0.0 -196.0
-1969-12-31 16:00:07.828 62.0 NULL NULL -200.0 -226.28 226.28 0.0 -1629.36 62.0 -62.0 -62.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 262.0 0.0 0.0 0.0 -262.0
-1969-12-31 16:00:07.907 57.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1497.9601 57.0 -57.0 -57.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7253.0 0.0 0.0 0.0 -7253.0
-1969-12-31 16:00:07.942 -20.0 NULL NULL -200.0 -226.28 226.28 0.0 525.60004 -20.0 20.0 20.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 180.0 0.0 0.0 0.0 -180.0
-1969-12-31 16:00:07.946 -11.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 289.08002 -11.0 11.0 11.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7185.0 0.0 0.0 0.0 -7185.0
-1969-12-31 16:00:08 32.0 NULL NULL -200.0 -226.28 226.28 0.0 -840.96 32.0 -32.0 -32.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 232.0 0.0 0.0 0.0 -232.0
-1969-12-31 16:00:08.001 29.0 NULL NULL -200.0 -226.28 226.28 0.0 -762.12 29.0 -29.0 -29.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 229.0 0.0 0.0 0.0 -229.0
-1969-12-31 16:00:08.007 -8.0 NULL NULL -200.0 -226.28 226.28 0.0 210.24 -8.0 8.0 8.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 192.0 0.0 0.0 0.0 -192.0
-1969-12-31 16:00:08.011 -41.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1077.48 -41.0 41.0 41.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7155.0 0.0 0.0 0.0 -7155.0
-1969-12-31 16:00:08.03 31.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -814.68 31.0 -31.0 -31.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7227.0 0.0 0.0 0.0 -7227.0
-1969-12-31 16:00:08.04 -38.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 998.64 -38.0 38.0 38.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7158.0 0.0 0.0 0.0 -7158.0
-1969-12-31 16:00:08.046 -33.0 NULL NULL -200.0 -226.28 226.28 0.0 867.24005 -33.0 33.0 33.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 167.0 0.0 0.0 0.0 -167.0
-1969-12-31 16:00:08.048 21.0 NULL NULL -200.0 -226.28 226.28 0.0 -551.88 21.0 -21.0 -21.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 221.0 0.0 0.0 0.0 -221.0
-1969-12-31 16:00:08.063 51.0 NULL NULL -200.0 -226.28 226.28 0.0 -1340.28 51.0 -51.0 -51.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 251.0 0.0 0.0 0.0 -251.0
-1969-12-31 16:00:08.091 -47.0 NULL NULL -200.0 -226.28 226.28 0.0 1235.16 -47.0 47.0 47.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 153.0 0.0 0.0 0.0 -153.0
-1969-12-31 16:00:08.191 55.0 NULL NULL -200.0 -226.28 226.28 0.0 -1445.4 55.0 -55.0 -55.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 255.0 0.0 0.0 0.0 -255.0
-1969-12-31 16:00:08.198 -62.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1629.36 -62.0 62.0 62.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7134.0 0.0 0.0 0.0 -7134.0
-1969-12-31 16:00:08.241 54.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1419.12 54.0 -54.0 -54.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7250.0 0.0 0.0 0.0 -7250.0
-1969-12-31 16:00:08.267 -5.0 NULL NULL -200.0 -226.28 226.28 0.0 131.40001 -5.0 5.0 5.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 195.0 0.0 0.0 0.0 -195.0
-1969-12-31 16:00:08.27 11.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -289.08002 11.0 -11.0 -11.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7207.0 0.0 0.0 0.0 -7207.0
-1969-12-31 16:00:08.292 28.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -735.84 28.0 -28.0 -28.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7224.0 0.0 0.0 0.0 -7224.0
-1969-12-31 16:00:08.307 23.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -604.44 23.0 -23.0 -23.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7219.0 0.0 0.0 0.0 -7219.0
-1969-12-31 16:00:08.33 -26.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 683.28 -26.0 26.0 26.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7170.0 0.0 0.0 0.0 -7170.0
-1969-12-31 16:00:08.351 -45.0 NULL NULL -200.0 -226.28 226.28 0.0 1182.6 -45.0 45.0 45.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 155.0 0.0 0.0 0.0 -155.0
-1969-12-31 16:00:08.378 45.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1182.6 45.0 -45.0 -45.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7241.0 0.0 0.0 0.0 -7241.0
-1969-12-31 16:00:08.38 50.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1314.0 50.0 -50.0 -50.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7246.0 0.0 0.0 0.0 -7246.0
-1969-12-31 16:00:08.408 41.0 NULL NULL -200.0 -226.28 226.28 0.0 -1077.48 41.0 -41.0 -41.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 241.0 0.0 0.0 0.0 -241.0
-1969-12-31 16:00:08.418 41.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1077.48 41.0 -41.0 -41.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7237.0 0.0 0.0 0.0 -7237.0
-1969-12-31 16:00:08.549 -14.0 NULL NULL -200.0 -226.28 226.28 0.0 367.92 -14.0 14.0 14.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 186.0 0.0 0.0 0.0 -186.0
-1969-12-31 16:00:08.554 30.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -788.4 30.0 -30.0 -30.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7226.0 0.0 0.0 0.0 -7226.0
-1969-12-31 16:00:08.58 35.0 NULL NULL -200.0 -226.28 226.28 0.0 -919.80005 35.0 -35.0 -35.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 235.0 0.0 0.0 0.0 -235.0
-1969-12-31 16:00:08.615 -36.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 946.08 -36.0 36.0 36.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7160.0 0.0 0.0 0.0 -7160.0
-1969-12-31 16:00:08.615 15.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -394.2 15.0 -15.0 -15.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7211.0 0.0 0.0 0.0 -7211.0
-1969-12-31 16:00:08.692 37.0 NULL NULL -200.0 -226.28 226.28 0.0 -972.36005 37.0 -37.0 -37.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 237.0 0.0 0.0 0.0 -237.0
-1969-12-31 16:00:08.693 -48.0 NULL NULL -200.0 -226.28 226.28 0.0 1261.4401 -48.0 48.0 48.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 152.0 0.0 0.0 0.0 -152.0
-1969-12-31 16:00:08.703 38.0 NULL NULL -200.0 -226.28 226.28 0.0 -998.64 38.0 -38.0 -38.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 238.0 0.0 0.0 0.0 -238.0
-1969-12-31 16:00:08.704 -14.0 NULL NULL -200.0 -226.28 226.28 0.0 367.92 -14.0 14.0 14.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 186.0 0.0 0.0 0.0 -186.0
-1969-12-31 16:00:08.726 49.0 NULL NULL -200.0 -226.28 226.28 0.0 -1287.7201 49.0 -49.0 -49.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 249.0 0.0 0.0 0.0 -249.0
-1969-12-31 16:00:08.74 -58.0 NULL NULL -200.0 -226.28 226.28 0.0 1524.24 -58.0 58.0 58.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 142.0 0.0 0.0 0.0 -142.0
-1969-12-31 16:00:08.745 11.0 NULL NULL -200.0 -226.28 226.28 0.0 -289.08002 11.0 -11.0 -11.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 211.0 0.0 0.0 0.0 -211.0
-1969-12-31 16:00:08.757 8.0 NULL NULL -200.0 -226.28 226.28 0.0 -210.24 8.0 -8.0 -8.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 208.0 0.0 0.0 0.0 -208.0
-1969-12-31 16:00:08.781 -6.0 NULL NULL -200.0 -226.28 226.28 0.0 157.68001 -6.0 6.0 6.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 194.0 0.0 0.0 0.0 -194.0
-1969-12-31 16:00:08.805 -54.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1419.12 -54.0 54.0 54.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7142.0 0.0 0.0 0.0 -7142.0
-1969-12-31 16:00:08.839 -24.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 630.72003 -24.0 24.0 24.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7172.0 0.0 0.0 0.0 -7172.0
-1969-12-31 16:00:08.852 -39.0 NULL NULL -200.0 -226.28 226.28 0.0 1024.92 -39.0 39.0 39.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 161.0 0.0 0.0 0.0 -161.0
-1969-12-31 16:00:08.884 -47.0 NULL NULL -200.0 -226.28 226.28 0.0 1235.16 -47.0 47.0 47.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 153.0 0.0 0.0 0.0 -153.0
-1969-12-31 16:00:08.896 -55.0 NULL NULL -200.0 -226.28 226.28 0.0 1445.4 -55.0 55.0 55.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 145.0 0.0 0.0 0.0 -145.0
-1969-12-31 16:00:09.001 19.0 NULL NULL -200.0 -226.28 226.28 0.0 -499.32 19.0 -19.0 -19.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 219.0 0.0 0.0 0.0 -219.0
-1969-12-31 16:00:09.061 -53.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1392.8401 -53.0 53.0 53.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7143.0 0.0 0.0 0.0 -7143.0
-1969-12-31 16:00:09.111 -37.0 NULL NULL -200.0 -226.28 226.28 0.0 972.36005 -37.0 37.0 37.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 163.0 0.0 0.0 0.0 -163.0
-1969-12-31 16:00:09.144 -42.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1103.76 -42.0 42.0 42.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7154.0 0.0 0.0 0.0 -7154.0
-1969-12-31 16:00:09.161 2.0 NULL NULL -200.0 -226.28 226.28 0.0 -52.56 2.0 -2.0 -2.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 202.0 0.0 0.0 0.0 -202.0
-1969-12-31 16:00:09.182 -21.0 NULL NULL -200.0 -226.28 226.28 0.0 551.88 -21.0 21.0 21.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 179.0 0.0 0.0 0.0 -179.0
-1969-12-31 16:00:09.21 14.0 NULL NULL -200.0 -226.28 226.28 0.0 -367.92 14.0 -14.0 -14.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 214.0 0.0 0.0 0.0 -214.0
-1969-12-31 16:00:09.22 10.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -262.80002 10.0 -10.0 -10.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7206.0 0.0 0.0 0.0 -7206.0
-1969-12-31 16:00:09.251 61.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1603.0801 61.0 -61.0 -61.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7257.0 0.0 0.0 0.0 -7257.0
-1969-12-31 16:00:09.387 13.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -341.64 13.0 -13.0 -13.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7209.0 0.0 0.0 0.0 -7209.0
-1969-12-31 16:00:09.416 -20.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 525.60004 -20.0 20.0 20.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7176.0 0.0 0.0 0.0 -7176.0
-1969-12-31 16:00:09.421 -55.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1445.4 -55.0 55.0 55.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7141.0 0.0 0.0 0.0 -7141.0
-1969-12-31 16:00:09.441 57.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1497.9601 57.0 -57.0 -57.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7253.0 0.0 0.0 0.0 -7253.0
-1969-12-31 16:00:09.452 56.0 NULL NULL -200.0 -226.28 226.28 0.0 -1471.68 56.0 -56.0 -56.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 256.0 0.0 0.0 0.0 -256.0
-1969-12-31 16:00:09.511 -1.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 26.28 -1.0 1.0 1.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7195.0 0.0 0.0 0.0 -7195.0
-1969-12-31 16:00:09.519 8.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -210.24 8.0 -8.0 -8.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7204.0 0.0 0.0 0.0 -7204.0
-1969-12-31 16:00:09.539 7.0 NULL NULL -200.0 -226.28 226.28 0.0 -183.96 7.0 -7.0 -7.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 207.0 0.0 0.0 0.0 -207.0
-1969-12-31 16:00:09.556 12.0 NULL NULL -200.0 -226.28 226.28 0.0 -315.36002 12.0 -12.0 -12.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 212.0 0.0 0.0 0.0 -212.0
-1969-12-31 16:00:09.622 -58.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1524.24 -58.0 58.0 58.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7138.0 0.0 0.0 0.0 -7138.0
-1969-12-31 16:00:09.65 34.0 NULL NULL -200.0 -226.28 226.28 0.0 -893.52 34.0 -34.0 -34.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 234.0 0.0 0.0 0.0 -234.0
-1969-12-31 16:00:09.819 -3.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 78.840004 -3.0 3.0 3.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7193.0 0.0 0.0 0.0 -7193.0
-1969-12-31 16:00:09.842 22.0 NULL NULL -200.0 -226.28 226.28 0.0 -578.16003 22.0 -22.0 -22.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 222.0 0.0 0.0 0.0 -222.0
-1969-12-31 16:00:09.907 37.0 NULL NULL -200.0 -226.28 226.28 0.0 -972.36005 37.0 -37.0 -37.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 237.0 0.0 0.0 0.0 -237.0
-1969-12-31 16:00:09.911 22.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -578.16003 22.0 -22.0 -22.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7218.0 0.0 0.0 0.0 -7218.0
-1969-12-31 16:00:09.93 -28.0 NULL NULL -200.0 -226.28 226.28 0.0 735.84 -28.0 28.0 28.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 172.0 0.0 0.0 0.0 -172.0
-1969-12-31 16:00:09.934 37.0 NULL NULL -200.0 -226.28 226.28 0.0 -972.36005 37.0 -37.0 -37.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 237.0 0.0 0.0 0.0 -237.0
-1969-12-31 16:00:09.974 -18.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 473.04 -18.0 18.0 18.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7178.0 0.0 0.0 0.0 -7178.0
-1969-12-31 16:00:09.995 8.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -210.24 8.0 -8.0 -8.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7204.0 0.0 0.0 0.0 -7204.0
-1969-12-31 16:00:10.096 -44.0 NULL NULL -200.0 -226.28 226.28 0.0 1156.3201 -44.0 44.0 44.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 156.0 0.0 0.0 0.0 -156.0
-1969-12-31 16:00:10.104 -17.0 NULL NULL -200.0 -226.28 226.28 0.0 446.76 -17.0 17.0 17.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 183.0 0.0 0.0 0.0 -183.0
-1969-12-31 16:00:10.104 -39.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1024.92 -39.0 39.0 39.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7157.0 0.0 0.0 0.0 -7157.0
-1969-12-31 16:00:10.139 0.0 NULL NULL -200.0 -226.28 226.28 0.0 -0.0 0.0 -0.0 -0.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 200.0 0.0 0.0 0.0 -200.0
-1969-12-31 16:00:10.14 25.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -657.0 25.0 -25.0 -25.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7221.0 0.0 0.0 0.0 -7221.0
-1969-12-31 16:00:10.187 54.0 NULL NULL -200.0 -226.28 226.28 0.0 -1419.12 54.0 -54.0 -54.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 254.0 0.0 0.0 0.0 -254.0
-1969-12-31 16:00:10.192 -26.28 NULL NULL -7196.0 -7222.28 7222.28 0.0 690.6384 -26.28 26.28 26.28 709.8063882063881 0.0 1 -709.8063882063881 NULL 7169.7199993133545 0.0 0.0 0.0 -7169.7199993133545
-1969-12-31 16:00:10.198 2.0 NULL NULL -200.0 -226.28 226.28 0.0 -52.56 2.0 -2.0 -2.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 202.0 0.0 0.0 0.0 -202.0
-1969-12-31 16:00:10.225 -62.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1629.36 -62.0 62.0 62.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7134.0 0.0 0.0 0.0 -7134.0
-1969-12-31 16:00:10.227 -9.0 NULL NULL -200.0 -226.28 226.28 0.0 236.52 -9.0 9.0 9.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 191.0 0.0 0.0 0.0 -191.0
-1969-12-31 16:00:10.274 -47.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1235.16 -47.0 47.0 47.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7149.0 0.0 0.0 0.0 -7149.0
-1969-12-31 16:00:10.285 26.0 NULL NULL -200.0 -226.28 226.28 0.0 -683.28 26.0 -26.0 -26.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 226.0 0.0 0.0 0.0 -226.0
-1969-12-31 16:00:10.321 51.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1340.28 51.0 -51.0 -51.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7247.0 0.0 0.0 0.0 -7247.0
-1969-12-31 16:00:10.364 1.0 NULL NULL -200.0 -226.28 226.28 0.0 -26.28 1.0 -1.0 -1.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 201.0 0.0 0.0 0.0 -201.0
-1969-12-31 16:00:10.383 -24.0 NULL NULL -200.0 -226.28 226.28 0.0 630.72003 -24.0 24.0 24.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 176.0 0.0 0.0 0.0 -176.0
-1969-12-31 16:00:10.421 24.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -630.72003 24.0 -24.0 -24.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7220.0 0.0 0.0 0.0 -7220.0
-1969-12-31 16:00:10.452 -9.0 NULL NULL -200.0 -226.28 226.28 0.0 236.52 -9.0 9.0 9.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 191.0 0.0 0.0 0.0 -191.0
-1969-12-31 16:00:10.467 36.0 NULL NULL -200.0 -226.28 226.28 0.0 -946.08 36.0 -36.0 -36.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 236.0 0.0 0.0 0.0 -236.0
-1969-12-31 16:00:10.485 -62.0 NULL NULL -200.0 -226.28 226.28 0.0 1629.36 -62.0 62.0 62.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 138.0 0.0 0.0 0.0 -138.0
-1969-12-31 16:00:10.496 -11.0 NULL NULL -200.0 -226.28 226.28 0.0 289.08002 -11.0 11.0 11.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 189.0 0.0 0.0 0.0 -189.0
-1969-12-31 16:00:10.551 -20.0 NULL NULL -200.0 -226.28 226.28 0.0 525.60004 -20.0 20.0 20.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 180.0 0.0 0.0 0.0 -180.0
-1969-12-31 16:00:10.573 13.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -341.64 13.0 -13.0 -13.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7209.0 0.0 0.0 0.0 -7209.0
-1969-12-31 16:00:10.601 55.0 NULL NULL -200.0 -226.28 226.28 0.0 -1445.4 55.0 -55.0 -55.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 255.0 0.0 0.0 0.0 -255.0
-1969-12-31 16:00:10.649 -32.0 NULL NULL -200.0 -226.28 226.28 0.0 840.96 -32.0 32.0 32.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 168.0 0.0 0.0 0.0 -168.0
-1969-12-31 16:00:10.652 21.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -551.88 21.0 -21.0 -21.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7217.0 0.0 0.0 0.0 -7217.0
-1969-12-31 16:00:10.669 48.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1261.4401 48.0 -48.0 -48.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7244.0 0.0 0.0 0.0 -7244.0
-1969-12-31 16:00:10.674 28.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -735.84 28.0 -28.0 -28.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7224.0 0.0 0.0 0.0 -7224.0
-1969-12-31 16:00:10.701 17.0 NULL NULL -200.0 -226.28 226.28 0.0 -446.76 17.0 -17.0 -17.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 217.0 0.0 0.0 0.0 -217.0
-1969-12-31 16:00:10.721 -29.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 762.12 -29.0 29.0 29.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7167.0 0.0 0.0 0.0 -7167.0
-1969-12-31 16:00:10.723 19.0 NULL NULL -200.0 -226.28 226.28 0.0 -499.32 19.0 -19.0 -19.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 219.0 0.0 0.0 0.0 -219.0
-1969-12-31 16:00:10.835 -43.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1130.04 -43.0 43.0 43.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7153.0 0.0 0.0 0.0 -7153.0
-1969-12-31 16:00:10.867 -50.0 NULL NULL -200.0 -226.28 226.28 0.0 1314.0 -50.0 50.0 50.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 150.0 0.0 0.0 0.0 -150.0
-1969-12-31 16:00:10.939 -17.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 446.76 -17.0 17.0 17.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7179.0 0.0 0.0 0.0 -7179.0
-1969-12-31 16:00:10.959 -33.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 867.24005 -33.0 33.0 33.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7163.0 0.0 0.0 0.0 -7163.0
-1969-12-31 16:00:11.059 -3.0 NULL NULL -200.0 -226.28 226.28 0.0 78.840004 -3.0 3.0 3.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 197.0 0.0 0.0 0.0 -197.0
-1969-12-31 16:00:11.061 -10.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 262.80002 -10.0 10.0 10.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7186.0 0.0 0.0 0.0 -7186.0
-1969-12-31 16:00:11.08 -9.0 NULL NULL -200.0 -226.28 226.28 0.0 236.52 -9.0 9.0 9.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 191.0 0.0 0.0 0.0 -191.0
-1969-12-31 16:00:11.089 0.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -0.0 0.0 -0.0 -0.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7196.0 0.0 0.0 0.0 -7196.0
-1969-12-31 16:00:11.132 61.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1603.0801 61.0 -61.0 -61.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7257.0 0.0 0.0 0.0 -7257.0
-1969-12-31 16:00:11.148 9.0 NULL NULL -200.0 -226.28 226.28 0.0 -236.52 9.0 -9.0 -9.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 209.0 0.0 0.0 0.0 -209.0
-1969-12-31 16:00:11.15 15.0 NULL NULL -200.0 -226.28 226.28 0.0 -394.2 15.0 -15.0 -15.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 215.0 0.0 0.0 0.0 -215.0
-1969-12-31 16:00:11.153 4.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -105.12 4.0 -4.0 -4.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7200.0 0.0 0.0 0.0 -7200.0
-1969-12-31 16:00:11.198 -54.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1419.12 -54.0 54.0 54.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7142.0 0.0 0.0 0.0 -7142.0
-1969-12-31 16:00:11.342 20.0 NULL NULL -200.0 -226.28 226.28 0.0 -525.60004 20.0 -20.0 -20.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 220.0 0.0 0.0 0.0 -220.0
-1969-12-31 16:00:11.356 35.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -919.80005 35.0 -35.0 -35.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7231.0 0.0 0.0 0.0 -7231.0
-1969-12-31 16:00:11.38 -55.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1445.4 -55.0 55.0 55.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7141.0 0.0 0.0 0.0 -7141.0
-1969-12-31 16:00:11.402 -23.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 604.44 -23.0 23.0 23.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7173.0 0.0 0.0 0.0 -7173.0
-1969-12-31 16:00:11.494 -2.0 NULL NULL -200.0 -226.28 226.28 0.0 52.56 -2.0 2.0 2.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 198.0 0.0 0.0 0.0 -198.0
-1969-12-31 16:00:11.515 45.0 NULL NULL -200.0 -226.28 226.28 0.0 -1182.6 45.0 -45.0 -45.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 245.0 0.0 0.0 0.0 -245.0
-1969-12-31 16:00:11.591 42.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1103.76 42.0 -42.0 -42.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7238.0 0.0 0.0 0.0 -7238.0
-1969-12-31 16:00:11.611 -34.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 893.52 -34.0 34.0 34.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7162.0 0.0 0.0 0.0 -7162.0
-1969-12-31 16:00:11.637 -3.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 78.840004 -3.0 3.0 3.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7193.0 0.0 0.0 0.0 -7193.0
-1969-12-31 16:00:11.681 25.0 NULL NULL -200.0 -226.28 226.28 0.0 -657.0 25.0 -25.0 -25.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 225.0 0.0 0.0 0.0 -225.0
-1969-12-31 16:00:11.749 25.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -657.0 25.0 -25.0 -25.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7221.0 0.0 0.0 0.0 -7221.0
-1969-12-31 16:00:11.758 -9.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 236.52 -9.0 9.0 9.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7187.0 0.0 0.0 0.0 -7187.0
-1969-12-31 16:00:11.758 42.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1103.76 42.0 -42.0 -42.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7238.0 0.0 0.0 0.0 -7238.0
-1969-12-31 16:00:11.847 -20.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 525.60004 -20.0 20.0 20.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7176.0 0.0 0.0 0.0 -7176.0
-1969-12-31 16:00:12.006 49.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1287.7201 49.0 -49.0 -49.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7245.0 0.0 0.0 0.0 -7245.0
-1969-12-31 16:00:12.06 2.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -52.56 2.0 -2.0 -2.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7198.0 0.0 0.0 0.0 -7198.0
-1969-12-31 16:00:12.065 -58.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1524.24 -58.0 58.0 58.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7138.0 0.0 0.0 0.0 -7138.0
-1969-12-31 16:00:12.104 10.0 NULL NULL -200.0 -226.28 226.28 0.0 -262.80002 10.0 -10.0 -10.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 210.0 0.0 0.0 0.0 -210.0
-1969-12-31 16:00:12.112 22.0 NULL NULL -200.0 -226.28 226.28 0.0 -578.16003 22.0 -22.0 -22.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 222.0 0.0 0.0 0.0 -222.0
-1969-12-31 16:00:12.163 25.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -657.0 25.0 -25.0 -25.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7221.0 0.0 0.0 0.0 -7221.0
-1969-12-31 16:00:12.183 47.0 NULL NULL -200.0 -226.28 226.28 0.0 -1235.16 47.0 -47.0 -47.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 247.0 0.0 0.0 0.0 -247.0
-1969-12-31 16:00:12.317 -23.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 604.44 -23.0 23.0 23.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7173.0 0.0 0.0 0.0 -7173.0
-1969-12-31 16:00:12.339 -64.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1681.92 -64.0 64.0 64.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7132.0 0.0 0.0 0.0 -7132.0
-1969-12-31 16:00:12.36 62.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1629.36 62.0 -62.0 -62.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7258.0 0.0 0.0 0.0 -7258.0
-1969-12-31 16:00:12.473 25.0 NULL NULL -200.0 -226.28 226.28 0.0 -657.0 25.0 -25.0 -25.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 225.0 0.0 0.0 0.0 -225.0
-1969-12-31 16:00:12.477 14.0 NULL NULL -200.0 -226.28 226.28 0.0 -367.92 14.0 -14.0 -14.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 214.0 0.0 0.0 0.0 -214.0
-1969-12-31 16:00:12.502 34.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -893.52 34.0 -34.0 -34.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7230.0 0.0 0.0 0.0 -7230.0
-1969-12-31 16:00:12.523 57.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1497.9601 57.0 -57.0 -57.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7253.0 0.0 0.0 0.0 -7253.0
-1969-12-31 16:00:12.538 -44.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1156.3201 -44.0 44.0 44.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7152.0 0.0 0.0 0.0 -7152.0
-1969-12-31 16:00:12.574 -16.0 NULL NULL -200.0 -226.28 226.28 0.0 420.48 -16.0 16.0 16.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 184.0 0.0 0.0 0.0 -184.0
-1969-12-31 16:00:12.58 15.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -394.2 15.0 -15.0 -15.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7211.0 0.0 0.0 0.0 -7211.0
-1969-12-31 16:00:12.626 -57.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1497.9601 -57.0 57.0 57.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7139.0 0.0 0.0 0.0 -7139.0
-1969-12-31 16:00:12.748 -42.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1103.76 -42.0 42.0 42.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7154.0 0.0 0.0 0.0 -7154.0
-1969-12-31 16:00:12.762 62.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1629.36 62.0 -62.0 -62.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7258.0 0.0 0.0 0.0 -7258.0
-1969-12-31 16:00:12.772 12.0 NULL NULL -200.0 -226.28 226.28 0.0 -315.36002 12.0 -12.0 -12.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 212.0 0.0 0.0 0.0 -212.0
-1969-12-31 16:00:12.901 -22.0 NULL NULL -200.0 -226.28 226.28 0.0 578.16003 -22.0 22.0 22.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 178.0 0.0 0.0 0.0 -178.0
-1969-12-31 16:00:12.921 -45.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1182.6 -45.0 45.0 45.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7151.0 0.0 0.0 0.0 -7151.0
-1969-12-31 16:00:12.935 -30.0 NULL NULL -200.0 -226.28 226.28 0.0 788.4 -30.0 30.0 30.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 170.0 0.0 0.0 0.0 -170.0
-1969-12-31 16:00:12.959 -35.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 919.80005 -35.0 35.0 35.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7161.0 0.0 0.0 0.0 -7161.0
-1969-12-31 16:00:13.046 -39.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1024.92 -39.0 39.0 39.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7157.0 0.0 0.0 0.0 -7157.0
-1969-12-31 16:00:13.064 3.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -78.840004 3.0 -3.0 -3.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7199.0 0.0 0.0 0.0 -7199.0
-1969-12-31 16:00:13.124 -30.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 788.4 -30.0 30.0 30.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7166.0 0.0 0.0 0.0 -7166.0
-1969-12-31 16:00:13.128 54.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 -1419.12 54.0 -54.0 -54.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7250.0 0.0 0.0 0.0 -7250.0
-1969-12-31 16:00:13.132 6.0 NULL NULL -200.0 -226.28 226.28 0.0 -157.68001 6.0 -6.0 -6.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 206.0 0.0 0.0 0.0 -206.0
-1969-12-31 16:00:13.153 21.0 NULL NULL -200.0 -226.28 226.28 0.0 -551.88 21.0 -21.0 -21.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 221.0 0.0 0.0 0.0 -221.0
-1969-12-31 16:00:13.197 -51.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1340.28 -51.0 51.0 51.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7145.0 0.0 0.0 0.0 -7145.0
-1969-12-31 16:00:13.253 -50.0 NULL NULL -200.0 -226.28 226.28 0.0 1314.0 -50.0 50.0 50.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 150.0 0.0 0.0 0.0 -150.0
-1969-12-31 16:00:13.324 -4.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 105.12 -4.0 4.0 4.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7192.0 0.0 0.0 0.0 -7192.0
-1969-12-31 16:00:13.358 -39.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1024.92 -39.0 39.0 39.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7157.0 0.0 0.0 0.0 -7157.0
-1969-12-31 16:00:13.374 -25.0 NULL NULL -200.0 -226.28 226.28 0.0 657.0 -25.0 25.0 25.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 175.0 0.0 0.0 0.0 -175.0
-1969-12-31 16:00:13.383 11.0 NULL NULL -200.0 -226.28 226.28 0.0 -289.08002 11.0 -11.0 -11.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 211.0 0.0 0.0 0.0 -211.0
-1969-12-31 16:00:13.396 0.0 NULL NULL -200.0 -226.28 226.28 0.0 -0.0 0.0 -0.0 -0.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 200.0 0.0 0.0 0.0 -200.0
-1969-12-31 16:00:13.404 15.0 NULL NULL -200.0 -226.28 226.28 0.0 -394.2 15.0 -15.0 -15.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 215.0 0.0 0.0 0.0 -215.0
-1969-12-31 16:00:13.438 -15.0 NULL NULL -200.0 -226.28 226.28 0.0 394.2 -15.0 15.0 15.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 185.0 0.0 0.0 0.0 -185.0
-1969-12-31 16:00:13.455 29.0 NULL NULL -200.0 -226.28 226.28 0.0 -762.12 29.0 -29.0 -29.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 229.0 0.0 0.0 0.0 -229.0
-1969-12-31 16:00:13.473 -9.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 236.52 -9.0 9.0 9.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7187.0 0.0 0.0 0.0 -7187.0
-1969-12-31 16:00:13.495 -19.0 NULL NULL -200.0 -226.28 226.28 0.0 499.32 -19.0 19.0 19.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 181.0 0.0 0.0 0.0 -181.0
-1969-12-31 16:00:13.602 -56.0 NULL NULL -7196.0 -7222.28 7222.28 0.0 1471.68 -56.0 56.0 56.0 709.8063882063881 0.0 1 -709.8063882063881 NULL 7140.0 0.0 0.0 0.0 -7140.0
-1969-12-31 16:00:13.605 -35.0 NULL NULL -200.0 -226.28 226.28 0.0 919.80005 -35.0 35.0 35.0 22.238820638820638 0.0 1 -22.238820638820638 NULL 165.0 0.0 0.0 0.0 -165.0
-1969-12-31 16:00:13.638
<TRUNCATED>
[48/50] [abbrv] hive git commit: HIVE-14671 : merge master into
hive-14535 (Wei Zheng)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
index 0000000,a1187c4..6354f77
mode 000000,100644..100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
@@@ -1,0 -1,243 +1,245 @@@
+ /*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+ package org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.table;
+
+ import org.apache.hadoop.fs.Path;
+ import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+ import org.apache.hadoop.hive.metastore.TableType;
+ import org.apache.hadoop.hive.metastore.api.Database;
+ import org.apache.hadoop.hive.metastore.api.MetaException;
+ import org.apache.hadoop.hive.ql.ErrorMsg;
+ import org.apache.hadoop.hive.ql.exec.ReplCopyTask;
+ import org.apache.hadoop.hive.ql.exec.Task;
+ import org.apache.hadoop.hive.ql.exec.TaskFactory;
+ import org.apache.hadoop.hive.ql.exec.Utilities;
+ import org.apache.hadoop.hive.ql.exec.repl.ReplStateLogWork;
+ import org.apache.hadoop.hive.ql.exec.repl.bootstrap.ReplLoadTask;
+ import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.TableEvent;
+ import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.TaskTracker;
+ import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.util.Context;
+ import org.apache.hadoop.hive.ql.metadata.Table;
+ import org.apache.hadoop.hive.ql.parse.EximUtil;
+ import org.apache.hadoop.hive.ql.parse.ImportSemanticAnalyzer;
+ import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
+ import org.apache.hadoop.hive.ql.parse.SemanticException;
+ import org.apache.hadoop.hive.ql.parse.repl.ReplLogger;
+ import org.apache.hadoop.hive.ql.plan.ImportTableDesc;
+ import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
+ import org.apache.hadoop.hive.ql.plan.MoveWork;
++import org.apache.hadoop.hive.ql.session.SessionState;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ import java.io.IOException;
+ import java.io.Serializable;
+ import java.util.ArrayList;
+ import java.util.HashSet;
+ import java.util.List;
+ import java.util.TreeMap;
+
+ import static org.apache.hadoop.hive.ql.parse.ImportSemanticAnalyzer.isPartitioned;
+
+ public class LoadTable {
+ private final static Logger LOG = LoggerFactory.getLogger(LoadTable.class);
+ // private final Helper helper;
+ private final Context context;
+ private final ReplLogger replLogger;
+ private final TableContext tableContext;
+ private final TaskTracker tracker;
+ private final TableEvent event;
+
+ public LoadTable(TableEvent event, Context context, ReplLogger replLogger,
+ TableContext tableContext, TaskTracker limiter)
+ throws SemanticException, IOException {
+ this.event = event;
+ this.context = context;
+ this.replLogger = replLogger;
+ this.tableContext = tableContext;
+ this.tracker = new TaskTracker(limiter);
+ }
+
+ private void createTableReplLogTask(String tableName, TableType tableType) throws SemanticException {
+ ReplStateLogWork replLogWork = new ReplStateLogWork(replLogger,tableName, tableType);
+ Task<ReplStateLogWork> replLogTask = TaskFactory.get(replLogWork, context.hiveConf);
+ ReplLoadTask.dependency(tracker.tasks(), replLogTask);
+
+ if (tracker.tasks().isEmpty()) {
+ tracker.addTask(replLogTask);
+ } else {
+ ReplLoadTask.dependency(tracker.tasks(), replLogTask);
+
+ List<Task<? extends Serializable>> visited = new ArrayList<>();
+ tracker.updateTaskCount(replLogTask, visited);
+ }
+ }
+
+ public TaskTracker tasks() throws SemanticException {
+ // Path being passed to us is a table dump location. We go ahead and load it in as needed.
+ // If tblName is null, then we default to the table name specified in _metadata, which is good.
+ // or are both specified, in which case, that's what we are intended to create the new table as.
+ try {
+ if (event.shouldNotReplicate()) {
+ return tracker;
+ }
+ String dbName = tableContext.dbNameToLoadIn; //this can never be null or empty;
+ // Create table associated with the import
+ // Executed if relevant, and used to contain all the other details about the table if not.
+ ImportTableDesc tableDesc = tableContext.overrideProperties(event.tableDesc(dbName));
+ Table table = ImportSemanticAnalyzer.tableIfExists(tableDesc, context.hiveDb);
+ ReplicationSpec replicationSpec = event.replicationSpec();
+
+ // Normally, on import, trying to create a table or a partition in a db that does not yet exist
+ // is a error condition. However, in the case of a REPL LOAD, it is possible that we are trying
+ // to create tasks to create a table inside a db that as-of-now does not exist, but there is
+ // a precursor Task waiting that will create it before this is encountered. Thus, we instantiate
+ // defaults and do not error out in that case.
+ // the above will change now since we are going to split replication load in multiple execution
+ // tasks and hence we could have created the database earlier in which case the waitOnPrecursor will
+ // be false and hence if db Not found we should error out.
+ Database parentDb = context.hiveDb.getDatabase(tableDesc.getDatabaseName());
+ if (parentDb == null) {
+ if (!tableContext.waitOnPrecursor()) {
+ throw new SemanticException(
+ ErrorMsg.DATABASE_NOT_EXISTS.getMsg(tableDesc.getDatabaseName()));
+ }
+ }
+
+ if (table == null) {
+ // If table doesn't exist, allow creating a new one only if the database state is older than the update.
+ if ((parentDb != null) && (!replicationSpec
+ .allowReplacementInto(parentDb.getParameters()))) {
+ // If the target table exists and is newer or same as current update based on repl.last.id, then just noop it.
+ return tracker;
+ }
+ } else {
+ if (!replicationSpec.allowReplacementInto(table.getParameters())) {
+ // If the target table exists and is newer or same as current update based on repl.last.id, then just noop it.
+ return tracker;
+ }
+ }
+
+ if (tableDesc.getLocation() == null) {
+ tableDesc.setLocation(location(tableDesc, parentDb));
+ }
+
+
+ /* Note: In the following section, Metadata-only import handling logic is
+ interleaved with regular repl-import logic. The rule of thumb being
+ followed here is that MD-only imports are essentially ALTERs. They do
+ not load data, and should not be "creating" any metadata - they should
+ be replacing instead. The only place it makes sense for a MD-only import
+ to create is in the case of a table that's been dropped and recreated,
+ or in the case of an unpartitioned table. In all other cases, it should
+ behave like a noop or a pure MD alter.
+ */
+ if (table == null) {
+ newTableTasks(tableDesc);
+ } else {
+ existingTableTasks(tableDesc, table, replicationSpec);
+ }
+
+ if (!isPartitioned(tableDesc)) {
+ createTableReplLogTask(tableDesc.getTableName(), tableDesc.tableType());
+ }
+ return tracker;
+ } catch (Exception e) {
+ throw new SemanticException(e);
+ }
+ }
+
+ private void existingTableTasks(ImportTableDesc tblDesc, Table table,
+ ReplicationSpec replicationSpec) {
+ if (!table.isPartitioned()) {
+
+ LOG.debug("table non-partitioned");
+ if (!replicationSpec.allowReplacementInto(table.getParameters())) {
+ return; // silently return, table is newer than our replacement.
+ }
+
+ Task<? extends Serializable> alterTableTask = alterTableTask(tblDesc, replicationSpec);
+ if (replicationSpec.isMetadataOnly()) {
+ tracker.addTask(alterTableTask);
+ } else {
+ Task<?> loadTableTask =
+ loadTableTask(table, replicationSpec, event.metadataPath(), event.metadataPath());
+ alterTableTask.addDependentTask(loadTableTask);
+ tracker.addTask(alterTableTask);
+ }
+ }
+ }
+
+ private void newTableTasks(ImportTableDesc tblDesc) throws SemanticException {
+ Table table;
+ table = new Table(tblDesc.getDatabaseName(), tblDesc.getTableName());
+ // Either we're dropping and re-creating, or the table didn't exist, and we're creating.
+ Task<?> createTableTask =
+ tblDesc.getCreateTableTask(new HashSet<>(), new HashSet<>(), context.hiveConf);
+ if (event.replicationSpec().isMetadataOnly()) {
+ tracker.addTask(createTableTask);
+ return;
+ }
+ if (!isPartitioned(tblDesc)) {
+ LOG.debug("adding dependent CopyWork/MoveWork for table");
+ Task<?> loadTableTask =
+ loadTableTask(table, event.replicationSpec(), new Path(tblDesc.getLocation()),
+ event.metadataPath());
+ createTableTask.addDependentTask(loadTableTask);
+ }
+ tracker.addTask(createTableTask);
+ }
+
+ private String location(ImportTableDesc tblDesc, Database parentDb)
+ throws MetaException, SemanticException {
+ if (!tableContext.waitOnPrecursor()) {
+ return context.warehouse.getDefaultTablePath(parentDb, tblDesc.getTableName()).toString();
+ } else {
+ Path tablePath = new Path(
+ context.warehouse.getDefaultDatabasePath(tblDesc.getDatabaseName()),
+ MetaStoreUtils.encodeTableName(tblDesc.getTableName().toLowerCase())
+ );
+ return context.warehouse.getDnsPath(tablePath).toString();
+ }
+ }
+
+ private Task<?> loadTableTask(Table table, ReplicationSpec replicationSpec, Path tgtPath,
+ Path fromURI) {
+ Path dataPath = new Path(fromURI, EximUtil.DATA_PATH_NAME);
+ Path tmpPath = context.utils.getExternalTmpPath(tgtPath);
+ Task<?> copyTask =
+ ReplCopyTask.getLoadCopyTask(replicationSpec, dataPath, tmpPath, context.hiveConf);
+
+ LoadTableDesc loadTableWork = new LoadTableDesc(
- tmpPath, Utilities.getTableDesc(table), new TreeMap<>(), replicationSpec.isReplace()
++ tmpPath, Utilities.getTableDesc(table), new TreeMap<>(), replicationSpec.isReplace(),
++ SessionState.get().getTxnMgr().getCurrentTxnId()
+ );
+ MoveWork moveWork =
+ new MoveWork(new HashSet<>(), new HashSet<>(), loadTableWork, null, false);
+ Task<?> loadTableTask = TaskFactory.get(moveWork, context.hiveConf);
+ copyTask.addDependentTask(loadTableTask);
+ return copyTask;
+ }
+
+ private Task<? extends Serializable> alterTableTask(ImportTableDesc tableDesc,
+ ReplicationSpec replicationSpec) {
+ tableDesc.setReplaceMode(true);
+ if ((replicationSpec != null) && (replicationSpec.isInReplicationScope())) {
+ tableDesc.setReplicationSpec(replicationSpec);
+ }
+ return tableDesc.getCreateTableTask(new HashSet<>(), new HashSet<>(), context.hiveConf);
+ }
+ }
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
index cfa4e2c,90d1372..b5733ec
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
@@@ -145,12 -162,18 +162,18 @@@ public class VectorizedRowBatchCtx
public static void getPartitionValues(VectorizedRowBatchCtx vrbCtx, Configuration hiveConf,
FileSplit split, Object[] partitionValues) throws IOException {
+ // TODO: this is invalid for SMB. Keep this for now for legacy reasons. See the other overload.
+ MapWork mapWork = Utilities.getMapWork(hiveConf);
+ getPartitionValues(vrbCtx, mapWork, split, partitionValues);
+ }
- Map<Path, PartitionDesc> pathToPartitionInfo = Utilities
- .getMapWork(hiveConf).getPathToPartitionInfo();
+ public static void getPartitionValues(VectorizedRowBatchCtx vrbCtx,
+ MapWork mapWork, FileSplit split, Object[] partitionValues)
+ throws IOException {
+ Map<Path, PartitionDesc> pathToPartitionInfo = mapWork.getPathToPartitionInfo();
PartitionDesc partDesc = HiveFileFormatUtils
- .getPartitionDescFromPathRecursively(pathToPartitionInfo,
+ .getFromPathRecursively(pathToPartitionInfo,
split.getPath(), IOPrepareCache.get().getPartitionDescMap());
getPartitionValues(vrbCtx, partDesc, partitionValues);
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index f325c0e,feacdd8..9c9a2d3
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@@ -36,9 -37,9 +37,10 @@@ import org.apache.hadoop.hive.conf.Hive
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.api.DataOperationType;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.TransactionalValidationListener;
import org.apache.hadoop.hive.ql.ErrorMsg;
+ import org.apache.hadoop.hive.ql.io.orc.OrcRecordUpdater;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId;
@@@ -345,11 -384,7 +385,10 @@@ public class AcidUtils
public static final String SPLIT_UPDATE_STRING = "split_update";
public static final int HASH_BASED_MERGE_BIT = 0x02;
public static final String HASH_BASED_MERGE_STRING = "hash_merge";
+ public static final int INSERT_ONLY_BIT = 0x04;
+ public static final String INSERT_ONLY_STRING = "insert_only";
public static final String DEFAULT_VALUE_STRING = TransactionalValidationListener.DEFAULT_TRANSACTIONAL_PROPERTY;
- public static final String LEGACY_VALUE_STRING = TransactionalValidationListener.LEGACY_TRANSACTIONAL_PROPERTY;
+ public static final String INSERTONLY_VALUE_STRING = TransactionalValidationListener.INSERTONLY_TRANSACTIONAL_PROPERTY;
private AcidOperationalProperties() {
}
@@@ -401,12 -414,6 +430,9 @@@
if (propertiesStr.equalsIgnoreCase(DEFAULT_VALUE_STRING)) {
return AcidOperationalProperties.getDefault();
}
- if (propertiesStr.equalsIgnoreCase(LEGACY_VALUE_STRING)) {
- return AcidOperationalProperties.getLegacy();
- }
+ if (propertiesStr.equalsIgnoreCase(INSERTONLY_VALUE_STRING)) {
+ return AcidOperationalProperties.getInsertOnly();
+ }
AcidOperationalProperties obj = new AcidOperationalProperties();
String[] options = propertiesStr.split("\\|");
for (String option : options) {
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
index a70fde6,38aaeed..6178de2
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
@@@ -14,16 -14,12 +14,20 @@@
package org.apache.hadoop.hive.ql.io.parquet;
import java.io.IOException;
+import java.util.Map;
+ import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+ import org.apache.hadoop.hive.common.io.DataCache;
+ import org.apache.hadoop.hive.common.io.FileMetadataCache;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface;
+import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
+ import org.apache.hadoop.hive.ql.io.LlapCacheOnlyInputFormatInterface;
+import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetTableUtils;
+import org.apache.hadoop.hive.ql.plan.MapWork;
+import org.apache.hadoop.hive.ql.plan.PartitionDesc;
+import org.apache.hadoop.mapred.FileSplit;
+import org.apache.hadoop.mapred.JobConf;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.ql.exec.Utilities;
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/ProjectionPusher.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
index 01e8a48,5dec791..82804b2
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
@@@ -423,11 -423,19 +423,19 @@@ public final class DbTxnManager extend
compBuilder.setExclusive();
compBuilder.setOperationType(DataOperationType.NO_TXN);
break;
-
+ case INSERT_OVERWRITE:
+ t = getTable(output);
+ if (AcidUtils.isAcidTable(t)) {
+ compBuilder.setSemiShared();
+ compBuilder.setOperationType(DataOperationType.UPDATE);
+ } else {
+ compBuilder.setExclusive();
+ compBuilder.setOperationType(DataOperationType.NO_TXN);
+ }
+ break;
case INSERT:
assert t != null;
- if(AcidUtils.isAcidTable(t)) {
+ if(AcidUtils.isFullAcidTable(t)) {
compBuilder.setShared();
}
else {
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index ae451e2,d661f10..a0b735c
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@@ -65,9 -62,9 +65,10 @@@ import org.apache.hadoop.fs.FileStatus
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
+ import org.apache.hadoop.hive.common.BlobStorageUtils;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.common.HiveStatsUtils;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.common.ObjectPair;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate;
@@@ -3338,8 -3250,9 +3389,9 @@@ private void constructOneLBLocationMap(
// (1) Do not delete the dest dir before doing the move operation.
// (2) It is assumed that subdir and dir are in same encryption zone.
// (3) Move individual files from scr dir to dest dir.
- boolean destIsSubDir = isSubDir(srcf, destf, srcFs, destFs, isSrcLocal);
+ boolean srcIsSubDirOfDest = isSubDir(srcf, destf, srcFs, destFs, isSrcLocal),
+ destIsSubDirOfSrc = isSubDir(destf, srcf, destFs, srcFs, false);
+ final String msg = "Unable to move source " + srcf + " to destination " + destf;
-
try {
if (replace) {
try{
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index 7fd8f04,a054abb..021507f
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@@ -1507,9 -1517,14 +1530,15 @@@ public class DDLSemanticAnalyzer extend
}
Table tab = getTable(tableName, true);
+ // cascade only occurs with partitioned table
+ if (isCascade && !tab.isPartitioned()) {
+ throw new SemanticException(
+ ErrorMsg.ALTER_TABLE_NON_PARTITIONED_TABLE_CASCADE_NOT_SUPPORTED);
+ }
+
// Determine the lock type to acquire
- WriteEntity.WriteType writeType = WriteEntity.determineAlterTableWriteType(op);
+ WriteEntity.WriteType writeType = doForceExclusive
+ ? WriteType.DDL_EXCLUSIVE : WriteEntity.determineAlterTableWriteType(op);
if (!alterPartitions) {
inputs.add(new ReadEntity(tab));
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
index 2401036,40c34bf..4109de4
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
@@@ -140,15 -141,15 +141,16 @@@ public class EximUtil
* Initialize the URI where the exported data collection is
* to created for export, or is present for import
*/
- static URI getValidatedURI(HiveConf conf, String dcPath) throws SemanticException {
+ public static URI getValidatedURI(HiveConf conf, String dcPath) throws SemanticException {
try {
- boolean testMode = conf.getBoolVar(HiveConf.ConfVars.HIVETESTMODE);
+ boolean testMode = conf.getBoolVar(HiveConf.ConfVars.HIVETESTMODE)
+ || conf.getBoolVar(HiveConf.ConfVars.HIVEEXIMTESTMODE);
URI uri = new Path(dcPath).toUri();
- String scheme = uri.getScheme();
+ FileSystem fs = FileSystem.get(uri, conf);
+ // Get scheme from FileSystem
+ String scheme = fs.getScheme();
String authority = uri.getAuthority();
String path = uri.getPath();
- FileSystem fs = FileSystem.get(uri, conf);
LOG.info("Path before norm :" + path);
// generate absolute path relative to home directory
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
index e534272,b8c6ea9..54ee7ae
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
@@@ -18,42 -18,12 +18,46 @@@
package org.apache.hadoop.hive.ql.parse;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URI;
+import java.util.HashSet;
+import java.util.List;
+
+import org.antlr.runtime.tree.Tree;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.common.ValidReadTxnList;
+import org.apache.hadoop.hive.common.ValidTxnList;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.QueryState;
+import org.apache.hadoop.hive.ql.exec.ReplCopyTask;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.TaskFactory;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.hooks.ReadEntity;
+import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+ import org.antlr.runtime.tree.Tree;
+ import org.apache.hadoop.hive.ql.ErrorMsg;
+ import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.PartitionIterable;
import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.CopyWork;
+import org.slf4j.Logger;
+ import org.apache.hadoop.hive.ql.parse.repl.dump.TableExport;
/**
* ExportSemanticAnalyzer.
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
index 5ac15af,7f3460f..387582b
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
@@@ -62,12 -59,10 +62,12 @@@ import org.apache.hadoop.hive.ql.metada
import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.repl.load.MetaData;
+ import org.apache.hadoop.hive.ql.parse.repl.load.UpdatedMetaDataTracker;
import org.apache.hadoop.hive.ql.plan.AddPartitionDesc;
+import org.apache.hadoop.hive.ql.plan.CopyWork;
+import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
import org.apache.hadoop.hive.ql.plan.ImportTableDesc;
import org.apache.hadoop.hive.ql.plan.DDLWork;
- import org.apache.hadoop.hive.ql.plan.DropTableDesc;
import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
import org.apache.hadoop.hive.ql.plan.MoveWork;
import org.apache.hadoop.hive.ql.session.SessionState;
@@@ -223,9 -228,8 +234,9 @@@ public class ImportSemanticAnalyzer ext
} catch (Exception e) {
throw new HiveException(e);
}
+ boolean isSourceMm = MetaStoreUtils.isInsertOnlyTable(tblDesc.getTblProps());
- if ((replicationSpec!= null) && replicationSpec.isInReplicationScope()){
+ if ((replicationSpec != null) && replicationSpec.isInReplicationScope()){
tblDesc.setReplicationSpec(replicationSpec);
}
@@@ -318,8 -306,8 +324,8 @@@
} else {
createReplImportTasks(
tblDesc, partitionDescs,
- isPartSpecSet, replicationSpec, waitOnPrecursor, table,
- fromURI, fs, wh, x, txnId, stmtId, isSourceMm);
+ replicationSpec, waitOnPrecursor, table,
- fromURI, fs, wh, x, updatedMetadata);
++ fromURI, fs, wh, x, txnId, stmtId, isSourceMm, updatedMetadata);
}
return tableExists;
}
@@@ -800,8 -734,8 +799,8 @@@
// ensure if destination is not empty only for regular import
Path tgtPath = new Path(table.getDataLocation().toString());
FileSystem tgtFs = FileSystem.get(tgtPath.toUri(), x.getConf());
- checkTargetLocationEmpty(tgtFs, tgtPath, replicationSpec, x);
+ checkTargetLocationEmpty(tgtFs, tgtPath, replicationSpec, x.getLOG());
- loadTable(fromURI, table, false, tgtPath, replicationSpec,x);
+ loadTable(fromURI, table, false, tgtPath, replicationSpec, x, txnId, stmtId, isSourceMm);
}
// Set this to read because we can't overwrite any existing partitions
x.getOutputs().add(new WriteEntity(table, WriteEntity.WriteType.DDL_NO_LOCK));
@@@ -838,14 -769,8 +837,14 @@@
tablePath = wh.getDefaultTablePath(parentDb, tblDesc.getTableName());
}
FileSystem tgtFs = FileSystem.get(tablePath.toUri(), x.getConf());
- checkTargetLocationEmpty(tgtFs, tablePath, replicationSpec,x);
+ checkTargetLocationEmpty(tgtFs, tablePath, replicationSpec,x.getLOG());
- t.addDependentTask(loadTable(fromURI, table, false, tablePath, replicationSpec, x));
+ if (isSourceMm) { // since target table doesn't exist, it should inherit soruce table's properties
+ Map<String, String> tblproperties = table.getParameters();
+ tblproperties.put("transactional", "true");
+ tblproperties.put("transactional_properties", "insert_only");
+ table.setParameters(tblproperties);
+ }
+ t.addDependentTask(loadTable(fromURI, table, false, tablePath, replicationSpec, x, txnId, stmtId, isSourceMm));
}
}
x.getTasks().add(t);
@@@ -866,29 -783,14 +865,15 @@@
private static void createReplImportTasks(
ImportTableDesc tblDesc,
List<AddPartitionDesc> partitionDescs,
- boolean isPartSpecSet, ReplicationSpec replicationSpec, boolean waitOnPrecursor,
+ ReplicationSpec replicationSpec, boolean waitOnPrecursor,
Table table, URI fromURI, FileSystem fs, Warehouse wh,
- EximUtil.SemanticAnalyzerWrapperContext x, Long txnId, int stmtId, boolean isSourceMm)
- EximUtil.SemanticAnalyzerWrapperContext x,
++ EximUtil.SemanticAnalyzerWrapperContext x, Long txnId, int stmtId, boolean isSourceMm,
+ UpdatedMetaDataTracker updatedMetadata)
throws HiveException, URISyntaxException, IOException, MetaException {
+ Task<?> dr = null;
WriteEntity.WriteType lockType = WriteEntity.WriteType.DDL_NO_LOCK;
- if ((table != null) && (isPartitioned(tblDesc) != table.isPartitioned())){
- // If destination table exists, but is partitioned, and we think we're writing to an unpartitioned
- // or if destination table exists, but is unpartitioned and we think we're writing to a partitioned
- // table, then this can only happen because there are drops in the queue that are yet to be processed.
- // So, we check the repl.last.id of the destination, and if it's newer, we no-op. If it's older, we
- // drop and re-create.
- if (replicationSpec.allowReplacementInto(table)){
- dr = dropTableTask(table, x);
- lockType = WriteEntity.WriteType.DDL_EXCLUSIVE;
- table = null; // null it out so we go into the table re-create flow.
- } else {
- return; // noop out of here.
- }
- }
-
// Normally, on import, trying to create a table or a partition in a db that does not yet exist
// is a error condition. However, in the case of a REPL LOAD, it is possible that we are trying
// to create tasks to create a table inside a db that as-of-now does not exist, but there is
@@@ -942,21 -863,18 +949,18 @@@
for (AddPartitionDesc addPartitionDesc : partitionDescs) {
addPartitionDesc.setReplicationSpec(replicationSpec);
t.addDependentTask(
- addSinglePartition(fromURI, fs, tblDesc, table, wh, addPartitionDesc, replicationSpec, x));
+ addSinglePartition(fromURI, fs, tblDesc, table, wh, addPartitionDesc, replicationSpec, x, txnId, stmtId, isSourceMm, ict));
+ if (updatedMetadata != null) {
+ updatedMetadata.addPartition(addPartitionDesc.getPartition(0).getPartSpec());
+ }
}
} else {
x.getLOG().debug("adding dependent CopyWork/MoveWork for table");
- t.addDependentTask(loadTable(fromURI, table, true, new Path(tblDesc.getLocation()),replicationSpec, x));
+ t.addDependentTask(loadTable(fromURI, table, true, new Path(tblDesc.getLocation()), replicationSpec, x, txnId, stmtId, isSourceMm));
}
}
- if (dr == null){
- // Simply create
- x.getTasks().add(t);
- } else {
- // Drop and recreate
- dr.addDependentTask(t);
- x.getTasks().add(dr);
- }
+ // Simply create
+ x.getTasks().add(t);
} else {
// Table existed, and is okay to replicate into, not dropping and re-creating.
if (table.isPartitioned()) {
@@@ -971,15 -887,18 +975,18 @@@
if ((ptn = x.getHive().getPartition(table, partSpec, false)) == null) {
if (!replicationSpec.isMetadataOnly()){
x.getTasks().add(addSinglePartition(
- fromURI, fs, tblDesc, table, wh, addPartitionDesc, replicationSpec, x));
+ fromURI, fs, tblDesc, table, wh, addPartitionDesc, replicationSpec, x, txnId, stmtId, isSourceMm, ict));
+ if (updatedMetadata != null) {
+ updatedMetadata.addPartition(addPartitionDesc.getPartition(0).getPartSpec());
+ }
}
} else {
// If replicating, then the partition already existing means we need to replace, maybe, if
// the destination ptn's repl.last.id is older than the replacement's.
- if (replicationSpec.allowReplacementInto(ptn)){
+ if (replicationSpec.allowReplacementInto(ptn.getParameters())){
if (!replicationSpec.isMetadataOnly()){
x.getTasks().add(addSinglePartition(
- fromURI, fs, tblDesc, table, wh, addPartitionDesc, replicationSpec, x));
+ fromURI, fs, tblDesc, table, wh, addPartitionDesc, replicationSpec, x, txnId, stmtId, isSourceMm, ict));
} else {
x.getTasks().add(alterSinglePartition(
fromURI, fs, tblDesc, table, wh, addPartitionDesc, replicationSpec, ptn, x));
@@@ -1002,13 -921,9 +1009,10 @@@
}
} else {
x.getLOG().debug("table non-partitioned");
- if (!replicationSpec.allowReplacementInto(table)){
- return; // silently return, table is newer than our replacement.
- }
if (!replicationSpec.isMetadataOnly()) {
// repl-imports are replace-into unless the event is insert-into
- loadTable(fromURI, table, replicationSpec.isReplace(), new Path(fromURI), replicationSpec, x);
+ loadTable(fromURI, table, replicationSpec.isReplace(), new Path(fromURI),
+ replicationSpec, x, txnId, stmtId, isSourceMm);
} else {
x.getTasks().add(alterTableTask(tblDesc, x, replicationSpec));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
index 9b39cd0,fa79700..5e70863
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
@@@ -227,6 -228,9 +229,9 @@@ public class LoadSemanticAnalyzer exten
+ " and use 'insert... select' to allow Hive to enforce bucketing. " + error);
}
- if(AcidUtils.isAcidTable(ts.tableHandle)) {
++ if(AcidUtils.isAcidTable(ts.tableHandle) && !MetaStoreUtils.isInsertOnlyTable(ts.tableHandle.getParameters())) {
+ throw new SemanticException(ErrorMsg.LOAD_DATA_ON_ACID_TABLE, ts.tableHandle.getCompleteName());
+ }
// make sure the arguments make sense
List<FileStatus> files = applyConstraintsAndGetFiles(fromURI, fromTree, isLocal);
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 19882eb,6f379da..ab3cbe1
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@@ -6851,17 -6896,13 +6855,21 @@@ public class SemanticAnalyzer extends B
AcidUtils.Operation acidOp = AcidUtils.Operation.NOT_ACID;
if (destTableIsAcid) {
acidOp = getAcidType(table_desc.getOutputFileFormatClass(), dest);
- checkAcidConstraints(qb, table_desc, dest_tab, acidOp);
+ checkAcidConstraints(qb, table_desc, dest_tab);
}
- ltd = new LoadTableDesc(queryTmpdir, table_desc, dpCtx, acidOp);
+ if (MetaStoreUtils.isInsertOnlyTable(table_desc.getProperties())) {
+ acidOp = getAcidType(table_desc.getOutputFileFormatClass(), dest);
+ }
+ if (isMmTable) {
+ txnId = SessionState.get().getTxnMgr().getCurrentTxnId();
+ }
+ boolean isReplace = !qb.getParseInfo().isInsertIntoTable(
+ dest_tab.getDbName(), dest_tab.getTableName());
+ ltd = new LoadTableDesc(queryTmpdir, table_desc, dpCtx, acidOp, isReplace, txnId);
+ // For Acid table, Insert Overwrite shouldn't replace the table content. We keep the old
+ // deltas and base and leave them up to the cleaner to clean up
+ ltd.setReplace(!qb.getParseInfo().isInsertIntoTable(dest_tab.getDbName(),
+ dest_tab.getTableName()) && !destTableIsAcid);
ltd.setLbCtx(lbCtx);
loadTableWork.add(ltd);
} else {
@@@ -6915,17 -7011,13 +6923,19 @@@
AcidUtils.Operation acidOp = AcidUtils.Operation.NOT_ACID;
if (destTableIsAcid) {
acidOp = getAcidType(table_desc.getOutputFileFormatClass(), dest);
- checkAcidConstraints(qb, table_desc, dest_tab, acidOp);
+ checkAcidConstraints(qb, table_desc, dest_tab);
}
- ltd = new LoadTableDesc(queryTmpdir, table_desc, dest_part.getSpec(), acidOp);
+ if (MetaStoreUtils.isInsertOnlyTable(dest_part.getTable().getParameters())) {
+ acidOp = getAcidType(table_desc.getOutputFileFormatClass(), dest);
+ }
+ if (isMmTable) {
+ txnId = SessionState.get().getTxnMgr().getCurrentTxnId();
+ }
+ ltd = new LoadTableDesc(queryTmpdir, table_desc, dest_part.getSpec(), acidOp, txnId);
+ // For Acid table, Insert Overwrite shouldn't replace the table content. We keep the old
+ // deltas and base and leave them up to the cleaner to clean up
ltd.setReplace(!qb.getParseInfo().isInsertIntoTable(dest_tab.getDbName(),
- dest_tab.getTableName()));
+ dest_tab.getTableName()) && !destTableIsAcid);
ltd.setLbCtx(lbCtx);
loadTableWork.add(ltd);
@@@ -7091,122 -7226,18 +7101,122 @@@
genPartnCols(dest, input, qb, table_desc, dest_tab, rsCtx);
}
- FileSinkDesc fileSinkDesc = new FileSinkDesc(
- queryTmpdir,
- table_desc,
- conf.getBoolVar(HiveConf.ConfVars.COMPRESSRESULT),
- currentTableId,
- rsCtx.isMultiFileSpray(),
- canBeMerged,
- rsCtx.getNumFiles(),
- rsCtx.getTotalFiles(),
- rsCtx.getPartnCols(),
- dpCtx,
- dest_path);
+ FileSinkDesc fileSinkDesc = createFileSinkDesc(dest, table_desc, dest_part,
+ dest_path, currentTableId, destTableIsAcid, destTableIsTemporary,
+ destTableIsMaterialization, queryTmpdir, rsCtx, dpCtx, lbCtx, fsRS,
- canBeMerged, txnId, isMmCtas);
++ canBeMerged, dest_tab, txnId, isMmCtas);
+ if (isMmCtas) {
+ // Add FSD so that the LoadTask compilation could fix up its path to avoid the move.
+ tableDesc.setWriter(fileSinkDesc);
+ }
+
+ if (SessionState.get().isHiveServerQuery() &&
+ null != table_desc &&
+ table_desc.getSerdeClassName().equalsIgnoreCase(ThriftJDBCBinarySerDe.class.getName()) &&
+ HiveConf.getBoolVar(conf,HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_SERIALIZE_IN_TASKS)) {
+ fileSinkDesc.setIsUsingThriftJDBCBinarySerDe(true);
+ } else {
+ fileSinkDesc.setIsUsingThriftJDBCBinarySerDe(false);
+ }
+
+ Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild(
+ fileSinkDesc, fsRS, input), inputRR);
+
+ handleLineage(ltd, output);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Created FileSink Plan for clause: " + dest + "dest_path: "
+ + dest_path + " row schema: " + inputRR.toString());
+ }
+
+ FileSinkOperator fso = (FileSinkOperator) output;
+ fso.getConf().setTable(dest_tab);
+ // the following code is used to collect column stats when
+ // hive.stats.autogather=true
+ // and it is an insert overwrite or insert into table
+ if (dest_tab != null && conf.getBoolVar(ConfVars.HIVESTATSAUTOGATHER)
+ && conf.getBoolVar(ConfVars.HIVESTATSCOLAUTOGATHER)
+ && ColumnStatsAutoGatherContext.canRunAutogatherStats(fso)) {
+ if (dest_type.intValue() == QBMetaData.DEST_TABLE) {
+ genAutoColumnStatsGatheringPipeline(qb, table_desc, partSpec, input, qb.getParseInfo()
+ .isInsertIntoTable(dest_tab.getDbName(), dest_tab.getTableName()));
+ } else if (dest_type.intValue() == QBMetaData.DEST_PARTITION) {
+ genAutoColumnStatsGatheringPipeline(qb, table_desc, dest_part.getSpec(), input, qb
+ .getParseInfo().isInsertIntoTable(dest_tab.getDbName(), dest_tab.getTableName()));
+
+ }
+ }
+ return output;
+ }
+
+ private ColsAndTypes deriveFileSinkColTypes(
+ RowResolver inputRR, List<FieldSchema> field_schemas) throws SemanticException {
+ ColsAndTypes result = new ColsAndTypes("", "");
+ ArrayList<ColumnInfo> colInfos = inputRR.getColumnInfos();
+ boolean first = true;
+ for (ColumnInfo colInfo : colInfos) {
+ String[] nm = inputRR.reverseLookup(colInfo.getInternalName());
+
+ if (nm[1] != null) { // non-null column alias
+ colInfo.setAlias(nm[1]);
+ }
+
+ String colName = colInfo.getInternalName(); //default column name
+ if (field_schemas != null) {
+ FieldSchema col = new FieldSchema();
+ if (!("".equals(nm[0])) && nm[1] != null) {
+ colName = unescapeIdentifier(colInfo.getAlias()).toLowerCase(); // remove ``
+ }
+ colName = fixCtasColumnName(colName);
+ col.setName(colName);
+ String typeName = colInfo.getType().getTypeName();
+ // CTAS should NOT create a VOID type
+ if (typeName.equals(serdeConstants.VOID_TYPE_NAME)) {
+ throw new SemanticException(ErrorMsg.CTAS_CREATES_VOID_TYPE.getMsg(colName));
+ }
+ col.setType(typeName);
+ field_schemas.add(col);
+ }
+
+ if (!first) {
+ result.cols = result.cols.concat(",");
+ result.colTypes = result.colTypes.concat(":");
+ }
+
+ first = false;
+ result.cols = result.cols.concat(colName);
+
+ // Replace VOID type with string when the output is a temp table or
+ // local files.
+ // A VOID type can be generated under the query:
+ //
+ // select NULL from tt;
+ // or
+ // insert overwrite local directory "abc" select NULL from tt;
+ //
+ // where there is no column type to which the NULL value should be
+ // converted.
+ //
+ String tName = colInfo.getType().getTypeName();
+ if (tName.equals(serdeConstants.VOID_TYPE_NAME)) {
+ result.colTypes = result.colTypes.concat(serdeConstants.STRING_TYPE_NAME);
+ } else {
+ result.colTypes = result.colTypes.concat(tName);
+ }
+ }
+ return result;
+ }
+
+ private FileSinkDesc createFileSinkDesc(String dest, TableDesc table_desc,
+ Partition dest_part, Path dest_path, int currentTableId,
+ boolean destTableIsAcid, boolean destTableIsTemporary,
+ boolean destTableIsMaterialization, Path queryTmpdir,
+ SortBucketRSCtx rsCtx, DynamicPartitionCtx dpCtx, ListBucketingCtx lbCtx,
- RowSchema fsRS, boolean canBeMerged, Long mmWriteId, boolean isMmCtas) throws SemanticException {
++ RowSchema fsRS, boolean canBeMerged, Table dest_tab, Long mmWriteId, boolean isMmCtas) throws SemanticException {
+ FileSinkDesc fileSinkDesc = new FileSinkDesc(queryTmpdir, table_desc,
+ conf.getBoolVar(HiveConf.ConfVars.COMPRESSRESULT), currentTableId, rsCtx.isMultiFileSpray(),
+ canBeMerged, rsCtx.getNumFiles(), rsCtx.getTotalFiles(), rsCtx.getPartnCols(), dpCtx,
+ dest_path, mmWriteId, isMmCtas);
boolean isHiveServerQuery = SessionState.get().isHiveServerQuery();
fileSinkDesc.setHiveServerQuery(isHiveServerQuery);
@@@ -7448,14 -7395,12 +7456,9 @@@
*/
conf.set(AcidUtils.CONF_ACID_KEY, "true");
- if (!Operation.NOT_ACID.equals(acidOp)) {
- if (table.getNumBuckets() < 1) {
- throw new SemanticException(ErrorMsg.ACID_OP_ON_NONACID_TABLE, table.getTableName());
- }
- if (table.getSortCols() != null && table.getSortCols().size() > 0) {
- throw new SemanticException(ErrorMsg.ACID_NO_SORTED_BUCKETS, table.getTableName());
- }
+ if (table.getSortCols() != null && table.getSortCols().size() > 0) {
+ throw new SemanticException(ErrorMsg.ACID_NO_SORTED_BUCKETS, table.getTableName());
}
-
-
-
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
index d0a82af,4f8ba6f..ffc2daf
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
@@@ -355,6 -396,52 +396,52 @@@ public class GenSparkUtils
}
/**
+ * Create and add any dependent move tasks.
+ *
+ * This is forked from {@link GenMapRedUtils}. The difference is that it doesn't check
+ * 'isLinkedFileSink' and does not set parent dir for the linked file sinks.
+ */
+ public static Path createMoveTask(Task<? extends Serializable> currTask, boolean chDir,
+ FileSinkOperator fsOp, ParseContext parseCtx, List<Task<MoveWork>> mvTasks,
+ HiveConf hconf, DependencyCollectionTask dependencyTask) {
+
+ Path dest = null;
++ FileSinkDesc fileSinkDesc = fsOp.getConf();
+
+ if (chDir) {
+ dest = fsOp.getConf().getFinalDirName();
+
+ // generate the temporary file
+ // it must be on the same file system as the current destination
+ Context baseCtx = parseCtx.getContext();
+
+ Path tmpDir = baseCtx.getExternalTmpPath(dest);
+
- FileSinkDesc fileSinkDesc = fsOp.getConf();
+ // Change all the linked file sink descriptors
+ if (fileSinkDesc.getLinkedFileSinkDesc() != null) {
+ for (FileSinkDesc fsConf : fileSinkDesc.getLinkedFileSinkDesc()) {
+ fsConf.setDirName(tmpDir);
+ }
+ } else {
+ fileSinkDesc.setDirName(tmpDir);
+ }
+ }
+
+ Task<MoveWork> mvTask = null;
+
+ if (!chDir) {
- mvTask = GenMapRedUtils.findMoveTask(mvTasks, fsOp);
++ mvTask = GenMapRedUtils.findMoveTaskForFsopOutput(mvTasks, fileSinkDesc.getFinalDirName(), false);
+ }
+
+ // Set the move task to be dependent on the current task
+ if (mvTask != null) {
+ GenMapRedUtils.addDependentMoveTasks(mvTask, hconf, currTask, dependencyTask);
+ }
+
+ return dest;
+ }
+
+ /**
* Populate partition pruning information from the pruning sink operator to the
* target MapWork (the MapWork for the big table side). The information include the source table
* name, column name, and partition key expression. It also set up the temporary path used to
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
index 955e0e5,a3df166..4732f0a
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
@@@ -20,9 -20,9 +20,10 @@@ package org.apache.hadoop.hive.ql.plan
import java.util.ArrayList;
import java.util.List;
+ import java.util.Objects;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
----------------------------------------------------------------------
diff --cc ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
index 3600a6b,0f129fc..5a8ac31
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
@@@ -487,79 -439,7 +439,6 @@@ public class TestTxnCommands extends Te
}
}
- /**
- * takes raw data and turns it into a string as if from Driver.getResults()
- * sorts rows in dictionary order
- */
- private List<String> stringifyValues(int[][] rowsIn) {
- assert rowsIn.length > 0;
- int[][] rows = rowsIn.clone();
- Arrays.sort(rows, new RowComp());
- List<String> rs = new ArrayList<String>();
- for(int[] row : rows) {
- assert row.length > 0;
- StringBuilder sb = new StringBuilder();
- for(int value : row) {
- sb.append(value).append("\t");
- }
- sb.setLength(sb.length() - 1);
- rs.add(sb.toString());
- }
- return rs;
- }
- private static final class RowComp implements Comparator<int[]> {
- @Override
- public int compare(int[] row1, int[] row2) {
- assert row1 != null && row2 != null && row1.length == row2.length;
- for(int i = 0; i < row1.length; i++) {
- int comp = Integer.compare(row1[i], row2[i]);
- if(comp != 0) {
- return comp;
- }
- }
- return 0;
- }
- }
- private String makeValuesClause(int[][] rows) {
- assert rows.length > 0;
- StringBuilder sb = new StringBuilder("values");
- for(int[] row : rows) {
- assert row.length > 0;
- if(row.length > 1) {
- sb.append("(");
- }
- for(int value : row) {
- sb.append(value).append(",");
- }
- sb.setLength(sb.length() - 1);//remove trailing comma
- if(row.length > 1) {
- sb.append(")");
- }
- sb.append(",");
- }
- sb.setLength(sb.length() - 1);//remove trailing comma
- return sb.toString();
- }
-
- private List<String> runStatementOnDriver(String stmt) throws Exception {
- LOG.info("Running " + stmt);
- CommandProcessorResponse cpr = d.run(stmt);
- if(cpr.getResponseCode() != 0) {
- throw new RuntimeException(stmt + " failed: " + cpr);
- }
- List<String> rs = new ArrayList<String>();
- d.getResults(rs);
- return rs;
- }
- private CommandProcessorResponse runStatementOnDriverNegative(String stmt) throws Exception {
- CommandProcessorResponse cpr = d.run(stmt);
- if(cpr.getResponseCode() != 0) {
- return cpr;
- }
- throw new RuntimeException("Didn't get expected failure!");
- }
--
- // @Ignore
@Test
public void exchangePartition() throws Exception {
runStatementOnDriver("create database ex1");
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
----------------------------------------------------------------------
diff --cc ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
index 7a73a17,21b4a2c..bd6e6a0
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
@@@ -85,16 -84,16 +84,17 @@@ public class TestTxnCommands2
protected HiveConf hiveConf;
protected Driver d;
- protected static enum Table {
+ protected enum Table {
ACIDTBL("acidTbl"),
- ACIDTBLPART("acidTblPart"),
+ ACIDTBLPART("acidTblPart", "p"),
NONACIDORCTBL("nonAcidOrcTbl"),
- NONACIDPART("nonAcidPart"),
- NONACIDPART2("nonAcidPart2"),
- ACIDNESTEDPART("acidNestedPart"),
+ NONACIDPART("nonAcidPart", "p"),
+ NONACIDPART2("nonAcidPart2", "p2"),
- ACIDNESTEDPART("acidNestedPart", "p,q");
++ ACIDNESTEDPART("acidNestedPart", "p,q"),
+ MMTBL("mmTbl");
private final String name;
+ private final String partitionColumns;
@Override
public String toString() {
return name;
@@@ -738,14 -841,13 +843,13 @@@
// There should be only 1 directory left: base_0000001.
// Original bucket files, delta directories and previous base directory should have been cleaned up.
status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
- (Table.NONACIDORCTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+ (Table.NONACIDORCTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
Assert.assertEquals(1, status.length);
- Assert.assertEquals("base_0000023", status[0].getPath().getName());
+ Assert.assertEquals("base_0000025", status[0].getPath().getName());
FileStatus[] buckets = fs.listStatus(status[0].getPath(), FileUtils.STAGING_DIR_PATH_FILTER);
Arrays.sort(buckets);
- Assert.assertEquals(BUCKET_COUNT, buckets.length);
- Assert.assertEquals("bucket_00000", buckets[0].getPath().getName());
- Assert.assertEquals("bucket_00001", buckets[1].getPath().getName());
+ Assert.assertEquals(1, buckets.length);
+ Assert.assertEquals("bucket_00001", buckets[0].getPath().getName());
rs = runStatementOnDriver("select a,b from " + Table.NONACIDORCTBL);
resultData = new int[][] {{1, 3}, {3, 4}};
Assert.assertEquals(stringifyValues(resultData), rs);
@@@ -1643,65 -1755,241 +1757,299 @@@
Assert.assertEquals(stringifyValues(rExpected), r);
}
+ @Test
+ public void testBucketCodec() throws Exception {
+ d.destroy();
+ //insert data in "legacy" format
+ hiveConf.setIntVar(HiveConf.ConfVars.TESTMODE_BUCKET_CODEC_VERSION, 0);
+ d = new Driver(hiveConf);
+
+ int[][] targetVals = {{2,1},{4,3},{5,6},{7,8}};
+ runStatementOnDriver("insert into " + Table.ACIDTBL + " " + makeValuesClause(targetVals));
+
+ d.destroy();
+ hiveConf.setIntVar(HiveConf.ConfVars.TESTMODE_BUCKET_CODEC_VERSION, 1);
+ d = new Driver(hiveConf);
+ //do some operations with new format
+ runStatementOnDriver("update " + Table.ACIDTBL + " set b=11 where a in (5,7)");
+ runStatementOnDriver("insert into " + Table.ACIDTBL + " values(11,11)");
+ runStatementOnDriver("delete from " + Table.ACIDTBL + " where a = 7");
+
+ //make sure we get the right data back before/after compactions
+ List<String> r = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
+ int[][] rExpected = {{2,1},{4,3},{5,11},{11,11}};
+ Assert.assertEquals(stringifyValues(rExpected), r);
+
+ runStatementOnDriver("ALTER TABLE " + Table.ACIDTBL + " COMPACT 'MINOR'");
+ runWorker(hiveConf);
+
+ r = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
+ Assert.assertEquals(stringifyValues(rExpected), r);
+
+ runStatementOnDriver("ALTER TABLE " + Table.ACIDTBL + " COMPACT 'MAJOR'");
+ runWorker(hiveConf);
+
+ r = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
+ Assert.assertEquals(stringifyValues(rExpected), r);
+ }
+ /**
+ * Test the scenario when IOW comes in before a MAJOR compaction happens
+ * @throws Exception
+ */
+ @Test
+ public void testInsertOverwrite1() throws Exception {
+ FileSystem fs = FileSystem.get(hiveConf);
+ FileStatus[] status;
+
+ // 1. Insert two rows to an ACID table
+ runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) values(1,2)");
+ runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) values(3,4)");
+ status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+ (Table.ACIDTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+ // There should be 2 delta dirs in the location
+ Assert.assertEquals(2, status.length);
+ for (int i = 0; i < status.length; i++) {
+ Assert.assertTrue(status[i].getPath().getName().matches("delta_.*"));
+ }
+
+ // 2. INSERT OVERWRITE
+ // Prepare data for the source table
+ runStatementOnDriver("insert into " + Table.NONACIDORCTBL + "(a,b) values(5,6),(7,8)");
+ // Insert overwrite ACID table from source table
+ runStatementOnDriver("insert overwrite table " + Table.ACIDTBL + " select a,b from " + Table.NONACIDORCTBL);
+ status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+ (Table.ACIDTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+ // There should be 2 delta dirs, plus a base dir in the location
+ Assert.assertEquals(3, status.length);
+ boolean sawBase = false;
+ String baseDir = "";
+ int deltaCount = 0;
+ for (int i = 0; i < status.length; i++) {
+ String dirName = status[i].getPath().getName();
+ if (dirName.matches("delta_.*")) {
+ deltaCount++;
+ } else {
+ sawBase = true;
+ baseDir = dirName;
+ Assert.assertTrue(baseDir.matches("base_.*"));
+ }
+ }
+ Assert.assertEquals(2, deltaCount);
+ Assert.assertTrue(sawBase);
+ // Verify query result
+ List<String> rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a");
+ int [][] resultData = new int[][] {{5,6},{7,8}};
+ Assert.assertEquals(stringifyValues(resultData), rs);
+
+ // 3. Perform a major compaction. Nothing should change. Both deltas and base dirs should have the same name.
+ // Re-verify directory layout and query result by using the same logic as above
+ runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MAJOR'");
+ runWorker(hiveConf);
+ status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+ (Table.ACIDTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+ // There should be 2 delta dirs, plus a base dir in the location
+ Assert.assertEquals(3, status.length);
+ sawBase = false;
+ deltaCount = 0;
+ for (int i = 0; i < status.length; i++) {
+ String dirName = status[i].getPath().getName();
+ if (dirName.matches("delta_.*")) {
+ deltaCount++;
+ } else {
+ sawBase = true;
+ Assert.assertTrue(dirName.matches("base_.*"));
+ Assert.assertEquals(baseDir, dirName);
+ }
+ }
+ Assert.assertEquals(2, deltaCount);
+ Assert.assertTrue(sawBase);
+ // Verify query result
+ rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a");
+ Assert.assertEquals(stringifyValues(resultData), rs);
+
+ // 4. Run Cleaner. It should remove the 2 delta dirs.
+ runCleaner(hiveConf);
+ // There should be only 1 directory left: base_xxxxxxx.
+ // The delta dirs should have been cleaned up.
+ status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+ (Table.ACIDTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+ Assert.assertEquals(1, status.length);
+ Assert.assertTrue(status[0].getPath().getName().matches("base_.*"));
+ Assert.assertEquals(baseDir, status[0].getPath().getName());
+ // Verify query result
+ rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a");
+ Assert.assertEquals(stringifyValues(resultData), rs);
+ }
+
+ /**
+ * Test the scenario when IOW comes in after a MAJOR compaction happens
+ * @throws Exception
+ */
+ @Test
+ public void testInsertOverwrite2() throws Exception {
+ FileSystem fs = FileSystem.get(hiveConf);
+ FileStatus[] status;
+
+ // 1. Insert two rows to an ACID table
+ runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) values(1,2)");
+ runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) values(3,4)");
+ status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+ (Table.ACIDTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+ // There should be 2 delta dirs in the location
+ Assert.assertEquals(2, status.length);
+ for (int i = 0; i < status.length; i++) {
+ Assert.assertTrue(status[i].getPath().getName().matches("delta_.*"));
+ }
+
+ // 2. Perform a major compaction. There should be an extra base dir now.
+ runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MAJOR'");
+ runWorker(hiveConf);
+ status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+ (Table.ACIDTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+ // There should be 2 delta dirs, plus a base dir in the location
+ Assert.assertEquals(3, status.length);
+ boolean sawBase = false;
+ int deltaCount = 0;
+ for (int i = 0; i < status.length; i++) {
+ String dirName = status[i].getPath().getName();
+ if (dirName.matches("delta_.*")) {
+ deltaCount++;
+ } else {
+ sawBase = true;
+ Assert.assertTrue(dirName.matches("base_.*"));
+ }
+ }
+ Assert.assertEquals(2, deltaCount);
+ Assert.assertTrue(sawBase);
+ // Verify query result
+ int [][] resultData = new int[][] {{1,2},{3,4}};
+ List<String> rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a");
+ Assert.assertEquals(stringifyValues(resultData), rs);
+
+ // 3. INSERT OVERWRITE
+ // Prepare data for the source table
+ runStatementOnDriver("insert into " + Table.NONACIDORCTBL + "(a,b) values(5,6),(7,8)");
+ // Insert overwrite ACID table from source table
+ runStatementOnDriver("insert overwrite table " + Table.ACIDTBL + " select a,b from " + Table.NONACIDORCTBL);
+ status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+ (Table.ACIDTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+ // There should be 2 delta dirs, plus 2 base dirs in the location
+ Assert.assertEquals(4, status.length);
+ int baseCount = 0;
+ deltaCount = 0;
+ for (int i = 0; i < status.length; i++) {
+ String dirName = status[i].getPath().getName();
+ if (dirName.matches("delta_.*")) {
+ deltaCount++;
+ } else {
+ baseCount++;
+ }
+ }
+ Assert.assertEquals(2, deltaCount);
+ Assert.assertEquals(2, baseCount);
+ // Verify query result
+ resultData = new int[][] {{5,6},{7,8}};
+ rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a");
+ Assert.assertEquals(stringifyValues(resultData), rs);
+
+ // 4. Perform another major compaction. Nothing should change. Both deltas and both base dirs
+ // should have the same name.
+ // Re-verify directory layout and query result by using the same logic as above
+ runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MAJOR'");
+ runWorker(hiveConf);
+ status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+ (Table.ACIDTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+ // There should be 2 delta dirs, plus 2 base dirs in the location
+ Assert.assertEquals(4, status.length);
+ baseCount = 0;
+ deltaCount = 0;
+ for (int i = 0; i < status.length; i++) {
+ String dirName = status[i].getPath().getName();
+ if (dirName.matches("delta_.*")) {
+ deltaCount++;
+ } else {
+ Assert.assertTrue(dirName.matches("base_.*"));
+ baseCount++;
+ }
+ }
+ Assert.assertEquals(2, deltaCount);
+ Assert.assertEquals(2, baseCount);
+ // Verify query result
+ rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a");
+ Assert.assertEquals(stringifyValues(resultData), rs);
+
+ // 5. Run Cleaner. It should remove the 2 delta dirs and 1 old base dir.
+ runCleaner(hiveConf);
+ // There should be only 1 directory left: base_xxxxxxx.
+ // The delta dirs should have been cleaned up.
+ status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+ (Table.ACIDTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+ Assert.assertEquals(1, status.length);
+ Assert.assertTrue(status[0].getPath().getName().matches("base_.*"));
+ // Verify query result
+ rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a");
+ Assert.assertEquals(stringifyValues(resultData), rs);
+ }
+
/**
+ * Test compaction for Micro-managed table
+ * 1. Regular compaction shouldn't impact any valid subdirectories of MM tables
+ * 2. Compactions will only remove subdirectories for aborted transactions of MM tables, if any
+ * @throws Exception
+ */
+ @Test
+ public void testMmTableCompaction() throws Exception {
+ // 1. Insert some rows into MM table
+ runStatementOnDriver("insert into " + Table.MMTBL + "(a,b) values(1,2)");
+ runStatementOnDriver("insert into " + Table.MMTBL + "(a,b) values(3,4)");
+ // There should be 2 delta directories
+ verifyDirAndResult(2);
+
+ // 2. Perform a MINOR compaction. Since nothing was aborted, subdirs should stay.
+ runStatementOnDriver("alter table "+ Table.MMTBL + " compact 'MINOR'");
+ runWorker(hiveConf);
+ verifyDirAndResult(2);
+
+ // 3. Let a transaction be aborted
+ hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, true);
+ runStatementOnDriver("insert into " + Table.MMTBL + "(a,b) values(5,6)");
+ hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, false);
+ // There should be 3 delta directories. The new one is the aborted one.
+ verifyDirAndResult(3);
+
+ // 4. Perform a MINOR compaction again. This time it will remove the subdir for aborted transaction.
+ runStatementOnDriver("alter table "+ Table.MMTBL + " compact 'MINOR'");
+ runWorker(hiveConf);
+ // The worker should remove the subdir for aborted transaction
+ verifyDirAndResult(2);
+
+ // 5. Run Cleaner. Shouldn't impact anything.
+ runCleaner(hiveConf);
+ verifyDirAndResult(2);
+ }
+
+ private void verifyDirAndResult(int expectedDeltas) throws Exception {
+ FileSystem fs = FileSystem.get(hiveConf);
+ // Verify the content of subdirs
+ FileStatus[] status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+ (Table.MMTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+ int sawDeltaTimes = 0;
+ for (int i = 0; i < status.length; i++) {
+ Assert.assertTrue(status[i].getPath().getName().matches("delta_.*"));
+ sawDeltaTimes++;
+ FileStatus[] files = fs.listStatus(status[i].getPath(), FileUtils.STAGING_DIR_PATH_FILTER);
+ Assert.assertEquals(1, files.length);
+ Assert.assertTrue(files[0].getPath().getName().equals("000000_0"));
+ }
+ Assert.assertEquals(expectedDeltas, sawDeltaTimes);
+
+ // Verify query result
+ int [][] resultData = new int[][] {{1,2}, {3,4}};
+ List<String> rs = runStatementOnDriver("select a,b from " + Table.MMTBL);
+ Assert.assertEquals(stringifyValues(resultData), rs);
+ }
+
+ /**
* takes raw data and turns it into a string as if from Driver.getResults()
* sorts rows in dictionary order
*/
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
----------------------------------------------------------------------
diff --cc ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
index 6b19eb1,4c30732..ccd7d8e
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
@@@ -40,10 -46,15 +46,7 @@@ import java.util.TreeSet
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.BlockLocation;
-import org.apache.hadoop.fs.FSDataInputStream;
--import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FSInputStream;
--import org.apache.hadoop.fs.FileStatus;
--import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocatedFileStatus;
--import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
++import org.apache.hadoop.fs.*;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.common.type.HiveDecimal;
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/test/org/apache/hadoop/hive/ql/optimizer/TestGenMapRedUtilsCreateConditionalTask.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/test/queries/clientpositive/mm_all.q
----------------------------------------------------------------------
diff --cc ql/src/test/queries/clientpositive/mm_all.q
index 6001e9f,0000000..e23260f
mode 100644,000000..100644
--- a/ql/src/test/queries/clientpositive/mm_all.q
+++ b/ql/src/test/queries/clientpositive/mm_all.q
@@@ -1,370 -1,0 +1,325 @@@
+set hive.mapred.mode=nonstrict;
+set hive.explain.user=false;
+set hive.fetch.task.conversion=none;
+set tez.grouping.min-size=1;
+set tez.grouping.max-size=2;
+set hive.exec.dynamic.partition.mode=nonstrict;
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+
+
+-- Force multiple writers when reading
+drop table intermediate;
+create table intermediate(key int) partitioned by (p int) stored as orc;
+insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2;
+insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2;
+insert into table intermediate partition(p='457') select distinct key from src where key >= 100 order by key asc limit 2;
+
+
+drop table part_mm;
+create table part_mm(key int) partitioned by (key_mm int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only");
+explain insert into table part_mm partition(key_mm=455) select key from intermediate;
+insert into table part_mm partition(key_mm=455) select key from intermediate;
+insert into table part_mm partition(key_mm=456) select key from intermediate;
+insert into table part_mm partition(key_mm=455) select key from intermediate;
+select * from part_mm order by key, key_mm;
+
+-- TODO: doesn't work truncate table part_mm partition(key_mm=455);
+select * from part_mm order by key, key_mm;
+truncate table part_mm;
+select * from part_mm order by key, key_mm;
+drop table part_mm;
+
+drop table simple_mm;
+create table simple_mm(key int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only");
+insert into table simple_mm select key from intermediate;
+select * from simple_mm order by key;
+insert into table simple_mm select key from intermediate;
+select * from simple_mm order by key;
+truncate table simple_mm;
+select * from simple_mm;
+drop table simple_mm;
+
+
+-- simple DP (no bucketing)
+drop table dp_mm;
+
+set hive.exec.dynamic.partition.mode=nonstrict;
+
+set hive.merge.mapredfiles=false;
+set hive.merge.sparkfiles=false;
+set hive.merge.tezfiles=false;
+
+create table dp_mm (key int) partitioned by (key1 string, key2 int) stored as orc
+ tblproperties ("transactional"="true", "transactional_properties"="insert_only");
+
+insert into table dp_mm partition (key1='123', key2) select key, key from intermediate;
+
+select * from dp_mm order by key;
+
+drop table dp_mm;
+
+
+-- union
+
+create table union_mm(id int) tblproperties ("transactional"="true", "transactional_properties"="insert_only");
+insert into table union_mm
+select temps.p from (
+select key as p from intermediate
+union all
+select key + 1 as p from intermediate ) temps;
+
+select * from union_mm order by id;
+
+insert into table union_mm
+select p from
+(
+select key + 1 as p from intermediate
+union all
+select key from intermediate
+) tab group by p
+union all
+select key + 2 as p from intermediate;
+
+select * from union_mm order by id;
+
+insert into table union_mm
+SELECT p FROM
+(
+ SELECT key + 1 as p FROM intermediate
+ UNION ALL
+ SELECT key as p FROM (
+ SELECT distinct key FROM (
+ SELECT key FROM (
+ SELECT key + 2 as key FROM intermediate
+ UNION ALL
+ SELECT key FROM intermediate
+ )t1
+ group by key)t2
+ )t3
+)t4
+group by p;
+
+
+select * from union_mm order by id;
+drop table union_mm;
+
+
+create table partunion_mm(id int) partitioned by (key int) tblproperties ("transactional"="true", "transactional_properties"="insert_only");
+insert into table partunion_mm partition(key)
+select temps.* from (
+select key as p, key from intermediate
+union all
+select key + 1 as p, key + 1 from intermediate ) temps;
+
+select * from partunion_mm order by id;
+drop table partunion_mm;
+
+
+
+create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3))
+ stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only");
+
+insert into table skew_mm
+select key, key, key from intermediate;
+
+select * from skew_mm order by k2, k1, k4;
+drop table skew_mm;
+
+
+create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int)
+skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only");
+
+insert into table skew_dp_union_mm partition (k3)
+select key as i, key as j, key as k, key as l from intermediate
+union all
+select key +1 as i, key +2 as j, key +3 as k, key +4 as l from intermediate;
+
+
+select * from skew_dp_union_mm order by k2, k1, k4;
+drop table skew_dp_union_mm;
+
+
+
+set hive.merge.orcfile.stripe.level=true;
+set hive.merge.tezfiles=true;
+set hive.merge.mapfiles=true;
+set hive.merge.mapredfiles=true;
+
+
+create table merge0_mm (id int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only");
+
+insert into table merge0_mm select key from intermediate;
+select * from merge0_mm;
+
+set tez.grouping.split-count=1;
+insert into table merge0_mm select key from intermediate;
+set tez.grouping.split-count=0;
+select * from merge0_mm;
+
+drop table merge0_mm;
+
+
+create table merge2_mm (id int) tblproperties("transactional"="true", "transactional_properties"="insert_only");
+
+insert into table merge2_mm select key from intermediate;
+select * from merge2_mm;
+
+set tez.grouping.split-count=1;
+insert into table merge2_mm select key from intermediate;
+set tez.grouping.split-count=0;
+select * from merge2_mm;
+
+drop table merge2_mm;
+
+
+create table merge1_mm (id int) partitioned by (key int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only");
+
+insert into table merge1_mm partition (key) select key, key from intermediate;
+select * from merge1_mm order by id, key;
+
+set tez.grouping.split-count=1;
+insert into table merge1_mm partition (key) select key, key from intermediate;
+set tez.grouping.split-count=0;
+select * from merge1_mm order by id, key;
+
+drop table merge1_mm;
+
+set hive.merge.tezfiles=false;
+set hive.merge.mapfiles=false;
+set hive.merge.mapredfiles=false;
+
+-- TODO: need to include merge+union+DP, but it's broken for now
+
+
+drop table ctas0_mm;
+create table ctas0_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as select * from intermediate;
+select * from ctas0_mm;
+drop table ctas0_mm;
+
+drop table ctas1_mm;
+create table ctas1_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as
+ select * from intermediate union all select * from intermediate;
+select * from ctas1_mm;
+drop table ctas1_mm;
+
+
- drop table load0_mm;
- create table load0_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only");
- load data local inpath '../../data/files/kv1.txt' into table load0_mm;
- select count(1) from load0_mm;
- load data local inpath '../../data/files/kv2.txt' into table load0_mm;
- select count(1) from load0_mm;
- load data local inpath '../../data/files/kv2.txt' overwrite into table load0_mm;
- select count(1) from load0_mm;
- drop table load0_mm;
-
-
- drop table intermediate2;
- create table intermediate2 (key string, value string) stored as textfile
- location 'file:${system:test.tmp.dir}/intermediate2';
- load data local inpath '../../data/files/kv1.txt' into table intermediate2;
- load data local inpath '../../data/files/kv2.txt' into table intermediate2;
- load data local inpath '../../data/files/kv3.txt' into table intermediate2;
-
- drop table load1_mm;
- create table load1_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only");
- load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv2.txt' into table load1_mm;
- load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv1.txt' into table load1_mm;
- select count(1) from load1_mm;
- load data local inpath '../../data/files/kv1.txt' into table intermediate2;
- load data local inpath '../../data/files/kv2.txt' into table intermediate2;
- load data local inpath '../../data/files/kv3.txt' into table intermediate2;
- load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv*.txt' overwrite into table load1_mm;
- select count(1) from load1_mm;
- load data local inpath '../../data/files/kv2.txt' into table intermediate2;
- load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv2.txt' overwrite into table load1_mm;
- select count(1) from load1_mm;
- drop table load1_mm;
-
- drop table load2_mm;
- create table load2_mm (key string, value string)
- partitioned by (k int, l int) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only");
- load data local inpath '../../data/files/kv1.txt' into table intermediate2;
- load data local inpath '../../data/files/kv2.txt' into table intermediate2;
- load data local inpath '../../data/files/kv3.txt' into table intermediate2;
- load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv*.txt' into table load2_mm partition(k=5, l=5);
- select count(1) from load2_mm;
- drop table load2_mm;
- drop table intermediate2;
-
-
+drop table multi0_1_mm;
+drop table multi0_2_mm;
+create table multi0_1_mm (key int, key2 int) tblproperties("transactional"="true", "transactional_properties"="insert_only");
+create table multi0_2_mm (key int, key2 int) tblproperties("transactional"="true", "transactional_properties"="insert_only");
+
+--from intermediate
+--insert overwrite table multi0_1_mm select key, p
+--insert overwrite table multi0_2_mm select p, key;
+insert into table multi0_1_mm select key, p from intermediate;
+insert into table multi0_2_mm select p, key from intermediate;
+
+select * from multi0_1_mm order by key, key2;
+select * from multi0_2_mm order by key, key2;
+
+set hive.merge.mapredfiles=true;
+set hive.merge.sparkfiles=true;
+set hive.merge.tezfiles=true;
+
+--from intermediate
+--insert into table multi0_1_mm select p, key
+--insert overwrite table multi0_2_mm select key, p;
+insert into table multi0_1_mm select p, key from intermediate;
+insert into table multi0_2_mm select key, p from intermediate;
+select * from multi0_1_mm order by key, key2;
+select * from multi0_2_mm order by key, key2;
+
+set hive.merge.mapredfiles=false;
+set hive.merge.sparkfiles=false;
+set hive.merge.tezfiles=false;
+
+drop table multi0_1_mm;
+drop table multi0_2_mm;
+
+
+drop table multi1_mm;
+create table multi1_mm (key int, key2 int) partitioned by (p int) tblproperties("transactional"="true", "transactional_properties"="insert_only");
+from intermediate
+insert into table multi1_mm partition(p=1) select p, key
+insert into table multi1_mm partition(p=2) select key, p;
+select * from multi1_mm order by key, key2, p;
+--from intermediate
+--insert into table multi1_mm partition(p=2) select p, key
+--insert overwrite table multi1_mm partition(p=1) select key, p;
+insert into table multi1_mm partition(p=2) select p, key from intermediate;
+insert into table multi1_mm partition(p=1) select key, p from intermediate;
+select * from multi1_mm order by key, key2, p;
+
+from intermediate
+insert into table multi1_mm partition(p) select p, key, p
+insert into table multi1_mm partition(p=1) select key, p;
+select key, key2, p from multi1_mm order by key, key2, p;
+
+from intermediate
+insert into table multi1_mm partition(p) select p, key, 1
+insert into table multi1_mm partition(p=1) select key, p;
+select key, key2, p from multi1_mm order by key, key2, p;
+drop table multi1_mm;
+
+
+
+
+set datanucleus.cache.collections=false;
+set hive.stats.autogather=true;
+
+drop table stats_mm;
+create table stats_mm(key int) tblproperties("transactional"="true", "transactional_properties"="insert_only");
+--insert overwrite table stats_mm select key from intermediate;
+insert into table stats_mm select key from intermediate;
+desc formatted stats_mm;
+
+insert into table stats_mm select key from intermediate;
+desc formatted stats_mm;
+drop table stats_mm;
+
+drop table stats2_mm;
+create table stats2_mm tblproperties("transactional"="true", "transactional_properties"="insert_only") as select array(key, value) from src;
+desc formatted stats2_mm;
+drop table stats2_mm;
+
+
+set hive.optimize.skewjoin=true;
+set hive.skewjoin.key=2;
+set hive.optimize.metadataonly=false;
+
+CREATE TABLE skewjoin_mm(key INT, value STRING) STORED AS TEXTFILE tblproperties ("transactional"="true", "transactional_properties"="insert_only");
+FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT into TABLE skewjoin_mm SELECT src1.key, src2.value;
+select count(distinct key) from skewjoin_mm;
+drop table skewjoin_mm;
+
+set hive.optimize.skewjoin=false;
+
+set hive.optimize.index.filter=true;
+set hive.auto.convert.join=false;
+CREATE TABLE parquet1_mm(id INT) STORED AS PARQUET tblproperties ("transactional"="true", "transactional_properties"="insert_only");
+INSERT INTO parquet1_mm VALUES(1), (2);
+CREATE TABLE parquet2_mm(id INT, value STRING) STORED AS PARQUET tblproperties ("transactional"="true", "transactional_properties"="insert_only");
+INSERT INTO parquet2_mm VALUES(1, 'value1');
+INSERT INTO parquet2_mm VALUES(1, 'value2');
+select parquet1_mm.id, t1.value, t2.value FROM parquet1_mm
+ JOIN parquet2_mm t1 ON parquet1_mm.id=t1.id
+ JOIN parquet2_mm t2 ON parquet1_mm.id=t2.id
+where t1.value = 'value1' and t2.value = 'value2';
+drop table parquet1_mm;
+drop table parquet2_mm;
+
+set hive.auto.convert.join=true;
+
+
+DROP TABLE IF EXISTS temp1;
+CREATE TEMPORARY TABLE temp1 (a int) TBLPROPERTIES ("transactional"="true", "transactional_properties"="insert_only");
+INSERT INTO temp1 SELECT key FROM intermediate;
+DESC EXTENDED temp1;
+SELECT * FROM temp1;
+
+
+drop table intermediate;
+
+
+
[15/50] [abbrv] hive git commit: HIVE-17375: stddev_samp,
var_samp standard compliance (Zoltan Haindrich, reviewed by Ashutosh Chauhan)
Posted by we...@apache.org.
HIVE-17375: stddev_samp,var_samp standard compliance (Zoltan Haindrich, reviewed by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/dd04a92f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/dd04a92f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/dd04a92f
Branch: refs/heads/hive-14535
Commit: dd04a92f7178df2ef26ca0b76bd2985c34a7dbaf
Parents: e352ef4
Author: Zoltan Haindrich <ki...@rxd.hu>
Authored: Mon Aug 28 22:19:53 2017 +0200
Committer: Zoltan Haindrich <ki...@rxd.hu>
Committed: Mon Aug 28 23:06:46 2017 +0200
----------------------------------------------------------------------
.../ql/udf/generic/GenericUDAFStdSample.java | 13 +-
.../udf/generic/GenericUDAFVarianceSample.java | 13 +-
.../queries/clientpositive/udf_stddev_samp.q | 9 +
.../test/queries/clientpositive/udf_var_samp.q | 9 +
.../results/clientpositive/decimal_udf.q.out | 24 +-
.../clientpositive/llap/vectorization_15.q.out | 94 +-
.../clientpositive/spark/vectorization_15.q.out | 94 +-
.../clientpositive/udf_stddev_samp.q.out | 66 +-
.../results/clientpositive/udf_var_samp.q.out | 66 +-
.../clientpositive/vectorization_12.q.out | 1064 +++++++--------
.../clientpositive/vectorization_14.q.out | 1244 +++++++++---------
.../clientpositive/vectorization_15.q.out | 94 +-
.../clientpositive/vectorization_16.q.out | 322 ++---
.../clientpositive/vectorization_9.q.out | 322 ++---
14 files changed, 1783 insertions(+), 1651 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/dd04a92f/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStdSample.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStdSample.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStdSample.java
index e85046c..e032982 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStdSample.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStdSample.java
@@ -31,7 +31,10 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
*
*/
@Description(name = "stddev_samp",
- value = "_FUNC_(x) - Returns the sample standard deviation of a set of numbers")
+ value = "_FUNC_(x) - Returns the sample standard deviation of a set of numbers.\n"
+ + "If applied to an empty set: NULL is returned.\n"
+ + "If applied to a set with a single element: NULL is returned.\n"
+ + "Otherwise it computes: sqrt(var_samp(x))")
public class GenericUDAFStdSample extends GenericUDAFVariance {
@Override
@@ -78,14 +81,10 @@ public class GenericUDAFStdSample extends GenericUDAFVariance {
public Object terminate(AggregationBuffer agg) throws HiveException {
StdAgg myagg = (StdAgg) agg;
- if (myagg.count == 0) { // SQL standard - return null for zero elements
+ if (myagg.count <= 1) { // SQL standard - return null for zero or one elements
return null;
} else {
- if (myagg.count > 1) {
- getResult().set(Math.sqrt(myagg.variance / (myagg.count - 1)));
- } else { // for one element the variance is always 0
- getResult().set(0);
- }
+ getResult().set(Math.sqrt(myagg.variance / (myagg.count - 1)));
return getResult();
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/dd04a92f/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java
index ab863be..8815086 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java
@@ -31,7 +31,10 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
*
*/
@Description(name = "var_samp",
- value = "_FUNC_(x) - Returns the sample variance of a set of numbers")
+ value = "_FUNC_(x) - Returns the sample variance of a set of numbers.\n"
+ + "If applied to an empty set: NULL is returned.\n"
+ + "If applied to a set with a single element: NULL is returned.\n"
+ + "Otherwise it computes: (S2-S1*S1/N)/(N-1)")
public class GenericUDAFVarianceSample extends GenericUDAFVariance {
@Override
@@ -78,14 +81,10 @@ public class GenericUDAFVarianceSample extends GenericUDAFVariance {
public Object terminate(AggregationBuffer agg) throws HiveException {
StdAgg myagg = (StdAgg) agg;
- if (myagg.count == 0) { // SQL standard - return null for zero elements
+ if (myagg.count <= 1) {
return null;
} else {
- if (myagg.count > 1) {
- getResult().set(myagg.variance / (myagg.count - 1));
- } else { // for one element the variance is always 0
- getResult().set(0);
- }
+ getResult().set(myagg.variance / (myagg.count - 1));
return getResult();
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/dd04a92f/ql/src/test/queries/clientpositive/udf_stddev_samp.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/udf_stddev_samp.q b/ql/src/test/queries/clientpositive/udf_stddev_samp.q
index 30d2f24..76e479d 100644
--- a/ql/src/test/queries/clientpositive/udf_stddev_samp.q
+++ b/ql/src/test/queries/clientpositive/udf_stddev_samp.q
@@ -2,3 +2,12 @@ DESCRIBE FUNCTION stddev_samp;
DESCRIBE FUNCTION EXTENDED stddev_samp;
DESCRIBE FUNCTION stddev_samp;
DESCRIBE FUNCTION EXTENDED stddev_samp;
+
+
+drop table if exists t;
+create table t (a int);
+insert into t values (1),(-1),(0);
+
+select stddev_samp(a) from t;
+select stddev_samp(a) from t where a=0;
+select round(stddev_samp(a),5) from t where a>=0;
http://git-wip-us.apache.org/repos/asf/hive/blob/dd04a92f/ql/src/test/queries/clientpositive/udf_var_samp.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/udf_var_samp.q b/ql/src/test/queries/clientpositive/udf_var_samp.q
index de68e49..710cbf0 100644
--- a/ql/src/test/queries/clientpositive/udf_var_samp.q
+++ b/ql/src/test/queries/clientpositive/udf_var_samp.q
@@ -2,3 +2,12 @@ DESCRIBE FUNCTION var_samp;
DESCRIBE FUNCTION EXTENDED var_samp;
DESCRIBE FUNCTION var_samp;
DESCRIBE FUNCTION EXTENDED var_samp;
+
+drop table if exists t;
+create table t (a int);
+insert into t values (1),(-1),(0);
+
+select var_samp(a) from t;
+select var_samp(a) from t where a=0;
+select round(var_samp(a),5) from t where a>=0;
+
http://git-wip-us.apache.org/repos/asf/hive/blob/dd04a92f/ql/src/test/results/clientpositive/decimal_udf.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/decimal_udf.q.out b/ql/src/test/results/clientpositive/decimal_udf.q.out
index 515fc7d..8bda27a 100644
--- a/ql/src/test/results/clientpositive/decimal_udf.q.out
+++ b/ql/src/test/results/clientpositive/decimal_udf.q.out
@@ -1970,23 +1970,23 @@ POSTHOOK: query: SELECT value, stddev_samp(key), var_samp(key) FROM DECIMAL_UDF
POSTHOOK: type: QUERY
POSTHOOK: Input: default@decimal_udf
#### A masked pattern was here ####
--1234567890 0.0 0.0
--1255 0.0 0.0
--11 0.0 0.0
+-1234567890 NULL NULL
+-1255 NULL NULL
+-11 NULL NULL
-1 0.0 0.0
0 0.2348228191855647 0.055141756410256405
1 0.06627820154470102 0.004392800000000008
2 0.0 0.0
3 0.0 0.0
-4 0.0 0.0
-10 0.0 0.0
-20 0.0 0.0
-100 0.0 0.0
-124 0.0 0.0
-125 0.0 0.0
-200 0.0 0.0
-4400 0.0 0.0
-1234567890 0.0 0.0
+4 NULL NULL
+10 NULL NULL
+20 NULL NULL
+100 NULL NULL
+124 NULL NULL
+125 NULL NULL
+200 NULL NULL
+4400 NULL NULL
+1234567890 NULL NULL
PREHOOK: query: EXPLAIN SELECT histogram_numeric(key, 3) FROM DECIMAL_UDF
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN SELECT histogram_numeric(key, 3) FROM DECIMAL_UDF
http://git-wip-us.apache.org/repos/asf/hive/blob/dd04a92f/ql/src/test/results/clientpositive/llap/vectorization_15.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vectorization_15.q.out b/ql/src/test/results/clientpositive/llap/vectorization_15.q.out
index 79c7084..9cc350d 100644
--- a/ql/src/test/results/clientpositive/llap/vectorization_15.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorization_15.q.out
@@ -259,50 +259,50 @@ ORDER BY cfloat, cboolean1, cdouble, cstring1, ctinyint, cint, ctimestamp1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
--48.0 NULL -7196.0 NULL -48 NULL 1969-12-31 16:00:06.337 0.0 NULL -7196.0 -572463.388 33.0 0.0 0.0 -23.0 48 NULL NULL -23 NULL NULL
--51.0 NULL -200.0 NULL -51 NULL 1969-12-31 15:59:55.423 0.0 NULL -200.0 -15910.599999999999 33.0 0.0 0.0 -23.0 51 NULL NULL -23 NULL NULL
--51.0 false NULL 10 -51 1058319346 1969-12-31 16:00:08.451 0.0 -1058319372.28 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 1.05831942E9 -23 1058319372.28 0.0
--51.0 false NULL 10TYIE5S35U6dj3N -51 -469581869 1969-12-31 16:00:08.451 0.0 469581842.72 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 -4.69581792E8 -23 -469581842.72 0.0
--51.0 false NULL 1Lh6Uoq3WhNtOqQHu7WN7U -51 -352637533 1969-12-31 16:00:08.451 0.0 352637506.72 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 -3.52637472E8 -23 -352637506.72 0.0
--51.0 true NULL 04Y1mA17 -51 -114647521 1969-12-31 16:00:08.451 0.0 114647494.72 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 -1.14647472E8 -23 -114647494.72 0.0
--51.0 true NULL 10Wu570aLPO0p02P17FeH -51 405338893 1969-12-31 16:00:08.451 0.0 -405338919.28 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 4.05338944E8 -23 405338919.28 0.0
--51.0 true NULL 3cQp060 -51 -226923315 1969-12-31 16:00:08.451 0.0 226923288.72 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 -2.26923264E8 -23 -226923288.72 0.0
--51.0 true NULL 8EPG0Xi307qd -51 -328662044 1969-12-31 16:00:08.451 0.0 328662017.72 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 -3.28661984E8 -23 -328662017.72 0.0
--51.0 true NULL 8iHtdkJ6d -51 1006818344 1969-12-31 16:00:08.451 0.0 -1006818370.28 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 1.00681843E9 -23 1006818370.28 0.0
--51.0 true NULL QiOcvR0kt6r7f0R7fiPxQTCU -51 266531954 1969-12-31 16:00:08.451 0.0 -266531980.28 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 2.66532E8 -23 266531980.28 0.0
--51.0 true NULL Ybpj38RTTYl7CnJXPNx1g4C -51 -370919370 1969-12-31 16:00:08.451 0.0 370919343.72 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 -3.70919296E8 -23 -370919343.72 0.0
--6.0 NULL -200.0 NULL -6 NULL 1969-12-31 15:59:56.094 0.0 NULL -200.0 -15910.599999999999 3.0 0.0 0.0 -23.0 6 NULL NULL -5 NULL NULL
--62.0 NULL 15601.0 NULL -62 NULL 1969-12-31 15:59:56.527 0.0 NULL 15601.0 1241106.353 33.0 0.0 0.0 -23.0 62 NULL NULL -23 NULL NULL
-11.0 false NULL 10pO8p1LNx4Y 11 271296824 1969-12-31 16:00:02.351 0.0 -271296850.28 NULL NULL 0.0 0.0 0.0 NULL -11 0.0 2.71296832E8 -1 271296850.28 0.0
-11.0 false NULL 1H6wGP 11 -560827082 1969-12-31 16:00:02.351 0.0 560827055.72 NULL NULL 0.0 0.0 0.0 NULL -11 0.0 -5.6082707E8 -1 -560827055.72 0.0
-11.0 false NULL 2a7V63IL7jK3o 11 -325931647 1969-12-31 16:00:02.351 0.0 325931620.72 NULL NULL 0.0 0.0 0.0 NULL -11 0.0 -3.25931648E8 -1 -325931620.72 0.0
-11.0 true NULL 10 11 92365813 1969-12-31 16:00:02.351 0.0 -92365839.28 NULL NULL 0.0 0.0 0.0 NULL -11 0.0 9.2365808E7 -1 92365839.28 0.0
-21.0 NULL 15601.0 NULL 21 NULL 1969-12-31 15:59:56.527 0.0 NULL 15601.0 1241106.353 12.0 0.0 0.0 -23.0 -21 NULL NULL -2 NULL NULL
-32.0 NULL -200.0 NULL 32 NULL 1969-12-31 16:00:02.445 0.0 NULL -200.0 -15910.599999999999 1.0 0.0 0.0 -23.0 -32 NULL NULL -23 NULL NULL
-36.0 NULL -200.0 NULL 36 NULL 1969-12-31 16:00:00.554 0.0 NULL -200.0 -15910.599999999999 33.0 0.0 0.0 -23.0 -36 NULL NULL -23 NULL NULL
-5.0 NULL 15601.0 NULL 5 NULL 1969-12-31 15:59:56.527 0.0 NULL 15601.0 1241106.353 3.0 0.0 0.0 -23.0 -5 NULL NULL -3 NULL NULL
-58.0 NULL 15601.0 NULL 58 NULL 1969-12-31 15:59:56.527 0.0 NULL 15601.0 1241106.353 33.0 0.0 0.0 -23.0 -58 NULL NULL -23 NULL NULL
-8.0 false NULL 10V3pN5r5lI2qWl2lG103 8 -362835731 1969-12-31 16:00:15.892 0.0 362835704.72 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 -3.62835744E8 -7 -362835704.72 0.0
-8.0 false NULL 10c4qt584m5y6uWT 8 -183000142 1969-12-31 16:00:15.892 0.0 183000115.72 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 -1.8300016E8 -7 -183000115.72 0.0
-8.0 false NULL 8GloEukQ0c68JDmnYL53 8 -722873402 1969-12-31 16:00:15.892 0.0 722873375.72 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 -7.2287341E8 -7 -722873375.72 0.0
-8.0 false NULL kA0XH5C5 8 -503903864 1969-12-31 16:00:15.892 0.0 503903837.72 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 -5.03903872E8 -7 -503903837.72 0.0
-8.0 true NULL 100VTM7PEW8GH1uE 8 88129338 1969-12-31 16:00:15.892 0.0 -88129364.28 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 8.8129328E7 -7 88129364.28 0.0
-8.0 true NULL 1062158y 8 -1005155523 1969-12-31 16:00:15.892 0.0 1005155496.72 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 -1.00515552E9 -7 -1005155496.72 0.0
-8.0 true NULL 1063cEnGjSal 8 -624769630 1969-12-31 16:00:15.892 0.0 624769603.72 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 -6.247696E8 -7 -624769603.72 0.0
-8.0 true NULL 4kMasVoB7lX1wc5i64bNk 8 683567667 1969-12-31 16:00:15.892 0.0 -683567693.28 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 6.8356768E8 -7 683567693.28 0.0
-8.0 true NULL XH6I7A417 8 436627202 1969-12-31 16:00:15.892 0.0 -436627228.28 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 4.366272E8 -7 436627228.28 0.0
-NULL false -13008.0 10 NULL -896629175 NULL NULL 896629148.72 -13008.0 -1034825.424 NULL NULL NULL -23.0 NULL 0.0 NULL NULL -896629148.72 0.0
-NULL false -7707.0 10vke853 NULL 424959354 NULL NULL -424959380.28 -7707.0 -613114.971 NULL NULL NULL -23.0 NULL 0.0 NULL NULL 424959380.28 0.0
-NULL false 14771.0 10 NULL 993788576 NULL NULL -993788602.28 14771.0 1175077.363 NULL NULL NULL -23.0 NULL 0.0 NULL NULL 993788602.28 0.0
-NULL false 2459.0 10 NULL -394064473 NULL NULL 394064446.72 2459.0 195620.827 NULL NULL NULL -23.0 NULL 0.0 NULL NULL -394064446.72 0.0
-NULL true -14640.0 101n6n461o NULL -819657767 NULL NULL 819657740.72 -14640.0 -1164655.92 NULL NULL NULL -23.0 NULL 0.0 NULL NULL -819657740.72 0.0
-NULL true -14674.0 105aFDAt30c4rI4U NULL -831072496 NULL NULL 831072469.72 -14674.0 -1167360.722 NULL NULL NULL -23.0 NULL 0.0 NULL NULL -831072469.72 0.0
-NULL true -7238.0 10ljXCFT6fG6Qi3S7414e NULL -256767096 NULL NULL 256767069.72 -7238.0 -575804.614 NULL NULL NULL -23.0 NULL 0.0 NULL NULL -256767069.72 0.0
-NULL true -9489.0 10Yr6 NULL 500274721 NULL NULL -500274747.28 -9489.0 -754878.417 NULL NULL NULL -23.0 NULL 0.0 NULL NULL 500274747.28 0.0
-NULL true 10419.0 10 NULL -721614386 NULL NULL 721614359.72 10419.0 828862.7069999999 NULL NULL NULL -23.0 NULL 0.0 NULL NULL -721614359.72 0.0
-NULL true 14519.0 100xJdkyc NULL 729277608 NULL NULL -729277634.28 14519.0 1155030.007 NULL NULL NULL -23.0 NULL 0.0 NULL NULL 729277634.28 0.0
-NULL true 2261.0 10Hr5oB07Ohu0622u NULL 612000160 NULL NULL -612000186.28 2261.0 179869.33299999998 NULL NULL NULL -23.0 NULL 0.0 NULL NULL 612000186.28 0.0
-NULL true 4516.0 10lL0XD6WP2x64f70N0fHmC1 NULL -974538365 NULL NULL 974538338.72 4516.0 359261.348 NULL NULL NULL -23.0 NULL 0.0 NULL NULL -974538338.72 0.0
-NULL true 4767.0 10dUdwyXp5XwgpkTxLffmv3x NULL 187503456 NULL NULL -187503482.28 4767.0 379229.151 NULL NULL NULL -23.0 NULL 0.0 NULL NULL 187503482.28 0.0
-NULL true 5658.0 10M3eGUsKVonbl70DyoCk25 NULL 486382507 NULL NULL -486382533.28 5658.0 450110.874 NULL NULL NULL -23.0 NULL 0.0 NULL NULL 486382533.28 0.0
-NULL true 8659.0 1047piRsT3c3r134I NULL 487236176 NULL NULL -487236202.28 8659.0 688849.427 NULL NULL NULL -23.0 NULL 0.0 NULL NULL 487236202.28 0.0
+-48.0 NULL -7196.0 NULL -48 NULL 1969-12-31 16:00:06.337 NULL NULL -7196.0 -572463.388 33.0 NULL 0.0 -23.0 48 NULL NULL -23 NULL NULL
+-51.0 NULL -200.0 NULL -51 NULL 1969-12-31 15:59:55.423 NULL NULL -200.0 -15910.599999999999 33.0 NULL 0.0 -23.0 51 NULL NULL -23 NULL NULL
+-51.0 false NULL 10 -51 1058319346 1969-12-31 16:00:08.451 NULL -1058319372.28 NULL NULL 33.0 NULL 0.0 NULL 51 NULL 1.05831942E9 -23 1058319372.28 0.0
+-51.0 false NULL 10TYIE5S35U6dj3N -51 -469581869 1969-12-31 16:00:08.451 NULL 469581842.72 NULL NULL 33.0 NULL 0.0 NULL 51 NULL -4.69581792E8 -23 -469581842.72 0.0
+-51.0 false NULL 1Lh6Uoq3WhNtOqQHu7WN7U -51 -352637533 1969-12-31 16:00:08.451 NULL 352637506.72 NULL NULL 33.0 NULL 0.0 NULL 51 NULL -3.52637472E8 -23 -352637506.72 0.0
+-51.0 true NULL 04Y1mA17 -51 -114647521 1969-12-31 16:00:08.451 NULL 114647494.72 NULL NULL 33.0 NULL 0.0 NULL 51 NULL -1.14647472E8 -23 -114647494.72 0.0
+-51.0 true NULL 10Wu570aLPO0p02P17FeH -51 405338893 1969-12-31 16:00:08.451 NULL -405338919.28 NULL NULL 33.0 NULL 0.0 NULL 51 NULL 4.05338944E8 -23 405338919.28 0.0
+-51.0 true NULL 3cQp060 -51 -226923315 1969-12-31 16:00:08.451 NULL 226923288.72 NULL NULL 33.0 NULL 0.0 NULL 51 NULL -2.26923264E8 -23 -226923288.72 0.0
+-51.0 true NULL 8EPG0Xi307qd -51 -328662044 1969-12-31 16:00:08.451 NULL 328662017.72 NULL NULL 33.0 NULL 0.0 NULL 51 NULL -3.28661984E8 -23 -328662017.72 0.0
+-51.0 true NULL 8iHtdkJ6d -51 1006818344 1969-12-31 16:00:08.451 NULL -1006818370.28 NULL NULL 33.0 NULL 0.0 NULL 51 NULL 1.00681843E9 -23 1006818370.28 0.0
+-51.0 true NULL QiOcvR0kt6r7f0R7fiPxQTCU -51 266531954 1969-12-31 16:00:08.451 NULL -266531980.28 NULL NULL 33.0 NULL 0.0 NULL 51 NULL 2.66532E8 -23 266531980.28 0.0
+-51.0 true NULL Ybpj38RTTYl7CnJXPNx1g4C -51 -370919370 1969-12-31 16:00:08.451 NULL 370919343.72 NULL NULL 33.0 NULL 0.0 NULL 51 NULL -3.70919296E8 -23 -370919343.72 0.0
+-6.0 NULL -200.0 NULL -6 NULL 1969-12-31 15:59:56.094 NULL NULL -200.0 -15910.599999999999 3.0 NULL 0.0 -23.0 6 NULL NULL -5 NULL NULL
+-62.0 NULL 15601.0 NULL -62 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 33.0 NULL 0.0 -23.0 62 NULL NULL -23 NULL NULL
+11.0 false NULL 10pO8p1LNx4Y 11 271296824 1969-12-31 16:00:02.351 NULL -271296850.28 NULL NULL 0.0 NULL 0.0 NULL -11 NULL 2.71296832E8 -1 271296850.28 0.0
+11.0 false NULL 1H6wGP 11 -560827082 1969-12-31 16:00:02.351 NULL 560827055.72 NULL NULL 0.0 NULL 0.0 NULL -11 NULL -5.6082707E8 -1 -560827055.72 0.0
+11.0 false NULL 2a7V63IL7jK3o 11 -325931647 1969-12-31 16:00:02.351 NULL 325931620.72 NULL NULL 0.0 NULL 0.0 NULL -11 NULL -3.25931648E8 -1 -325931620.72 0.0
+11.0 true NULL 10 11 92365813 1969-12-31 16:00:02.351 NULL -92365839.28 NULL NULL 0.0 NULL 0.0 NULL -11 NULL 9.2365808E7 -1 92365839.28 0.0
+21.0 NULL 15601.0 NULL 21 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 12.0 NULL 0.0 -23.0 -21 NULL NULL -2 NULL NULL
+32.0 NULL -200.0 NULL 32 NULL 1969-12-31 16:00:02.445 NULL NULL -200.0 -15910.599999999999 1.0 NULL 0.0 -23.0 -32 NULL NULL -23 NULL NULL
+36.0 NULL -200.0 NULL 36 NULL 1969-12-31 16:00:00.554 NULL NULL -200.0 -15910.599999999999 33.0 NULL 0.0 -23.0 -36 NULL NULL -23 NULL NULL
+5.0 NULL 15601.0 NULL 5 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 3.0 NULL 0.0 -23.0 -5 NULL NULL -3 NULL NULL
+58.0 NULL 15601.0 NULL 58 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 33.0 NULL 0.0 -23.0 -58 NULL NULL -23 NULL NULL
+8.0 false NULL 10V3pN5r5lI2qWl2lG103 8 -362835731 1969-12-31 16:00:15.892 NULL 362835704.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -3.62835744E8 -7 -362835704.72 0.0
+8.0 false NULL 10c4qt584m5y6uWT 8 -183000142 1969-12-31 16:00:15.892 NULL 183000115.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -1.8300016E8 -7 -183000115.72 0.0
+8.0 false NULL 8GloEukQ0c68JDmnYL53 8 -722873402 1969-12-31 16:00:15.892 NULL 722873375.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -7.2287341E8 -7 -722873375.72 0.0
+8.0 false NULL kA0XH5C5 8 -503903864 1969-12-31 16:00:15.892 NULL 503903837.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -5.03903872E8 -7 -503903837.72 0.0
+8.0 true NULL 100VTM7PEW8GH1uE 8 88129338 1969-12-31 16:00:15.892 NULL -88129364.28 NULL NULL 1.0 NULL 0.0 NULL -8 NULL 8.8129328E7 -7 88129364.28 0.0
+8.0 true NULL 1062158y 8 -1005155523 1969-12-31 16:00:15.892 NULL 1005155496.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -1.00515552E9 -7 -1005155496.72 0.0
+8.0 true NULL 1063cEnGjSal 8 -624769630 1969-12-31 16:00:15.892 NULL 624769603.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -6.247696E8 -7 -624769603.72 0.0
+8.0 true NULL 4kMasVoB7lX1wc5i64bNk 8 683567667 1969-12-31 16:00:15.892 NULL -683567693.28 NULL NULL 1.0 NULL 0.0 NULL -8 NULL 6.8356768E8 -7 683567693.28 0.0
+8.0 true NULL XH6I7A417 8 436627202 1969-12-31 16:00:15.892 NULL -436627228.28 NULL NULL 1.0 NULL 0.0 NULL -8 NULL 4.366272E8 -7 436627228.28 0.0
+NULL false -13008.0 10 NULL -896629175 NULL NULL 896629148.72 -13008.0 -1034825.424 NULL NULL NULL -23.0 NULL NULL NULL NULL -896629148.72 0.0
+NULL false -7707.0 10vke853 NULL 424959354 NULL NULL -424959380.28 -7707.0 -613114.971 NULL NULL NULL -23.0 NULL NULL NULL NULL 424959380.28 0.0
+NULL false 14771.0 10 NULL 993788576 NULL NULL -993788602.28 14771.0 1175077.363 NULL NULL NULL -23.0 NULL NULL NULL NULL 993788602.28 0.0
+NULL false 2459.0 10 NULL -394064473 NULL NULL 394064446.72 2459.0 195620.827 NULL NULL NULL -23.0 NULL NULL NULL NULL -394064446.72 0.0
+NULL true -14640.0 101n6n461o NULL -819657767 NULL NULL 819657740.72 -14640.0 -1164655.92 NULL NULL NULL -23.0 NULL NULL NULL NULL -819657740.72 0.0
+NULL true -14674.0 105aFDAt30c4rI4U NULL -831072496 NULL NULL 831072469.72 -14674.0 -1167360.722 NULL NULL NULL -23.0 NULL NULL NULL NULL -831072469.72 0.0
+NULL true -7238.0 10ljXCFT6fG6Qi3S7414e NULL -256767096 NULL NULL 256767069.72 -7238.0 -575804.614 NULL NULL NULL -23.0 NULL NULL NULL NULL -256767069.72 0.0
+NULL true -9489.0 10Yr6 NULL 500274721 NULL NULL -500274747.28 -9489.0 -754878.417 NULL NULL NULL -23.0 NULL NULL NULL NULL 500274747.28 0.0
+NULL true 10419.0 10 NULL -721614386 NULL NULL 721614359.72 10419.0 828862.7069999999 NULL NULL NULL -23.0 NULL NULL NULL NULL -721614359.72 0.0
+NULL true 14519.0 100xJdkyc NULL 729277608 NULL NULL -729277634.28 14519.0 1155030.007 NULL NULL NULL -23.0 NULL NULL NULL NULL 729277634.28 0.0
+NULL true 2261.0 10Hr5oB07Ohu0622u NULL 612000160 NULL NULL -612000186.28 2261.0 179869.33299999998 NULL NULL NULL -23.0 NULL NULL NULL NULL 612000186.28 0.0
+NULL true 4516.0 10lL0XD6WP2x64f70N0fHmC1 NULL -974538365 NULL NULL 974538338.72 4516.0 359261.348 NULL NULL NULL -23.0 NULL NULL NULL NULL -974538338.72 0.0
+NULL true 4767.0 10dUdwyXp5XwgpkTxLffmv3x NULL 187503456 NULL NULL -187503482.28 4767.0 379229.151 NULL NULL NULL -23.0 NULL NULL NULL NULL 187503482.28 0.0
+NULL true 5658.0 10M3eGUsKVonbl70DyoCk25 NULL 486382507 NULL NULL -486382533.28 5658.0 450110.874 NULL NULL NULL -23.0 NULL NULL NULL NULL 486382533.28 0.0
+NULL true 8659.0 1047piRsT3c3r134I NULL 487236176 NULL NULL -487236202.28 8659.0 688849.427 NULL NULL NULL -23.0 NULL NULL NULL NULL 487236202.28 0.0
http://git-wip-us.apache.org/repos/asf/hive/blob/dd04a92f/ql/src/test/results/clientpositive/spark/vectorization_15.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_15.q.out b/ql/src/test/results/clientpositive/spark/vectorization_15.q.out
index 6ef97c2..1b589f7 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_15.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_15.q.out
@@ -255,50 +255,50 @@ ORDER BY cfloat, cboolean1, cdouble, cstring1, ctinyint, cint, ctimestamp1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
--48.0 NULL -7196.0 NULL -48 NULL 1969-12-31 16:00:06.337 0.0 NULL -7196.0 -572463.388 33.0 0.0 0.0 -23.0 48 NULL NULL -23 NULL NULL
--51.0 NULL -200.0 NULL -51 NULL 1969-12-31 15:59:55.423 0.0 NULL -200.0 -15910.599999999999 33.0 0.0 0.0 -23.0 51 NULL NULL -23 NULL NULL
--51.0 false NULL 10 -51 1058319346 1969-12-31 16:00:08.451 0.0 -1058319372.28 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 1.05831942E9 -23 1058319372.28 0.0
--51.0 false NULL 10TYIE5S35U6dj3N -51 -469581869 1969-12-31 16:00:08.451 0.0 469581842.72 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 -4.69581792E8 -23 -469581842.72 0.0
--51.0 false NULL 1Lh6Uoq3WhNtOqQHu7WN7U -51 -352637533 1969-12-31 16:00:08.451 0.0 352637506.72 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 -3.52637472E8 -23 -352637506.72 0.0
--51.0 true NULL 04Y1mA17 -51 -114647521 1969-12-31 16:00:08.451 0.0 114647494.72 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 -1.14647472E8 -23 -114647494.72 0.0
--51.0 true NULL 10Wu570aLPO0p02P17FeH -51 405338893 1969-12-31 16:00:08.451 0.0 -405338919.28 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 4.05338944E8 -23 405338919.28 0.0
--51.0 true NULL 3cQp060 -51 -226923315 1969-12-31 16:00:08.451 0.0 226923288.72 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 -2.26923264E8 -23 -226923288.72 0.0
--51.0 true NULL 8EPG0Xi307qd -51 -328662044 1969-12-31 16:00:08.451 0.0 328662017.72 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 -3.28661984E8 -23 -328662017.72 0.0
--51.0 true NULL 8iHtdkJ6d -51 1006818344 1969-12-31 16:00:08.451 0.0 -1006818370.28 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 1.00681843E9 -23 1006818370.28 0.0
--51.0 true NULL QiOcvR0kt6r7f0R7fiPxQTCU -51 266531954 1969-12-31 16:00:08.451 0.0 -266531980.28 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 2.66532E8 -23 266531980.28 0.0
--51.0 true NULL Ybpj38RTTYl7CnJXPNx1g4C -51 -370919370 1969-12-31 16:00:08.451 0.0 370919343.72 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 -3.70919296E8 -23 -370919343.72 0.0
--6.0 NULL -200.0 NULL -6 NULL 1969-12-31 15:59:56.094 0.0 NULL -200.0 -15910.599999999999 3.0 0.0 0.0 -23.0 6 NULL NULL -5 NULL NULL
--62.0 NULL 15601.0 NULL -62 NULL 1969-12-31 15:59:56.527 0.0 NULL 15601.0 1241106.353 33.0 0.0 0.0 -23.0 62 NULL NULL -23 NULL NULL
-11.0 false NULL 10pO8p1LNx4Y 11 271296824 1969-12-31 16:00:02.351 0.0 -271296850.28 NULL NULL 0.0 0.0 0.0 NULL -11 0.0 2.71296832E8 -1 271296850.28 0.0
-11.0 false NULL 1H6wGP 11 -560827082 1969-12-31 16:00:02.351 0.0 560827055.72 NULL NULL 0.0 0.0 0.0 NULL -11 0.0 -5.6082707E8 -1 -560827055.72 0.0
-11.0 false NULL 2a7V63IL7jK3o 11 -325931647 1969-12-31 16:00:02.351 0.0 325931620.72 NULL NULL 0.0 0.0 0.0 NULL -11 0.0 -3.25931648E8 -1 -325931620.72 0.0
-11.0 true NULL 10 11 92365813 1969-12-31 16:00:02.351 0.0 -92365839.28 NULL NULL 0.0 0.0 0.0 NULL -11 0.0 9.2365808E7 -1 92365839.28 0.0
-21.0 NULL 15601.0 NULL 21 NULL 1969-12-31 15:59:56.527 0.0 NULL 15601.0 1241106.353 12.0 0.0 0.0 -23.0 -21 NULL NULL -2 NULL NULL
-32.0 NULL -200.0 NULL 32 NULL 1969-12-31 16:00:02.445 0.0 NULL -200.0 -15910.599999999999 1.0 0.0 0.0 -23.0 -32 NULL NULL -23 NULL NULL
-36.0 NULL -200.0 NULL 36 NULL 1969-12-31 16:00:00.554 0.0 NULL -200.0 -15910.599999999999 33.0 0.0 0.0 -23.0 -36 NULL NULL -23 NULL NULL
-5.0 NULL 15601.0 NULL 5 NULL 1969-12-31 15:59:56.527 0.0 NULL 15601.0 1241106.353 3.0 0.0 0.0 -23.0 -5 NULL NULL -3 NULL NULL
-58.0 NULL 15601.0 NULL 58 NULL 1969-12-31 15:59:56.527 0.0 NULL 15601.0 1241106.353 33.0 0.0 0.0 -23.0 -58 NULL NULL -23 NULL NULL
-8.0 false NULL 10V3pN5r5lI2qWl2lG103 8 -362835731 1969-12-31 16:00:15.892 0.0 362835704.72 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 -3.62835744E8 -7 -362835704.72 0.0
-8.0 false NULL 10c4qt584m5y6uWT 8 -183000142 1969-12-31 16:00:15.892 0.0 183000115.72 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 -1.8300016E8 -7 -183000115.72 0.0
-8.0 false NULL 8GloEukQ0c68JDmnYL53 8 -722873402 1969-12-31 16:00:15.892 0.0 722873375.72 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 -7.2287341E8 -7 -722873375.72 0.0
-8.0 false NULL kA0XH5C5 8 -503903864 1969-12-31 16:00:15.892 0.0 503903837.72 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 -5.03903872E8 -7 -503903837.72 0.0
-8.0 true NULL 100VTM7PEW8GH1uE 8 88129338 1969-12-31 16:00:15.892 0.0 -88129364.28 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 8.8129328E7 -7 88129364.28 0.0
-8.0 true NULL 1062158y 8 -1005155523 1969-12-31 16:00:15.892 0.0 1005155496.72 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 -1.00515552E9 -7 -1005155496.72 0.0
-8.0 true NULL 1063cEnGjSal 8 -624769630 1969-12-31 16:00:15.892 0.0 624769603.72 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 -6.247696E8 -7 -624769603.72 0.0
-8.0 true NULL 4kMasVoB7lX1wc5i64bNk 8 683567667 1969-12-31 16:00:15.892 0.0 -683567693.28 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 6.8356768E8 -7 683567693.28 0.0
-8.0 true NULL XH6I7A417 8 436627202 1969-12-31 16:00:15.892 0.0 -436627228.28 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 4.366272E8 -7 436627228.28 0.0
-NULL false -13008.0 10 NULL -896629175 NULL NULL 896629148.72 -13008.0 -1034825.424 NULL NULL NULL -23.0 NULL 0.0 NULL NULL -896629148.72 0.0
-NULL false -7707.0 10vke853 NULL 424959354 NULL NULL -424959380.28 -7707.0 -613114.971 NULL NULL NULL -23.0 NULL 0.0 NULL NULL 424959380.28 0.0
-NULL false 14771.0 10 NULL 993788576 NULL NULL -993788602.28 14771.0 1175077.363 NULL NULL NULL -23.0 NULL 0.0 NULL NULL 993788602.28 0.0
-NULL false 2459.0 10 NULL -394064473 NULL NULL 394064446.72 2459.0 195620.827 NULL NULL NULL -23.0 NULL 0.0 NULL NULL -394064446.72 0.0
-NULL true -14640.0 101n6n461o NULL -819657767 NULL NULL 819657740.72 -14640.0 -1164655.92 NULL NULL NULL -23.0 NULL 0.0 NULL NULL -819657740.72 0.0
-NULL true -14674.0 105aFDAt30c4rI4U NULL -831072496 NULL NULL 831072469.72 -14674.0 -1167360.722 NULL NULL NULL -23.0 NULL 0.0 NULL NULL -831072469.72 0.0
-NULL true -7238.0 10ljXCFT6fG6Qi3S7414e NULL -256767096 NULL NULL 256767069.72 -7238.0 -575804.614 NULL NULL NULL -23.0 NULL 0.0 NULL NULL -256767069.72 0.0
-NULL true -9489.0 10Yr6 NULL 500274721 NULL NULL -500274747.28 -9489.0 -754878.417 NULL NULL NULL -23.0 NULL 0.0 NULL NULL 500274747.28 0.0
-NULL true 10419.0 10 NULL -721614386 NULL NULL 721614359.72 10419.0 828862.7069999999 NULL NULL NULL -23.0 NULL 0.0 NULL NULL -721614359.72 0.0
-NULL true 14519.0 100xJdkyc NULL 729277608 NULL NULL -729277634.28 14519.0 1155030.007 NULL NULL NULL -23.0 NULL 0.0 NULL NULL 729277634.28 0.0
-NULL true 2261.0 10Hr5oB07Ohu0622u NULL 612000160 NULL NULL -612000186.28 2261.0 179869.33299999998 NULL NULL NULL -23.0 NULL 0.0 NULL NULL 612000186.28 0.0
-NULL true 4516.0 10lL0XD6WP2x64f70N0fHmC1 NULL -974538365 NULL NULL 974538338.72 4516.0 359261.348 NULL NULL NULL -23.0 NULL 0.0 NULL NULL -974538338.72 0.0
-NULL true 4767.0 10dUdwyXp5XwgpkTxLffmv3x NULL 187503456 NULL NULL -187503482.28 4767.0 379229.151 NULL NULL NULL -23.0 NULL 0.0 NULL NULL 187503482.28 0.0
-NULL true 5658.0 10M3eGUsKVonbl70DyoCk25 NULL 486382507 NULL NULL -486382533.28 5658.0 450110.874 NULL NULL NULL -23.0 NULL 0.0 NULL NULL 486382533.28 0.0
-NULL true 8659.0 1047piRsT3c3r134I NULL 487236176 NULL NULL -487236202.28 8659.0 688849.427 NULL NULL NULL -23.0 NULL 0.0 NULL NULL 487236202.28 0.0
+-48.0 NULL -7196.0 NULL -48 NULL 1969-12-31 16:00:06.337 NULL NULL -7196.0 -572463.388 33.0 NULL 0.0 -23.0 48 NULL NULL -23 NULL NULL
+-51.0 NULL -200.0 NULL -51 NULL 1969-12-31 15:59:55.423 NULL NULL -200.0 -15910.599999999999 33.0 NULL 0.0 -23.0 51 NULL NULL -23 NULL NULL
+-51.0 false NULL 10 -51 1058319346 1969-12-31 16:00:08.451 NULL -1058319372.28 NULL NULL 33.0 NULL 0.0 NULL 51 NULL 1.05831942E9 -23 1058319372.28 0.0
+-51.0 false NULL 10TYIE5S35U6dj3N -51 -469581869 1969-12-31 16:00:08.451 NULL 469581842.72 NULL NULL 33.0 NULL 0.0 NULL 51 NULL -4.69581792E8 -23 -469581842.72 0.0
+-51.0 false NULL 1Lh6Uoq3WhNtOqQHu7WN7U -51 -352637533 1969-12-31 16:00:08.451 NULL 352637506.72 NULL NULL 33.0 NULL 0.0 NULL 51 NULL -3.52637472E8 -23 -352637506.72 0.0
+-51.0 true NULL 04Y1mA17 -51 -114647521 1969-12-31 16:00:08.451 NULL 114647494.72 NULL NULL 33.0 NULL 0.0 NULL 51 NULL -1.14647472E8 -23 -114647494.72 0.0
+-51.0 true NULL 10Wu570aLPO0p02P17FeH -51 405338893 1969-12-31 16:00:08.451 NULL -405338919.28 NULL NULL 33.0 NULL 0.0 NULL 51 NULL 4.05338944E8 -23 405338919.28 0.0
+-51.0 true NULL 3cQp060 -51 -226923315 1969-12-31 16:00:08.451 NULL 226923288.72 NULL NULL 33.0 NULL 0.0 NULL 51 NULL -2.26923264E8 -23 -226923288.72 0.0
+-51.0 true NULL 8EPG0Xi307qd -51 -328662044 1969-12-31 16:00:08.451 NULL 328662017.72 NULL NULL 33.0 NULL 0.0 NULL 51 NULL -3.28661984E8 -23 -328662017.72 0.0
+-51.0 true NULL 8iHtdkJ6d -51 1006818344 1969-12-31 16:00:08.451 NULL -1006818370.28 NULL NULL 33.0 NULL 0.0 NULL 51 NULL 1.00681843E9 -23 1006818370.28 0.0
+-51.0 true NULL QiOcvR0kt6r7f0R7fiPxQTCU -51 266531954 1969-12-31 16:00:08.451 NULL -266531980.28 NULL NULL 33.0 NULL 0.0 NULL 51 NULL 2.66532E8 -23 266531980.28 0.0
+-51.0 true NULL Ybpj38RTTYl7CnJXPNx1g4C -51 -370919370 1969-12-31 16:00:08.451 NULL 370919343.72 NULL NULL 33.0 NULL 0.0 NULL 51 NULL -3.70919296E8 -23 -370919343.72 0.0
+-6.0 NULL -200.0 NULL -6 NULL 1969-12-31 15:59:56.094 NULL NULL -200.0 -15910.599999999999 3.0 NULL 0.0 -23.0 6 NULL NULL -5 NULL NULL
+-62.0 NULL 15601.0 NULL -62 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 33.0 NULL 0.0 -23.0 62 NULL NULL -23 NULL NULL
+11.0 false NULL 10pO8p1LNx4Y 11 271296824 1969-12-31 16:00:02.351 NULL -271296850.28 NULL NULL 0.0 NULL 0.0 NULL -11 NULL 2.71296832E8 -1 271296850.28 0.0
+11.0 false NULL 1H6wGP 11 -560827082 1969-12-31 16:00:02.351 NULL 560827055.72 NULL NULL 0.0 NULL 0.0 NULL -11 NULL -5.6082707E8 -1 -560827055.72 0.0
+11.0 false NULL 2a7V63IL7jK3o 11 -325931647 1969-12-31 16:00:02.351 NULL 325931620.72 NULL NULL 0.0 NULL 0.0 NULL -11 NULL -3.25931648E8 -1 -325931620.72 0.0
+11.0 true NULL 10 11 92365813 1969-12-31 16:00:02.351 NULL -92365839.28 NULL NULL 0.0 NULL 0.0 NULL -11 NULL 9.2365808E7 -1 92365839.28 0.0
+21.0 NULL 15601.0 NULL 21 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 12.0 NULL 0.0 -23.0 -21 NULL NULL -2 NULL NULL
+32.0 NULL -200.0 NULL 32 NULL 1969-12-31 16:00:02.445 NULL NULL -200.0 -15910.599999999999 1.0 NULL 0.0 -23.0 -32 NULL NULL -23 NULL NULL
+36.0 NULL -200.0 NULL 36 NULL 1969-12-31 16:00:00.554 NULL NULL -200.0 -15910.599999999999 33.0 NULL 0.0 -23.0 -36 NULL NULL -23 NULL NULL
+5.0 NULL 15601.0 NULL 5 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 3.0 NULL 0.0 -23.0 -5 NULL NULL -3 NULL NULL
+58.0 NULL 15601.0 NULL 58 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 33.0 NULL 0.0 -23.0 -58 NULL NULL -23 NULL NULL
+8.0 false NULL 10V3pN5r5lI2qWl2lG103 8 -362835731 1969-12-31 16:00:15.892 NULL 362835704.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -3.62835744E8 -7 -362835704.72 0.0
+8.0 false NULL 10c4qt584m5y6uWT 8 -183000142 1969-12-31 16:00:15.892 NULL 183000115.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -1.8300016E8 -7 -183000115.72 0.0
+8.0 false NULL 8GloEukQ0c68JDmnYL53 8 -722873402 1969-12-31 16:00:15.892 NULL 722873375.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -7.2287341E8 -7 -722873375.72 0.0
+8.0 false NULL kA0XH5C5 8 -503903864 1969-12-31 16:00:15.892 NULL 503903837.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -5.03903872E8 -7 -503903837.72 0.0
+8.0 true NULL 100VTM7PEW8GH1uE 8 88129338 1969-12-31 16:00:15.892 NULL -88129364.28 NULL NULL 1.0 NULL 0.0 NULL -8 NULL 8.8129328E7 -7 88129364.28 0.0
+8.0 true NULL 1062158y 8 -1005155523 1969-12-31 16:00:15.892 NULL 1005155496.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -1.00515552E9 -7 -1005155496.72 0.0
+8.0 true NULL 1063cEnGjSal 8 -624769630 1969-12-31 16:00:15.892 NULL 624769603.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -6.247696E8 -7 -624769603.72 0.0
+8.0 true NULL 4kMasVoB7lX1wc5i64bNk 8 683567667 1969-12-31 16:00:15.892 NULL -683567693.28 NULL NULL 1.0 NULL 0.0 NULL -8 NULL 6.8356768E8 -7 683567693.28 0.0
+8.0 true NULL XH6I7A417 8 436627202 1969-12-31 16:00:15.892 NULL -436627228.28 NULL NULL 1.0 NULL 0.0 NULL -8 NULL 4.366272E8 -7 436627228.28 0.0
+NULL false -13008.0 10 NULL -896629175 NULL NULL 896629148.72 -13008.0 -1034825.424 NULL NULL NULL -23.0 NULL NULL NULL NULL -896629148.72 0.0
+NULL false -7707.0 10vke853 NULL 424959354 NULL NULL -424959380.28 -7707.0 -613114.971 NULL NULL NULL -23.0 NULL NULL NULL NULL 424959380.28 0.0
+NULL false 14771.0 10 NULL 993788576 NULL NULL -993788602.28 14771.0 1175077.363 NULL NULL NULL -23.0 NULL NULL NULL NULL 993788602.28 0.0
+NULL false 2459.0 10 NULL -394064473 NULL NULL 394064446.72 2459.0 195620.827 NULL NULL NULL -23.0 NULL NULL NULL NULL -394064446.72 0.0
+NULL true -14640.0 101n6n461o NULL -819657767 NULL NULL 819657740.72 -14640.0 -1164655.92 NULL NULL NULL -23.0 NULL NULL NULL NULL -819657740.72 0.0
+NULL true -14674.0 105aFDAt30c4rI4U NULL -831072496 NULL NULL 831072469.72 -14674.0 -1167360.722 NULL NULL NULL -23.0 NULL NULL NULL NULL -831072469.72 0.0
+NULL true -7238.0 10ljXCFT6fG6Qi3S7414e NULL -256767096 NULL NULL 256767069.72 -7238.0 -575804.614 NULL NULL NULL -23.0 NULL NULL NULL NULL -256767069.72 0.0
+NULL true -9489.0 10Yr6 NULL 500274721 NULL NULL -500274747.28 -9489.0 -754878.417 NULL NULL NULL -23.0 NULL NULL NULL NULL 500274747.28 0.0
+NULL true 10419.0 10 NULL -721614386 NULL NULL 721614359.72 10419.0 828862.7069999999 NULL NULL NULL -23.0 NULL NULL NULL NULL -721614359.72 0.0
+NULL true 14519.0 100xJdkyc NULL 729277608 NULL NULL -729277634.28 14519.0 1155030.007 NULL NULL NULL -23.0 NULL NULL NULL NULL 729277634.28 0.0
+NULL true 2261.0 10Hr5oB07Ohu0622u NULL 612000160 NULL NULL -612000186.28 2261.0 179869.33299999998 NULL NULL NULL -23.0 NULL NULL NULL NULL 612000186.28 0.0
+NULL true 4516.0 10lL0XD6WP2x64f70N0fHmC1 NULL -974538365 NULL NULL 974538338.72 4516.0 359261.348 NULL NULL NULL -23.0 NULL NULL NULL NULL -974538338.72 0.0
+NULL true 4767.0 10dUdwyXp5XwgpkTxLffmv3x NULL 187503456 NULL NULL -187503482.28 4767.0 379229.151 NULL NULL NULL -23.0 NULL NULL NULL NULL 187503482.28 0.0
+NULL true 5658.0 10M3eGUsKVonbl70DyoCk25 NULL 486382507 NULL NULL -486382533.28 5658.0 450110.874 NULL NULL NULL -23.0 NULL NULL NULL NULL 486382533.28 0.0
+NULL true 8659.0 1047piRsT3c3r134I NULL 487236176 NULL NULL -487236202.28 8659.0 688849.427 NULL NULL NULL -23.0 NULL NULL NULL NULL 487236202.28 0.0
http://git-wip-us.apache.org/repos/asf/hive/blob/dd04a92f/ql/src/test/results/clientpositive/udf_stddev_samp.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_stddev_samp.q.out b/ql/src/test/results/clientpositive/udf_stddev_samp.q.out
index c6b7adf..674de04 100644
--- a/ql/src/test/results/clientpositive/udf_stddev_samp.q.out
+++ b/ql/src/test/results/clientpositive/udf_stddev_samp.q.out
@@ -2,23 +2,81 @@ PREHOOK: query: DESCRIBE FUNCTION stddev_samp
PREHOOK: type: DESCFUNCTION
POSTHOOK: query: DESCRIBE FUNCTION stddev_samp
POSTHOOK: type: DESCFUNCTION
-stddev_samp(x) - Returns the sample standard deviation of a set of numbers
+stddev_samp(x) - Returns the sample standard deviation of a set of numbers.
+If applied to an empty set: NULL is returned.
+If applied to a set with a single element: NULL is returned.
+Otherwise it computes: sqrt(var_samp(x))
PREHOOK: query: DESCRIBE FUNCTION EXTENDED stddev_samp
PREHOOK: type: DESCFUNCTION
POSTHOOK: query: DESCRIBE FUNCTION EXTENDED stddev_samp
POSTHOOK: type: DESCFUNCTION
-stddev_samp(x) - Returns the sample standard deviation of a set of numbers
+stddev_samp(x) - Returns the sample standard deviation of a set of numbers.
+If applied to an empty set: NULL is returned.
+If applied to a set with a single element: NULL is returned.
+Otherwise it computes: sqrt(var_samp(x))
Function class:org.apache.hadoop.hive.ql.udf.generic.GenericUDAFStdSample
Function type:BUILTIN
PREHOOK: query: DESCRIBE FUNCTION stddev_samp
PREHOOK: type: DESCFUNCTION
POSTHOOK: query: DESCRIBE FUNCTION stddev_samp
POSTHOOK: type: DESCFUNCTION
-stddev_samp(x) - Returns the sample standard deviation of a set of numbers
+stddev_samp(x) - Returns the sample standard deviation of a set of numbers.
+If applied to an empty set: NULL is returned.
+If applied to a set with a single element: NULL is returned.
+Otherwise it computes: sqrt(var_samp(x))
PREHOOK: query: DESCRIBE FUNCTION EXTENDED stddev_samp
PREHOOK: type: DESCFUNCTION
POSTHOOK: query: DESCRIBE FUNCTION EXTENDED stddev_samp
POSTHOOK: type: DESCFUNCTION
-stddev_samp(x) - Returns the sample standard deviation of a set of numbers
+stddev_samp(x) - Returns the sample standard deviation of a set of numbers.
+If applied to an empty set: NULL is returned.
+If applied to a set with a single element: NULL is returned.
+Otherwise it computes: sqrt(var_samp(x))
Function class:org.apache.hadoop.hive.ql.udf.generic.GenericUDAFStdSample
Function type:BUILTIN
+PREHOOK: query: drop table if exists t
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists t
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table t (a int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t
+POSTHOOK: query: create table t (a int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t
+PREHOOK: query: insert into t values (1),(-1),(0)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@t
+POSTHOOK: query: insert into t values (1),(-1),(0)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@t
+POSTHOOK: Lineage: t.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: select stddev_samp(a) from t
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: select stddev_samp(a) from t
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+1.0
+PREHOOK: query: select stddev_samp(a) from t where a=0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: select stddev_samp(a) from t where a=0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+NULL
+PREHOOK: query: select round(stddev_samp(a),5) from t where a>=0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: select round(stddev_samp(a),5) from t where a>=0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+0.70711
http://git-wip-us.apache.org/repos/asf/hive/blob/dd04a92f/ql/src/test/results/clientpositive/udf_var_samp.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_var_samp.q.out b/ql/src/test/results/clientpositive/udf_var_samp.q.out
index 4a5c9b4..c1d23d0 100644
--- a/ql/src/test/results/clientpositive/udf_var_samp.q.out
+++ b/ql/src/test/results/clientpositive/udf_var_samp.q.out
@@ -2,23 +2,81 @@ PREHOOK: query: DESCRIBE FUNCTION var_samp
PREHOOK: type: DESCFUNCTION
POSTHOOK: query: DESCRIBE FUNCTION var_samp
POSTHOOK: type: DESCFUNCTION
-var_samp(x) - Returns the sample variance of a set of numbers
+var_samp(x) - Returns the sample variance of a set of numbers.
+If applied to an empty set: NULL is returned.
+If applied to a set with a single element: NULL is returned.
+Otherwise it computes: (S2-S1*S1/N)/(N-1)
PREHOOK: query: DESCRIBE FUNCTION EXTENDED var_samp
PREHOOK: type: DESCFUNCTION
POSTHOOK: query: DESCRIBE FUNCTION EXTENDED var_samp
POSTHOOK: type: DESCFUNCTION
-var_samp(x) - Returns the sample variance of a set of numbers
+var_samp(x) - Returns the sample variance of a set of numbers.
+If applied to an empty set: NULL is returned.
+If applied to a set with a single element: NULL is returned.
+Otherwise it computes: (S2-S1*S1/N)/(N-1)
Function class:org.apache.hadoop.hive.ql.udf.generic.GenericUDAFVarianceSample
Function type:BUILTIN
PREHOOK: query: DESCRIBE FUNCTION var_samp
PREHOOK: type: DESCFUNCTION
POSTHOOK: query: DESCRIBE FUNCTION var_samp
POSTHOOK: type: DESCFUNCTION
-var_samp(x) - Returns the sample variance of a set of numbers
+var_samp(x) - Returns the sample variance of a set of numbers.
+If applied to an empty set: NULL is returned.
+If applied to a set with a single element: NULL is returned.
+Otherwise it computes: (S2-S1*S1/N)/(N-1)
PREHOOK: query: DESCRIBE FUNCTION EXTENDED var_samp
PREHOOK: type: DESCFUNCTION
POSTHOOK: query: DESCRIBE FUNCTION EXTENDED var_samp
POSTHOOK: type: DESCFUNCTION
-var_samp(x) - Returns the sample variance of a set of numbers
+var_samp(x) - Returns the sample variance of a set of numbers.
+If applied to an empty set: NULL is returned.
+If applied to a set with a single element: NULL is returned.
+Otherwise it computes: (S2-S1*S1/N)/(N-1)
Function class:org.apache.hadoop.hive.ql.udf.generic.GenericUDAFVarianceSample
Function type:BUILTIN
+PREHOOK: query: drop table if exists t
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists t
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table t (a int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t
+POSTHOOK: query: create table t (a int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t
+PREHOOK: query: insert into t values (1),(-1),(0)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@t
+POSTHOOK: query: insert into t values (1),(-1),(0)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@t
+POSTHOOK: Lineage: t.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: select var_samp(a) from t
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: select var_samp(a) from t
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+1.0
+PREHOOK: query: select var_samp(a) from t where a=0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: select var_samp(a) from t where a=0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+NULL
+PREHOOK: query: select round(var_samp(a),5) from t where a>=0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: select round(var_samp(a),5) from t where a>=0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+0.5
[25/50] [abbrv] hive git commit: HIVE-17100 : Improve HS2 operation
logs for REPL commands (Sankar Hariappan, reviewed by Anishek Agarwal,
Thejas Nair)
Posted by we...@apache.org.
HIVE-17100 : Improve HS2 operation logs for REPL commands (Sankar Hariappan, reviewed by Anishek Agarwal, Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/70cb7f0b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/70cb7f0b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/70cb7f0b
Branch: refs/heads/hive-14535
Commit: 70cb7f0b22a7beb353a27576683ad824c4a4c007
Parents: a6b9cd5
Author: Sankar Hariappan <ma...@gmail.com>
Authored: Wed Aug 30 17:22:04 2017 -0700
Committer: Thejas M Nair <th...@hortonworks.com>
Committed: Wed Aug 30 17:22:04 2017 -0700
----------------------------------------------------------------------
.../listener/DummyRawStoreFailEvent.java | 7 +
.../hadoop/hive/metastore/HiveMetaStore.java | 7 +
.../hive/metastore/HiveMetaStoreClient.java | 7 +
.../hadoop/hive/metastore/IMetaStoreClient.java | 11 +
.../hadoop/hive/metastore/ObjectStore.java | 23 +
.../apache/hadoop/hive/metastore/RawStore.java | 9 +
.../hive/metastore/cache/CachedStore.java | 7 +
.../hive/metastore/messaging/EventUtils.java | 13 +
.../DummyRawStoreControlledCommit.java | 7 +
.../DummyRawStoreForJdoConnection.java | 6 +
ql/if/queryplan.thrift | 1 +
ql/src/gen/thrift/gen-cpp/queryplan_types.cpp | 8 +-
ql/src/gen/thrift/gen-cpp/queryplan_types.h | 3 +-
.../hadoop/hive/ql/plan/api/StageType.java | 5 +-
ql/src/gen/thrift/gen-php/Types.php | 2 +
ql/src/gen/thrift/gen-py/queryplan/ttypes.py | 3 +
ql/src/gen/thrift/gen-rb/queryplan_types.rb | 5 +-
.../apache/hadoop/hive/ql/exec/TaskFactory.java | 3 +
.../hadoop/hive/ql/exec/repl/ReplDumpTask.java | 39 +-
.../hive/ql/exec/repl/ReplStateLogTask.java | 51 +
.../hive/ql/exec/repl/ReplStateLogWork.java | 100 +
.../ql/exec/repl/bootstrap/ReplLoadTask.java | 53 +-
.../ql/exec/repl/bootstrap/ReplLoadWork.java | 3 +-
.../filesystem/BootstrapEventsIterator.java | 56 +-
.../filesystem/DatabaseEventsIterator.java | 4 +
.../exec/repl/bootstrap/load/LoadFunction.java | 18 +-
.../exec/repl/bootstrap/load/TaskTracker.java | 13 +-
.../bootstrap/load/table/LoadPartitions.java | 65 +-
.../repl/bootstrap/load/table/LoadTable.java | 45 +-
.../ql/parse/ReplicationSemanticAnalyzer.java | 70 +-
.../hadoop/hive/ql/parse/ReplicationSpec.java | 14 +-
.../hadoop/hive/ql/parse/repl/ReplLogger.java | 36 +
.../hadoop/hive/ql/parse/repl/ReplState.java | 57 +
.../hadoop/hive/ql/parse/repl/dump/Utils.java | 17 +-
.../repl/dump/log/BootstrapDumpLogger.java | 71 +
.../repl/dump/log/IncrementalDumpLogger.java | 55 +
.../repl/dump/log/state/BootstrapDumpBegin.java | 47 +
.../repl/dump/log/state/BootstrapDumpEnd.java | 59 +
.../dump/log/state/BootstrapDumpFunction.java | 44 +
.../repl/dump/log/state/BootstrapDumpTable.java | 52 +
.../dump/log/state/IncrementalDumpBegin.java | 43 +
.../repl/dump/log/state/IncrementalDumpEnd.java | 54 +
.../dump/log/state/IncrementalDumpEvent.java | 51 +
.../repl/load/log/BootstrapLoadLogger.java | 66 +
.../repl/load/log/IncrementalLoadLogger.java | 55 +
.../repl/load/log/state/BootstrapLoadBegin.java | 51 +
.../repl/load/log/state/BootstrapLoadEnd.java | 59 +
.../load/log/state/BootstrapLoadFunction.java | 44 +
.../repl/load/log/state/BootstrapLoadTable.java | 52 +
.../load/log/state/IncrementalLoadBegin.java | 47 +
.../repl/load/log/state/IncrementalLoadEnd.java | 54 +
.../load/log/state/IncrementalLoadEvent.java | 51 +
.../load/message/CreateFunctionHandler.java | 8 +
.../parse/repl/load/message/MessageHandler.java | 9 +-
.../hadoop/hive/ql/plan/ImportTableDesc.java | 53 +-
.../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 2515 ++++++++++--------
.../gen/thrift/gen-cpp/ThriftHiveMetastore.h | 126 +
.../ThriftHiveMetastore_server.skeleton.cpp | 5 +
.../gen/thrift/gen-cpp/hive_metastore_types.cpp | 1037 +++++---
.../gen/thrift/gen-cpp/hive_metastore_types.h | 89 +
.../api/NotificationEventsCountRequest.java | 488 ++++
.../api/NotificationEventsCountResponse.java | 387 +++
.../hive/metastore/api/ThriftHiveMetastore.java | 858 ++++++
.../gen-php/metastore/ThriftHiveMetastore.php | 216 ++
.../src/gen/thrift/gen-php/metastore/Types.php | 173 ++
.../hive_metastore/ThriftHiveMetastore-remote | 7 +
.../hive_metastore/ThriftHiveMetastore.py | 185 ++
.../gen/thrift/gen-py/hive_metastore/ttypes.py | 149 ++
.../gen/thrift/gen-rb/hive_metastore_types.rb | 37 +
.../gen/thrift/gen-rb/thrift_hive_metastore.rb | 54 +
.../src/main/thrift/hive_metastore.thrift | 10 +
71 files changed, 6491 insertions(+), 1638 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
----------------------------------------------------------------------
diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
index 5d7cfad..8d861e4 100644
--- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
+++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
@@ -47,6 +47,8 @@ import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
import org.apache.hadoop.hive.metastore.api.NotificationEventRequest;
import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
+import org.apache.hadoop.hive.metastore.api.NotificationEventsCountRequest;
+import org.apache.hadoop.hive.metastore.api.NotificationEventsCountResponse;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.PartitionEventType;
import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
@@ -841,6 +843,11 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable {
}
@Override
+ public NotificationEventsCountResponse getNotificationEventsCount(NotificationEventsCountRequest rqst) {
+ return objectStore.getNotificationEventsCount(rqst);
+ }
+
+ @Override
public void flushCache() {
objectStore.flushCache();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 55def04..5812a1b 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -7026,6 +7026,13 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
@Override
+ public NotificationEventsCountResponse get_notification_events_count(NotificationEventsCountRequest rqst)
+ throws TException {
+ RawStore ms = getMS();
+ return ms.getNotificationEventsCount(rqst);
+ }
+
+ @Override
public FireEventResponse fire_listener_event(FireEventRequest rqst) throws TException {
switch (rqst.getData().getSetField()) {
case INSERT_DATA:
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 688f181..70451c4 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -2356,6 +2356,13 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
return client.get_current_notificationEventId();
}
+ @InterfaceAudience.LimitedPrivate({"HCatalog"})
+ @Override
+ public NotificationEventsCountResponse getNotificationEventsCount(NotificationEventsCountRequest rqst)
+ throws TException {
+ return client.get_notification_events_count(rqst);
+ }
+
@InterfaceAudience.LimitedPrivate({"Apache Hive, HCatalog"})
@Override
public FireEventResponse fireListenerEvent(FireEventRequest rqst) throws TException {
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
index 813a283..69a845c 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
@@ -76,6 +76,8 @@ import org.apache.hadoop.hive.metastore.api.NoSuchTxnException;
import org.apache.hadoop.hive.metastore.api.NotNullConstraintsRequest;
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
+import org.apache.hadoop.hive.metastore.api.NotificationEventsCountResponse;
+import org.apache.hadoop.hive.metastore.api.NotificationEventsCountRequest;
import org.apache.hadoop.hive.metastore.api.OpenTxnsResponse;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.PartitionEventType;
@@ -1634,6 +1636,15 @@ public interface IMetaStoreClient {
CurrentNotificationEventId getCurrentNotificationEventId() throws TException;
/**
+ * Get the number of events from given eventID for the input database.
+ * @return number of events
+ * @throws TException
+ */
+ @InterfaceAudience.LimitedPrivate({"HCatalog"})
+ NotificationEventsCountResponse getNotificationEventsCount(NotificationEventsCountRequest rqst)
+ throws TException;
+
+ /**
* Request that the metastore fire an event. Currently this is only supported for DML
* operations, since the metastore knows when DDL operations happen.
* @param request
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index efbdb8f..0db1bc0 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -98,6 +98,8 @@ import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
import org.apache.hadoop.hive.metastore.api.NotificationEventRequest;
import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
+import org.apache.hadoop.hive.metastore.api.NotificationEventsCountRequest;
+import org.apache.hadoop.hive.metastore.api.NotificationEventsCountResponse;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.PartitionEventType;
@@ -8339,6 +8341,27 @@ public class ObjectStore implements RawStore, Configurable {
}
}
+ @Override
+ public NotificationEventsCountResponse getNotificationEventsCount(NotificationEventsCountRequest rqst) {
+ Long result = 0L;
+ boolean commited = false;
+ Query query = null;
+ try {
+ openTransaction();
+ long fromEventId = rqst.getFromEventId();
+ String inputDbName = rqst.getDbName();
+ String queryStr = "select count(eventId) from " + MNotificationLog.class.getName()
+ + " where eventId > fromEventId && dbName == inputDbName";
+ query = pm.newQuery(queryStr);
+ query.declareParameters("java.lang.Long fromEventId, java.lang.String inputDbName");
+ result = (Long) query.execute(fromEventId, inputDbName);
+ commited = commitTransaction();
+ return new NotificationEventsCountResponse(result.longValue());
+ } finally {
+ rollbackAndCleanup(commited, query);
+ }
+ }
+
private MNotificationLog translateThriftToDb(NotificationEvent entry) {
MNotificationLog dbEntry = new MNotificationLog();
dbEntry.setEventId(entry.getEventId());
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
index a2ae4c5..71982a0 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
@@ -46,6 +46,8 @@ import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
import org.apache.hadoop.hive.metastore.api.NotificationEventRequest;
import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
+import org.apache.hadoop.hive.metastore.api.NotificationEventsCountRequest;
+import org.apache.hadoop.hive.metastore.api.NotificationEventsCountResponse;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.PartitionEventType;
import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
@@ -621,6 +623,13 @@ public interface RawStore extends Configurable {
*/
public CurrentNotificationEventId getCurrentNotificationEventId();
+ /**
+ * Get the number of events corresponding to given database with fromEventId.
+ * This is intended for use by the repl commands to track the progress of incremental dump.
+ * @return
+ */
+ public NotificationEventsCountResponse getNotificationEventsCount(NotificationEventsCountRequest rqst);
+
/*
* Flush any catalog objects held by the metastore implementation. Note that this does not
* flush statistics objects. This should be called at the beginning of each query.
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
index 697cc2e..93d1ba6 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
@@ -63,6 +63,8 @@ import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
import org.apache.hadoop.hive.metastore.api.NotificationEventRequest;
import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
+import org.apache.hadoop.hive.metastore.api.NotificationEventsCountRequest;
+import org.apache.hadoop.hive.metastore.api.NotificationEventsCountResponse;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.PartitionEventType;
import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
@@ -1803,6 +1805,11 @@ public class CachedStore implements RawStore, Configurable {
}
@Override
+ public NotificationEventsCountResponse getNotificationEventsCount(NotificationEventsCountRequest rqst) {
+ return rawStore.getNotificationEventsCount(rqst);
+ }
+
+ @Override
public void flushCache() {
rawStore.flushCache();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/EventUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/EventUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/EventUtils.java
index 8205c25..8f90c7a 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/EventUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/EventUtils.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.metastore.messaging;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+import org.apache.hadoop.hive.metastore.api.NotificationEventsCountRequest;
import org.apache.hadoop.hive.metastore.messaging.event.filters.DatabaseAndTableFilter;
import org.apache.thrift.TException;
@@ -34,6 +35,7 @@ public class EventUtils {
public interface NotificationFetcher {
int getBatchSize() throws IOException;
long getCurrentNotificationEventId() throws IOException;
+ long getDbNotificationEventsCount(long fromEventId, String dbName) throws IOException;
List<NotificationEvent> getNextNotificationEvents(
long pos, IMetaStoreClient.NotificationFilter filter) throws IOException;
}
@@ -75,6 +77,17 @@ public class EventUtils {
}
@Override
+ public long getDbNotificationEventsCount(long fromEventId, String dbName) throws IOException {
+ try {
+ NotificationEventsCountRequest rqst
+ = new NotificationEventsCountRequest(fromEventId, dbName);
+ return msc.getNotificationEventsCount(rqst).getEventsCount();
+ } catch (TException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
public List<NotificationEvent> getNextNotificationEvents(
long pos, IMetaStoreClient.NotificationFilter filter) throws IOException {
try {
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
index fdb2866..4db203d 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
@@ -44,6 +44,8 @@ import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
import org.apache.hadoop.hive.metastore.api.NotificationEventRequest;
import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
+import org.apache.hadoop.hive.metastore.api.NotificationEventsCountRequest;
+import org.apache.hadoop.hive.metastore.api.NotificationEventsCountResponse;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.PartitionEventType;
import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
@@ -793,6 +795,11 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable {
}
@Override
+ public NotificationEventsCountResponse getNotificationEventsCount(NotificationEventsCountRequest rqst) {
+ return objectStore.getNotificationEventsCount(rqst);
+ }
+
+ @Override
public void flushCache() {
objectStore.flushCache();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
index f422c4e..fb16cfc 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
@@ -45,6 +45,8 @@ import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
import org.apache.hadoop.hive.metastore.api.NotificationEventRequest;
import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
+import org.apache.hadoop.hive.metastore.api.NotificationEventsCountRequest;
+import org.apache.hadoop.hive.metastore.api.NotificationEventsCountResponse;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.PartitionEventType;
import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
@@ -809,6 +811,10 @@ public class DummyRawStoreForJdoConnection implements RawStore {
return null;
}
+ @Override
+ public NotificationEventsCountResponse getNotificationEventsCount(NotificationEventsCountRequest rqst) {
+ return null;
+ }
public void flushCache() {
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/if/queryplan.thrift
----------------------------------------------------------------------
diff --git a/ql/if/queryplan.thrift b/ql/if/queryplan.thrift
index 00b0200..aaf644a 100644
--- a/ql/if/queryplan.thrift
+++ b/ql/if/queryplan.thrift
@@ -102,6 +102,7 @@ enum StageType {
COLUMNSTATS,
REPL_DUMP,
REPL_BOOTSTRAP_LOAD,
+ REPL_STATE_LOG
}
struct Stage {
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/gen/thrift/gen-cpp/queryplan_types.cpp
----------------------------------------------------------------------
diff --git a/ql/src/gen/thrift/gen-cpp/queryplan_types.cpp b/ql/src/gen/thrift/gen-cpp/queryplan_types.cpp
index f467da2..7262017 100644
--- a/ql/src/gen/thrift/gen-cpp/queryplan_types.cpp
+++ b/ql/src/gen/thrift/gen-cpp/queryplan_types.cpp
@@ -117,7 +117,8 @@ int _kStageTypeValues[] = {
StageType::DEPENDENCY_COLLECTION,
StageType::COLUMNSTATS,
StageType::REPL_DUMP,
- StageType::REPL_BOOTSTRAP_LOAD
+ StageType::REPL_BOOTSTRAP_LOAD,
+ StageType::REPL_STATE_LOG
};
const char* _kStageTypeNames[] = {
"CONDITIONAL",
@@ -133,9 +134,10 @@ const char* _kStageTypeNames[] = {
"DEPENDENCY_COLLECTION",
"COLUMNSTATS",
"REPL_DUMP",
- "REPL_BOOTSTRAP_LOAD"
+ "REPL_BOOTSTRAP_LOAD",
+ "REPL_STATE_LOG"
};
-const std::map<int, const char*> _StageType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(14, _kStageTypeValues, _kStageTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+const std::map<int, const char*> _StageType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(15, _kStageTypeValues, _kStageTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
Adjacency::~Adjacency() throw() {
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/gen/thrift/gen-cpp/queryplan_types.h
----------------------------------------------------------------------
diff --git a/ql/src/gen/thrift/gen-cpp/queryplan_types.h b/ql/src/gen/thrift/gen-cpp/queryplan_types.h
index ac87ef7..18dc867 100644
--- a/ql/src/gen/thrift/gen-cpp/queryplan_types.h
+++ b/ql/src/gen/thrift/gen-cpp/queryplan_types.h
@@ -95,7 +95,8 @@ struct StageType {
DEPENDENCY_COLLECTION = 10,
COLUMNSTATS = 11,
REPL_DUMP = 12,
- REPL_BOOTSTRAP_LOAD = 13
+ REPL_BOOTSTRAP_LOAD = 13,
+ REPL_STATE_LOG = 14
};
};
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/StageType.java
----------------------------------------------------------------------
diff --git a/ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/StageType.java b/ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/StageType.java
index 11a8f6d..ed408d2 100644
--- a/ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/StageType.java
+++ b/ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/StageType.java
@@ -25,7 +25,8 @@ public enum StageType implements org.apache.thrift.TEnum {
DEPENDENCY_COLLECTION(10),
COLUMNSTATS(11),
REPL_DUMP(12),
- REPL_BOOTSTRAP_LOAD(13);
+ REPL_BOOTSTRAP_LOAD(13),
+ REPL_STATE_LOG(14);
private final int value;
@@ -74,6 +75,8 @@ public enum StageType implements org.apache.thrift.TEnum {
return REPL_DUMP;
case 13:
return REPL_BOOTSTRAP_LOAD;
+ case 14:
+ return REPL_STATE_LOG;
default:
return null;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/gen/thrift/gen-php/Types.php
----------------------------------------------------------------------
diff --git a/ql/src/gen/thrift/gen-php/Types.php b/ql/src/gen/thrift/gen-php/Types.php
index 68edfcd..bca2eee 100644
--- a/ql/src/gen/thrift/gen-php/Types.php
+++ b/ql/src/gen/thrift/gen-php/Types.php
@@ -116,6 +116,7 @@ final class StageType {
const COLUMNSTATS = 11;
const REPL_DUMP = 12;
const REPL_BOOTSTRAP_LOAD = 13;
+ const REPL_STATE_LOG = 14;
static public $__names = array(
0 => 'CONDITIONAL',
1 => 'COPY',
@@ -131,6 +132,7 @@ final class StageType {
11 => 'COLUMNSTATS',
12 => 'REPL_DUMP',
13 => 'REPL_BOOTSTRAP_LOAD',
+ 14 => 'REPL_STATE_LOG',
);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/gen/thrift/gen-py/queryplan/ttypes.py
----------------------------------------------------------------------
diff --git a/ql/src/gen/thrift/gen-py/queryplan/ttypes.py b/ql/src/gen/thrift/gen-py/queryplan/ttypes.py
index 6bf65af..1f0d627 100644
--- a/ql/src/gen/thrift/gen-py/queryplan/ttypes.py
+++ b/ql/src/gen/thrift/gen-py/queryplan/ttypes.py
@@ -162,6 +162,7 @@ class StageType:
COLUMNSTATS = 11
REPL_DUMP = 12
REPL_BOOTSTRAP_LOAD = 13
+ REPL_STATE_LOG = 14
_VALUES_TO_NAMES = {
0: "CONDITIONAL",
@@ -178,6 +179,7 @@ class StageType:
11: "COLUMNSTATS",
12: "REPL_DUMP",
13: "REPL_BOOTSTRAP_LOAD",
+ 14: "REPL_STATE_LOG",
}
_NAMES_TO_VALUES = {
@@ -195,6 +197,7 @@ class StageType:
"COLUMNSTATS": 11,
"REPL_DUMP": 12,
"REPL_BOOTSTRAP_LOAD": 13,
+ "REPL_STATE_LOG": 14,
}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/gen/thrift/gen-rb/queryplan_types.rb
----------------------------------------------------------------------
diff --git a/ql/src/gen/thrift/gen-rb/queryplan_types.rb b/ql/src/gen/thrift/gen-rb/queryplan_types.rb
index 2730dde..88d9c17 100644
--- a/ql/src/gen/thrift/gen-rb/queryplan_types.rb
+++ b/ql/src/gen/thrift/gen-rb/queryplan_types.rb
@@ -74,8 +74,9 @@ module StageType
COLUMNSTATS = 11
REPL_DUMP = 12
REPL_BOOTSTRAP_LOAD = 13
- VALUE_MAP = {0 => "CONDITIONAL", 1 => "COPY", 2 => "DDL", 3 => "MAPRED", 4 => "EXPLAIN", 5 => "FETCH", 6 => "FUNC", 7 => "MAPREDLOCAL", 8 => "MOVE", 9 => "STATS", 10 => "DEPENDENCY_COLLECTION", 11 => "COLUMNSTATS", 12 => "REPL_DUMP", 13 => "REPL_BOOTSTRAP_LOAD"}
- VALID_VALUES = Set.new([CONDITIONAL, COPY, DDL, MAPRED, EXPLAIN, FETCH, FUNC, MAPREDLOCAL, MOVE, STATS, DEPENDENCY_COLLECTION, COLUMNSTATS, REPL_DUMP, REPL_BOOTSTRAP_LOAD]).freeze
+ REPL_STATE_LOG = 14
+ VALUE_MAP = {0 => "CONDITIONAL", 1 => "COPY", 2 => "DDL", 3 => "MAPRED", 4 => "EXPLAIN", 5 => "FETCH", 6 => "FUNC", 7 => "MAPREDLOCAL", 8 => "MOVE", 9 => "STATS", 10 => "DEPENDENCY_COLLECTION", 11 => "COLUMNSTATS", 12 => "REPL_DUMP", 13 => "REPL_BOOTSTRAP_LOAD", 14 => "REPL_STATE_LOG"}
+ VALID_VALUES = Set.new([CONDITIONAL, COPY, DDL, MAPRED, EXPLAIN, FETCH, FUNC, MAPREDLOCAL, MOVE, STATS, DEPENDENCY_COLLECTION, COLUMNSTATS, REPL_DUMP, REPL_BOOTSTRAP_LOAD, REPL_STATE_LOG]).freeze
end
class Adjacency
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
index 91ac4bf..fe9b624 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
@@ -27,6 +27,8 @@ import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
import org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask;
import org.apache.hadoop.hive.ql.exec.repl.ReplDumpTask;
import org.apache.hadoop.hive.ql.exec.repl.ReplDumpWork;
+import org.apache.hadoop.hive.ql.exec.repl.ReplStateLogTask;
+import org.apache.hadoop.hive.ql.exec.repl.ReplStateLogWork;
import org.apache.hadoop.hive.ql.exec.repl.bootstrap.ReplLoadTask;
import org.apache.hadoop.hive.ql.exec.repl.bootstrap.ReplLoadWork;
import org.apache.hadoop.hive.ql.exec.spark.SparkTask;
@@ -114,6 +116,7 @@ public final class TaskFactory {
taskvec.add(new TaskTuple<SparkWork>(SparkWork.class, SparkTask.class));
taskvec.add(new TaskTuple<>(ReplDumpWork.class, ReplDumpTask.class));
taskvec.add(new TaskTuple<>(ReplLoadWork.class, ReplLoadTask.class));
+ taskvec.add(new TaskTuple<>(ReplStateLogWork.class, ReplStateLogTask.class));
}
private static ThreadLocal<Integer> tid = new ThreadLocal<Integer>() {
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
index 67a67fd..d3af0ed 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hive.ql.parse.EximUtil;
import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.apache.hadoop.hive.ql.parse.repl.ReplLogger;
import org.apache.hadoop.hive.ql.parse.repl.dump.HiveWrapper;
import org.apache.hadoop.hive.ql.parse.repl.dump.TableExport;
import org.apache.hadoop.hive.ql.parse.repl.dump.Utils;
@@ -49,6 +50,8 @@ import org.apache.hadoop.hive.ql.parse.repl.dump.events.EventHandlerFactory;
import org.apache.hadoop.hive.ql.parse.repl.dump.io.FunctionSerializer;
import org.apache.hadoop.hive.ql.parse.repl.dump.io.JsonWriter;
import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
+import org.apache.hadoop.hive.ql.parse.repl.dump.log.BootstrapDumpLogger;
+import org.apache.hadoop.hive.ql.parse.repl.dump.log.IncrementalDumpLogger;
import org.apache.hadoop.hive.ql.plan.api.StageType;
import org.apache.thrift.TException;
import org.slf4j.Logger;
@@ -64,7 +67,7 @@ public class ReplDumpTask extends Task<ReplDumpWork> implements Serializable {
private static final String FUNCTION_METADATA_FILE_NAME = "_metadata";
private Logger LOG = LoggerFactory.getLogger(ReplDumpTask.class);
- private Logger REPL_STATE_LOG = LoggerFactory.getLogger("ReplState");
+ private ReplLogger replLogger;
@Override
public String getName() {
@@ -127,8 +130,9 @@ public class ReplDumpTask extends Task<ReplDumpWork> implements Serializable {
String dbName = (null != work.dbNameOrPattern && !work.dbNameOrPattern.isEmpty())
? work.dbNameOrPattern
: "?";
- REPL_STATE_LOG
- .info("Repl Dump: Started Repl Dump for DB: {}, Dump Type: INCREMENTAL", dbName);
+ replLogger = new IncrementalDumpLogger(dbName, dumpRoot.toString(),
+ evFetcher.getDbNotificationEventsCount(work.eventFrom, dbName));
+ replLogger.startLog();
while (evIter.hasNext()) {
NotificationEvent ev = evIter.next();
lastReplId = ev.getEventId();
@@ -136,7 +140,7 @@ public class ReplDumpTask extends Task<ReplDumpWork> implements Serializable {
dumpEvent(ev, evRoot, cmRoot);
}
- REPL_STATE_LOG.info("Repl Dump: Completed Repl Dump for DB: {}", dbName);
+ replLogger.endLog(lastReplId.toString());
LOG.info("Done dumping events, preparing to return {},{}", dumpRoot.toUri(), lastReplId);
Utils.writeOutput(
@@ -159,10 +163,9 @@ public class ReplDumpTask extends Task<ReplDumpWork> implements Serializable {
conf,
getNewEventOnlyReplicationSpec(ev.getEventId())
);
- EventHandlerFactory.handlerFor(ev).handle(context);
- REPL_STATE_LOG.info(
- "Repl Dump: Dumped event with ID: {}, Type: {} and dumped metadata and data to path {}",
- String.valueOf(ev.getEventId()), ev.getEventType(), evRoot.toUri().toString());
+ EventHandler eventHandler = EventHandlerFactory.handlerFor(ev);
+ eventHandler.handle(context);
+ replLogger.eventLog(String.valueOf(ev.getEventId()), eventHandler.dumpType().toString());
}
private ReplicationSpec getNewEventOnlyReplicationSpec(Long eventId) throws SemanticException {
@@ -174,12 +177,13 @@ public class ReplDumpTask extends Task<ReplDumpWork> implements Serializable {
private Long bootStrapDump(Path dumpRoot, DumpMetaData dmd, Path cmRoot) throws Exception {
// bootstrap case
Long bootDumpBeginReplId = getHive().getMSC().getCurrentNotificationEventId().getEventId();
+
for (String dbName : Utils.matchesDb(getHive(), work.dbNameOrPattern)) {
- REPL_STATE_LOG
- .info("Repl Dump: Started analyzing Repl Dump for DB: {}, Dump Type: BOOTSTRAP",
- dbName);
LOG.debug("ReplicationSemanticAnalyzer: analyzeReplDump dumping db: " + dbName);
-
+ replLogger = new BootstrapDumpLogger(dbName, dumpRoot.toString(),
+ Utils.getAllTables(getHive(), dbName).size(),
+ getHive().getAllFunctions().size());
+ replLogger.startLog();
Path dbRoot = dumpDbMetadata(dbName, dumpRoot);
dumpFunctionMetadata(dbName, dumpRoot);
for (String tblName : Utils.matchesTbl(getHive(), dbName, work.tableNameOrPattern)) {
@@ -187,6 +191,7 @@ public class ReplDumpTask extends Task<ReplDumpWork> implements Serializable {
"analyzeReplDump dumping table: " + tblName + " to db root " + dbRoot.toUri());
dumpTable(dbName, tblName, dbRoot);
}
+ replLogger.endLog(bootDumpBeginReplId.toString());
}
Long bootDumpEndReplId = getHive().getMSC().getCurrentNotificationEventId().getEventId();
LOG.info("Bootstrap object dump phase took from {} to {}", bootDumpBeginReplId,
@@ -228,7 +233,6 @@ public class ReplDumpTask extends Task<ReplDumpWork> implements Serializable {
Path dumpPath = new Path(dbRoot, EximUtil.METADATA_NAME);
HiveWrapper.Tuple<Database> database = new HiveWrapper(getHive(), dbName).database();
EximUtil.createDbExportDump(fs, dumpPath, database.object, database.replicationSpec);
- REPL_STATE_LOG.info("Repl Dump: Dumped DB metadata");
return dbRoot;
}
@@ -240,9 +244,7 @@ public class ReplDumpTask extends Task<ReplDumpWork> implements Serializable {
new TableExport.Paths(work.astRepresentationForErrorMsg, dbRoot, tblName, conf);
String distCpDoAsUser = conf.getVar(HiveConf.ConfVars.HIVE_DISTCP_DOAS_USER);
new TableExport(exportPaths, ts, getNewReplicationSpec(), db, distCpDoAsUser, conf).write();
- REPL_STATE_LOG.info(
- "Repl Dump: Analyzed dump for table/view: {}.{} and dumping metadata and data to path {}",
- dbName, tblName, exportPaths.exportRootDir.toString());
+ replLogger.tableLog(tblName, ts.tableHandle.getTableType());
} catch (InvalidTableException te) {
// Bootstrap dump shouldn't fail if the table is dropped/renamed while dumping it.
// Just log a debug message and skip it.
@@ -295,7 +297,7 @@ public class ReplDumpTask extends Task<ReplDumpWork> implements Serializable {
FunctionSerializer serializer = new FunctionSerializer(tuple.object, conf);
serializer.writeTo(jsonWriter, tuple.replicationSpec);
}
- REPL_STATE_LOG.info("Repl Dump: Dumped metadata for function: {}", functionName);
+ replLogger.functionLog(functionName);
}
}
@@ -303,8 +305,7 @@ public class ReplDumpTask extends Task<ReplDumpWork> implements Serializable {
try {
HiveWrapper.Tuple<Function> tuple = new HiveWrapper(getHive(), dbName).function(functionName);
if (tuple.object.getResourceUris().isEmpty()) {
- REPL_STATE_LOG.warn(
- "Not replicating function: " + functionName + " as it seems to have been created "
+ LOG.warn("Not replicating function: " + functionName + " as it seems to have been created "
+ "without USING clause");
return null;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplStateLogTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplStateLogTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplStateLogTask.java
new file mode 100644
index 0000000..614af54
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplStateLogTask.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.repl;
+
+import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.plan.api.StageType;
+
+import java.io.Serializable;
+
+/**
+ * ReplStateLogTask.
+ *
+ * Exists for the sole purpose of reducing the number of dependency edges in the task graph.
+ **/
+public class ReplStateLogTask extends Task<ReplStateLogWork> implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public int execute(DriverContext driverContext) {
+ work.replStateLog();
+ return 0;
+ }
+
+ @Override
+ public StageType getType() {
+ return StageType.REPL_STATE_LOG;
+ }
+
+ @Override
+ public String getName() {
+ return "REPL_STATE_LOG";
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplStateLogWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplStateLogWork.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplStateLogWork.java
new file mode 100644
index 0000000..6462fe5
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplStateLogWork.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.repl;
+
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
+import org.apache.hadoop.hive.ql.parse.repl.ReplLogger;
+import org.apache.hadoop.hive.ql.plan.Explain;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
+
+import java.io.Serializable;
+import java.util.Map;
+
+
+/**
+ * ReplStateLogWork
+ *
+ */
+@Explain(displayName = "Repl State Log", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+public class ReplStateLogWork implements Serializable {
+ private static final long serialVersionUID = 1L;
+ private final ReplLogger replLogger;
+ private final LOG_TYPE logType;
+ private String eventId;
+ private String eventType;
+ private String tableName;
+ private TableType tableType;
+ private String functionName;
+ private String lastReplId;
+
+ private enum LOG_TYPE {
+ TABLE,
+ FUNCTION,
+ EVENT,
+ END
+ }
+
+ public ReplStateLogWork(ReplLogger replLogger, String eventId, String eventType) {
+ this.logType = LOG_TYPE.EVENT;
+ this.replLogger = replLogger;
+ this.eventId = eventId;
+ this.eventType = eventType;
+ }
+
+ public ReplStateLogWork(ReplLogger replLogger, String tableName, TableType tableType) {
+ this.logType = LOG_TYPE.TABLE;
+ this.replLogger = replLogger;
+ this.tableName = tableName;
+ this.tableType = tableType;
+ }
+
+ public ReplStateLogWork(ReplLogger replLogger, String functionName) {
+ this.logType = LOG_TYPE.FUNCTION;
+ this.replLogger = replLogger;
+ this.functionName = functionName;
+ }
+
+ public ReplStateLogWork(ReplLogger replLogger, Map<String,String> dbProps) {
+ this.logType = LOG_TYPE.END;
+ this.replLogger = replLogger;
+ this.lastReplId = ReplicationSpec.getLastReplicatedStateFromParameters(dbProps);
+ }
+
+ public void replStateLog() {
+ switch (logType) {
+ case TABLE: {
+ replLogger.tableLog(tableName, tableType);
+ break;
+ }
+ case FUNCTION: {
+ replLogger.functionLog(functionName);
+ break;
+ }
+ case EVENT: {
+ replLogger.eventLog(eventId, eventType);
+ break;
+ }
+ case END: {
+ replLogger.endLog(lastReplId);
+ break;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadTask.java
index 6ea1754..cd31b17 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadTask.java
@@ -18,9 +18,12 @@
package org.apache.hadoop.hive.ql.exec.repl.bootstrap;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
+import org.apache.hadoop.hive.ql.exec.repl.ReplStateLogWork;
import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.BootstrapEvent;
import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.DatabaseEvent;
import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.FunctionEvent;
@@ -35,6 +38,8 @@ import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.table.LoadTable;
import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.table.TableContext;
import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.util.Context;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.repl.ReplLogger;
+import org.apache.hadoop.hive.ql.plan.ImportTableDesc;
import org.apache.hadoop.hive.ql.plan.api.StageType;
import java.io.Serializable;
@@ -107,7 +112,8 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
TableContext tableContext =
new TableContext(dbTracker, work.dbNameToLoadIn, work.tableNameToLoadIn);
TableEvent tableEvent = (TableEvent) next;
- LoadTable loadTable = new LoadTable(tableEvent, context, tableContext, loadTaskTracker);
+ LoadTable loadTable = new LoadTable(tableEvent, context, iterator.replLogger(),
+ tableContext, loadTaskTracker);
tableTracker = loadTable.tasks();
if (!scope.database) {
scope.rootTasks.addAll(tableTracker.tasks());
@@ -124,8 +130,8 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
// for a table we explicitly try to load partitions as there is no separate partitions events.
LoadPartitions loadPartitions =
- new LoadPartitions(context, loadTaskTracker, tableEvent, work.dbNameToLoadIn,
- tableContext);
+ new LoadPartitions(context, iterator.replLogger(), loadTaskTracker, tableEvent,
+ work.dbNameToLoadIn, tableContext);
TaskTracker partitionsTracker = loadPartitions.tasks();
partitionsPostProcessing(iterator, scope, loadTaskTracker, tableTracker,
partitionsTracker);
@@ -142,9 +148,8 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
TableContext tableContext = new TableContext(dbTracker, work.dbNameToLoadIn,
work.tableNameToLoadIn);
LoadPartitions loadPartitions =
- new LoadPartitions(context, tableContext, loadTaskTracker, event.asTableEvent(),
- work.dbNameToLoadIn,
- event.lastPartitionReplicated());
+ new LoadPartitions(context, iterator.replLogger(), tableContext, loadTaskTracker,
+ event.asTableEvent(), work.dbNameToLoadIn, event.lastPartitionReplicated());
/*
the tableTracker here should be a new instance and not an existing one as this can
only happen when we break in between loading partitions.
@@ -156,8 +161,8 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
break;
}
case Function: {
- LoadFunction loadFunction =
- new LoadFunction(context, (FunctionEvent) next, work.dbNameToLoadIn, dbTracker);
+ LoadFunction loadFunction = new LoadFunction(context, iterator.replLogger(),
+ (FunctionEvent) next, work.dbNameToLoadIn, dbTracker);
TaskTracker functionsTracker = loadFunction.tasks();
if (!scope.database) {
scope.rootTasks.addAll(functionsTracker.tasks());
@@ -169,11 +174,16 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
break;
}
}
+
+ if (!iterator.currentDbHasNext()) {
+ createEndReplLogTask(context, scope, iterator.replLogger());
+ }
}
boolean addAnotherLoadTask = iterator.hasNext() || loadTaskTracker.hasReplicationState();
createBuilderTask(scope.rootTasks, addAnotherLoadTask);
if (!iterator.hasNext()) {
loadTaskTracker.update(updateDatabaseLastReplID(maxTasks, context, scope));
+ work.updateDbEventState(null);
}
this.childTasks = scope.rootTasks;
LOG.info("Root Tasks / Total Tasks : {} / {} ", childTasks.size(), loadTaskTracker.numberOfTasks());
@@ -186,6 +196,19 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
return 0;
}
+ private Task<? extends Serializable> createEndReplLogTask(Context context, Scope scope,
+ ReplLogger replLogger) throws SemanticException {
+ Database dbInMetadata = work.databaseEvent(context.hiveConf).dbInMetadata(work.dbNameToLoadIn);
+ ReplStateLogWork replLogWork = new ReplStateLogWork(replLogger, dbInMetadata.getParameters());
+ Task<ReplStateLogWork> replLogTask = TaskFactory.get(replLogWork, conf);
+ if (null == scope.rootTasks) {
+ scope.rootTasks.add(replLogTask);
+ } else {
+ dependency(scope.rootTasks, replLogTask);
+ }
+ return replLogTask;
+ }
+
/**
* There was a database update done before and we want to make sure we update the last repl
* id on this database as we are now going to switch to processing a new database.
@@ -243,18 +266,20 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
/**
* add the dependency to the leaf node
*/
- private boolean dependency(List<Task<? extends Serializable>> tasks,
- Task<ReplLoadWork> loadTask) {
+ public static boolean dependency(List<Task<? extends Serializable>> tasks, Task<?> tailTask) {
if (tasks == null || tasks.isEmpty()) {
return true;
}
for (Task<? extends Serializable> task : tasks) {
- boolean dependency = dependency(task.getChildTasks(), loadTask);
- if (dependency) {
- task.addDependentTask(loadTask);
+ if (task == tailTask) {
+ continue;
+ }
+ boolean leafNode = dependency(task.getChildTasks(), tailTask);
+ if (leafNode) {
+ task.addDependentTask(tailTask);
}
}
- return true;
+ return false;
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadWork.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadWork.java
index eb18e5f..f51afe1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadWork.java
@@ -38,7 +38,7 @@ public class ReplLoadWork implements Serializable {
public ReplLoadWork(HiveConf hiveConf, String dumpDirectory, String dbNameToLoadIn,
String tableNameToLoadIn) throws IOException {
this.tableNameToLoadIn = tableNameToLoadIn;
- this.iterator = new BootstrapEventsIterator(dumpDirectory, hiveConf);
+ this.iterator = new BootstrapEventsIterator(dumpDirectory, dbNameToLoadIn, hiveConf);
this.dbNameToLoadIn = dbNameToLoadIn;
}
@@ -61,7 +61,6 @@ public class ReplLoadWork implements Serializable {
DatabaseEvent databaseEvent(HiveConf hiveConf) {
DatabaseEvent databaseEvent = state.toEvent(hiveConf);
- state = null;
return databaseEvent;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/BootstrapEventsIterator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/BootstrapEventsIterator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/BootstrapEventsIterator.java
index 4e635ad..43a85f2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/BootstrapEventsIterator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/BootstrapEventsIterator.java
@@ -17,13 +17,15 @@
*/
package org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.filesystem;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.*;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.ReplicationState;
import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.BootstrapEvent;
import org.apache.hadoop.hive.ql.parse.EximUtil;
+import org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.repl.load.log.BootstrapLoadLogger;
+import org.apache.hadoop.hive.ql.parse.repl.ReplLogger;
import java.io.IOException;
import java.util.Arrays;
@@ -69,8 +71,13 @@ public class BootstrapEventsIterator implements Iterator<BootstrapEvent> {
warehouse.
*/
private Iterator<DatabaseEventsIterator> dbEventsIterator;
+ private final String dumpDirectory;
+ private final String dbNameToLoadIn;
+ private final HiveConf hiveConf;
+ private ReplLogger replLogger;
- public BootstrapEventsIterator(String dumpDirectory, HiveConf hiveConf) throws IOException {
+ public BootstrapEventsIterator(String dumpDirectory, String dbNameToLoadIn, HiveConf hiveConf)
+ throws IOException {
Path path = new Path(dumpDirectory);
FileSystem fileSystem = path.getFileSystem(hiveConf);
FileStatus[] fileStatuses =
@@ -93,6 +100,9 @@ public class BootstrapEventsIterator implements Iterator<BootstrapEvent> {
}
}).collect(Collectors.toList()).iterator();
+ this.dumpDirectory = dumpDirectory;
+ this.dbNameToLoadIn = dbNameToLoadIn;
+ this.hiveConf = hiveConf;
}
@Override
@@ -101,6 +111,7 @@ public class BootstrapEventsIterator implements Iterator<BootstrapEvent> {
if (currentDatabaseIterator == null) {
if (dbEventsIterator.hasNext()) {
currentDatabaseIterator = dbEventsIterator.next();
+ initReplLogger();
} else {
return false;
}
@@ -127,7 +138,44 @@ public class BootstrapEventsIterator implements Iterator<BootstrapEvent> {
throw new UnsupportedOperationException("This operation is not supported");
}
+ public boolean currentDbHasNext() {
+ return ((currentDatabaseIterator != null) && (currentDatabaseIterator.hasNext()));
+ }
+
public void setReplicationState(ReplicationState replicationState) {
this.currentDatabaseIterator.replicationState = replicationState;
}
+
+ public ReplLogger replLogger() {
+ return replLogger;
+ }
+
+ private void initReplLogger() {
+ try {
+ Path dbDumpPath = currentDatabaseIterator.dbLevelPath();
+ FileSystem fs = dbDumpPath.getFileSystem(hiveConf);
+
+ long numTables = getSubDirs(fs, dbDumpPath).length;
+ long numFunctions = 0;
+ Path funcPath = new Path(dbDumpPath, ReplicationSemanticAnalyzer.FUNCTIONS_ROOT_DIR_NAME);
+ if (fs.exists(funcPath)) {
+ numFunctions = getSubDirs(fs, funcPath).length;
+ }
+ String dbName = StringUtils.isBlank(dbNameToLoadIn) ? dbDumpPath.getName() : dbNameToLoadIn;
+ replLogger = new BootstrapLoadLogger(dbName, dumpDirectory, numTables, numFunctions);
+ replLogger.startLog();
+ } catch (IOException e) {
+ // Ignore the exception
+ }
+ }
+
+ FileStatus[] getSubDirs(FileSystem fs, Path dirPath) throws IOException {
+ return fs.listStatus(dirPath, new PathFilter() {
+ @Override
+ public boolean accept(Path p) {
+ String name = p.getName();
+ return !name.startsWith("_") && !name.startsWith(".");
+ }
+ });
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/DatabaseEventsIterator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/DatabaseEventsIterator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/DatabaseEventsIterator.java
index 3100875..dc0e192 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/DatabaseEventsIterator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/DatabaseEventsIterator.java
@@ -58,6 +58,10 @@ class DatabaseEventsIterator implements Iterator<BootstrapEvent> {
remoteIterator = fileSystem.listFiles(dbLevelPath, true);
}
+ public Path dbLevelPath() {
+ return this.dbLevelPath;
+ }
+
@Override
public boolean hasNext() {
try {
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadFunction.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadFunction.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadFunction.java
index e9b8711..8852a60 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadFunction.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadFunction.java
@@ -20,10 +20,14 @@ package org.apache.hadoop.hive.ql.exec.repl.bootstrap.load;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.TaskFactory;
+import org.apache.hadoop.hive.ql.exec.repl.ReplStateLogWork;
import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.FunctionEvent;
import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.util.Context;
import org.apache.hadoop.hive.ql.parse.EximUtil;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.repl.ReplLogger;
+import org.apache.hadoop.hive.ql.exec.repl.bootstrap.ReplLoadTask;
import org.apache.hadoop.hive.ql.parse.repl.load.message.CreateFunctionHandler;
import org.apache.hadoop.hive.ql.parse.repl.load.message.MessageHandler;
import org.slf4j.Logger;
@@ -39,18 +43,27 @@ import static org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.stripQuotes;
public class LoadFunction {
private static final Logger LOG = LoggerFactory.getLogger(LoadFunction.class);
private Context context;
+ private ReplLogger replLogger;
private final FunctionEvent event;
private final String dbNameToLoadIn;
private final TaskTracker tracker;
- public LoadFunction(Context context, FunctionEvent event, String dbNameToLoadIn,
- TaskTracker existingTracker) {
+ public LoadFunction(Context context, ReplLogger replLogger, FunctionEvent event,
+ String dbNameToLoadIn, TaskTracker existingTracker) {
this.context = context;
+ this.replLogger = replLogger;
this.event = event;
this.dbNameToLoadIn = dbNameToLoadIn;
this.tracker = new TaskTracker(existingTracker);
}
+ private void createFunctionReplLogTask(List<Task<? extends Serializable>> functionTasks,
+ String functionName) {
+ ReplStateLogWork replLogWork = new ReplStateLogWork(replLogger, functionName);
+ Task<ReplStateLogWork> replLogTask = TaskFactory.get(replLogWork, context.hiveConf);
+ ReplLoadTask.dependency(functionTasks, replLogTask);
+ }
+
public TaskTracker tasks() throws IOException, SemanticException {
URI fromURI = EximUtil
.getValidatedURI(context.hiveConf, stripQuotes(event.rootDir().toUri().toString()));
@@ -63,6 +76,7 @@ public class LoadFunction {
dbNameToLoadIn, null, fromPath.toString(), null, null, context.hiveConf,
context.hiveDb, null, LOG)
);
+ createFunctionReplLogTask(tasks, handler.getFunctionName());
tasks.forEach(tracker::addTask);
return tracker;
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/TaskTracker.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/TaskTracker.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/TaskTracker.java
index f246b8a..374a03b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/TaskTracker.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/TaskTracker.java
@@ -59,14 +59,21 @@ public class TaskTracker {
*/
public void addTask(Task<? extends Serializable> task) {
tasks.add(task);
- updateTaskCount(task);
+
+ List <Task<? extends Serializable>> visited = new ArrayList<>();
+ updateTaskCount(task, visited);
}
- private void updateTaskCount(Task<? extends Serializable> task) {
+ public void updateTaskCount(Task<? extends Serializable> task,
+ List <Task<? extends Serializable>> visited) {
numberOfTasks += 1;
+ visited.add(task);
if (task.getChildTasks() != null) {
for (Task<? extends Serializable> childTask : task.getChildTasks()) {
- updateTaskCount(childTask);
+ if (visited.contains(childTask)) {
+ continue;
+ }
+ updateTaskCount(childTask, visited);
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
index f088ba9..c944a13 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
@@ -26,8 +26,10 @@ import org.apache.hadoop.hive.ql.exec.ReplCopyTask;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.exec.repl.ReplStateLogWork;
import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.TableEvent;
import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.ReplicationState;
+import org.apache.hadoop.hive.ql.exec.repl.bootstrap.ReplLoadTask;
import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.TaskTracker;
import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.util.Context;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -36,6 +38,7 @@ import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.ImportSemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.repl.ReplLogger;
import org.apache.hadoop.hive.ql.plan.AddPartitionDesc;
import org.apache.hadoop.hive.ql.plan.DDLWork;
import org.apache.hadoop.hive.ql.plan.ImportTableDesc;
@@ -46,10 +49,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.Serializable;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import static org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.ReplicationState.PartitionState;
import static org.apache.hadoop.hive.ql.parse.ImportSemanticAnalyzer.isPartitioned;
@@ -59,6 +59,7 @@ public class LoadPartitions {
private static Logger LOG = LoggerFactory.getLogger(LoadPartitions.class);
private final Context context;
+ private final ReplLogger replLogger;
private final TableContext tableContext;
private final TableEvent event;
private final TaskTracker tracker;
@@ -67,17 +68,19 @@ public class LoadPartitions {
private final ImportTableDesc tableDesc;
private Table table;
- public LoadPartitions(Context context, TaskTracker tableTracker, TableEvent event,
- String dbNameToLoadIn, TableContext tableContext) throws HiveException, IOException {
- this(context, tableContext, tableTracker, event, dbNameToLoadIn, null);
+ public LoadPartitions(Context context, ReplLogger replLogger, TaskTracker tableTracker,
+ TableEvent event, String dbNameToLoadIn,
+ TableContext tableContext) throws HiveException, IOException {
+ this(context, replLogger, tableContext, tableTracker, event, dbNameToLoadIn, null);
}
- public LoadPartitions(Context context, TableContext tableContext, TaskTracker limiter,
- TableEvent event, String dbNameToLoadIn, AddPartitionDesc lastReplicatedPartition)
- throws HiveException, IOException {
+ public LoadPartitions(Context context, ReplLogger replLogger, TableContext tableContext,
+ TaskTracker limiter, TableEvent event, String dbNameToLoadIn,
+ AddPartitionDesc lastReplicatedPartition) throws HiveException, IOException {
this.tracker = new TaskTracker(limiter);
this.event = event;
this.context = context;
+ this.replLogger = replLogger;
this.lastReplicatedPartition = lastReplicatedPartition;
this.tableContext = tableContext;
@@ -98,6 +101,21 @@ public class LoadPartitions {
}
}
+ private void createTableReplLogTask() throws SemanticException {
+ ReplStateLogWork replLogWork = new ReplStateLogWork(replLogger,
+ tableDesc.getTableName(), tableDesc.tableType());
+ Task<ReplStateLogWork> replLogTask = TaskFactory.get(replLogWork, context.hiveConf);
+
+ if (tracker.tasks().isEmpty()) {
+ tracker.addTask(replLogTask);
+ } else {
+ ReplLoadTask.dependency(tracker.tasks(), replLogTask);
+
+ List<Task<? extends Serializable>> visited = new ArrayList<>();
+ tracker.updateTaskCount(replLogTask, visited);
+ }
+ }
+
public TaskTracker tasks() throws SemanticException {
try {
/*
@@ -113,7 +131,11 @@ public class LoadPartitions {
table = new Table(tableDesc.getDatabaseName(), tableDesc.getTableName());
if (isPartitioned(tableDesc)) {
updateReplicationState(initialReplicationState());
- return forNewTable();
+ if (!forNewTable().hasReplicationState()) {
+ // Add ReplStateLogTask only if no pending table load tasks left for next cycle
+ createTableReplLogTask();
+ }
+ return tracker;
}
} else {
// existing
@@ -122,7 +144,11 @@ public class LoadPartitions {
List<AddPartitionDesc> partitionDescs = event.partitionDescriptions(tableDesc);
if (!event.replicationSpec().isMetadataOnly() && !partitionDescs.isEmpty()) {
updateReplicationState(initialReplicationState());
- return forExistingTable(lastReplicatedPartition);
+ if (!forExistingTable(lastReplicatedPartition).hasReplicationState()) {
+ // Add ReplStateLogTask only if no pending table load tasks left for next cycle
+ createTableReplLogTask();
+ }
+ return tracker;
}
}
}
@@ -149,9 +175,11 @@ public class LoadPartitions {
while (iterator.hasNext() && tracker.canAddMoreTasks()) {
AddPartitionDesc addPartitionDesc = iterator.next();
tracker.addTask(addSinglePartition(table, addPartitionDesc));
- ReplicationState currentReplicationState =
- new ReplicationState(new PartitionState(table.getTableName(), addPartitionDesc));
- updateReplicationState(currentReplicationState);
+ if (iterator.hasNext() && !tracker.canAddMoreTasks()) {
+ ReplicationState currentReplicationState =
+ new ReplicationState(new PartitionState(table.getTableName(), addPartitionDesc));
+ updateReplicationState(currentReplicationState);
+ }
}
return tracker;
}
@@ -236,13 +264,16 @@ public class LoadPartitions {
boolean encounteredTheLastReplicatedPartition = (lastPartitionReplicated == null);
ReplicationSpec replicationSpec = event.replicationSpec();
LOG.debug("table partitioned");
- for (AddPartitionDesc addPartitionDesc : event.partitionDescriptions(tableDesc)) {
+
+ Iterator<AddPartitionDesc> iterator = event.partitionDescriptions(tableDesc).iterator();
+ while (iterator.hasNext()) {
/*
encounteredTheLastReplicatedPartition will be set, when we break creation of partition tasks
for a table, as we have reached the limit of number of tasks we should create for execution.
in this case on the next run we have to iterate over the partitions desc to reach the last replicated
partition so that we can start replicating partitions after that.
*/
+ AddPartitionDesc addPartitionDesc = iterator.next();
if (encounteredTheLastReplicatedPartition && tracker.canAddMoreTasks()) {
Map<String, String> partSpec = addPartitionDesc.getPartition(0).getPartSpec();
Partition ptn;
@@ -257,7 +288,7 @@ public class LoadPartitions {
if (replicationSpec.allowReplacementInto(ptn.getParameters())) {
if (replicationSpec.isMetadataOnly()) {
tracker.addTask(alterSinglePartition(addPartitionDesc, replicationSpec, ptn));
- if (!tracker.canAddMoreTasks()) {
+ if (iterator.hasNext() && !tracker.canAddMoreTasks()) {
tracker.setReplicationState(
new ReplicationState(new PartitionState(table.getTableName(), addPartitionDesc)
)
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
index cbb964a..a1187c4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.table;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.ql.ErrorMsg;
@@ -26,6 +27,8 @@ import org.apache.hadoop.hive.ql.exec.ReplCopyTask;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.exec.repl.ReplStateLogWork;
+import org.apache.hadoop.hive.ql.exec.repl.bootstrap.ReplLoadTask;
import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.TableEvent;
import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.TaskTracker;
import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.util.Context;
@@ -34,6 +37,7 @@ import org.apache.hadoop.hive.ql.parse.EximUtil;
import org.apache.hadoop.hive.ql.parse.ImportSemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.repl.ReplLogger;
import org.apache.hadoop.hive.ql.plan.ImportTableDesc;
import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
import org.apache.hadoop.hive.ql.plan.MoveWork;
@@ -42,7 +46,9 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.Serializable;
+import java.util.ArrayList;
import java.util.HashSet;
+import java.util.List;
import java.util.TreeMap;
import static org.apache.hadoop.hive.ql.parse.ImportSemanticAnalyzer.isPartitioned;
@@ -51,18 +57,36 @@ public class LoadTable {
private final static Logger LOG = LoggerFactory.getLogger(LoadTable.class);
// private final Helper helper;
private final Context context;
+ private final ReplLogger replLogger;
private final TableContext tableContext;
private final TaskTracker tracker;
private final TableEvent event;
- public LoadTable(TableEvent event, Context context, TableContext tableContext, TaskTracker limiter)
+ public LoadTable(TableEvent event, Context context, ReplLogger replLogger,
+ TableContext tableContext, TaskTracker limiter)
throws SemanticException, IOException {
this.event = event;
this.context = context;
+ this.replLogger = replLogger;
this.tableContext = tableContext;
this.tracker = new TaskTracker(limiter);
}
+ private void createTableReplLogTask(String tableName, TableType tableType) throws SemanticException {
+ ReplStateLogWork replLogWork = new ReplStateLogWork(replLogger,tableName, tableType);
+ Task<ReplStateLogWork> replLogTask = TaskFactory.get(replLogWork, context.hiveConf);
+ ReplLoadTask.dependency(tracker.tasks(), replLogTask);
+
+ if (tracker.tasks().isEmpty()) {
+ tracker.addTask(replLogTask);
+ } else {
+ ReplLoadTask.dependency(tracker.tasks(), replLogTask);
+
+ List<Task<? extends Serializable>> visited = new ArrayList<>();
+ tracker.updateTaskCount(replLogTask, visited);
+ }
+ }
+
public TaskTracker tasks() throws SemanticException {
// Path being passed to us is a table dump location. We go ahead and load it in as needed.
// If tblName is null, then we default to the table name specified in _metadata, which is good.
@@ -123,22 +147,27 @@ public class LoadTable {
behave like a noop or a pure MD alter.
*/
if (table == null) {
- return newTableTasks(tableDesc);
+ newTableTasks(tableDesc);
} else {
- return existingTableTasks(tableDesc, table, replicationSpec);
+ existingTableTasks(tableDesc, table, replicationSpec);
+ }
+
+ if (!isPartitioned(tableDesc)) {
+ createTableReplLogTask(tableDesc.getTableName(), tableDesc.tableType());
}
+ return tracker;
} catch (Exception e) {
throw new SemanticException(e);
}
}
- private TaskTracker existingTableTasks(ImportTableDesc tblDesc, Table table,
+ private void existingTableTasks(ImportTableDesc tblDesc, Table table,
ReplicationSpec replicationSpec) {
if (!table.isPartitioned()) {
LOG.debug("table non-partitioned");
if (!replicationSpec.allowReplacementInto(table.getParameters())) {
- return tracker; // silently return, table is newer than our replacement.
+ return; // silently return, table is newer than our replacement.
}
Task<? extends Serializable> alterTableTask = alterTableTask(tblDesc, replicationSpec);
@@ -151,10 +180,9 @@ public class LoadTable {
tracker.addTask(alterTableTask);
}
}
- return tracker;
}
- private TaskTracker newTableTasks(ImportTableDesc tblDesc) throws SemanticException {
+ private void newTableTasks(ImportTableDesc tblDesc) throws SemanticException {
Table table;
table = new Table(tblDesc.getDatabaseName(), tblDesc.getTableName());
// Either we're dropping and re-creating, or the table didn't exist, and we're creating.
@@ -162,7 +190,7 @@ public class LoadTable {
tblDesc.getCreateTableTask(new HashSet<>(), new HashSet<>(), context.hiveConf);
if (event.replicationSpec().isMetadataOnly()) {
tracker.addTask(createTableTask);
- return tracker;
+ return;
}
if (!isPartitioned(tblDesc)) {
LOG.debug("adding dependent CopyWork/MoveWork for table");
@@ -172,7 +200,6 @@ public class LoadTable {
createTableTask.addDependentTask(loadTableTask);
}
tracker.addTask(createTableTask);
- return tracker;
}
private String location(ImportTableDesc tblDesc, Database parentDb)
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
index 3e2c513..7794d3e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hive.ql.parse;
+import io.netty.util.internal.StringUtil;
import org.antlr.runtime.tree.Tree;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.FileStatus;
@@ -28,6 +29,7 @@ import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
import org.apache.hadoop.hive.ql.exec.repl.ReplDumpWork;
+import org.apache.hadoop.hive.ql.exec.repl.ReplStateLogWork;
import org.apache.hadoop.hive.ql.exec.repl.bootstrap.ReplLoadWork;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -37,13 +39,13 @@ import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
import org.apache.hadoop.hive.ql.parse.repl.load.EventDumpDirComparator;
import org.apache.hadoop.hive.ql.parse.repl.load.UpdatedMetaDataTracker;
import org.apache.hadoop.hive.ql.parse.repl.load.message.MessageHandler;
+import org.apache.hadoop.hive.ql.parse.repl.load.log.IncrementalLoadLogger;
+import org.apache.hadoop.hive.ql.parse.repl.ReplLogger;
import org.apache.hadoop.hive.ql.plan.AlterDatabaseDesc;
import org.apache.hadoop.hive.ql.plan.AlterTableDesc;
import org.apache.hadoop.hive.ql.plan.DDLWork;
import org.apache.hadoop.hive.ql.plan.DependencyCollectionWork;
import org.apache.hadoop.hive.ql.plan.PlanUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.io.FileNotFoundException;
import java.io.Serializable;
@@ -76,7 +78,6 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
private static final String dumpSchema = "dump_dir,last_repl_id#string,string";
public static final String FUNCTIONS_ROOT_DIR_NAME = "_functions";
- private final static Logger REPL_STATE_LOG = LoggerFactory.getLogger("ReplState");
ReplicationSemanticAnalyzer(QueryState queryState) throws SemanticException {
super(queryState);
@@ -329,12 +330,9 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
Task<? extends Serializable> evTaskRoot = TaskFactory.get(new DependencyCollectionWork(), conf);
Task<? extends Serializable> taskChainTail = evTaskRoot;
- int evstage = 0;
- int evIter = 0;
+ ReplLogger replLogger = new IncrementalLoadLogger(dbNameOrPattern,
+ loadPath.toString(), dirsInLoadPath.length);
- REPL_STATE_LOG.info("Repl Load: Started analyzing Repl load for DB: {} from path {}, Dump Type: INCREMENTAL",
- (null != dbNameOrPattern && !dbNameOrPattern.isEmpty()) ? dbNameOrPattern : "?",
- loadPath.toUri().toString());
for (FileStatus dir : dirsInLoadPath){
LOG.debug("Loading event from {} to {}.{}", dir.getPath().toUri(), dbNameOrPattern, tblNameOrPattern);
@@ -359,17 +357,16 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
String locn = dir.getPath().toUri().toString();
DumpMetaData eventDmd = new DumpMetaData(new Path(locn), conf);
- List<Task<? extends Serializable>> evTasks = analyzeEventLoad(
- dbNameOrPattern, tblNameOrPattern, locn, taskChainTail, eventDmd);
- evIter++;
- REPL_STATE_LOG.info("Repl Load: Analyzed load for event {}/{} " +
- "with ID: {}, Type: {}, Path: {}",
- evIter, dirsInLoadPath.length,
- dir.getPath().getName(), eventDmd.getDumpType().toString(), locn);
-
- LOG.debug("evstage#{} got {} tasks", evstage, evTasks!=null ? evTasks.size() : 0);
+ MessageHandler.Context context = new MessageHandler.Context(dbNameOrPattern,
+ tblNameOrPattern, locn, taskChainTail,
+ eventDmd, conf, db, ctx, LOG);
+ List<Task<? extends Serializable>> evTasks = analyzeEventLoad(context);
+
if ((evTasks != null) && (!evTasks.isEmpty())){
- Task<? extends Serializable> barrierTask = TaskFactory.get(new DependencyCollectionWork(), conf);
+ ReplStateLogWork replStateLogWork = new ReplStateLogWork(replLogger,
+ dir.getPath().getName(),
+ eventDmd.getDumpType().toString());
+ Task<? extends Serializable> barrierTask = TaskFactory.get(replStateLogWork, conf);
for (Task<? extends Serializable> t : evTasks){
t.addDependentTask(barrierTask);
LOG.debug("Added {}:{} as a precursor of barrier task {}:{}",
@@ -378,14 +375,23 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
LOG.debug("Updated taskChainTail from {}{} to {}{}",
taskChainTail.getClass(), taskChainTail.getId(), barrierTask.getClass(), barrierTask.getId());
taskChainTail = barrierTask;
- evstage++;
}
}
+
+ // If any event is there and db name is known, then dump the start and end logs
+ if (!evTaskRoot.equals(taskChainTail)) {
+ Map<String, String> dbProps = new HashMap<>();
+ dbProps.put(ReplicationSpec.KEY.CURR_STATE_ID.toString(), String.valueOf(dmd.getEventTo()));
+ ReplStateLogWork replStateLogWork = new ReplStateLogWork(replLogger, dbProps);
+ Task<? extends Serializable> barrierTask = TaskFactory.get(replStateLogWork, conf);
+ taskChainTail.addDependentTask(barrierTask);
+ LOG.debug("Added {}:{} as a precursor of barrier task {}:{}",
+ taskChainTail.getClass(), taskChainTail.getId(),
+ barrierTask.getClass(), barrierTask.getId());
+
+ replLogger.startLog();
+ }
rootTasks.add(evTaskRoot);
- REPL_STATE_LOG.info("Repl Load: Completed analyzing Repl load for DB: {} from path {} and created import " +
- "(DDL/COPY/MOVE) tasks",
- (null != dbNameOrPattern && !dbNameOrPattern.isEmpty()) ? dbNameOrPattern : "?",
- loadPath.toUri().toString());
}
} catch (Exception e) {
@@ -395,26 +401,22 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
}
private List<Task<? extends Serializable>> analyzeEventLoad(
- String dbName, String tblName,
- String location,
- Task<? extends Serializable> precursor,
- DumpMetaData dmd)
+ MessageHandler.Context context)
throws SemanticException {
- MessageHandler.Context context =
- new MessageHandler.Context(dbName, tblName, location, precursor, dmd, conf, db, ctx, LOG);
- MessageHandler messageHandler = dmd.getDumpType().handler();
+ MessageHandler messageHandler = context.dmd.getDumpType().handler();
List<Task<? extends Serializable>> tasks = messageHandler.handle(context);
- if (precursor != null) {
+ if (context.precursor != null) {
for (Task<? extends Serializable> t : tasks) {
- precursor.addDependentTask(t);
+ context.precursor.addDependentTask(t);
LOG.debug("Added {}:{} as a precursor of {}:{}",
- precursor.getClass(), precursor.getId(), t.getClass(), t.getId());
+ context.precursor.getClass(), context.precursor.getId(), t.getClass(), t.getId());
}
}
+
inputs.addAll(messageHandler.readEntities());
outputs.addAll(messageHandler.writeEntities());
- return addUpdateReplStateTasks(StringUtils.isEmpty(tblName),
+ return addUpdateReplStateTasks(StringUtils.isEmpty(context.tableName),
messageHandler.getUpdatedMetadata(), tasks);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java
index 1c54d29..235a44c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java
@@ -207,13 +207,6 @@ public class ReplicationSpec {
};
}
- private static String getLastReplicatedStateFromParameters(Map<String, String> m) {
- if ((m != null) && (m.containsKey(KEY.CURR_STATE_ID.toString()))){
- return m.get(KEY.CURR_STATE_ID.toString());
- }
- return null;
- }
-
private void init(ASTNode node){
// -> ^(TOK_REPLICATION $replId $isMetadataOnly)
isInReplicationScope = true;
@@ -225,6 +218,13 @@ public class ReplicationSpec {
}
}
+ public static String getLastReplicatedStateFromParameters(Map<String, String> m) {
+ if ((m != null) && (m.containsKey(KEY.CURR_STATE_ID.toString()))){
+ return m.get(KEY.CURR_STATE_ID.toString());
+ }
+ return null;
+ }
+
/**
* @return true if this statement is being run for the purposes of replication
*/
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/ReplLogger.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/ReplLogger.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/ReplLogger.java
new file mode 100644
index 0000000..6f8d5e0
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/ReplLogger.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl;
+
+import org.apache.hadoop.hive.metastore.TableType;
+
+public abstract class ReplLogger {
+
+ public ReplLogger() {
+ }
+
+ public abstract void startLog();
+ public abstract void endLog(String lastReplId);
+
+ public void tableLog(String tableName, TableType tableType) {
+ }
+ public void functionLog(String funcName){
+ }
+ public void eventLog(String eventId, String eventType) {
+ }
+}
[22/50] [abbrv] hive git commit: HIVE-17100 : Improve HS2 operation
logs for REPL commands (Sankar Hariappan, reviewed by Anishek Agarwal,
Thejas Nair)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
index 8406b42..3094394 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
@@ -173,6 +173,7 @@ class ThriftHiveMetastoreIf : virtual public ::facebook::fb303::FacebookService
virtual void add_dynamic_partitions(const AddDynamicPartitions& rqst) = 0;
virtual void get_next_notification(NotificationEventResponse& _return, const NotificationEventRequest& rqst) = 0;
virtual void get_current_notificationEventId(CurrentNotificationEventId& _return) = 0;
+ virtual void get_notification_events_count(NotificationEventsCountResponse& _return, const NotificationEventsCountRequest& rqst) = 0;
virtual void fire_listener_event(FireEventResponse& _return, const FireEventRequest& rqst) = 0;
virtual void flushCache() = 0;
virtual void cm_recycle(CmRecycleResponse& _return, const CmRecycleRequest& request) = 0;
@@ -692,6 +693,9 @@ class ThriftHiveMetastoreNull : virtual public ThriftHiveMetastoreIf , virtual p
void get_current_notificationEventId(CurrentNotificationEventId& /* _return */) {
return;
}
+ void get_notification_events_count(NotificationEventsCountResponse& /* _return */, const NotificationEventsCountRequest& /* rqst */) {
+ return;
+ }
void fire_listener_event(FireEventResponse& /* _return */, const FireEventRequest& /* rqst */) {
return;
}
@@ -19527,6 +19531,110 @@ class ThriftHiveMetastore_get_current_notificationEventId_presult {
};
+typedef struct _ThriftHiveMetastore_get_notification_events_count_args__isset {
+ _ThriftHiveMetastore_get_notification_events_count_args__isset() : rqst(false) {}
+ bool rqst :1;
+} _ThriftHiveMetastore_get_notification_events_count_args__isset;
+
+class ThriftHiveMetastore_get_notification_events_count_args {
+ public:
+
+ ThriftHiveMetastore_get_notification_events_count_args(const ThriftHiveMetastore_get_notification_events_count_args&);
+ ThriftHiveMetastore_get_notification_events_count_args& operator=(const ThriftHiveMetastore_get_notification_events_count_args&);
+ ThriftHiveMetastore_get_notification_events_count_args() {
+ }
+
+ virtual ~ThriftHiveMetastore_get_notification_events_count_args() throw();
+ NotificationEventsCountRequest rqst;
+
+ _ThriftHiveMetastore_get_notification_events_count_args__isset __isset;
+
+ void __set_rqst(const NotificationEventsCountRequest& val);
+
+ bool operator == (const ThriftHiveMetastore_get_notification_events_count_args & rhs) const
+ {
+ if (!(rqst == rhs.rqst))
+ return false;
+ return true;
+ }
+ bool operator != (const ThriftHiveMetastore_get_notification_events_count_args &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const ThriftHiveMetastore_get_notification_events_count_args & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+};
+
+
+class ThriftHiveMetastore_get_notification_events_count_pargs {
+ public:
+
+
+ virtual ~ThriftHiveMetastore_get_notification_events_count_pargs() throw();
+ const NotificationEventsCountRequest* rqst;
+
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+};
+
+typedef struct _ThriftHiveMetastore_get_notification_events_count_result__isset {
+ _ThriftHiveMetastore_get_notification_events_count_result__isset() : success(false) {}
+ bool success :1;
+} _ThriftHiveMetastore_get_notification_events_count_result__isset;
+
+class ThriftHiveMetastore_get_notification_events_count_result {
+ public:
+
+ ThriftHiveMetastore_get_notification_events_count_result(const ThriftHiveMetastore_get_notification_events_count_result&);
+ ThriftHiveMetastore_get_notification_events_count_result& operator=(const ThriftHiveMetastore_get_notification_events_count_result&);
+ ThriftHiveMetastore_get_notification_events_count_result() {
+ }
+
+ virtual ~ThriftHiveMetastore_get_notification_events_count_result() throw();
+ NotificationEventsCountResponse success;
+
+ _ThriftHiveMetastore_get_notification_events_count_result__isset __isset;
+
+ void __set_success(const NotificationEventsCountResponse& val);
+
+ bool operator == (const ThriftHiveMetastore_get_notification_events_count_result & rhs) const
+ {
+ if (!(success == rhs.success))
+ return false;
+ return true;
+ }
+ bool operator != (const ThriftHiveMetastore_get_notification_events_count_result &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const ThriftHiveMetastore_get_notification_events_count_result & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+};
+
+typedef struct _ThriftHiveMetastore_get_notification_events_count_presult__isset {
+ _ThriftHiveMetastore_get_notification_events_count_presult__isset() : success(false) {}
+ bool success :1;
+} _ThriftHiveMetastore_get_notification_events_count_presult__isset;
+
+class ThriftHiveMetastore_get_notification_events_count_presult {
+ public:
+
+
+ virtual ~ThriftHiveMetastore_get_notification_events_count_presult() throw();
+ NotificationEventsCountResponse* success;
+
+ _ThriftHiveMetastore_get_notification_events_count_presult__isset __isset;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+
+};
+
typedef struct _ThriftHiveMetastore_fire_listener_event_args__isset {
_ThriftHiveMetastore_fire_listener_event_args__isset() : rqst(false) {}
bool rqst :1;
@@ -20901,6 +21009,9 @@ class ThriftHiveMetastoreClient : virtual public ThriftHiveMetastoreIf, public
void get_current_notificationEventId(CurrentNotificationEventId& _return);
void send_get_current_notificationEventId();
void recv_get_current_notificationEventId(CurrentNotificationEventId& _return);
+ void get_notification_events_count(NotificationEventsCountResponse& _return, const NotificationEventsCountRequest& rqst);
+ void send_get_notification_events_count(const NotificationEventsCountRequest& rqst);
+ void recv_get_notification_events_count(NotificationEventsCountResponse& _return);
void fire_listener_event(FireEventResponse& _return, const FireEventRequest& rqst);
void send_fire_listener_event(const FireEventRequest& rqst);
void recv_fire_listener_event(FireEventResponse& _return);
@@ -21089,6 +21200,7 @@ class ThriftHiveMetastoreProcessor : public ::facebook::fb303::FacebookServiceP
void process_add_dynamic_partitions(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
void process_get_next_notification(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
void process_get_current_notificationEventId(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
+ void process_get_notification_events_count(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
void process_fire_listener_event(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
void process_flushCache(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
void process_cm_recycle(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
@@ -21253,6 +21365,7 @@ class ThriftHiveMetastoreProcessor : public ::facebook::fb303::FacebookServiceP
processMap_["add_dynamic_partitions"] = &ThriftHiveMetastoreProcessor::process_add_dynamic_partitions;
processMap_["get_next_notification"] = &ThriftHiveMetastoreProcessor::process_get_next_notification;
processMap_["get_current_notificationEventId"] = &ThriftHiveMetastoreProcessor::process_get_current_notificationEventId;
+ processMap_["get_notification_events_count"] = &ThriftHiveMetastoreProcessor::process_get_notification_events_count;
processMap_["fire_listener_event"] = &ThriftHiveMetastoreProcessor::process_fire_listener_event;
processMap_["flushCache"] = &ThriftHiveMetastoreProcessor::process_flushCache;
processMap_["cm_recycle"] = &ThriftHiveMetastoreProcessor::process_cm_recycle;
@@ -22740,6 +22853,16 @@ class ThriftHiveMetastoreMultiface : virtual public ThriftHiveMetastoreIf, publi
return;
}
+ void get_notification_events_count(NotificationEventsCountResponse& _return, const NotificationEventsCountRequest& rqst) {
+ size_t sz = ifaces_.size();
+ size_t i = 0;
+ for (; i < (sz - 1); ++i) {
+ ifaces_[i]->get_notification_events_count(_return, rqst);
+ }
+ ifaces_[i]->get_notification_events_count(_return, rqst);
+ return;
+ }
+
void fire_listener_event(FireEventResponse& _return, const FireEventRequest& rqst) {
size_t sz = ifaces_.size();
size_t i = 0;
@@ -23298,6 +23421,9 @@ class ThriftHiveMetastoreConcurrentClient : virtual public ThriftHiveMetastoreIf
void get_current_notificationEventId(CurrentNotificationEventId& _return);
int32_t send_get_current_notificationEventId();
void recv_get_current_notificationEventId(CurrentNotificationEventId& _return, const int32_t seqid);
+ void get_notification_events_count(NotificationEventsCountResponse& _return, const NotificationEventsCountRequest& rqst);
+ int32_t send_get_notification_events_count(const NotificationEventsCountRequest& rqst);
+ void recv_get_notification_events_count(NotificationEventsCountResponse& _return, const int32_t seqid);
void fire_listener_event(FireEventResponse& _return, const FireEventRequest& rqst);
int32_t send_fire_listener_event(const FireEventRequest& rqst);
void recv_fire_listener_event(FireEventResponse& _return, const int32_t seqid);
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore_server.skeleton.cpp
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore_server.skeleton.cpp b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore_server.skeleton.cpp
index ec2071a..c656d5f 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore_server.skeleton.cpp
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore_server.skeleton.cpp
@@ -777,6 +777,11 @@ class ThriftHiveMetastoreHandler : virtual public ThriftHiveMetastoreIf {
printf("get_current_notificationEventId\n");
}
+ void get_notification_events_count(NotificationEventsCountResponse& _return, const NotificationEventsCountRequest& rqst) {
+ // Your implementation goes here
+ printf("get_notification_events_count\n");
+ }
+
void fire_listener_event(FireEventResponse& _return, const FireEventRequest& rqst) {
// Your implementation goes here
printf("fire_listener_event\n");
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
index a4f72b6..5c09fdd 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
@@ -17218,6 +17218,201 @@ void CurrentNotificationEventId::printTo(std::ostream& out) const {
}
+NotificationEventsCountRequest::~NotificationEventsCountRequest() throw() {
+}
+
+
+void NotificationEventsCountRequest::__set_fromEventId(const int64_t val) {
+ this->fromEventId = val;
+}
+
+void NotificationEventsCountRequest::__set_dbName(const std::string& val) {
+ this->dbName = val;
+}
+
+uint32_t NotificationEventsCountRequest::read(::apache::thrift::protocol::TProtocol* iprot) {
+
+ apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+ uint32_t xfer = 0;
+ std::string fname;
+ ::apache::thrift::protocol::TType ftype;
+ int16_t fid;
+
+ xfer += iprot->readStructBegin(fname);
+
+ using ::apache::thrift::protocol::TProtocolException;
+
+ bool isset_fromEventId = false;
+ bool isset_dbName = false;
+
+ while (true)
+ {
+ xfer += iprot->readFieldBegin(fname, ftype, fid);
+ if (ftype == ::apache::thrift::protocol::T_STOP) {
+ break;
+ }
+ switch (fid)
+ {
+ case 1:
+ if (ftype == ::apache::thrift::protocol::T_I64) {
+ xfer += iprot->readI64(this->fromEventId);
+ isset_fromEventId = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 2:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->dbName);
+ isset_dbName = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ default:
+ xfer += iprot->skip(ftype);
+ break;
+ }
+ xfer += iprot->readFieldEnd();
+ }
+
+ xfer += iprot->readStructEnd();
+
+ if (!isset_fromEventId)
+ throw TProtocolException(TProtocolException::INVALID_DATA);
+ if (!isset_dbName)
+ throw TProtocolException(TProtocolException::INVALID_DATA);
+ return xfer;
+}
+
+uint32_t NotificationEventsCountRequest::write(::apache::thrift::protocol::TProtocol* oprot) const {
+ uint32_t xfer = 0;
+ apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+ xfer += oprot->writeStructBegin("NotificationEventsCountRequest");
+
+ xfer += oprot->writeFieldBegin("fromEventId", ::apache::thrift::protocol::T_I64, 1);
+ xfer += oprot->writeI64(this->fromEventId);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("dbName", ::apache::thrift::protocol::T_STRING, 2);
+ xfer += oprot->writeString(this->dbName);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldStop();
+ xfer += oprot->writeStructEnd();
+ return xfer;
+}
+
+void swap(NotificationEventsCountRequest &a, NotificationEventsCountRequest &b) {
+ using ::std::swap;
+ swap(a.fromEventId, b.fromEventId);
+ swap(a.dbName, b.dbName);
+}
+
+NotificationEventsCountRequest::NotificationEventsCountRequest(const NotificationEventsCountRequest& other690) {
+ fromEventId = other690.fromEventId;
+ dbName = other690.dbName;
+}
+NotificationEventsCountRequest& NotificationEventsCountRequest::operator=(const NotificationEventsCountRequest& other691) {
+ fromEventId = other691.fromEventId;
+ dbName = other691.dbName;
+ return *this;
+}
+void NotificationEventsCountRequest::printTo(std::ostream& out) const {
+ using ::apache::thrift::to_string;
+ out << "NotificationEventsCountRequest(";
+ out << "fromEventId=" << to_string(fromEventId);
+ out << ", " << "dbName=" << to_string(dbName);
+ out << ")";
+}
+
+
+NotificationEventsCountResponse::~NotificationEventsCountResponse() throw() {
+}
+
+
+void NotificationEventsCountResponse::__set_eventsCount(const int64_t val) {
+ this->eventsCount = val;
+}
+
+uint32_t NotificationEventsCountResponse::read(::apache::thrift::protocol::TProtocol* iprot) {
+
+ apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+ uint32_t xfer = 0;
+ std::string fname;
+ ::apache::thrift::protocol::TType ftype;
+ int16_t fid;
+
+ xfer += iprot->readStructBegin(fname);
+
+ using ::apache::thrift::protocol::TProtocolException;
+
+ bool isset_eventsCount = false;
+
+ while (true)
+ {
+ xfer += iprot->readFieldBegin(fname, ftype, fid);
+ if (ftype == ::apache::thrift::protocol::T_STOP) {
+ break;
+ }
+ switch (fid)
+ {
+ case 1:
+ if (ftype == ::apache::thrift::protocol::T_I64) {
+ xfer += iprot->readI64(this->eventsCount);
+ isset_eventsCount = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ default:
+ xfer += iprot->skip(ftype);
+ break;
+ }
+ xfer += iprot->readFieldEnd();
+ }
+
+ xfer += iprot->readStructEnd();
+
+ if (!isset_eventsCount)
+ throw TProtocolException(TProtocolException::INVALID_DATA);
+ return xfer;
+}
+
+uint32_t NotificationEventsCountResponse::write(::apache::thrift::protocol::TProtocol* oprot) const {
+ uint32_t xfer = 0;
+ apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+ xfer += oprot->writeStructBegin("NotificationEventsCountResponse");
+
+ xfer += oprot->writeFieldBegin("eventsCount", ::apache::thrift::protocol::T_I64, 1);
+ xfer += oprot->writeI64(this->eventsCount);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldStop();
+ xfer += oprot->writeStructEnd();
+ return xfer;
+}
+
+void swap(NotificationEventsCountResponse &a, NotificationEventsCountResponse &b) {
+ using ::std::swap;
+ swap(a.eventsCount, b.eventsCount);
+}
+
+NotificationEventsCountResponse::NotificationEventsCountResponse(const NotificationEventsCountResponse& other692) {
+ eventsCount = other692.eventsCount;
+}
+NotificationEventsCountResponse& NotificationEventsCountResponse::operator=(const NotificationEventsCountResponse& other693) {
+ eventsCount = other693.eventsCount;
+ return *this;
+}
+void NotificationEventsCountResponse::printTo(std::ostream& out) const {
+ using ::apache::thrift::to_string;
+ out << "NotificationEventsCountResponse(";
+ out << "eventsCount=" << to_string(eventsCount);
+ out << ")";
+}
+
+
InsertEventRequestData::~InsertEventRequestData() throw() {
}
@@ -17270,14 +17465,14 @@ uint32_t InsertEventRequestData::read(::apache::thrift::protocol::TProtocol* ipr
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->filesAdded.clear();
- uint32_t _size690;
- ::apache::thrift::protocol::TType _etype693;
- xfer += iprot->readListBegin(_etype693, _size690);
- this->filesAdded.resize(_size690);
- uint32_t _i694;
- for (_i694 = 0; _i694 < _size690; ++_i694)
+ uint32_t _size694;
+ ::apache::thrift::protocol::TType _etype697;
+ xfer += iprot->readListBegin(_etype697, _size694);
+ this->filesAdded.resize(_size694);
+ uint32_t _i698;
+ for (_i698 = 0; _i698 < _size694; ++_i698)
{
- xfer += iprot->readString(this->filesAdded[_i694]);
+ xfer += iprot->readString(this->filesAdded[_i698]);
}
xfer += iprot->readListEnd();
}
@@ -17290,14 +17485,14 @@ uint32_t InsertEventRequestData::read(::apache::thrift::protocol::TProtocol* ipr
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->filesAddedChecksum.clear();
- uint32_t _size695;
- ::apache::thrift::protocol::TType _etype698;
- xfer += iprot->readListBegin(_etype698, _size695);
- this->filesAddedChecksum.resize(_size695);
- uint32_t _i699;
- for (_i699 = 0; _i699 < _size695; ++_i699)
+ uint32_t _size699;
+ ::apache::thrift::protocol::TType _etype702;
+ xfer += iprot->readListBegin(_etype702, _size699);
+ this->filesAddedChecksum.resize(_size699);
+ uint32_t _i703;
+ for (_i703 = 0; _i703 < _size699; ++_i703)
{
- xfer += iprot->readString(this->filesAddedChecksum[_i699]);
+ xfer += iprot->readString(this->filesAddedChecksum[_i703]);
}
xfer += iprot->readListEnd();
}
@@ -17333,10 +17528,10 @@ uint32_t InsertEventRequestData::write(::apache::thrift::protocol::TProtocol* op
xfer += oprot->writeFieldBegin("filesAdded", ::apache::thrift::protocol::T_LIST, 2);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->filesAdded.size()));
- std::vector<std::string> ::const_iterator _iter700;
- for (_iter700 = this->filesAdded.begin(); _iter700 != this->filesAdded.end(); ++_iter700)
+ std::vector<std::string> ::const_iterator _iter704;
+ for (_iter704 = this->filesAdded.begin(); _iter704 != this->filesAdded.end(); ++_iter704)
{
- xfer += oprot->writeString((*_iter700));
+ xfer += oprot->writeString((*_iter704));
}
xfer += oprot->writeListEnd();
}
@@ -17346,10 +17541,10 @@ uint32_t InsertEventRequestData::write(::apache::thrift::protocol::TProtocol* op
xfer += oprot->writeFieldBegin("filesAddedChecksum", ::apache::thrift::protocol::T_LIST, 3);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->filesAddedChecksum.size()));
- std::vector<std::string> ::const_iterator _iter701;
- for (_iter701 = this->filesAddedChecksum.begin(); _iter701 != this->filesAddedChecksum.end(); ++_iter701)
+ std::vector<std::string> ::const_iterator _iter705;
+ for (_iter705 = this->filesAddedChecksum.begin(); _iter705 != this->filesAddedChecksum.end(); ++_iter705)
{
- xfer += oprot->writeString((*_iter701));
+ xfer += oprot->writeString((*_iter705));
}
xfer += oprot->writeListEnd();
}
@@ -17368,17 +17563,17 @@ void swap(InsertEventRequestData &a, InsertEventRequestData &b) {
swap(a.__isset, b.__isset);
}
-InsertEventRequestData::InsertEventRequestData(const InsertEventRequestData& other702) {
- replace = other702.replace;
- filesAdded = other702.filesAdded;
- filesAddedChecksum = other702.filesAddedChecksum;
- __isset = other702.__isset;
+InsertEventRequestData::InsertEventRequestData(const InsertEventRequestData& other706) {
+ replace = other706.replace;
+ filesAdded = other706.filesAdded;
+ filesAddedChecksum = other706.filesAddedChecksum;
+ __isset = other706.__isset;
}
-InsertEventRequestData& InsertEventRequestData::operator=(const InsertEventRequestData& other703) {
- replace = other703.replace;
- filesAdded = other703.filesAdded;
- filesAddedChecksum = other703.filesAddedChecksum;
- __isset = other703.__isset;
+InsertEventRequestData& InsertEventRequestData::operator=(const InsertEventRequestData& other707) {
+ replace = other707.replace;
+ filesAdded = other707.filesAdded;
+ filesAddedChecksum = other707.filesAddedChecksum;
+ __isset = other707.__isset;
return *this;
}
void InsertEventRequestData::printTo(std::ostream& out) const {
@@ -17460,13 +17655,13 @@ void swap(FireEventRequestData &a, FireEventRequestData &b) {
swap(a.__isset, b.__isset);
}
-FireEventRequestData::FireEventRequestData(const FireEventRequestData& other704) {
- insertData = other704.insertData;
- __isset = other704.__isset;
+FireEventRequestData::FireEventRequestData(const FireEventRequestData& other708) {
+ insertData = other708.insertData;
+ __isset = other708.__isset;
}
-FireEventRequestData& FireEventRequestData::operator=(const FireEventRequestData& other705) {
- insertData = other705.insertData;
- __isset = other705.__isset;
+FireEventRequestData& FireEventRequestData::operator=(const FireEventRequestData& other709) {
+ insertData = other709.insertData;
+ __isset = other709.__isset;
return *this;
}
void FireEventRequestData::printTo(std::ostream& out) const {
@@ -17563,14 +17758,14 @@ uint32_t FireEventRequest::read(::apache::thrift::protocol::TProtocol* iprot) {
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->partitionVals.clear();
- uint32_t _size706;
- ::apache::thrift::protocol::TType _etype709;
- xfer += iprot->readListBegin(_etype709, _size706);
- this->partitionVals.resize(_size706);
- uint32_t _i710;
- for (_i710 = 0; _i710 < _size706; ++_i710)
+ uint32_t _size710;
+ ::apache::thrift::protocol::TType _etype713;
+ xfer += iprot->readListBegin(_etype713, _size710);
+ this->partitionVals.resize(_size710);
+ uint32_t _i714;
+ for (_i714 = 0; _i714 < _size710; ++_i714)
{
- xfer += iprot->readString(this->partitionVals[_i710]);
+ xfer += iprot->readString(this->partitionVals[_i714]);
}
xfer += iprot->readListEnd();
}
@@ -17622,10 +17817,10 @@ uint32_t FireEventRequest::write(::apache::thrift::protocol::TProtocol* oprot) c
xfer += oprot->writeFieldBegin("partitionVals", ::apache::thrift::protocol::T_LIST, 5);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->partitionVals.size()));
- std::vector<std::string> ::const_iterator _iter711;
- for (_iter711 = this->partitionVals.begin(); _iter711 != this->partitionVals.end(); ++_iter711)
+ std::vector<std::string> ::const_iterator _iter715;
+ for (_iter715 = this->partitionVals.begin(); _iter715 != this->partitionVals.end(); ++_iter715)
{
- xfer += oprot->writeString((*_iter711));
+ xfer += oprot->writeString((*_iter715));
}
xfer += oprot->writeListEnd();
}
@@ -17646,21 +17841,21 @@ void swap(FireEventRequest &a, FireEventRequest &b) {
swap(a.__isset, b.__isset);
}
-FireEventRequest::FireEventRequest(const FireEventRequest& other712) {
- successful = other712.successful;
- data = other712.data;
- dbName = other712.dbName;
- tableName = other712.tableName;
- partitionVals = other712.partitionVals;
- __isset = other712.__isset;
-}
-FireEventRequest& FireEventRequest::operator=(const FireEventRequest& other713) {
- successful = other713.successful;
- data = other713.data;
- dbName = other713.dbName;
- tableName = other713.tableName;
- partitionVals = other713.partitionVals;
- __isset = other713.__isset;
+FireEventRequest::FireEventRequest(const FireEventRequest& other716) {
+ successful = other716.successful;
+ data = other716.data;
+ dbName = other716.dbName;
+ tableName = other716.tableName;
+ partitionVals = other716.partitionVals;
+ __isset = other716.__isset;
+}
+FireEventRequest& FireEventRequest::operator=(const FireEventRequest& other717) {
+ successful = other717.successful;
+ data = other717.data;
+ dbName = other717.dbName;
+ tableName = other717.tableName;
+ partitionVals = other717.partitionVals;
+ __isset = other717.__isset;
return *this;
}
void FireEventRequest::printTo(std::ostream& out) const {
@@ -17723,11 +17918,11 @@ void swap(FireEventResponse &a, FireEventResponse &b) {
(void) b;
}
-FireEventResponse::FireEventResponse(const FireEventResponse& other714) {
- (void) other714;
+FireEventResponse::FireEventResponse(const FireEventResponse& other718) {
+ (void) other718;
}
-FireEventResponse& FireEventResponse::operator=(const FireEventResponse& other715) {
- (void) other715;
+FireEventResponse& FireEventResponse::operator=(const FireEventResponse& other719) {
+ (void) other719;
return *this;
}
void FireEventResponse::printTo(std::ostream& out) const {
@@ -17827,15 +18022,15 @@ void swap(MetadataPpdResult &a, MetadataPpdResult &b) {
swap(a.__isset, b.__isset);
}
-MetadataPpdResult::MetadataPpdResult(const MetadataPpdResult& other716) {
- metadata = other716.metadata;
- includeBitset = other716.includeBitset;
- __isset = other716.__isset;
+MetadataPpdResult::MetadataPpdResult(const MetadataPpdResult& other720) {
+ metadata = other720.metadata;
+ includeBitset = other720.includeBitset;
+ __isset = other720.__isset;
}
-MetadataPpdResult& MetadataPpdResult::operator=(const MetadataPpdResult& other717) {
- metadata = other717.metadata;
- includeBitset = other717.includeBitset;
- __isset = other717.__isset;
+MetadataPpdResult& MetadataPpdResult::operator=(const MetadataPpdResult& other721) {
+ metadata = other721.metadata;
+ includeBitset = other721.includeBitset;
+ __isset = other721.__isset;
return *this;
}
void MetadataPpdResult::printTo(std::ostream& out) const {
@@ -17886,17 +18081,17 @@ uint32_t GetFileMetadataByExprResult::read(::apache::thrift::protocol::TProtocol
if (ftype == ::apache::thrift::protocol::T_MAP) {
{
this->metadata.clear();
- uint32_t _size718;
- ::apache::thrift::protocol::TType _ktype719;
- ::apache::thrift::protocol::TType _vtype720;
- xfer += iprot->readMapBegin(_ktype719, _vtype720, _size718);
- uint32_t _i722;
- for (_i722 = 0; _i722 < _size718; ++_i722)
+ uint32_t _size722;
+ ::apache::thrift::protocol::TType _ktype723;
+ ::apache::thrift::protocol::TType _vtype724;
+ xfer += iprot->readMapBegin(_ktype723, _vtype724, _size722);
+ uint32_t _i726;
+ for (_i726 = 0; _i726 < _size722; ++_i726)
{
- int64_t _key723;
- xfer += iprot->readI64(_key723);
- MetadataPpdResult& _val724 = this->metadata[_key723];
- xfer += _val724.read(iprot);
+ int64_t _key727;
+ xfer += iprot->readI64(_key727);
+ MetadataPpdResult& _val728 = this->metadata[_key727];
+ xfer += _val728.read(iprot);
}
xfer += iprot->readMapEnd();
}
@@ -17937,11 +18132,11 @@ uint32_t GetFileMetadataByExprResult::write(::apache::thrift::protocol::TProtoco
xfer += oprot->writeFieldBegin("metadata", ::apache::thrift::protocol::T_MAP, 1);
{
xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_I64, ::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(this->metadata.size()));
- std::map<int64_t, MetadataPpdResult> ::const_iterator _iter725;
- for (_iter725 = this->metadata.begin(); _iter725 != this->metadata.end(); ++_iter725)
+ std::map<int64_t, MetadataPpdResult> ::const_iterator _iter729;
+ for (_iter729 = this->metadata.begin(); _iter729 != this->metadata.end(); ++_iter729)
{
- xfer += oprot->writeI64(_iter725->first);
- xfer += _iter725->second.write(oprot);
+ xfer += oprot->writeI64(_iter729->first);
+ xfer += _iter729->second.write(oprot);
}
xfer += oprot->writeMapEnd();
}
@@ -17962,13 +18157,13 @@ void swap(GetFileMetadataByExprResult &a, GetFileMetadataByExprResult &b) {
swap(a.isSupported, b.isSupported);
}
-GetFileMetadataByExprResult::GetFileMetadataByExprResult(const GetFileMetadataByExprResult& other726) {
- metadata = other726.metadata;
- isSupported = other726.isSupported;
+GetFileMetadataByExprResult::GetFileMetadataByExprResult(const GetFileMetadataByExprResult& other730) {
+ metadata = other730.metadata;
+ isSupported = other730.isSupported;
}
-GetFileMetadataByExprResult& GetFileMetadataByExprResult::operator=(const GetFileMetadataByExprResult& other727) {
- metadata = other727.metadata;
- isSupported = other727.isSupported;
+GetFileMetadataByExprResult& GetFileMetadataByExprResult::operator=(const GetFileMetadataByExprResult& other731) {
+ metadata = other731.metadata;
+ isSupported = other731.isSupported;
return *this;
}
void GetFileMetadataByExprResult::printTo(std::ostream& out) const {
@@ -18029,14 +18224,14 @@ uint32_t GetFileMetadataByExprRequest::read(::apache::thrift::protocol::TProtoco
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->fileIds.clear();
- uint32_t _size728;
- ::apache::thrift::protocol::TType _etype731;
- xfer += iprot->readListBegin(_etype731, _size728);
- this->fileIds.resize(_size728);
- uint32_t _i732;
- for (_i732 = 0; _i732 < _size728; ++_i732)
+ uint32_t _size732;
+ ::apache::thrift::protocol::TType _etype735;
+ xfer += iprot->readListBegin(_etype735, _size732);
+ this->fileIds.resize(_size732);
+ uint32_t _i736;
+ for (_i736 = 0; _i736 < _size732; ++_i736)
{
- xfer += iprot->readI64(this->fileIds[_i732]);
+ xfer += iprot->readI64(this->fileIds[_i736]);
}
xfer += iprot->readListEnd();
}
@@ -18063,9 +18258,9 @@ uint32_t GetFileMetadataByExprRequest::read(::apache::thrift::protocol::TProtoco
break;
case 4:
if (ftype == ::apache::thrift::protocol::T_I32) {
- int32_t ecast733;
- xfer += iprot->readI32(ecast733);
- this->type = (FileMetadataExprType::type)ecast733;
+ int32_t ecast737;
+ xfer += iprot->readI32(ecast737);
+ this->type = (FileMetadataExprType::type)ecast737;
this->__isset.type = true;
} else {
xfer += iprot->skip(ftype);
@@ -18095,10 +18290,10 @@ uint32_t GetFileMetadataByExprRequest::write(::apache::thrift::protocol::TProtoc
xfer += oprot->writeFieldBegin("fileIds", ::apache::thrift::protocol::T_LIST, 1);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, static_cast<uint32_t>(this->fileIds.size()));
- std::vector<int64_t> ::const_iterator _iter734;
- for (_iter734 = this->fileIds.begin(); _iter734 != this->fileIds.end(); ++_iter734)
+ std::vector<int64_t> ::const_iterator _iter738;
+ for (_iter738 = this->fileIds.begin(); _iter738 != this->fileIds.end(); ++_iter738)
{
- xfer += oprot->writeI64((*_iter734));
+ xfer += oprot->writeI64((*_iter738));
}
xfer += oprot->writeListEnd();
}
@@ -18132,19 +18327,19 @@ void swap(GetFileMetadataByExprRequest &a, GetFileMetadataByExprRequest &b) {
swap(a.__isset, b.__isset);
}
-GetFileMetadataByExprRequest::GetFileMetadataByExprRequest(const GetFileMetadataByExprRequest& other735) {
- fileIds = other735.fileIds;
- expr = other735.expr;
- doGetFooters = other735.doGetFooters;
- type = other735.type;
- __isset = other735.__isset;
+GetFileMetadataByExprRequest::GetFileMetadataByExprRequest(const GetFileMetadataByExprRequest& other739) {
+ fileIds = other739.fileIds;
+ expr = other739.expr;
+ doGetFooters = other739.doGetFooters;
+ type = other739.type;
+ __isset = other739.__isset;
}
-GetFileMetadataByExprRequest& GetFileMetadataByExprRequest::operator=(const GetFileMetadataByExprRequest& other736) {
- fileIds = other736.fileIds;
- expr = other736.expr;
- doGetFooters = other736.doGetFooters;
- type = other736.type;
- __isset = other736.__isset;
+GetFileMetadataByExprRequest& GetFileMetadataByExprRequest::operator=(const GetFileMetadataByExprRequest& other740) {
+ fileIds = other740.fileIds;
+ expr = other740.expr;
+ doGetFooters = other740.doGetFooters;
+ type = other740.type;
+ __isset = other740.__isset;
return *this;
}
void GetFileMetadataByExprRequest::printTo(std::ostream& out) const {
@@ -18197,17 +18392,17 @@ uint32_t GetFileMetadataResult::read(::apache::thrift::protocol::TProtocol* ipro
if (ftype == ::apache::thrift::protocol::T_MAP) {
{
this->metadata.clear();
- uint32_t _size737;
- ::apache::thrift::protocol::TType _ktype738;
- ::apache::thrift::protocol::TType _vtype739;
- xfer += iprot->readMapBegin(_ktype738, _vtype739, _size737);
- uint32_t _i741;
- for (_i741 = 0; _i741 < _size737; ++_i741)
+ uint32_t _size741;
+ ::apache::thrift::protocol::TType _ktype742;
+ ::apache::thrift::protocol::TType _vtype743;
+ xfer += iprot->readMapBegin(_ktype742, _vtype743, _size741);
+ uint32_t _i745;
+ for (_i745 = 0; _i745 < _size741; ++_i745)
{
- int64_t _key742;
- xfer += iprot->readI64(_key742);
- std::string& _val743 = this->metadata[_key742];
- xfer += iprot->readBinary(_val743);
+ int64_t _key746;
+ xfer += iprot->readI64(_key746);
+ std::string& _val747 = this->metadata[_key746];
+ xfer += iprot->readBinary(_val747);
}
xfer += iprot->readMapEnd();
}
@@ -18248,11 +18443,11 @@ uint32_t GetFileMetadataResult::write(::apache::thrift::protocol::TProtocol* opr
xfer += oprot->writeFieldBegin("metadata", ::apache::thrift::protocol::T_MAP, 1);
{
xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_I64, ::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->metadata.size()));
- std::map<int64_t, std::string> ::const_iterator _iter744;
- for (_iter744 = this->metadata.begin(); _iter744 != this->metadata.end(); ++_iter744)
+ std::map<int64_t, std::string> ::const_iterator _iter748;
+ for (_iter748 = this->metadata.begin(); _iter748 != this->metadata.end(); ++_iter748)
{
- xfer += oprot->writeI64(_iter744->first);
- xfer += oprot->writeBinary(_iter744->second);
+ xfer += oprot->writeI64(_iter748->first);
+ xfer += oprot->writeBinary(_iter748->second);
}
xfer += oprot->writeMapEnd();
}
@@ -18273,13 +18468,13 @@ void swap(GetFileMetadataResult &a, GetFileMetadataResult &b) {
swap(a.isSupported, b.isSupported);
}
-GetFileMetadataResult::GetFileMetadataResult(const GetFileMetadataResult& other745) {
- metadata = other745.metadata;
- isSupported = other745.isSupported;
+GetFileMetadataResult::GetFileMetadataResult(const GetFileMetadataResult& other749) {
+ metadata = other749.metadata;
+ isSupported = other749.isSupported;
}
-GetFileMetadataResult& GetFileMetadataResult::operator=(const GetFileMetadataResult& other746) {
- metadata = other746.metadata;
- isSupported = other746.isSupported;
+GetFileMetadataResult& GetFileMetadataResult::operator=(const GetFileMetadataResult& other750) {
+ metadata = other750.metadata;
+ isSupported = other750.isSupported;
return *this;
}
void GetFileMetadataResult::printTo(std::ostream& out) const {
@@ -18325,14 +18520,14 @@ uint32_t GetFileMetadataRequest::read(::apache::thrift::protocol::TProtocol* ipr
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->fileIds.clear();
- uint32_t _size747;
- ::apache::thrift::protocol::TType _etype750;
- xfer += iprot->readListBegin(_etype750, _size747);
- this->fileIds.resize(_size747);
- uint32_t _i751;
- for (_i751 = 0; _i751 < _size747; ++_i751)
+ uint32_t _size751;
+ ::apache::thrift::protocol::TType _etype754;
+ xfer += iprot->readListBegin(_etype754, _size751);
+ this->fileIds.resize(_size751);
+ uint32_t _i755;
+ for (_i755 = 0; _i755 < _size751; ++_i755)
{
- xfer += iprot->readI64(this->fileIds[_i751]);
+ xfer += iprot->readI64(this->fileIds[_i755]);
}
xfer += iprot->readListEnd();
}
@@ -18363,10 +18558,10 @@ uint32_t GetFileMetadataRequest::write(::apache::thrift::protocol::TProtocol* op
xfer += oprot->writeFieldBegin("fileIds", ::apache::thrift::protocol::T_LIST, 1);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, static_cast<uint32_t>(this->fileIds.size()));
- std::vector<int64_t> ::const_iterator _iter752;
- for (_iter752 = this->fileIds.begin(); _iter752 != this->fileIds.end(); ++_iter752)
+ std::vector<int64_t> ::const_iterator _iter756;
+ for (_iter756 = this->fileIds.begin(); _iter756 != this->fileIds.end(); ++_iter756)
{
- xfer += oprot->writeI64((*_iter752));
+ xfer += oprot->writeI64((*_iter756));
}
xfer += oprot->writeListEnd();
}
@@ -18382,11 +18577,11 @@ void swap(GetFileMetadataRequest &a, GetFileMetadataRequest &b) {
swap(a.fileIds, b.fileIds);
}
-GetFileMetadataRequest::GetFileMetadataRequest(const GetFileMetadataRequest& other753) {
- fileIds = other753.fileIds;
+GetFileMetadataRequest::GetFileMetadataRequest(const GetFileMetadataRequest& other757) {
+ fileIds = other757.fileIds;
}
-GetFileMetadataRequest& GetFileMetadataRequest::operator=(const GetFileMetadataRequest& other754) {
- fileIds = other754.fileIds;
+GetFileMetadataRequest& GetFileMetadataRequest::operator=(const GetFileMetadataRequest& other758) {
+ fileIds = other758.fileIds;
return *this;
}
void GetFileMetadataRequest::printTo(std::ostream& out) const {
@@ -18445,11 +18640,11 @@ void swap(PutFileMetadataResult &a, PutFileMetadataResult &b) {
(void) b;
}
-PutFileMetadataResult::PutFileMetadataResult(const PutFileMetadataResult& other755) {
- (void) other755;
+PutFileMetadataResult::PutFileMetadataResult(const PutFileMetadataResult& other759) {
+ (void) other759;
}
-PutFileMetadataResult& PutFileMetadataResult::operator=(const PutFileMetadataResult& other756) {
- (void) other756;
+PutFileMetadataResult& PutFileMetadataResult::operator=(const PutFileMetadataResult& other760) {
+ (void) other760;
return *this;
}
void PutFileMetadataResult::printTo(std::ostream& out) const {
@@ -18503,14 +18698,14 @@ uint32_t PutFileMetadataRequest::read(::apache::thrift::protocol::TProtocol* ipr
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->fileIds.clear();
- uint32_t _size757;
- ::apache::thrift::protocol::TType _etype760;
- xfer += iprot->readListBegin(_etype760, _size757);
- this->fileIds.resize(_size757);
- uint32_t _i761;
- for (_i761 = 0; _i761 < _size757; ++_i761)
+ uint32_t _size761;
+ ::apache::thrift::protocol::TType _etype764;
+ xfer += iprot->readListBegin(_etype764, _size761);
+ this->fileIds.resize(_size761);
+ uint32_t _i765;
+ for (_i765 = 0; _i765 < _size761; ++_i765)
{
- xfer += iprot->readI64(this->fileIds[_i761]);
+ xfer += iprot->readI64(this->fileIds[_i765]);
}
xfer += iprot->readListEnd();
}
@@ -18523,14 +18718,14 @@ uint32_t PutFileMetadataRequest::read(::apache::thrift::protocol::TProtocol* ipr
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->metadata.clear();
- uint32_t _size762;
- ::apache::thrift::protocol::TType _etype765;
- xfer += iprot->readListBegin(_etype765, _size762);
- this->metadata.resize(_size762);
- uint32_t _i766;
- for (_i766 = 0; _i766 < _size762; ++_i766)
+ uint32_t _size766;
+ ::apache::thrift::protocol::TType _etype769;
+ xfer += iprot->readListBegin(_etype769, _size766);
+ this->metadata.resize(_size766);
+ uint32_t _i770;
+ for (_i770 = 0; _i770 < _size766; ++_i770)
{
- xfer += iprot->readBinary(this->metadata[_i766]);
+ xfer += iprot->readBinary(this->metadata[_i770]);
}
xfer += iprot->readListEnd();
}
@@ -18541,9 +18736,9 @@ uint32_t PutFileMetadataRequest::read(::apache::thrift::protocol::TProtocol* ipr
break;
case 3:
if (ftype == ::apache::thrift::protocol::T_I32) {
- int32_t ecast767;
- xfer += iprot->readI32(ecast767);
- this->type = (FileMetadataExprType::type)ecast767;
+ int32_t ecast771;
+ xfer += iprot->readI32(ecast771);
+ this->type = (FileMetadataExprType::type)ecast771;
this->__isset.type = true;
} else {
xfer += iprot->skip(ftype);
@@ -18573,10 +18768,10 @@ uint32_t PutFileMetadataRequest::write(::apache::thrift::protocol::TProtocol* op
xfer += oprot->writeFieldBegin("fileIds", ::apache::thrift::protocol::T_LIST, 1);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, static_cast<uint32_t>(this->fileIds.size()));
- std::vector<int64_t> ::const_iterator _iter768;
- for (_iter768 = this->fileIds.begin(); _iter768 != this->fileIds.end(); ++_iter768)
+ std::vector<int64_t> ::const_iterator _iter772;
+ for (_iter772 = this->fileIds.begin(); _iter772 != this->fileIds.end(); ++_iter772)
{
- xfer += oprot->writeI64((*_iter768));
+ xfer += oprot->writeI64((*_iter772));
}
xfer += oprot->writeListEnd();
}
@@ -18585,10 +18780,10 @@ uint32_t PutFileMetadataRequest::write(::apache::thrift::protocol::TProtocol* op
xfer += oprot->writeFieldBegin("metadata", ::apache::thrift::protocol::T_LIST, 2);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->metadata.size()));
- std::vector<std::string> ::const_iterator _iter769;
- for (_iter769 = this->metadata.begin(); _iter769 != this->metadata.end(); ++_iter769)
+ std::vector<std::string> ::const_iterator _iter773;
+ for (_iter773 = this->metadata.begin(); _iter773 != this->metadata.end(); ++_iter773)
{
- xfer += oprot->writeBinary((*_iter769));
+ xfer += oprot->writeBinary((*_iter773));
}
xfer += oprot->writeListEnd();
}
@@ -18612,17 +18807,17 @@ void swap(PutFileMetadataRequest &a, PutFileMetadataRequest &b) {
swap(a.__isset, b.__isset);
}
-PutFileMetadataRequest::PutFileMetadataRequest(const PutFileMetadataRequest& other770) {
- fileIds = other770.fileIds;
- metadata = other770.metadata;
- type = other770.type;
- __isset = other770.__isset;
+PutFileMetadataRequest::PutFileMetadataRequest(const PutFileMetadataRequest& other774) {
+ fileIds = other774.fileIds;
+ metadata = other774.metadata;
+ type = other774.type;
+ __isset = other774.__isset;
}
-PutFileMetadataRequest& PutFileMetadataRequest::operator=(const PutFileMetadataRequest& other771) {
- fileIds = other771.fileIds;
- metadata = other771.metadata;
- type = other771.type;
- __isset = other771.__isset;
+PutFileMetadataRequest& PutFileMetadataRequest::operator=(const PutFileMetadataRequest& other775) {
+ fileIds = other775.fileIds;
+ metadata = other775.metadata;
+ type = other775.type;
+ __isset = other775.__isset;
return *this;
}
void PutFileMetadataRequest::printTo(std::ostream& out) const {
@@ -18683,11 +18878,11 @@ void swap(ClearFileMetadataResult &a, ClearFileMetadataResult &b) {
(void) b;
}
-ClearFileMetadataResult::ClearFileMetadataResult(const ClearFileMetadataResult& other772) {
- (void) other772;
+ClearFileMetadataResult::ClearFileMetadataResult(const ClearFileMetadataResult& other776) {
+ (void) other776;
}
-ClearFileMetadataResult& ClearFileMetadataResult::operator=(const ClearFileMetadataResult& other773) {
- (void) other773;
+ClearFileMetadataResult& ClearFileMetadataResult::operator=(const ClearFileMetadataResult& other777) {
+ (void) other777;
return *this;
}
void ClearFileMetadataResult::printTo(std::ostream& out) const {
@@ -18731,14 +18926,14 @@ uint32_t ClearFileMetadataRequest::read(::apache::thrift::protocol::TProtocol* i
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->fileIds.clear();
- uint32_t _size774;
- ::apache::thrift::protocol::TType _etype777;
- xfer += iprot->readListBegin(_etype777, _size774);
- this->fileIds.resize(_size774);
- uint32_t _i778;
- for (_i778 = 0; _i778 < _size774; ++_i778)
+ uint32_t _size778;
+ ::apache::thrift::protocol::TType _etype781;
+ xfer += iprot->readListBegin(_etype781, _size778);
+ this->fileIds.resize(_size778);
+ uint32_t _i782;
+ for (_i782 = 0; _i782 < _size778; ++_i782)
{
- xfer += iprot->readI64(this->fileIds[_i778]);
+ xfer += iprot->readI64(this->fileIds[_i782]);
}
xfer += iprot->readListEnd();
}
@@ -18769,10 +18964,10 @@ uint32_t ClearFileMetadataRequest::write(::apache::thrift::protocol::TProtocol*
xfer += oprot->writeFieldBegin("fileIds", ::apache::thrift::protocol::T_LIST, 1);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, static_cast<uint32_t>(this->fileIds.size()));
- std::vector<int64_t> ::const_iterator _iter779;
- for (_iter779 = this->fileIds.begin(); _iter779 != this->fileIds.end(); ++_iter779)
+ std::vector<int64_t> ::const_iterator _iter783;
+ for (_iter783 = this->fileIds.begin(); _iter783 != this->fileIds.end(); ++_iter783)
{
- xfer += oprot->writeI64((*_iter779));
+ xfer += oprot->writeI64((*_iter783));
}
xfer += oprot->writeListEnd();
}
@@ -18788,11 +18983,11 @@ void swap(ClearFileMetadataRequest &a, ClearFileMetadataRequest &b) {
swap(a.fileIds, b.fileIds);
}
-ClearFileMetadataRequest::ClearFileMetadataRequest(const ClearFileMetadataRequest& other780) {
- fileIds = other780.fileIds;
+ClearFileMetadataRequest::ClearFileMetadataRequest(const ClearFileMetadataRequest& other784) {
+ fileIds = other784.fileIds;
}
-ClearFileMetadataRequest& ClearFileMetadataRequest::operator=(const ClearFileMetadataRequest& other781) {
- fileIds = other781.fileIds;
+ClearFileMetadataRequest& ClearFileMetadataRequest::operator=(const ClearFileMetadataRequest& other785) {
+ fileIds = other785.fileIds;
return *this;
}
void ClearFileMetadataRequest::printTo(std::ostream& out) const {
@@ -18874,11 +19069,11 @@ void swap(CacheFileMetadataResult &a, CacheFileMetadataResult &b) {
swap(a.isSupported, b.isSupported);
}
-CacheFileMetadataResult::CacheFileMetadataResult(const CacheFileMetadataResult& other782) {
- isSupported = other782.isSupported;
+CacheFileMetadataResult::CacheFileMetadataResult(const CacheFileMetadataResult& other786) {
+ isSupported = other786.isSupported;
}
-CacheFileMetadataResult& CacheFileMetadataResult::operator=(const CacheFileMetadataResult& other783) {
- isSupported = other783.isSupported;
+CacheFileMetadataResult& CacheFileMetadataResult::operator=(const CacheFileMetadataResult& other787) {
+ isSupported = other787.isSupported;
return *this;
}
void CacheFileMetadataResult::printTo(std::ostream& out) const {
@@ -19019,19 +19214,19 @@ void swap(CacheFileMetadataRequest &a, CacheFileMetadataRequest &b) {
swap(a.__isset, b.__isset);
}
-CacheFileMetadataRequest::CacheFileMetadataRequest(const CacheFileMetadataRequest& other784) {
- dbName = other784.dbName;
- tblName = other784.tblName;
- partName = other784.partName;
- isAllParts = other784.isAllParts;
- __isset = other784.__isset;
+CacheFileMetadataRequest::CacheFileMetadataRequest(const CacheFileMetadataRequest& other788) {
+ dbName = other788.dbName;
+ tblName = other788.tblName;
+ partName = other788.partName;
+ isAllParts = other788.isAllParts;
+ __isset = other788.__isset;
}
-CacheFileMetadataRequest& CacheFileMetadataRequest::operator=(const CacheFileMetadataRequest& other785) {
- dbName = other785.dbName;
- tblName = other785.tblName;
- partName = other785.partName;
- isAllParts = other785.isAllParts;
- __isset = other785.__isset;
+CacheFileMetadataRequest& CacheFileMetadataRequest::operator=(const CacheFileMetadataRequest& other789) {
+ dbName = other789.dbName;
+ tblName = other789.tblName;
+ partName = other789.partName;
+ isAllParts = other789.isAllParts;
+ __isset = other789.__isset;
return *this;
}
void CacheFileMetadataRequest::printTo(std::ostream& out) const {
@@ -19079,14 +19274,14 @@ uint32_t GetAllFunctionsResponse::read(::apache::thrift::protocol::TProtocol* ip
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->functions.clear();
- uint32_t _size786;
- ::apache::thrift::protocol::TType _etype789;
- xfer += iprot->readListBegin(_etype789, _size786);
- this->functions.resize(_size786);
- uint32_t _i790;
- for (_i790 = 0; _i790 < _size786; ++_i790)
+ uint32_t _size790;
+ ::apache::thrift::protocol::TType _etype793;
+ xfer += iprot->readListBegin(_etype793, _size790);
+ this->functions.resize(_size790);
+ uint32_t _i794;
+ for (_i794 = 0; _i794 < _size790; ++_i794)
{
- xfer += this->functions[_i790].read(iprot);
+ xfer += this->functions[_i794].read(iprot);
}
xfer += iprot->readListEnd();
}
@@ -19116,10 +19311,10 @@ uint32_t GetAllFunctionsResponse::write(::apache::thrift::protocol::TProtocol* o
xfer += oprot->writeFieldBegin("functions", ::apache::thrift::protocol::T_LIST, 1);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(this->functions.size()));
- std::vector<Function> ::const_iterator _iter791;
- for (_iter791 = this->functions.begin(); _iter791 != this->functions.end(); ++_iter791)
+ std::vector<Function> ::const_iterator _iter795;
+ for (_iter795 = this->functions.begin(); _iter795 != this->functions.end(); ++_iter795)
{
- xfer += (*_iter791).write(oprot);
+ xfer += (*_iter795).write(oprot);
}
xfer += oprot->writeListEnd();
}
@@ -19136,13 +19331,13 @@ void swap(GetAllFunctionsResponse &a, GetAllFunctionsResponse &b) {
swap(a.__isset, b.__isset);
}
-GetAllFunctionsResponse::GetAllFunctionsResponse(const GetAllFunctionsResponse& other792) {
- functions = other792.functions;
- __isset = other792.__isset;
+GetAllFunctionsResponse::GetAllFunctionsResponse(const GetAllFunctionsResponse& other796) {
+ functions = other796.functions;
+ __isset = other796.__isset;
}
-GetAllFunctionsResponse& GetAllFunctionsResponse::operator=(const GetAllFunctionsResponse& other793) {
- functions = other793.functions;
- __isset = other793.__isset;
+GetAllFunctionsResponse& GetAllFunctionsResponse::operator=(const GetAllFunctionsResponse& other797) {
+ functions = other797.functions;
+ __isset = other797.__isset;
return *this;
}
void GetAllFunctionsResponse::printTo(std::ostream& out) const {
@@ -19187,16 +19382,16 @@ uint32_t ClientCapabilities::read(::apache::thrift::protocol::TProtocol* iprot)
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->values.clear();
- uint32_t _size794;
- ::apache::thrift::protocol::TType _etype797;
- xfer += iprot->readListBegin(_etype797, _size794);
- this->values.resize(_size794);
- uint32_t _i798;
- for (_i798 = 0; _i798 < _size794; ++_i798)
+ uint32_t _size798;
+ ::apache::thrift::protocol::TType _etype801;
+ xfer += iprot->readListBegin(_etype801, _size798);
+ this->values.resize(_size798);
+ uint32_t _i802;
+ for (_i802 = 0; _i802 < _size798; ++_i802)
{
- int32_t ecast799;
- xfer += iprot->readI32(ecast799);
- this->values[_i798] = (ClientCapability::type)ecast799;
+ int32_t ecast803;
+ xfer += iprot->readI32(ecast803);
+ this->values[_i802] = (ClientCapability::type)ecast803;
}
xfer += iprot->readListEnd();
}
@@ -19227,10 +19422,10 @@ uint32_t ClientCapabilities::write(::apache::thrift::protocol::TProtocol* oprot)
xfer += oprot->writeFieldBegin("values", ::apache::thrift::protocol::T_LIST, 1);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I32, static_cast<uint32_t>(this->values.size()));
- std::vector<ClientCapability::type> ::const_iterator _iter800;
- for (_iter800 = this->values.begin(); _iter800 != this->values.end(); ++_iter800)
+ std::vector<ClientCapability::type> ::const_iterator _iter804;
+ for (_iter804 = this->values.begin(); _iter804 != this->values.end(); ++_iter804)
{
- xfer += oprot->writeI32((int32_t)(*_iter800));
+ xfer += oprot->writeI32((int32_t)(*_iter804));
}
xfer += oprot->writeListEnd();
}
@@ -19246,11 +19441,11 @@ void swap(ClientCapabilities &a, ClientCapabilities &b) {
swap(a.values, b.values);
}
-ClientCapabilities::ClientCapabilities(const ClientCapabilities& other801) {
- values = other801.values;
+ClientCapabilities::ClientCapabilities(const ClientCapabilities& other805) {
+ values = other805.values;
}
-ClientCapabilities& ClientCapabilities::operator=(const ClientCapabilities& other802) {
- values = other802.values;
+ClientCapabilities& ClientCapabilities::operator=(const ClientCapabilities& other806) {
+ values = other806.values;
return *this;
}
void ClientCapabilities::printTo(std::ostream& out) const {
@@ -19372,17 +19567,17 @@ void swap(GetTableRequest &a, GetTableRequest &b) {
swap(a.__isset, b.__isset);
}
-GetTableRequest::GetTableRequest(const GetTableRequest& other803) {
- dbName = other803.dbName;
- tblName = other803.tblName;
- capabilities = other803.capabilities;
- __isset = other803.__isset;
+GetTableRequest::GetTableRequest(const GetTableRequest& other807) {
+ dbName = other807.dbName;
+ tblName = other807.tblName;
+ capabilities = other807.capabilities;
+ __isset = other807.__isset;
}
-GetTableRequest& GetTableRequest::operator=(const GetTableRequest& other804) {
- dbName = other804.dbName;
- tblName = other804.tblName;
- capabilities = other804.capabilities;
- __isset = other804.__isset;
+GetTableRequest& GetTableRequest::operator=(const GetTableRequest& other808) {
+ dbName = other808.dbName;
+ tblName = other808.tblName;
+ capabilities = other808.capabilities;
+ __isset = other808.__isset;
return *this;
}
void GetTableRequest::printTo(std::ostream& out) const {
@@ -19466,11 +19661,11 @@ void swap(GetTableResult &a, GetTableResult &b) {
swap(a.table, b.table);
}
-GetTableResult::GetTableResult(const GetTableResult& other805) {
- table = other805.table;
+GetTableResult::GetTableResult(const GetTableResult& other809) {
+ table = other809.table;
}
-GetTableResult& GetTableResult::operator=(const GetTableResult& other806) {
- table = other806.table;
+GetTableResult& GetTableResult::operator=(const GetTableResult& other810) {
+ table = other810.table;
return *this;
}
void GetTableResult::printTo(std::ostream& out) const {
@@ -19533,14 +19728,14 @@ uint32_t GetTablesRequest::read(::apache::thrift::protocol::TProtocol* iprot) {
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->tblNames.clear();
- uint32_t _size807;
- ::apache::thrift::protocol::TType _etype810;
- xfer += iprot->readListBegin(_etype810, _size807);
- this->tblNames.resize(_size807);
- uint32_t _i811;
- for (_i811 = 0; _i811 < _size807; ++_i811)
+ uint32_t _size811;
+ ::apache::thrift::protocol::TType _etype814;
+ xfer += iprot->readListBegin(_etype814, _size811);
+ this->tblNames.resize(_size811);
+ uint32_t _i815;
+ for (_i815 = 0; _i815 < _size811; ++_i815)
{
- xfer += iprot->readString(this->tblNames[_i811]);
+ xfer += iprot->readString(this->tblNames[_i815]);
}
xfer += iprot->readListEnd();
}
@@ -19584,10 +19779,10 @@ uint32_t GetTablesRequest::write(::apache::thrift::protocol::TProtocol* oprot) c
xfer += oprot->writeFieldBegin("tblNames", ::apache::thrift::protocol::T_LIST, 2);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->tblNames.size()));
- std::vector<std::string> ::const_iterator _iter812;
- for (_iter812 = this->tblNames.begin(); _iter812 != this->tblNames.end(); ++_iter812)
+ std::vector<std::string> ::const_iterator _iter816;
+ for (_iter816 = this->tblNames.begin(); _iter816 != this->tblNames.end(); ++_iter816)
{
- xfer += oprot->writeString((*_iter812));
+ xfer += oprot->writeString((*_iter816));
}
xfer += oprot->writeListEnd();
}
@@ -19611,17 +19806,17 @@ void swap(GetTablesRequest &a, GetTablesRequest &b) {
swap(a.__isset, b.__isset);
}
-GetTablesRequest::GetTablesRequest(const GetTablesRequest& other813) {
- dbName = other813.dbName;
- tblNames = other813.tblNames;
- capabilities = other813.capabilities;
- __isset = other813.__isset;
+GetTablesRequest::GetTablesRequest(const GetTablesRequest& other817) {
+ dbName = other817.dbName;
+ tblNames = other817.tblNames;
+ capabilities = other817.capabilities;
+ __isset = other817.__isset;
}
-GetTablesRequest& GetTablesRequest::operator=(const GetTablesRequest& other814) {
- dbName = other814.dbName;
- tblNames = other814.tblNames;
- capabilities = other814.capabilities;
- __isset = other814.__isset;
+GetTablesRequest& GetTablesRequest::operator=(const GetTablesRequest& other818) {
+ dbName = other818.dbName;
+ tblNames = other818.tblNames;
+ capabilities = other818.capabilities;
+ __isset = other818.__isset;
return *this;
}
void GetTablesRequest::printTo(std::ostream& out) const {
@@ -19668,14 +19863,14 @@ uint32_t GetTablesResult::read(::apache::thrift::protocol::TProtocol* iprot) {
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->tables.clear();
- uint32_t _size815;
- ::apache::thrift::protocol::TType _etype818;
- xfer += iprot->readListBegin(_etype818, _size815);
- this->tables.resize(_size815);
- uint32_t _i819;
- for (_i819 = 0; _i819 < _size815; ++_i819)
+ uint32_t _size819;
+ ::apache::thrift::protocol::TType _etype822;
+ xfer += iprot->readListBegin(_etype822, _size819);
+ this->tables.resize(_size819);
+ uint32_t _i823;
+ for (_i823 = 0; _i823 < _size819; ++_i823)
{
- xfer += this->tables[_i819].read(iprot);
+ xfer += this->tables[_i823].read(iprot);
}
xfer += iprot->readListEnd();
}
@@ -19706,10 +19901,10 @@ uint32_t GetTablesResult::write(::apache::thrift::protocol::TProtocol* oprot) co
xfer += oprot->writeFieldBegin("tables", ::apache::thrift::protocol::T_LIST, 1);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(this->tables.size()));
- std::vector<Table> ::const_iterator _iter820;
- for (_iter820 = this->tables.begin(); _iter820 != this->tables.end(); ++_iter820)
+ std::vector<Table> ::const_iterator _iter824;
+ for (_iter824 = this->tables.begin(); _iter824 != this->tables.end(); ++_iter824)
{
- xfer += (*_iter820).write(oprot);
+ xfer += (*_iter824).write(oprot);
}
xfer += oprot->writeListEnd();
}
@@ -19725,11 +19920,11 @@ void swap(GetTablesResult &a, GetTablesResult &b) {
swap(a.tables, b.tables);
}
-GetTablesResult::GetTablesResult(const GetTablesResult& other821) {
- tables = other821.tables;
+GetTablesResult::GetTablesResult(const GetTablesResult& other825) {
+ tables = other825.tables;
}
-GetTablesResult& GetTablesResult::operator=(const GetTablesResult& other822) {
- tables = other822.tables;
+GetTablesResult& GetTablesResult::operator=(const GetTablesResult& other826) {
+ tables = other826.tables;
return *this;
}
void GetTablesResult::printTo(std::ostream& out) const {
@@ -19831,13 +20026,13 @@ void swap(CmRecycleRequest &a, CmRecycleRequest &b) {
swap(a.purge, b.purge);
}
-CmRecycleRequest::CmRecycleRequest(const CmRecycleRequest& other823) {
- dataPath = other823.dataPath;
- purge = other823.purge;
+CmRecycleRequest::CmRecycleRequest(const CmRecycleRequest& other827) {
+ dataPath = other827.dataPath;
+ purge = other827.purge;
}
-CmRecycleRequest& CmRecycleRequest::operator=(const CmRecycleRequest& other824) {
- dataPath = other824.dataPath;
- purge = other824.purge;
+CmRecycleRequest& CmRecycleRequest::operator=(const CmRecycleRequest& other828) {
+ dataPath = other828.dataPath;
+ purge = other828.purge;
return *this;
}
void CmRecycleRequest::printTo(std::ostream& out) const {
@@ -19897,11 +20092,11 @@ void swap(CmRecycleResponse &a, CmRecycleResponse &b) {
(void) b;
}
-CmRecycleResponse::CmRecycleResponse(const CmRecycleResponse& other825) {
- (void) other825;
+CmRecycleResponse::CmRecycleResponse(const CmRecycleResponse& other829) {
+ (void) other829;
}
-CmRecycleResponse& CmRecycleResponse::operator=(const CmRecycleResponse& other826) {
- (void) other826;
+CmRecycleResponse& CmRecycleResponse::operator=(const CmRecycleResponse& other830) {
+ (void) other830;
return *this;
}
void CmRecycleResponse::printTo(std::ostream& out) const {
@@ -20042,19 +20237,19 @@ void swap(TableMeta &a, TableMeta &b) {
swap(a.__isset, b.__isset);
}
-TableMeta::TableMeta(const TableMeta& other827) {
- dbName = other827.dbName;
- tableName = other827.tableName;
- tableType = other827.tableType;
- comments = other827.comments;
- __isset = other827.__isset;
+TableMeta::TableMeta(const TableMeta& other831) {
+ dbName = other831.dbName;
+ tableName = other831.tableName;
+ tableType = other831.tableType;
+ comments = other831.comments;
+ __isset = other831.__isset;
}
-TableMeta& TableMeta::operator=(const TableMeta& other828) {
- dbName = other828.dbName;
- tableName = other828.tableName;
- tableType = other828.tableType;
- comments = other828.comments;
- __isset = other828.__isset;
+TableMeta& TableMeta::operator=(const TableMeta& other832) {
+ dbName = other832.dbName;
+ tableName = other832.tableName;
+ tableType = other832.tableType;
+ comments = other832.comments;
+ __isset = other832.__isset;
return *this;
}
void TableMeta::printTo(std::ostream& out) const {
@@ -20137,13 +20332,13 @@ void swap(MetaException &a, MetaException &b) {
swap(a.__isset, b.__isset);
}
-MetaException::MetaException(const MetaException& other829) : TException() {
- message = other829.message;
- __isset = other829.__isset;
+MetaException::MetaException(const MetaException& other833) : TException() {
+ message = other833.message;
+ __isset = other833.__isset;
}
-MetaException& MetaException::operator=(const MetaException& other830) {
- message = other830.message;
- __isset = other830.__isset;
+MetaException& MetaException::operator=(const MetaException& other834) {
+ message = other834.message;
+ __isset = other834.__isset;
return *this;
}
void MetaException::printTo(std::ostream& out) const {
@@ -20234,13 +20429,13 @@ void swap(UnknownTableException &a, UnknownTableException &b) {
swap(a.__isset, b.__isset);
}
-UnknownTableException::UnknownTableException(const UnknownTableException& other831) : TException() {
- message = other831.message;
- __isset = other831.__isset;
+UnknownTableException::UnknownTableException(const UnknownTableException& other835) : TException() {
+ message = other835.message;
+ __isset = other835.__isset;
}
-UnknownTableException& UnknownTableException::operator=(const UnknownTableException& other832) {
- message = other832.message;
- __isset = other832.__isset;
+UnknownTableException& UnknownTableException::operator=(const UnknownTableException& other836) {
+ message = other836.message;
+ __isset = other836.__isset;
return *this;
}
void UnknownTableException::printTo(std::ostream& out) const {
@@ -20331,13 +20526,13 @@ void swap(UnknownDBException &a, UnknownDBException &b) {
swap(a.__isset, b.__isset);
}
-UnknownDBException::UnknownDBException(const UnknownDBException& other833) : TException() {
- message = other833.message;
- __isset = other833.__isset;
+UnknownDBException::UnknownDBException(const UnknownDBException& other837) : TException() {
+ message = other837.message;
+ __isset = other837.__isset;
}
-UnknownDBException& UnknownDBException::operator=(const UnknownDBException& other834) {
- message = other834.message;
- __isset = other834.__isset;
+UnknownDBException& UnknownDBException::operator=(const UnknownDBException& other838) {
+ message = other838.message;
+ __isset = other838.__isset;
return *this;
}
void UnknownDBException::printTo(std::ostream& out) const {
@@ -20428,13 +20623,13 @@ void swap(AlreadyExistsException &a, AlreadyExistsException &b) {
swap(a.__isset, b.__isset);
}
-AlreadyExistsException::AlreadyExistsException(const AlreadyExistsException& other835) : TException() {
- message = other835.message;
- __isset = other835.__isset;
+AlreadyExistsException::AlreadyExistsException(const AlreadyExistsException& other839) : TException() {
+ message = other839.message;
+ __isset = other839.__isset;
}
-AlreadyExistsException& AlreadyExistsException::operator=(const AlreadyExistsException& other836) {
- message = other836.message;
- __isset = other836.__isset;
+AlreadyExistsException& AlreadyExistsException::operator=(const AlreadyExistsException& other840) {
+ message = other840.message;
+ __isset = other840.__isset;
return *this;
}
void AlreadyExistsException::printTo(std::ostream& out) const {
@@ -20525,13 +20720,13 @@ void swap(InvalidPartitionException &a, InvalidPartitionException &b) {
swap(a.__isset, b.__isset);
}
-InvalidPartitionException::InvalidPartitionException(const InvalidPartitionException& other837) : TException() {
- message = other837.message;
- __isset = other837.__isset;
+InvalidPartitionException::InvalidPartitionException(const InvalidPartitionException& other841) : TException() {
+ message = other841.message;
+ __isset = other841.__isset;
}
-InvalidPartitionException& InvalidPartitionException::operator=(const InvalidPartitionException& other838) {
- message = other838.message;
- __isset = other838.__isset;
+InvalidPartitionException& InvalidPartitionException::operator=(const InvalidPartitionException& other842) {
+ message = other842.message;
+ __isset = other842.__isset;
return *this;
}
void InvalidPartitionException::printTo(std::ostream& out) const {
@@ -20622,13 +20817,13 @@ void swap(UnknownPartitionException &a, UnknownPartitionException &b) {
swap(a.__isset, b.__isset);
}
-UnknownPartitionException::UnknownPartitionException(const UnknownPartitionException& other839) : TException() {
- message = other839.message;
- __isset = other839.__isset;
+UnknownPartitionException::UnknownPartitionException(const UnknownPartitionException& other843) : TException() {
+ message = other843.message;
+ __isset = other843.__isset;
}
-UnknownPartitionException& UnknownPartitionException::operator=(const UnknownPartitionException& other840) {
- message = other840.message;
- __isset = other840.__isset;
+UnknownPartitionException& UnknownPartitionException::operator=(const UnknownPartitionException& other844) {
+ message = other844.message;
+ __isset = other844.__isset;
return *this;
}
void UnknownPartitionException::printTo(std::ostream& out) const {
@@ -20719,13 +20914,13 @@ void swap(InvalidObjectException &a, InvalidObjectException &b) {
swap(a.__isset, b.__isset);
}
-InvalidObjectException::InvalidObjectException(const InvalidObjectException& other841) : TException() {
- message = other841.message;
- __isset = other841.__isset;
+InvalidObjectException::InvalidObjectException(const InvalidObjectException& other845) : TException() {
+ message = other845.message;
+ __isset = other845.__isset;
}
-InvalidObjectException& InvalidObjectException::operator=(const InvalidObjectException& other842) {
- message = other842.message;
- __isset = other842.__isset;
+InvalidObjectException& InvalidObjectException::operator=(const InvalidObjectException& other846) {
+ message = other846.message;
+ __isset = other846.__isset;
return *this;
}
void InvalidObjectException::printTo(std::ostream& out) const {
@@ -20816,13 +21011,13 @@ void swap(NoSuchObjectException &a, NoSuchObjectException &b) {
swap(a.__isset, b.__isset);
}
-NoSuchObjectException::NoSuchObjectException(const NoSuchObjectException& other843) : TException() {
- message = other843.message;
- __isset = other843.__isset;
+NoSuchObjectException::NoSuchObjectException(const NoSuchObjectException& other847) : TException() {
+ message = other847.message;
+ __isset = other847.__isset;
}
-NoSuchObjectException& NoSuchObjectException::operator=(const NoSuchObjectException& other844) {
- message = other844.message;
- __isset = other844.__isset;
+NoSuchObjectException& NoSuchObjectException::operator=(const NoSuchObjectException& other848) {
+ message = other848.message;
+ __isset = other848.__isset;
return *this;
}
void NoSuchObjectException::printTo(std::ostream& out) const {
@@ -20913,13 +21108,13 @@ void swap(IndexAlreadyExistsException &a, IndexAlreadyExistsException &b) {
swap(a.__isset, b.__isset);
}
-IndexAlreadyExistsException::IndexAlreadyExistsException(const IndexAlreadyExistsException& other845) : TException() {
- message = other845.message;
- __isset = other845.__isset;
+IndexAlreadyExistsException::IndexAlreadyExistsException(const IndexAlreadyExistsException& other849) : TException() {
+ message = other849.message;
+ __isset = other849.__isset;
}
-IndexAlreadyExistsException& IndexAlreadyExistsException::operator=(const IndexAlreadyExistsException& other846) {
- message = other846.message;
- __isset = other846.__isset;
+IndexAlreadyExistsException& IndexAlreadyExistsException::operator=(const IndexAlreadyExistsException& other850) {
+ message = other850.message;
+ __isset = other850.__isset;
return *this;
}
void IndexAlreadyExistsException::printTo(std::ostream& out) const {
@@ -21010,13 +21205,13 @@ void swap(InvalidOperationException &a, InvalidOperationException &b) {
swap(a.__isset, b.__isset);
}
-InvalidOperationException::InvalidOperationException(const InvalidOperationException& other847) : TException() {
- message = other847.message;
- __isset = other847.__isset;
+InvalidOperationException::InvalidOperationException(const InvalidOperationException& other851) : TException() {
+ message = other851.message;
+ __isset = other851.__isset;
}
-InvalidOperationException& InvalidOperationException::operator=(const InvalidOperationException& other848) {
- message = other848.message;
- __isset = other848.__isset;
+InvalidOperationException& InvalidOperationException::operator=(const InvalidOperationException& other852) {
+ message = other852.message;
+ __isset = other852.__isset;
return *this;
}
void InvalidOperationException::printTo(std::ostream& out) const {
@@ -21107,13 +21302,13 @@ void swap(ConfigValSecurityException &a, ConfigValSecurityException &b) {
swap(a.__isset, b.__isset);
}
-ConfigValSecurityException::ConfigValSecurityException(const ConfigValSecurityException& other849) : TException() {
- message = other849.message;
- __isset = other849.__isset;
+ConfigValSecurityException::ConfigValSecurityException(const ConfigValSecurityException& other853) : TException() {
+ message = other853.message;
+ __isset = other853.__isset;
}
-ConfigValSecurityException& ConfigValSecurityException::operator=(const ConfigValSecurityException& other850) {
- message = other850.message;
- __isset = other850.__isset;
+ConfigValSecurityException& ConfigValSecurityException::operator=(const ConfigValSecurityException& other854) {
+ message = other854.message;
+ __isset = other854.__isset;
return *this;
}
void ConfigValSecurityException::printTo(std::ostream& out) const {
@@ -21204,13 +21399,13 @@ void swap(InvalidInputException &a, InvalidInputException &b) {
swap(a.__isset, b.__isset);
}
-InvalidInputException::InvalidInputException(const InvalidInputException& other851) : TException() {
- message = other851.message;
- __isset = other851.__isset;
+InvalidInputException::InvalidInputException(const InvalidInputException& other855) : TException() {
+ message = other855.message;
+ __isset = other855.__isset;
}
-InvalidInputException& InvalidInputException::operator=(const InvalidInputException& other852) {
- message = other852.message;
- __isset = other852.__isset;
+InvalidInputException& InvalidInputException::operator=(const InvalidInputException& other856) {
+ message = other856.message;
+ __isset = other856.__isset;
return *this;
}
void InvalidInputException::printTo(std::ostream& out) const {
@@ -21301,13 +21496,13 @@ void swap(NoSuchTxnException &a, NoSuchTxnException &b) {
swap(a.__isset, b.__isset);
}
-NoSuchTxnException::NoSuchTxnException(const NoSuchTxnException& other853) : TException() {
- message = other853.message;
- __isset = other853.__isset;
+NoSuchTxnException::NoSuchTxnException(const NoSuchTxnException& other857) : TException() {
+ message = other857.message;
+ __isset = other857.__isset;
}
-NoSuchTxnException& NoSuchTxnException::operator=(const NoSuchTxnException& other854) {
- message = other854.message;
- __isset = other854.__isset;
+NoSuchTxnException& NoSuchTxnException::operator=(const NoSuchTxnException& other858) {
+ message = other858.message;
+ __isset = other858.__isset;
return *this;
}
void NoSuchTxnException::printTo(std::ostream& out) const {
@@ -21398,13 +21593,13 @@ void swap(TxnAbortedException &a, TxnAbortedException &b) {
swap(a.__isset, b.__isset);
}
-TxnAbortedException::TxnAbortedException(const TxnAbortedException& other855) : TException() {
- message = other855.message;
- __isset = other855.__isset;
+TxnAbortedException::TxnAbortedException(const TxnAbortedException& other859) : TException() {
+ message = other859.message;
+ __isset = other859.__isset;
}
-TxnAbortedException& TxnAbortedException::operator=(const TxnAbortedException& other856) {
- message = other856.message;
- __isset = other856.__isset;
+TxnAbortedException& TxnAbortedException::operator=(const TxnAbortedException& other860) {
+ message = other860.message;
+ __isset = other860.__isset;
return *this;
}
void TxnAbortedException::printTo(std::ostream& out) const {
@@ -21495,13 +21690,13 @@ void swap(TxnOpenException &a, TxnOpenException &b) {
swap(a.__isset, b.__isset);
}
-TxnOpenException::TxnOpenException(const TxnOpenException& other857) : TException() {
- message = other857.message;
- __isset = other857.__isset;
+TxnOpenException::TxnOpenException(const TxnOpenException& other861) : TException() {
+ message = other861.message;
+ __isset = other861.__isset;
}
-TxnOpenException& TxnOpenException::operator=(const TxnOpenException& other858) {
- message = other858.message;
- __isset = other858.__isset;
+TxnOpenException& TxnOpenException::operator=(const TxnOpenException& other862) {
+ message = other862.message;
+ __isset = other862.__isset;
return *this;
}
void TxnOpenException::printTo(std::ostream& out) const {
@@ -21592,13 +21787,13 @@ void swap(NoSuchLockException &a, NoSuchLockException &b) {
swap(a.__isset, b.__isset);
}
-NoSuchLockException::NoSuchLockException(const NoSuchLockException& other859) : TException() {
- message = other859.message;
- __isset = other859.__isset;
+NoSuchLockException::NoSuchLockException(const NoSuchLockException& other863) : TException() {
+ message = other863.message;
+ __isset = other863.__isset;
}
-NoSuchLockException& NoSuchLockException::operator=(const NoSuchLockException& other860) {
- message = other860.message;
- __isset = other860.__isset;
+NoSuchLockException& NoSuchLockException::operator=(const NoSuchLockException& other864) {
+ message = other864.message;
+ __isset = other864.__isset;
return *this;
}
void NoSuchLockException::printTo(std::ostream& out) const {
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
index 2bba534..d5963f3 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
@@ -380,6 +380,10 @@ class NotificationEventResponse;
class CurrentNotificationEventId;
+class NotificationEventsCountRequest;
+
+class NotificationEventsCountResponse;
+
class InsertEventRequestData;
class FireEventRequestData;
@@ -6999,6 +7003,91 @@ inline std::ostream& operator<<(std::ostream& out, const CurrentNotificationEven
return out;
}
+
+class NotificationEventsCountRequest {
+ public:
+
+ NotificationEventsCountRequest(const NotificationEventsCountRequest&);
+ NotificationEventsCountRequest& operator=(const NotificationEventsCountRequest&);
+ NotificationEventsCountRequest() : fromEventId(0), dbName() {
+ }
+
+ virtual ~NotificationEventsCountRequest() throw();
+ int64_t fromEventId;
+ std::string dbName;
+
+ void __set_fromEventId(const int64_t val);
+
+ void __set_dbName(const std::string& val);
+
+ bool operator == (const NotificationEventsCountRequest & rhs) const
+ {
+ if (!(fromEventId == rhs.fromEventId))
+ return false;
+ if (!(dbName == rhs.dbName))
+ return false;
+ return true;
+ }
+ bool operator != (const NotificationEventsCountRequest &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const NotificationEventsCountRequest & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+};
+
+void swap(NotificationEventsCountRequest &a, NotificationEventsCountRequest &b);
+
+inline std::ostream& operator<<(std::ostream& out, const NotificationEventsCountRequest& obj)
+{
+ obj.printTo(out);
+ return out;
+}
+
+
+class NotificationEventsCountResponse {
+ public:
+
+ NotificationEventsCountResponse(const NotificationEventsCountResponse&);
+ NotificationEventsCountResponse& operator=(const NotificationEventsCountResponse&);
+ NotificationEventsCountResponse() : eventsCount(0) {
+ }
+
+ virtual ~NotificationEventsCountResponse() throw();
+ int64_t eventsCount;
+
+ void __set_eventsCount(const int64_t val);
+
+ bool operator == (const NotificationEventsCountResponse & rhs) const
+ {
+ if (!(eventsCount == rhs.eventsCount))
+ return false;
+ return true;
+ }
+ bool operator != (const NotificationEventsCountResponse &rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator < (const NotificationEventsCountResponse & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+ virtual void printTo(std::ostream& out) const;
+};
+
+void swap(NotificationEventsCountResponse &a, NotificationEventsCountResponse &b);
+
+inline std::ostream& operator<<(std::ostream& out, const NotificationEventsCountResponse& obj)
+{
+ obj.printTo(out);
+ return out;
+}
+
typedef struct _InsertEventRequestData__isset {
_InsertEventRequestData__isset() : replace(false), filesAddedChecksum(false) {}
bool replace :1;
[11/50] [abbrv] hive git commit: HIVE-17375: stddev_samp,
var_samp standard compliance (Zoltan Haindrich, reviewed by Ashutosh Chauhan)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/dd04a92f/ql/src/test/results/clientpositive/vectorization_9.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorization_9.q.out b/ql/src/test/results/clientpositive/vectorization_9.q.out
index 930b476..7af0bbd 100644
--- a/ql/src/test/results/clientpositive/vectorization_9.q.out
+++ b/ql/src/test/results/clientpositive/vectorization_9.q.out
@@ -474,168 +474,168 @@ N6BMOr83ecL NULL 1969-12-31 16:00:08.451 NULL NULL 0 NULL NULL NULL NULL NULL 0.
N6Dh6XreCWb0aA4nmDnFOO NULL 1969-12-31 16:00:02.351 NULL NULL 0 NULL NULL NULL NULL NULL 0.000000 NULL
N8222wByj NULL 1969-12-31 16:00:08.451 NULL NULL 0 NULL NULL NULL NULL NULL 0.000000 NULL
NABd3KhjjaVfcj2Q7SJ46 NULL 1969-12-31 16:00:02.351 NULL NULL 0 NULL NULL NULL NULL NULL 0.000000 NULL
-NULL 15601.0 1969-12-31 15:59:43.919 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:44.07 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:44.179 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:44.394 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:44.477 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:44.568 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:44.571 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:44.708 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:44.782 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:45.816 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:46.114 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:46.82 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:46.953 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:47.134 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:47.406 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:47.511 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:47.616 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:47.975 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:48.052 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:48.299 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:48.429 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:48.552 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:48.679 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:48.943 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:49.331 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:49.896 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:50.345 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:50.66 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:51.104 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:51.265 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:51.413 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:51.596 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:51.637 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:52.076 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:52.311 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:52.326 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:52.357 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:52.587 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:53.038 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:53.583 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:53.584 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:53.635 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:54.024 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:54.116 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:54.334 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:54.342 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:54.454 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:54.583 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:54.994 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:55.411 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:55.847 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:55.989 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:55.998 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:56.068 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:56.338 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:56.806 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:56.858 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:56.913 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:56.97 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:57.215 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:57.261 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:57.28 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:57.386 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:57.524 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:57.678 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:57.729 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:57.932 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:58.134 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:58.279 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:58.343 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:58.752 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.025 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.054 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.108 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.122 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.123 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.182 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.206 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.326 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.396 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.476 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.523 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.545 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.547 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.648 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.699 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.708 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.741 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.88 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.931 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.953 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:01.057 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:01.153 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:01.714 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:02.12 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:02.215 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:02.285 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:02.6 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:02.742 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:02.894 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:02.92 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:02.925 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:03.174 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:03.273 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:03.351 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:03.366 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:03.512 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:03.722 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:04.063 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:04.149 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:04.254 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:04.259 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:04.52 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:04.687 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:04.745 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:04.964 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:05.027 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:05.132 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:05.327 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:05.334 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
+NULL 15601.0 1969-12-31 15:59:43.919 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:44.07 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:44.179 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:44.394 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:44.477 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:44.568 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:44.571 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:44.708 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:44.782 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:45.816 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:46.114 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:46.82 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:46.953 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:47.134 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:47.406 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:47.511 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:47.616 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:47.975 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:48.052 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:48.299 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:48.429 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:48.552 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:48.679 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:48.943 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:49.331 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:49.896 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:50.345 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:50.66 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:51.104 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:51.265 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:51.413 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:51.596 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:51.637 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:52.076 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:52.311 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:52.326 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:52.357 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:52.587 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:53.038 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:53.583 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:53.584 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:53.635 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:54.024 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:54.116 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:54.334 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:54.342 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:54.454 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:54.583 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:54.994 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:55.411 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:55.847 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:55.989 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:55.998 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:56.068 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:56.338 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:56.806 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:56.858 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:56.913 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:56.97 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:57.215 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:57.261 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:57.28 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:57.386 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:57.524 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:57.678 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:57.729 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:57.932 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:58.134 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:58.279 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:58.343 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:58.752 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.025 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.054 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.108 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.122 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.123 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.182 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.206 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.326 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.396 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.476 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.523 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.545 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.547 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.648 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.699 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.708 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.741 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.88 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.931 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.953 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:01.057 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:01.153 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:01.714 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:02.12 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:02.215 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:02.285 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:02.6 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:02.742 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:02.894 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:02.92 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:02.925 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:03.174 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:03.273 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:03.351 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:03.366 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:03.512 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:03.722 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:04.063 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:04.149 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:04.254 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:04.259 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:04.52 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:04.687 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:04.745 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:04.964 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:05.027 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:05.132 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:05.327 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:05.334 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
NULL 15601.0 1969-12-31 16:00:05.617 -9747614.5639 9747614.5639 2 0.0 -0.0 0.0 15601.0 625.8070356964297 -1.439885 0.0
-NULL 15601.0 1969-12-31 16:00:05.83 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:06.051 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:06.692 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:07.844 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:08.176 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:08.252 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:08.368 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:08.607 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:08.868 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:08.948 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:09.357 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:09.473 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:09.582 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:09.697 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:10.045 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:10.132 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:10.173 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:10.259 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:10.649 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:10.738 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:10.898 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:10.957 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:10.983 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:12.205 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:12.498 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:12.848 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:12.853 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:12.948 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:13.029 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:13.183 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:13.503 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:13.801 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:13.955 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:14.452 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:14.565 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:14.733 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:14.747 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:14.903 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:15.39 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:15.805 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:16.076 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
+NULL 15601.0 1969-12-31 16:00:05.83 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:06.051 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:06.692 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:07.844 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:08.176 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:08.252 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:08.368 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:08.607 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:08.868 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:08.948 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:09.357 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:09.473 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:09.582 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:09.697 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:10.045 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:10.132 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:10.173 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:10.259 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:10.649 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:10.738 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:10.898 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:10.957 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:10.983 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:12.205 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:12.498 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:12.848 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:12.853 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:12.948 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:13.029 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:13.183 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:13.503 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:13.801 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:13.955 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:14.452 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:14.565 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:14.733 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:14.747 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:14.903 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:15.39 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:15.805 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:16.076 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
NULL 15601.0 1969-12-31 16:00:16.279 -9747614.5639 9747614.5639 2 0.0 -0.0 0.0 15601.0 625.8070356964297 -1.439885 0.0
Nmt6E360X6dpX58CR2 NULL 1969-12-31 16:00:02.351 NULL NULL 0 NULL NULL NULL NULL NULL 0.000000 NULL
O2U2c43Dx4QtYQ3ynA1CLGI3 NULL 1969-12-31 16:00:15.892 NULL NULL 0 NULL NULL NULL NULL NULL 0.000000 NULL
[41/50] [abbrv] hive git commit: HIVE-14671 : merge master into
hive-14535 (Wei Zheng)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
----------------------------------------------------------------------
diff --cc standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
index 0000000,f0f0a57..71c2cef
mode 000000,100644..100644
--- a/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
+++ b/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
@@@ -1,0 -1,21260 +1,21262 @@@
+ <?php
+ namespace metastore;
+
+ /**
+ * Autogenerated by Thrift Compiler (0.9.3)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+ use Thrift\Base\TBase;
+ use Thrift\Type\TType;
+ use Thrift\Type\TMessageType;
+ use Thrift\Exception\TException;
+ use Thrift\Exception\TProtocolException;
+ use Thrift\Protocol\TProtocol;
+ use Thrift\Protocol\TBinaryProtocolAccelerated;
+ use Thrift\Exception\TApplicationException;
+
+
+ final class HiveObjectType {
+ const GLOBAL = 1;
+ const DATABASE = 2;
+ const TABLE = 3;
+ const PARTITION = 4;
+ const COLUMN = 5;
+ static public $__names = array(
+ 1 => 'GLOBAL',
+ 2 => 'DATABASE',
+ 3 => 'TABLE',
+ 4 => 'PARTITION',
+ 5 => 'COLUMN',
+ );
+ }
+
+ final class PrincipalType {
+ const USER = 1;
+ const ROLE = 2;
+ const GROUP = 3;
+ static public $__names = array(
+ 1 => 'USER',
+ 2 => 'ROLE',
+ 3 => 'GROUP',
+ );
+ }
+
+ final class PartitionEventType {
+ const LOAD_DONE = 1;
+ static public $__names = array(
+ 1 => 'LOAD_DONE',
+ );
+ }
+
+ final class TxnState {
+ const COMMITTED = 1;
+ const ABORTED = 2;
+ const OPEN = 3;
+ static public $__names = array(
+ 1 => 'COMMITTED',
+ 2 => 'ABORTED',
+ 3 => 'OPEN',
+ );
+ }
+
+ final class LockLevel {
+ const DB = 1;
+ const TABLE = 2;
+ const PARTITION = 3;
+ static public $__names = array(
+ 1 => 'DB',
+ 2 => 'TABLE',
+ 3 => 'PARTITION',
+ );
+ }
+
+ final class LockState {
+ const ACQUIRED = 1;
+ const WAITING = 2;
+ const ABORT = 3;
+ const NOT_ACQUIRED = 4;
+ static public $__names = array(
+ 1 => 'ACQUIRED',
+ 2 => 'WAITING',
+ 3 => 'ABORT',
+ 4 => 'NOT_ACQUIRED',
+ );
+ }
+
+ final class LockType {
+ const SHARED_READ = 1;
+ const SHARED_WRITE = 2;
+ const EXCLUSIVE = 3;
+ static public $__names = array(
+ 1 => 'SHARED_READ',
+ 2 => 'SHARED_WRITE',
+ 3 => 'EXCLUSIVE',
+ );
+ }
+
+ final class CompactionType {
+ const MINOR = 1;
+ const MAJOR = 2;
+ static public $__names = array(
+ 1 => 'MINOR',
+ 2 => 'MAJOR',
+ );
+ }
+
+ final class GrantRevokeType {
+ const GRANT = 1;
+ const REVOKE = 2;
+ static public $__names = array(
+ 1 => 'GRANT',
+ 2 => 'REVOKE',
+ );
+ }
+
+ final class DataOperationType {
+ const SELECT = 1;
+ const INSERT = 2;
+ const UPDATE = 3;
+ const DELETE = 4;
+ const UNSET = 5;
+ const NO_TXN = 6;
+ static public $__names = array(
+ 1 => 'SELECT',
+ 2 => 'INSERT',
+ 3 => 'UPDATE',
+ 4 => 'DELETE',
+ 5 => 'UNSET',
+ 6 => 'NO_TXN',
+ );
+ }
+
+ final class EventRequestType {
+ const INSERT = 1;
+ const UPDATE = 2;
+ const DELETE = 3;
+ static public $__names = array(
+ 1 => 'INSERT',
+ 2 => 'UPDATE',
+ 3 => 'DELETE',
+ );
+ }
+
+ final class FunctionType {
+ const JAVA = 1;
+ static public $__names = array(
+ 1 => 'JAVA',
+ );
+ }
+
+ final class ResourceType {
+ const JAR = 1;
+ const FILE = 2;
+ const ARCHIVE = 3;
+ static public $__names = array(
+ 1 => 'JAR',
+ 2 => 'FILE',
+ 3 => 'ARCHIVE',
+ );
+ }
+
+ final class FileMetadataExprType {
+ const ORC_SARG = 1;
+ static public $__names = array(
+ 1 => 'ORC_SARG',
+ );
+ }
+
+ final class ClientCapability {
+ const TEST_CAPABILITY = 1;
++ const INSERT_ONLY_TABLES = 2;
+ static public $__names = array(
+ 1 => 'TEST_CAPABILITY',
++ 2 => 'INSERT_ONLY_TABLES',
+ );
+ }
+
+ class Version {
+ static $_TSPEC;
+
+ /**
+ * @var string
+ */
+ public $version = null;
+ /**
+ * @var string
+ */
+ public $comments = null;
+
+ public function __construct($vals=null) {
+ if (!isset(self::$_TSPEC)) {
+ self::$_TSPEC = array(
+ 1 => array(
+ 'var' => 'version',
+ 'type' => TType::STRING,
+ ),
+ 2 => array(
+ 'var' => 'comments',
+ 'type' => TType::STRING,
+ ),
+ );
+ }
+ if (is_array($vals)) {
+ if (isset($vals['version'])) {
+ $this->version = $vals['version'];
+ }
+ if (isset($vals['comments'])) {
+ $this->comments = $vals['comments'];
+ }
+ }
+ }
+
+ public function getName() {
+ return 'Version';
+ }
+
+ public function read($input)
+ {
+ $xfer = 0;
+ $fname = null;
+ $ftype = 0;
+ $fid = 0;
+ $xfer += $input->readStructBegin($fname);
+ while (true)
+ {
+ $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+ if ($ftype == TType::STOP) {
+ break;
+ }
+ switch ($fid)
+ {
+ case 1:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->version);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 2:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->comments);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ default:
+ $xfer += $input->skip($ftype);
+ break;
+ }
+ $xfer += $input->readFieldEnd();
+ }
+ $xfer += $input->readStructEnd();
+ return $xfer;
+ }
+
+ public function write($output) {
+ $xfer = 0;
+ $xfer += $output->writeStructBegin('Version');
+ if ($this->version !== null) {
+ $xfer += $output->writeFieldBegin('version', TType::STRING, 1);
+ $xfer += $output->writeString($this->version);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->comments !== null) {
+ $xfer += $output->writeFieldBegin('comments', TType::STRING, 2);
+ $xfer += $output->writeString($this->comments);
+ $xfer += $output->writeFieldEnd();
+ }
+ $xfer += $output->writeFieldStop();
+ $xfer += $output->writeStructEnd();
+ return $xfer;
+ }
+
+ }
+
+ class FieldSchema {
+ static $_TSPEC;
+
+ /**
+ * @var string
+ */
+ public $name = null;
+ /**
+ * @var string
+ */
+ public $type = null;
+ /**
+ * @var string
+ */
+ public $comment = null;
+
+ public function __construct($vals=null) {
+ if (!isset(self::$_TSPEC)) {
+ self::$_TSPEC = array(
+ 1 => array(
+ 'var' => 'name',
+ 'type' => TType::STRING,
+ ),
+ 2 => array(
+ 'var' => 'type',
+ 'type' => TType::STRING,
+ ),
+ 3 => array(
+ 'var' => 'comment',
+ 'type' => TType::STRING,
+ ),
+ );
+ }
+ if (is_array($vals)) {
+ if (isset($vals['name'])) {
+ $this->name = $vals['name'];
+ }
+ if (isset($vals['type'])) {
+ $this->type = $vals['type'];
+ }
+ if (isset($vals['comment'])) {
+ $this->comment = $vals['comment'];
+ }
+ }
+ }
+
+ public function getName() {
+ return 'FieldSchema';
+ }
+
+ public function read($input)
+ {
+ $xfer = 0;
+ $fname = null;
+ $ftype = 0;
+ $fid = 0;
+ $xfer += $input->readStructBegin($fname);
+ while (true)
+ {
+ $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+ if ($ftype == TType::STOP) {
+ break;
+ }
+ switch ($fid)
+ {
+ case 1:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->name);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 2:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->type);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 3:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->comment);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ default:
+ $xfer += $input->skip($ftype);
+ break;
+ }
+ $xfer += $input->readFieldEnd();
+ }
+ $xfer += $input->readStructEnd();
+ return $xfer;
+ }
+
+ public function write($output) {
+ $xfer = 0;
+ $xfer += $output->writeStructBegin('FieldSchema');
+ if ($this->name !== null) {
+ $xfer += $output->writeFieldBegin('name', TType::STRING, 1);
+ $xfer += $output->writeString($this->name);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->type !== null) {
+ $xfer += $output->writeFieldBegin('type', TType::STRING, 2);
+ $xfer += $output->writeString($this->type);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->comment !== null) {
+ $xfer += $output->writeFieldBegin('comment', TType::STRING, 3);
+ $xfer += $output->writeString($this->comment);
+ $xfer += $output->writeFieldEnd();
+ }
+ $xfer += $output->writeFieldStop();
+ $xfer += $output->writeStructEnd();
+ return $xfer;
+ }
+
+ }
+
+ class SQLPrimaryKey {
+ static $_TSPEC;
+
+ /**
+ * @var string
+ */
+ public $table_db = null;
+ /**
+ * @var string
+ */
+ public $table_name = null;
+ /**
+ * @var string
+ */
+ public $column_name = null;
+ /**
+ * @var int
+ */
+ public $key_seq = null;
+ /**
+ * @var string
+ */
+ public $pk_name = null;
+ /**
+ * @var bool
+ */
+ public $enable_cstr = null;
+ /**
+ * @var bool
+ */
+ public $validate_cstr = null;
+ /**
+ * @var bool
+ */
+ public $rely_cstr = null;
+
+ public function __construct($vals=null) {
+ if (!isset(self::$_TSPEC)) {
+ self::$_TSPEC = array(
+ 1 => array(
+ 'var' => 'table_db',
+ 'type' => TType::STRING,
+ ),
+ 2 => array(
+ 'var' => 'table_name',
+ 'type' => TType::STRING,
+ ),
+ 3 => array(
+ 'var' => 'column_name',
+ 'type' => TType::STRING,
+ ),
+ 4 => array(
+ 'var' => 'key_seq',
+ 'type' => TType::I32,
+ ),
+ 5 => array(
+ 'var' => 'pk_name',
+ 'type' => TType::STRING,
+ ),
+ 6 => array(
+ 'var' => 'enable_cstr',
+ 'type' => TType::BOOL,
+ ),
+ 7 => array(
+ 'var' => 'validate_cstr',
+ 'type' => TType::BOOL,
+ ),
+ 8 => array(
+ 'var' => 'rely_cstr',
+ 'type' => TType::BOOL,
+ ),
+ );
+ }
+ if (is_array($vals)) {
+ if (isset($vals['table_db'])) {
+ $this->table_db = $vals['table_db'];
+ }
+ if (isset($vals['table_name'])) {
+ $this->table_name = $vals['table_name'];
+ }
+ if (isset($vals['column_name'])) {
+ $this->column_name = $vals['column_name'];
+ }
+ if (isset($vals['key_seq'])) {
+ $this->key_seq = $vals['key_seq'];
+ }
+ if (isset($vals['pk_name'])) {
+ $this->pk_name = $vals['pk_name'];
+ }
+ if (isset($vals['enable_cstr'])) {
+ $this->enable_cstr = $vals['enable_cstr'];
+ }
+ if (isset($vals['validate_cstr'])) {
+ $this->validate_cstr = $vals['validate_cstr'];
+ }
+ if (isset($vals['rely_cstr'])) {
+ $this->rely_cstr = $vals['rely_cstr'];
+ }
+ }
+ }
+
+ public function getName() {
+ return 'SQLPrimaryKey';
+ }
+
+ public function read($input)
+ {
+ $xfer = 0;
+ $fname = null;
+ $ftype = 0;
+ $fid = 0;
+ $xfer += $input->readStructBegin($fname);
+ while (true)
+ {
+ $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+ if ($ftype == TType::STOP) {
+ break;
+ }
+ switch ($fid)
+ {
+ case 1:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->table_db);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 2:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->table_name);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 3:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->column_name);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 4:
+ if ($ftype == TType::I32) {
+ $xfer += $input->readI32($this->key_seq);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 5:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->pk_name);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 6:
+ if ($ftype == TType::BOOL) {
+ $xfer += $input->readBool($this->enable_cstr);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 7:
+ if ($ftype == TType::BOOL) {
+ $xfer += $input->readBool($this->validate_cstr);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 8:
+ if ($ftype == TType::BOOL) {
+ $xfer += $input->readBool($this->rely_cstr);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ default:
+ $xfer += $input->skip($ftype);
+ break;
+ }
+ $xfer += $input->readFieldEnd();
+ }
+ $xfer += $input->readStructEnd();
+ return $xfer;
+ }
+
+ public function write($output) {
+ $xfer = 0;
+ $xfer += $output->writeStructBegin('SQLPrimaryKey');
+ if ($this->table_db !== null) {
+ $xfer += $output->writeFieldBegin('table_db', TType::STRING, 1);
+ $xfer += $output->writeString($this->table_db);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->table_name !== null) {
+ $xfer += $output->writeFieldBegin('table_name', TType::STRING, 2);
+ $xfer += $output->writeString($this->table_name);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->column_name !== null) {
+ $xfer += $output->writeFieldBegin('column_name', TType::STRING, 3);
+ $xfer += $output->writeString($this->column_name);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->key_seq !== null) {
+ $xfer += $output->writeFieldBegin('key_seq', TType::I32, 4);
+ $xfer += $output->writeI32($this->key_seq);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->pk_name !== null) {
+ $xfer += $output->writeFieldBegin('pk_name', TType::STRING, 5);
+ $xfer += $output->writeString($this->pk_name);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->enable_cstr !== null) {
+ $xfer += $output->writeFieldBegin('enable_cstr', TType::BOOL, 6);
+ $xfer += $output->writeBool($this->enable_cstr);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->validate_cstr !== null) {
+ $xfer += $output->writeFieldBegin('validate_cstr', TType::BOOL, 7);
+ $xfer += $output->writeBool($this->validate_cstr);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->rely_cstr !== null) {
+ $xfer += $output->writeFieldBegin('rely_cstr', TType::BOOL, 8);
+ $xfer += $output->writeBool($this->rely_cstr);
+ $xfer += $output->writeFieldEnd();
+ }
+ $xfer += $output->writeFieldStop();
+ $xfer += $output->writeStructEnd();
+ return $xfer;
+ }
+
+ }
+
+ class SQLForeignKey {
+ static $_TSPEC;
+
+ /**
+ * @var string
+ */
+ public $pktable_db = null;
+ /**
+ * @var string
+ */
+ public $pktable_name = null;
+ /**
+ * @var string
+ */
+ public $pkcolumn_name = null;
+ /**
+ * @var string
+ */
+ public $fktable_db = null;
+ /**
+ * @var string
+ */
+ public $fktable_name = null;
+ /**
+ * @var string
+ */
+ public $fkcolumn_name = null;
+ /**
+ * @var int
+ */
+ public $key_seq = null;
+ /**
+ * @var int
+ */
+ public $update_rule = null;
+ /**
+ * @var int
+ */
+ public $delete_rule = null;
+ /**
+ * @var string
+ */
+ public $fk_name = null;
+ /**
+ * @var string
+ */
+ public $pk_name = null;
+ /**
+ * @var bool
+ */
+ public $enable_cstr = null;
+ /**
+ * @var bool
+ */
+ public $validate_cstr = null;
+ /**
+ * @var bool
+ */
+ public $rely_cstr = null;
+
+ public function __construct($vals=null) {
+ if (!isset(self::$_TSPEC)) {
+ self::$_TSPEC = array(
+ 1 => array(
+ 'var' => 'pktable_db',
+ 'type' => TType::STRING,
+ ),
+ 2 => array(
+ 'var' => 'pktable_name',
+ 'type' => TType::STRING,
+ ),
+ 3 => array(
+ 'var' => 'pkcolumn_name',
+ 'type' => TType::STRING,
+ ),
+ 4 => array(
+ 'var' => 'fktable_db',
+ 'type' => TType::STRING,
+ ),
+ 5 => array(
+ 'var' => 'fktable_name',
+ 'type' => TType::STRING,
+ ),
+ 6 => array(
+ 'var' => 'fkcolumn_name',
+ 'type' => TType::STRING,
+ ),
+ 7 => array(
+ 'var' => 'key_seq',
+ 'type' => TType::I32,
+ ),
+ 8 => array(
+ 'var' => 'update_rule',
+ 'type' => TType::I32,
+ ),
+ 9 => array(
+ 'var' => 'delete_rule',
+ 'type' => TType::I32,
+ ),
+ 10 => array(
+ 'var' => 'fk_name',
+ 'type' => TType::STRING,
+ ),
+ 11 => array(
+ 'var' => 'pk_name',
+ 'type' => TType::STRING,
+ ),
+ 12 => array(
+ 'var' => 'enable_cstr',
+ 'type' => TType::BOOL,
+ ),
+ 13 => array(
+ 'var' => 'validate_cstr',
+ 'type' => TType::BOOL,
+ ),
+ 14 => array(
+ 'var' => 'rely_cstr',
+ 'type' => TType::BOOL,
+ ),
+ );
+ }
+ if (is_array($vals)) {
+ if (isset($vals['pktable_db'])) {
+ $this->pktable_db = $vals['pktable_db'];
+ }
+ if (isset($vals['pktable_name'])) {
+ $this->pktable_name = $vals['pktable_name'];
+ }
+ if (isset($vals['pkcolumn_name'])) {
+ $this->pkcolumn_name = $vals['pkcolumn_name'];
+ }
+ if (isset($vals['fktable_db'])) {
+ $this->fktable_db = $vals['fktable_db'];
+ }
+ if (isset($vals['fktable_name'])) {
+ $this->fktable_name = $vals['fktable_name'];
+ }
+ if (isset($vals['fkcolumn_name'])) {
+ $this->fkcolumn_name = $vals['fkcolumn_name'];
+ }
+ if (isset($vals['key_seq'])) {
+ $this->key_seq = $vals['key_seq'];
+ }
+ if (isset($vals['update_rule'])) {
+ $this->update_rule = $vals['update_rule'];
+ }
+ if (isset($vals['delete_rule'])) {
+ $this->delete_rule = $vals['delete_rule'];
+ }
+ if (isset($vals['fk_name'])) {
+ $this->fk_name = $vals['fk_name'];
+ }
+ if (isset($vals['pk_name'])) {
+ $this->pk_name = $vals['pk_name'];
+ }
+ if (isset($vals['enable_cstr'])) {
+ $this->enable_cstr = $vals['enable_cstr'];
+ }
+ if (isset($vals['validate_cstr'])) {
+ $this->validate_cstr = $vals['validate_cstr'];
+ }
+ if (isset($vals['rely_cstr'])) {
+ $this->rely_cstr = $vals['rely_cstr'];
+ }
+ }
+ }
+
+ public function getName() {
+ return 'SQLForeignKey';
+ }
+
+ public function read($input)
+ {
+ $xfer = 0;
+ $fname = null;
+ $ftype = 0;
+ $fid = 0;
+ $xfer += $input->readStructBegin($fname);
+ while (true)
+ {
+ $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+ if ($ftype == TType::STOP) {
+ break;
+ }
+ switch ($fid)
+ {
+ case 1:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->pktable_db);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 2:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->pktable_name);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 3:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->pkcolumn_name);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 4:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->fktable_db);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 5:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->fktable_name);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 6:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->fkcolumn_name);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 7:
+ if ($ftype == TType::I32) {
+ $xfer += $input->readI32($this->key_seq);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 8:
+ if ($ftype == TType::I32) {
+ $xfer += $input->readI32($this->update_rule);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 9:
+ if ($ftype == TType::I32) {
+ $xfer += $input->readI32($this->delete_rule);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 10:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->fk_name);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 11:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->pk_name);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 12:
+ if ($ftype == TType::BOOL) {
+ $xfer += $input->readBool($this->enable_cstr);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 13:
+ if ($ftype == TType::BOOL) {
+ $xfer += $input->readBool($this->validate_cstr);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 14:
+ if ($ftype == TType::BOOL) {
+ $xfer += $input->readBool($this->rely_cstr);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ default:
+ $xfer += $input->skip($ftype);
+ break;
+ }
+ $xfer += $input->readFieldEnd();
+ }
+ $xfer += $input->readStructEnd();
+ return $xfer;
+ }
+
+ public function write($output) {
+ $xfer = 0;
+ $xfer += $output->writeStructBegin('SQLForeignKey');
+ if ($this->pktable_db !== null) {
+ $xfer += $output->writeFieldBegin('pktable_db', TType::STRING, 1);
+ $xfer += $output->writeString($this->pktable_db);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->pktable_name !== null) {
+ $xfer += $output->writeFieldBegin('pktable_name', TType::STRING, 2);
+ $xfer += $output->writeString($this->pktable_name);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->pkcolumn_name !== null) {
+ $xfer += $output->writeFieldBegin('pkcolumn_name', TType::STRING, 3);
+ $xfer += $output->writeString($this->pkcolumn_name);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->fktable_db !== null) {
+ $xfer += $output->writeFieldBegin('fktable_db', TType::STRING, 4);
+ $xfer += $output->writeString($this->fktable_db);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->fktable_name !== null) {
+ $xfer += $output->writeFieldBegin('fktable_name', TType::STRING, 5);
+ $xfer += $output->writeString($this->fktable_name);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->fkcolumn_name !== null) {
+ $xfer += $output->writeFieldBegin('fkcolumn_name', TType::STRING, 6);
+ $xfer += $output->writeString($this->fkcolumn_name);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->key_seq !== null) {
+ $xfer += $output->writeFieldBegin('key_seq', TType::I32, 7);
+ $xfer += $output->writeI32($this->key_seq);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->update_rule !== null) {
+ $xfer += $output->writeFieldBegin('update_rule', TType::I32, 8);
+ $xfer += $output->writeI32($this->update_rule);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->delete_rule !== null) {
+ $xfer += $output->writeFieldBegin('delete_rule', TType::I32, 9);
+ $xfer += $output->writeI32($this->delete_rule);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->fk_name !== null) {
+ $xfer += $output->writeFieldBegin('fk_name', TType::STRING, 10);
+ $xfer += $output->writeString($this->fk_name);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->pk_name !== null) {
+ $xfer += $output->writeFieldBegin('pk_name', TType::STRING, 11);
+ $xfer += $output->writeString($this->pk_name);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->enable_cstr !== null) {
+ $xfer += $output->writeFieldBegin('enable_cstr', TType::BOOL, 12);
+ $xfer += $output->writeBool($this->enable_cstr);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->validate_cstr !== null) {
+ $xfer += $output->writeFieldBegin('validate_cstr', TType::BOOL, 13);
+ $xfer += $output->writeBool($this->validate_cstr);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->rely_cstr !== null) {
+ $xfer += $output->writeFieldBegin('rely_cstr', TType::BOOL, 14);
+ $xfer += $output->writeBool($this->rely_cstr);
+ $xfer += $output->writeFieldEnd();
+ }
+ $xfer += $output->writeFieldStop();
+ $xfer += $output->writeStructEnd();
+ return $xfer;
+ }
+
+ }
+
+ class SQLUniqueConstraint {
+ static $_TSPEC;
+
+ /**
+ * @var string
+ */
+ public $table_db = null;
+ /**
+ * @var string
+ */
+ public $table_name = null;
+ /**
+ * @var string
+ */
+ public $column_name = null;
+ /**
+ * @var int
+ */
+ public $key_seq = null;
+ /**
+ * @var string
+ */
+ public $uk_name = null;
+ /**
+ * @var bool
+ */
+ public $enable_cstr = null;
+ /**
+ * @var bool
+ */
+ public $validate_cstr = null;
+ /**
+ * @var bool
+ */
+ public $rely_cstr = null;
+
+ public function __construct($vals=null) {
+ if (!isset(self::$_TSPEC)) {
+ self::$_TSPEC = array(
+ 1 => array(
+ 'var' => 'table_db',
+ 'type' => TType::STRING,
+ ),
+ 2 => array(
+ 'var' => 'table_name',
+ 'type' => TType::STRING,
+ ),
+ 3 => array(
+ 'var' => 'column_name',
+ 'type' => TType::STRING,
+ ),
+ 4 => array(
+ 'var' => 'key_seq',
+ 'type' => TType::I32,
+ ),
+ 5 => array(
+ 'var' => 'uk_name',
+ 'type' => TType::STRING,
+ ),
+ 6 => array(
+ 'var' => 'enable_cstr',
+ 'type' => TType::BOOL,
+ ),
+ 7 => array(
+ 'var' => 'validate_cstr',
+ 'type' => TType::BOOL,
+ ),
+ 8 => array(
+ 'var' => 'rely_cstr',
+ 'type' => TType::BOOL,
+ ),
+ );
+ }
+ if (is_array($vals)) {
+ if (isset($vals['table_db'])) {
+ $this->table_db = $vals['table_db'];
+ }
+ if (isset($vals['table_name'])) {
+ $this->table_name = $vals['table_name'];
+ }
+ if (isset($vals['column_name'])) {
+ $this->column_name = $vals['column_name'];
+ }
+ if (isset($vals['key_seq'])) {
+ $this->key_seq = $vals['key_seq'];
+ }
+ if (isset($vals['uk_name'])) {
+ $this->uk_name = $vals['uk_name'];
+ }
+ if (isset($vals['enable_cstr'])) {
+ $this->enable_cstr = $vals['enable_cstr'];
+ }
+ if (isset($vals['validate_cstr'])) {
+ $this->validate_cstr = $vals['validate_cstr'];
+ }
+ if (isset($vals['rely_cstr'])) {
+ $this->rely_cstr = $vals['rely_cstr'];
+ }
+ }
+ }
+
+ public function getName() {
+ return 'SQLUniqueConstraint';
+ }
+
+ public function read($input)
+ {
+ $xfer = 0;
+ $fname = null;
+ $ftype = 0;
+ $fid = 0;
+ $xfer += $input->readStructBegin($fname);
+ while (true)
+ {
+ $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+ if ($ftype == TType::STOP) {
+ break;
+ }
+ switch ($fid)
+ {
+ case 1:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->table_db);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 2:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->table_name);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 3:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->column_name);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 4:
+ if ($ftype == TType::I32) {
+ $xfer += $input->readI32($this->key_seq);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 5:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->uk_name);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 6:
+ if ($ftype == TType::BOOL) {
+ $xfer += $input->readBool($this->enable_cstr);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 7:
+ if ($ftype == TType::BOOL) {
+ $xfer += $input->readBool($this->validate_cstr);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 8:
+ if ($ftype == TType::BOOL) {
+ $xfer += $input->readBool($this->rely_cstr);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ default:
+ $xfer += $input->skip($ftype);
+ break;
+ }
+ $xfer += $input->readFieldEnd();
+ }
+ $xfer += $input->readStructEnd();
+ return $xfer;
+ }
+
+ public function write($output) {
+ $xfer = 0;
+ $xfer += $output->writeStructBegin('SQLUniqueConstraint');
+ if ($this->table_db !== null) {
+ $xfer += $output->writeFieldBegin('table_db', TType::STRING, 1);
+ $xfer += $output->writeString($this->table_db);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->table_name !== null) {
+ $xfer += $output->writeFieldBegin('table_name', TType::STRING, 2);
+ $xfer += $output->writeString($this->table_name);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->column_name !== null) {
+ $xfer += $output->writeFieldBegin('column_name', TType::STRING, 3);
+ $xfer += $output->writeString($this->column_name);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->key_seq !== null) {
+ $xfer += $output->writeFieldBegin('key_seq', TType::I32, 4);
+ $xfer += $output->writeI32($this->key_seq);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->uk_name !== null) {
+ $xfer += $output->writeFieldBegin('uk_name', TType::STRING, 5);
+ $xfer += $output->writeString($this->uk_name);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->enable_cstr !== null) {
+ $xfer += $output->writeFieldBegin('enable_cstr', TType::BOOL, 6);
+ $xfer += $output->writeBool($this->enable_cstr);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->validate_cstr !== null) {
+ $xfer += $output->writeFieldBegin('validate_cstr', TType::BOOL, 7);
+ $xfer += $output->writeBool($this->validate_cstr);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->rely_cstr !== null) {
+ $xfer += $output->writeFieldBegin('rely_cstr', TType::BOOL, 8);
+ $xfer += $output->writeBool($this->rely_cstr);
+ $xfer += $output->writeFieldEnd();
+ }
+ $xfer += $output->writeFieldStop();
+ $xfer += $output->writeStructEnd();
+ return $xfer;
+ }
+
+ }
+
+ class SQLNotNullConstraint {
+ static $_TSPEC;
+
+ /**
+ * @var string
+ */
+ public $table_db = null;
+ /**
+ * @var string
+ */
+ public $table_name = null;
+ /**
+ * @var string
+ */
+ public $column_name = null;
+ /**
+ * @var string
+ */
+ public $nn_name = null;
+ /**
+ * @var bool
+ */
+ public $enable_cstr = null;
+ /**
+ * @var bool
+ */
+ public $validate_cstr = null;
+ /**
+ * @var bool
+ */
+ public $rely_cstr = null;
+
+ public function __construct($vals=null) {
+ if (!isset(self::$_TSPEC)) {
+ self::$_TSPEC = array(
+ 1 => array(
+ 'var' => 'table_db',
+ 'type' => TType::STRING,
+ ),
+ 2 => array(
+ 'var' => 'table_name',
+ 'type' => TType::STRING,
+ ),
+ 3 => array(
+ 'var' => 'column_name',
+ 'type' => TType::STRING,
+ ),
+ 4 => array(
+ 'var' => 'nn_name',
+ 'type' => TType::STRING,
+ ),
+ 5 => array(
+ 'var' => 'enable_cstr',
+ 'type' => TType::BOOL,
+ ),
+ 6 => array(
+ 'var' => 'validate_cstr',
+ 'type' => TType::BOOL,
+ ),
+ 7 => array(
+ 'var' => 'rely_cstr',
+ 'type' => TType::BOOL,
+ ),
+ );
+ }
+ if (is_array($vals)) {
+ if (isset($vals['table_db'])) {
+ $this->table_db = $vals['table_db'];
+ }
+ if (isset($vals['table_name'])) {
+ $this->table_name = $vals['table_name'];
+ }
+ if (isset($vals['column_name'])) {
+ $this->column_name = $vals['column_name'];
+ }
+ if (isset($vals['nn_name'])) {
+ $this->nn_name = $vals['nn_name'];
+ }
+ if (isset($vals['enable_cstr'])) {
+ $this->enable_cstr = $vals['enable_cstr'];
+ }
+ if (isset($vals['validate_cstr'])) {
+ $this->validate_cstr = $vals['validate_cstr'];
+ }
+ if (isset($vals['rely_cstr'])) {
+ $this->rely_cstr = $vals['rely_cstr'];
+ }
+ }
+ }
+
+ public function getName() {
+ return 'SQLNotNullConstraint';
+ }
+
+ public function read($input)
+ {
+ $xfer = 0;
+ $fname = null;
+ $ftype = 0;
+ $fid = 0;
+ $xfer += $input->readStructBegin($fname);
+ while (true)
+ {
+ $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+ if ($ftype == TType::STOP) {
+ break;
+ }
+ switch ($fid)
+ {
+ case 1:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->table_db);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 2:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->table_name);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 3:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->column_name);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 4:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->nn_name);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 5:
+ if ($ftype == TType::BOOL) {
+ $xfer += $input->readBool($this->enable_cstr);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 6:
+ if ($ftype == TType::BOOL) {
+ $xfer += $input->readBool($this->validate_cstr);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 7:
+ if ($ftype == TType::BOOL) {
+ $xfer += $input->readBool($this->rely_cstr);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ default:
+ $xfer += $input->skip($ftype);
+ break;
+ }
+ $xfer += $input->readFieldEnd();
+ }
+ $xfer += $input->readStructEnd();
+ return $xfer;
+ }
+
+ public function write($output) {
+ $xfer = 0;
+ $xfer += $output->writeStructBegin('SQLNotNullConstraint');
+ if ($this->table_db !== null) {
+ $xfer += $output->writeFieldBegin('table_db', TType::STRING, 1);
+ $xfer += $output->writeString($this->table_db);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->table_name !== null) {
+ $xfer += $output->writeFieldBegin('table_name', TType::STRING, 2);
+ $xfer += $output->writeString($this->table_name);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->column_name !== null) {
+ $xfer += $output->writeFieldBegin('column_name', TType::STRING, 3);
+ $xfer += $output->writeString($this->column_name);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->nn_name !== null) {
+ $xfer += $output->writeFieldBegin('nn_name', TType::STRING, 4);
+ $xfer += $output->writeString($this->nn_name);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->enable_cstr !== null) {
+ $xfer += $output->writeFieldBegin('enable_cstr', TType::BOOL, 5);
+ $xfer += $output->writeBool($this->enable_cstr);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->validate_cstr !== null) {
+ $xfer += $output->writeFieldBegin('validate_cstr', TType::BOOL, 6);
+ $xfer += $output->writeBool($this->validate_cstr);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->rely_cstr !== null) {
+ $xfer += $output->writeFieldBegin('rely_cstr', TType::BOOL, 7);
+ $xfer += $output->writeBool($this->rely_cstr);
+ $xfer += $output->writeFieldEnd();
+ }
+ $xfer += $output->writeFieldStop();
+ $xfer += $output->writeStructEnd();
+ return $xfer;
+ }
+
+ }
+
+ class Type {
+ static $_TSPEC;
+
+ /**
+ * @var string
+ */
+ public $name = null;
+ /**
+ * @var string
+ */
+ public $type1 = null;
+ /**
+ * @var string
+ */
+ public $type2 = null;
+ /**
+ * @var \metastore\FieldSchema[]
+ */
+ public $fields = null;
+
+ public function __construct($vals=null) {
+ if (!isset(self::$_TSPEC)) {
+ self::$_TSPEC = array(
+ 1 => array(
+ 'var' => 'name',
+ 'type' => TType::STRING,
+ ),
+ 2 => array(
+ 'var' => 'type1',
+ 'type' => TType::STRING,
+ ),
+ 3 => array(
+ 'var' => 'type2',
+ 'type' => TType::STRING,
+ ),
+ 4 => array(
+ 'var' => 'fields',
+ 'type' => TType::LST,
+ 'etype' => TType::STRUCT,
+ 'elem' => array(
+ 'type' => TType::STRUCT,
+ 'class' => '\metastore\FieldSchema',
+ ),
+ ),
+ );
+ }
+ if (is_array($vals)) {
+ if (isset($vals['name'])) {
+ $this->name = $vals['name'];
+ }
+ if (isset($vals['type1'])) {
+ $this->type1 = $vals['type1'];
+ }
+ if (isset($vals['type2'])) {
+ $this->type2 = $vals['type2'];
+ }
+ if (isset($vals['fields'])) {
+ $this->fields = $vals['fields'];
+ }
+ }
+ }
+
+ public function getName() {
+ return 'Type';
+ }
+
+ public function read($input)
+ {
+ $xfer = 0;
+ $fname = null;
+ $ftype = 0;
+ $fid = 0;
+ $xfer += $input->readStructBegin($fname);
+ while (true)
+ {
+ $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+ if ($ftype == TType::STOP) {
+ break;
+ }
+ switch ($fid)
+ {
+ case 1:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->name);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 2:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->type1);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 3:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->type2);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 4:
+ if ($ftype == TType::LST) {
+ $this->fields = array();
+ $_size0 = 0;
+ $_etype3 = 0;
+ $xfer += $input->readListBegin($_etype3, $_size0);
+ for ($_i4 = 0; $_i4 < $_size0; ++$_i4)
+ {
+ $elem5 = null;
+ $elem5 = new \metastore\FieldSchema();
+ $xfer += $elem5->read($input);
+ $this->fields []= $elem5;
+ }
+ $xfer += $input->readListEnd();
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ default:
+ $xfer += $input->skip($ftype);
+ break;
+ }
+ $xfer += $input->readFieldEnd();
+ }
+ $xfer += $input->readStructEnd();
+ return $xfer;
+ }
+
+ public function write($output) {
+ $xfer = 0;
+ $xfer += $output->writeStructBegin('Type');
+ if ($this->name !== null) {
+ $xfer += $output->writeFieldBegin('name', TType::STRING, 1);
+ $xfer += $output->writeString($this->name);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->type1 !== null) {
+ $xfer += $output->writeFieldBegin('type1', TType::STRING, 2);
+ $xfer += $output->writeString($this->type1);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->type2 !== null) {
+ $xfer += $output->writeFieldBegin('type2', TType::STRING, 3);
+ $xfer += $output->writeString($this->type2);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->fields !== null) {
+ if (!is_array($this->fields)) {
+ throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA);
+ }
+ $xfer += $output->writeFieldBegin('fields', TType::LST, 4);
+ {
+ $output->writeListBegin(TType::STRUCT, count($this->fields));
+ {
+ foreach ($this->fields as $iter6)
+ {
+ $xfer += $iter6->write($output);
+ }
+ }
+ $output->writeListEnd();
+ }
+ $xfer += $output->writeFieldEnd();
+ }
+ $xfer += $output->writeFieldStop();
+ $xfer += $output->writeStructEnd();
+ return $xfer;
+ }
+
+ }
+
+ class HiveObjectRef {
+ static $_TSPEC;
+
+ /**
+ * @var int
+ */
+ public $objectType = null;
+ /**
+ * @var string
+ */
+ public $dbName = null;
+ /**
+ * @var string
+ */
+ public $objectName = null;
+ /**
+ * @var string[]
+ */
+ public $partValues = null;
+ /**
+ * @var string
+ */
+ public $columnName = null;
+
+ public function __construct($vals=null) {
+ if (!isset(self::$_TSPEC)) {
+ self::$_TSPEC = array(
+ 1 => array(
+ 'var' => 'objectType',
+ 'type' => TType::I32,
+ ),
+ 2 => array(
+ 'var' => 'dbName',
+ 'type' => TType::STRING,
+ ),
+ 3 => array(
+ 'var' => 'objectName',
+ 'type' => TType::STRING,
+ ),
+ 4 => array(
+ 'var' => 'partValues',
+ 'type' => TType::LST,
+ 'etype' => TType::STRING,
+ 'elem' => array(
+ 'type' => TType::STRING,
+ ),
+ ),
+ 5 => array(
+ 'var' => 'columnName',
+ 'type' => TType::STRING,
+ ),
+ );
+ }
+ if (is_array($vals)) {
+ if (isset($vals['objectType'])) {
+ $this->objectType = $vals['objectType'];
+ }
+ if (isset($vals['dbName'])) {
+ $this->dbName = $vals['dbName'];
+ }
+ if (isset($vals['objectName'])) {
+ $this->objectName = $vals['objectName'];
+ }
+ if (isset($vals['partValues'])) {
+ $this->partValues = $vals['partValues'];
+ }
+ if (isset($vals['columnName'])) {
+ $this->columnName = $vals['columnName'];
+ }
+ }
+ }
+
+ public function getName() {
+ return 'HiveObjectRef';
+ }
+
+ public function read($input)
+ {
+ $xfer = 0;
+ $fname = null;
+ $ftype = 0;
+ $fid = 0;
+ $xfer += $input->readStructBegin($fname);
+ while (true)
+ {
+ $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+ if ($ftype == TType::STOP) {
+ break;
+ }
+ switch ($fid)
+ {
+ case 1:
+ if ($ftype == TType::I32) {
+ $xfer += $input->readI32($this->objectType);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 2:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->dbName);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 3:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->objectName);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 4:
+ if ($ftype == TType::LST) {
+ $this->partValues = array();
+ $_size7 = 0;
+ $_etype10 = 0;
+ $xfer += $input->readListBegin($_etype10, $_size7);
+ for ($_i11 = 0; $_i11 < $_size7; ++$_i11)
+ {
+ $elem12 = null;
+ $xfer += $input->readString($elem12);
+ $this->partValues []= $elem12;
+ }
+ $xfer += $input->readListEnd();
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 5:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->columnName);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ default:
+ $xfer += $input->skip($ftype);
+ break;
+ }
+ $xfer += $input->readFieldEnd();
+ }
+ $xfer += $input->readStructEnd();
+ return $xfer;
+ }
+
+ public function write($output) {
+ $xfer = 0;
+ $xfer += $output->writeStructBegin('HiveObjectRef');
+ if ($this->objectType !== null) {
+ $xfer += $output->writeFieldBegin('objectType', TType::I32, 1);
+ $xfer += $output->writeI32($this->objectType);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->dbName !== null) {
+ $xfer += $output->writeFieldBegin('dbName', TType::STRING, 2);
+ $xfer += $output->writeString($this->dbName);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->objectName !== null) {
+ $xfer += $output->writeFieldBegin('objectName', TType::STRING, 3);
+ $xfer += $output->writeString($this->objectName);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->partValues !== null) {
+ if (!is_array($this->partValues)) {
+ throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA);
+ }
+ $xfer += $output->writeFieldBegin('partValues', TType::LST, 4);
+ {
+ $output->writeListBegin(TType::STRING, count($this->partValues));
+ {
+ foreach ($this->partValues as $iter13)
+ {
+ $xfer += $output->writeString($iter13);
+ }
+ }
+ $output->writeListEnd();
+ }
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->columnName !== null) {
+ $xfer += $output->writeFieldBegin('columnName', TType::STRING, 5);
+ $xfer += $output->writeString($this->columnName);
+ $xfer += $output->writeFieldEnd();
+ }
+ $xfer += $output->writeFieldStop();
+ $xfer += $output->writeStructEnd();
+ return $xfer;
+ }
+
+ }
+
+ class PrivilegeGrantInfo {
+ static $_TSPEC;
+
+ /**
+ * @var string
+ */
+ public $privilege = null;
+ /**
+ * @var int
+ */
+ public $createTime = null;
+ /**
+ * @var string
+ */
+ public $grantor = null;
+ /**
+ * @var int
+ */
+ public $grantorType = null;
+ /**
+ * @var bool
+ */
+ public $grantOption = null;
+
+ public function __construct($vals=null) {
+ if (!isset(self::$_TSPEC)) {
+ self::$_TSPEC = array(
+ 1 => array(
+ 'var' => 'privilege',
+ 'type' => TType::STRING,
+ ),
+ 2 => array(
+ 'var' => 'createTime',
+ 'type' => TType::I32,
+ ),
+ 3 => array(
+ 'var' => 'grantor',
+ 'type' => TType::STRING,
+ ),
+ 4 => array(
+ 'var' => 'grantorType',
+ 'type' => TType::I32,
+ ),
+ 5 => array(
+ 'var' => 'grantOption',
+ 'type' => TType::BOOL,
+ ),
+ );
+ }
+ if (is_array($vals)) {
+ if (isset($vals['privilege'])) {
+ $this->privilege = $vals['privilege'];
+ }
+ if (isset($vals['createTime'])) {
+ $this->createTime = $vals['createTime'];
+ }
+ if (isset($vals['grantor'])) {
+ $this->grantor = $vals['grantor'];
+ }
+ if (isset($vals['grantorType'])) {
+ $this->grantorType = $vals['grantorType'];
+ }
+ if (isset($vals['grantOption'])) {
+ $this->grantOption = $vals['grantOption'];
+ }
+ }
+ }
+
+ public function getName() {
+ return 'PrivilegeGrantInfo';
+ }
+
+ public function read($input)
+ {
+ $xfer = 0;
+ $fname = null;
+ $ftype = 0;
+ $fid = 0;
+ $xfer += $input->readStructBegin($fname);
+ while (true)
+ {
+ $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+ if ($ftype == TType::STOP) {
+ break;
+ }
+ switch ($fid)
+ {
+ case 1:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->privilege);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 2:
+ if ($ftype == TType::I32) {
+ $xfer += $input->readI32($this->createTime);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 3:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->grantor);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 4:
+ if ($ftype == TType::I32) {
+ $xfer += $input->readI32($this->grantorType);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 5:
+ if ($ftype == TType::BOOL) {
+ $xfer += $input->readBool($this->grantOption);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ default:
+ $xfer += $input->skip($ftype);
+ break;
+ }
+ $xfer += $input->readFieldEnd();
+ }
+ $xfer += $input->readStructEnd();
+ return $xfer;
+ }
+
+ public function write($output) {
+ $xfer = 0;
+ $xfer += $output->writeStructBegin('PrivilegeGrantInfo');
+ if ($this->privilege !== null) {
+ $xfer += $output->writeFieldBegin('privilege', TType::STRING, 1);
+ $xfer += $output->writeString($this->privilege);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->createTime !== null) {
+ $xfer += $output->writeFieldBegin('createTime', TType::I32, 2);
+ $xfer += $output->writeI32($this->createTime);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->grantor !== null) {
+ $xfer += $output->writeFieldBegin('grantor', TType::STRING, 3);
+ $xfer += $output->writeString($this->grantor);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->grantorType !== null) {
+ $xfer += $output->writeFieldBegin('grantorType', TType::I32, 4);
+ $xfer += $output->writeI32($this->grantorType);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->grantOption !== null) {
+ $xfer += $output->writeFieldBegin('grantOption', TType::BOOL, 5);
+ $xfer += $output->writeBool($this->grantOption);
+ $xfer += $output->writeFieldEnd();
+ }
+ $xfer += $output->writeFieldStop();
+ $xfer += $output->writeStructEnd();
+ return $xfer;
+ }
+
+ }
+
+ class HiveObjectPrivilege {
+ static $_TSPEC;
+
+ /**
+ * @var \metastore\HiveObjectRef
+ */
+ public $hiveObject = null;
+ /**
+ * @var string
+ */
+ public $principalName = null;
+ /**
+ * @var int
+ */
+ public $principalType = null;
+ /**
+ * @var \metastore\PrivilegeGrantInfo
+ */
+ public $grantInfo = null;
+
+ public function __construct($vals=null) {
+ if (!isset(self::$_TSPEC)) {
+ self::$_TSPEC = array(
+ 1 => array(
+ 'var' => 'hiveObject',
+ 'type' => TType::STRUCT,
+ 'class' => '\metastore\HiveObjectRef',
+ ),
+ 2 => array(
+ 'var' => 'principalName',
+ 'type' => TType::STRING,
+ ),
+ 3 => array(
+ 'var' => 'principalType',
+ 'type' => TType::I32,
+ ),
+ 4 => array(
+ 'var' => 'grantInfo',
+ 'type' => TType::STRUCT,
+ 'class' => '\metastore\PrivilegeGrantInfo',
+ ),
+ );
+ }
+ if (is_array($vals)) {
+ if (isset($vals['hiveObject'])) {
+ $this->hiveObject = $vals['hiveObject'];
+ }
+ if (isset($vals['principalName'])) {
+ $this->principalName = $vals['principalName'];
+ }
+ if (isset($vals['principalType'])) {
+ $this->principalType = $vals['principalType'];
+ }
+ if (isset($vals['grantInfo'])) {
+ $this->grantInfo = $vals['grantInfo'];
+ }
+ }
+ }
+
+ public function getName() {
+ return 'HiveObjectPrivilege';
+ }
+
+ public function read($input)
+ {
+ $xfer = 0;
+ $fname = null;
+ $ftype = 0;
+ $fid = 0;
+ $xfer += $input->readStructBegin($fname);
+ while (true)
+ {
+ $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+ if ($ftype == TType::STOP) {
+ break;
+ }
+ switch ($fid)
+ {
+ case 1:
+ if ($ftype == TType::STRUCT) {
+ $this->hiveObject = new \metastore\HiveObjectRef();
+ $xfer += $this->hiveObject->read($input);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 2:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->principalName);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 3:
+ if ($ftype == TType::I32) {
+ $xfer += $input->readI32($this->principalType);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 4:
+ if ($ftype == TType::STRUCT) {
+ $this->grantInfo = new \metastore\PrivilegeGrantInfo();
+ $xfer += $this->grantInfo->read($input);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ default:
+ $xfer += $input->skip($ftype);
+ break;
+ }
+ $xfer += $input->readFieldEnd();
+ }
+ $xfer += $input->readStructEnd();
+ return $xfer;
+ }
+
+ public function write($output) {
+ $xfer = 0;
+ $xfer += $output->writeStructBegin('HiveObjectPrivilege');
+ if ($this->hiveObject !== null) {
+ if (!is_object($this->hiveObject)) {
+ throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA);
+ }
+ $xfer += $output->writeFieldBegin('hiveObject', TType::STRUCT, 1);
+ $xfer += $this->hiveObject->write($output);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->principalName !== null) {
+ $xfer += $output->writeFieldBegin('principalName', TType::STRING, 2);
+ $xfer += $output->writeString($this->principalName);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->principalType !== null) {
+ $xfer += $output->writeFieldBegin('principalType', TType::I32, 3);
+ $xfer += $output->writeI32($this->principalType);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->grantInfo !== null) {
+ if (!is_object($this->grantInfo)) {
+ throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA);
+ }
+ $xfer += $output->writeFieldBegin('grantInfo', TType::STRUCT, 4);
+ $xfer += $this->grantInfo->write($output);
+ $xfer += $output->writeFieldEnd();
+ }
+ $xfer += $output->writeFieldStop();
+ $xfer += $output->writeStructEnd();
+ return $xfer;
+ }
+
+ }
+
+ class PrivilegeBag {
+ static $_TSPEC;
+
+ /**
+ * @var \metastore\HiveObjectPrivilege[]
+ */
+ public $privileges = null;
+
+ public function __construct($vals=null) {
+ if (!isset(self::$_TSPEC)) {
+ self::$_TSPEC = array(
+ 1 => array(
+ 'var' => 'privileges',
+ 'type' => TType::LST,
+ 'etype' => TType::STRUCT,
+ 'elem' => array(
+ 'type' => TType::STRUCT,
+ 'class' => '\metastore\HiveObjectPrivilege',
+ ),
+ ),
+ );
+ }
+ if (is_array($vals)) {
+ if (isset($vals['privileges'])) {
+ $this->privileges = $vals['privileges'];
+ }
+ }
+ }
+
+ public function getName() {
+ return 'PrivilegeBag';
+ }
+
+ public function read($input)
+ {
+ $xfer = 0;
+ $fname = null;
+ $ftype = 0;
+ $fid = 0;
+ $xfer += $input->readStructBegin($fname);
+ while (true)
+ {
+ $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+ if ($ftype == TType::STOP) {
+ break;
+ }
+ switch ($fid)
+ {
+ case 1:
+ if ($ftype == TType::LST) {
+ $this->privileges = array();
+ $_size14 = 0;
+ $_etype17 = 0;
+ $xfer += $input->readListBegin($_etype17, $_size14);
+ for ($_i18 = 0; $_i18 < $_size14; ++$_i18)
+ {
+ $elem19 = null;
+ $elem19 = new \metastore\HiveObjectPrivilege();
+ $xfer += $elem19->read($input);
+ $this->privileges []= $elem19;
+ }
+ $xfer += $input->readListEnd();
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ default:
+ $xfer += $input->skip($ftype);
+ break;
+ }
+ $xfer += $input->readFieldEnd();
+ }
+ $xfer += $input->readStructEnd();
+ return $xfer;
+ }
+
+ public function write($output) {
+ $xfer = 0;
+ $xfer += $output->writeStructBegin('PrivilegeBag');
+ if ($this->privileges !== null) {
+ if (!is_array($this->privileges)) {
+ throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA);
+ }
+ $xfer += $output->writeFieldBegin('privileges', TType::LST, 1);
+ {
+ $output->writeListBegin(TType::STRUCT, count($this->privileges));
+ {
+ foreach ($this->privileges as $iter20)
+ {
+ $xfer += $iter20->write($output);
+ }
+ }
+ $output->writeListEnd();
+ }
+ $xfer += $output->writeFieldEnd();
+ }
+ $xfer += $output->writeFieldStop();
+ $xfer += $output->writeStructEnd();
+ return $xfer;
+ }
+
+ }
+
+ class PrincipalPrivilegeSet {
+ static $_TSPEC;
+
+ /**
+ * @var array
+ */
+ public $userPrivileges = null;
+ /**
+ * @var array
+ */
+ public $groupPrivileges = null;
+ /**
+ * @var array
+ */
+ public $rolePrivileges = null;
+
+ public function __construct($vals=null) {
+ if (!isset(self::$_TSPEC)) {
+ self::$_TSPEC = array(
+ 1 => array(
+ 'var' => 'userPrivileges',
+ 'type' => TType::MAP,
+ 'ktype' => TType::STRING,
+ 'vtype' => TType::LST,
+ 'key' => array(
+ 'type' => TType::STRING,
+ ),
+ 'val' => array(
+ 'type' => TType::LST,
+ 'etype' => TType::STRUCT,
+ 'elem' => array(
+ 'type' => TType::STRUCT,
+ 'class' => '\metastore\PrivilegeGrantInfo',
+ ),
+ ),
+ ),
+ 2 => array(
+ 'var' => 'groupPrivileges',
+ 'type' => TType::MAP,
+ 'ktype' => TType::STRING,
+ 'vtype' => TType::LST,
+ 'key' => array(
+ 'type' => TType::STRING,
+ ),
+ 'val' => array(
+ 'type' => TType::LST,
+ 'etype' => TType::STRUCT,
+ 'elem' => array(
+ 'type' => TType::STRUCT,
+ 'class' => '\metastore\PrivilegeGrantInfo',
+ ),
+ ),
+ ),
+ 3 => array(
+ 'var' => 'rolePrivileges',
+ 'type' => TType::MAP,
+ 'ktype' => TType::STRING,
+ 'vtype' => TType::LST,
+ 'key' => array(
+ 'type' => TType::STRING,
+ ),
+ 'val' => array(
+ 'type' => TType::LST,
+ 'etype' => TType::STRUCT,
+ 'elem' => array(
+ 'type' => TType::STRUCT,
+ 'class' => '\metastore\PrivilegeGrantInfo',
+ ),
+ ),
+ ),
+ );
+ }
+ if (is_array($vals)) {
+ if (isset($vals['userPrivileges'])) {
+ $this->userPrivileges = $vals['userPrivileges'];
+ }
+ if (isset($vals['groupPrivileges'])) {
+ $this->groupPrivileges = $vals['groupPrivileges'];
+ }
+ if (isset($vals['rolePrivileges'])) {
+ $this->rolePrivileges = $vals['rolePrivileges'];
+ }
+ }
+ }
+
+ public function getName() {
+ return 'PrincipalPrivilegeSet';
+ }
+
+ public function read($input)
+ {
+ $xfer = 0;
+ $fname = null;
+ $ftype = 0;
+ $fid = 0;
+ $xfer += $input->readStructBegin($fname);
+ while (true)
+ {
+ $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+ if ($ftype == TType::STOP) {
+ break;
+ }
+ switch ($fid)
+ {
+ case 1:
+ if ($ftype == TType::MAP) {
+ $this->userPrivileges = array();
+ $_size21 = 0;
+ $_ktype22 = 0;
+ $_vtype23 = 0;
+ $xfer += $input->readMapBegin($_ktype22, $_vtype23, $_size21);
+ for ($_i25 = 0; $_i25 < $_size21; ++$_i25)
+ {
+ $key26 = '';
+ $val27 = array();
+ $xfer += $input->readString($key26);
+ $val27 = array();
+ $_size28 = 0;
+ $_etype31 = 0;
+ $xfer += $input->readListBegin($_etype31, $_size28);
+ for ($_i32 = 0; $_i32 < $_size28; ++$_i32)
+ {
+ $elem33 = null;
+ $elem33 = new \metastore\PrivilegeGrantInfo();
+ $xfer += $elem33->read($input);
+ $val27 []= $elem33;
+ }
+ $xfer += $input->readListEnd();
+ $this->userPrivileges[$key26] = $val27;
+ }
+ $xfer += $input->readMapEnd();
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 2:
+ if ($ftype == TType::MAP) {
+ $this->groupPrivileges = array();
+ $_size34 = 0;
+ $_ktype35 = 0;
+ $_vtype36 = 0;
+ $xfer += $input->readMapBegin($_ktype35, $_vtype36, $_size34);
+ for ($_i38 = 0; $_i38 < $_size34; ++$_i38)
+ {
+ $key39 = '';
+ $val40 = array();
+ $xfer += $input->readString($key39);
+ $val40 = array();
+ $_size41 = 0;
+ $_etype44 = 0;
+ $xfer += $input->readListBegin($_etype44, $_size41);
+ for ($_i45 = 0; $_i45 < $_size41; ++$_i45)
+ {
+ $elem46 = null;
+ $elem46 = new \metastore\PrivilegeGrantInfo();
+ $xfer += $elem46->read($input);
+ $val40 []= $elem46;
+ }
+ $xfer += $input->readListEnd();
+ $this->groupPrivileges[$key39] = $val40;
+ }
+ $xfer += $input->readMapEnd();
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 3:
+ if ($ftype == TType::MAP) {
+ $this->rolePrivileges = array();
+ $_size47 = 0;
+ $_ktype48 = 0;
+ $_vtype49 = 0;
+ $xfer += $input->readMapBegin($_ktype48, $_vtype49, $_size47);
+ for ($_i51 = 0; $_i51 < $_size47; ++$_i51)
+ {
+ $key52 = '';
+ $val53 = array();
+ $xfer += $input->readString($key52);
+ $val53 = array();
+ $_size54 = 0;
+ $_etype57 = 0;
+ $xfer += $input->readListBegin($_etype57, $_size54);
+ for ($_i58 = 0; $_i58 < $_size54; ++$_i58)
+ {
+ $elem59 = null;
+ $elem59 = new \metastore\PrivilegeGrantInfo();
+ $xfer += $elem59->read($input);
+ $val53 []= $elem59;
+ }
+ $xfer += $input->readListEnd();
+ $this->rolePrivileges[$key52] = $val53;
+ }
+ $xfer += $input->readMapEnd();
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ default:
+ $xfer += $input->skip($ftype);
+ break;
+ }
+ $xfer += $input->readFieldEnd();
+ }
+ $xfer += $input->readStructEnd();
+ return $xfer;
+ }
+
+ public function write($output) {
+ $xfer = 0;
+ $xfer += $output->writeStructBegin('PrincipalPrivilegeSet');
+ if ($this->userPrivileges !== null) {
+ if (!is_array($this->userPrivileges)) {
+ throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA);
+ }
+ $xfer += $output->writeFieldBegin('userPrivileges', TType::MAP, 1);
+ {
+ $output->writeMapBegin(TType::STRING, TType::LST, count($this->userPrivileges));
+ {
+ foreach ($this->userPrivileges as $kiter60 => $viter61)
+ {
+ $xfer += $output->writeString($kiter60);
+ {
+ $output->writeListBegin(TType::STRUCT, count($viter61));
+ {
+ foreach ($viter61 as $iter62)
+ {
+ $xfer += $iter62->write($output);
+ }
+ }
+ $output->writeListEnd();
+ }
+ }
+ }
+ $output->writeMapEnd();
+ }
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->groupPrivileges !== null) {
+ if (!is_array($this->groupPrivileges)) {
+ throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA);
+ }
+ $xfer += $output->writeFieldBegin('groupPrivileges', TType::MAP, 2);
+ {
+ $output->writeMapBegin(TType::STRING, TType::LST, count($this->groupPrivileges));
+ {
+ foreach ($this->groupPrivileges as $kiter63 => $viter64)
+ {
+ $xfer += $output->writeString($kiter63);
+ {
+ $output->writeListBegin(TType::STRUCT, count($viter64));
+ {
+ foreach ($viter64 as $iter65)
+ {
+ $xfer += $iter65->write($output);
+ }
+ }
+ $output->writeListEnd();
+ }
+ }
+ }
+ $output->writeMapEnd();
+ }
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->rolePrivileges !== null) {
+ if (!is_array($this->rolePrivileges)) {
+ throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA);
+ }
+ $xfer += $output->writeFieldBegin('rolePrivileges', TType::MAP, 3);
+ {
+ $output->writeMapBegin(TType::STRING, TType::LST, count($this->rolePrivileges));
+ {
+ foreach ($this->rolePrivileges as $kiter66 => $viter67)
+ {
+ $xfer += $output->writeString($kiter66);
+ {
+ $output->writeListBegin(TType::STRUCT, count($viter67));
+ {
+ foreach ($viter67 as $iter68)
+ {
+ $xfer += $iter68->write($output);
+ }
+ }
+ $output->writeListEnd();
+ }
+ }
+ }
+ $output->writeMapEnd();
+ }
+ $xfer += $output->writeFieldEnd();
+ }
+ $xfer += $output->writeFieldStop();
+ $xfer += $output->writeStructEnd();
+ return $xfer;
+ }
+
+ }
+
+ class GrantRevokePrivilegeRequest {
+ static $_TSPEC;
+
+ /**
+ * @var int
+ */
+ public $requestType = null;
+ /**
+ * @var \metastore\PrivilegeBag
+ */
+ public $privileges = null;
+ /**
+ * @var bool
+ */
+ public $revokeGrantOption = null;
+
+ public function __construct($vals=null) {
+ if (!isset(self::$_TSPEC)) {
+ self::$_TSPEC = array(
+ 1 => array(
+ 'var' => 'requestType',
+ 'type' => TType::I32,
+ ),
+ 2 => array(
+ 'var' => 'privileges',
+ 'type' => TType::STRUCT,
+ 'class' => '\metastore\PrivilegeBag',
+ ),
+ 3 => array(
+ 'var' => 'revokeGrantOption',
+ 'type' => TType::BOOL,
+ ),
+ );
+ }
+ if (is_array($vals)) {
+ if (isset($vals['requestType'])) {
+ $this->requestType = $vals['requestType'];
+ }
+ if (isset($vals['privileges'])) {
+ $this->privileges = $vals['privileges'];
+ }
+ if (isset($vals['revokeGrantOption'])) {
+ $this->revokeGrantOption = $vals['revokeGrantOption'];
+ }
+ }
+ }
+
+ public function getName() {
+ return 'GrantRevokePrivilegeRequest';
+ }
+
+ public function read($input)
+ {
+ $xfer = 0;
+ $fname = null;
+ $ftype = 0;
+ $fid = 0;
+ $xfer += $input->readStructBegin($fname);
+ while (true)
+ {
+ $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+ if ($ftype == TType::STOP) {
+ break;
+ }
+ switch ($fid)
+ {
+ case 1:
+ if ($ftype == TType::I32) {
+ $xfer += $input->readI32($this->requestType);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 2:
+ if ($ftype == TType::STRUCT) {
+ $this->privileges = new \metastore\PrivilegeBag();
+ $xfer += $this->privileges->read($input);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 3:
+ if ($ftype == TType::BOOL) {
+ $xfer += $input->readBool($this->revokeGrantOption);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ default:
+ $xfer += $input->skip($ftype);
+ break;
+ }
+ $xfer += $input->readFieldEnd();
+ }
+ $xfer += $input->readStructEnd();
+ return $xfer;
+ }
+
+ public function write($output) {
+ $xfer = 0;
+ $xfer += $output->writeStructBegin('GrantRevokePrivilegeRequest');
+ if ($this->requestType !== null) {
+ $xfer += $output->writeFieldBegin('requestType', TType::I32, 1);
+ $xfer += $output->writeI32($this->requestType);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->privileges !== null) {
+ if (!is_object($this->privileges)) {
+ throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA);
+ }
+ $xfer += $output->writeFieldBegin('privileges', TType::STRUCT, 2);
+ $xfer += $this->privileges->write($output);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->revokeGrantOption !== null) {
+ $xfer += $output->writeFieldBegin('revokeGrantOption', TType::BOOL, 3);
+ $xfer += $output->writeBool($this->revokeGrantOption);
+ $xfer += $output->writeFieldEnd();
+ }
+ $xfer += $output->writeFieldStop();
+ $xfer += $output->writeStructEnd();
+ return $xfer;
+ }
+
+ }
+
+ class GrantRevokePrivilegeResponse {
+ static $_TSPEC;
+
+ /**
+ * @var bool
+ */
+ public $success = null;
+
+ public function __construct($vals=null) {
+ if (!isset(self::$_TSPEC)) {
+ self::$_TSPEC = array(
+ 1 => array(
+ 'var' => 'success',
+ 'type' => TType::BOOL,
+ ),
+ );
+ }
+ if (is_array($vals)) {
+ if (isset($vals['success'])) {
+ $this->success = $vals['success'];
+ }
+ }
+ }
+
+ public function getName() {
+ return 'GrantRevokePrivilegeResponse';
+ }
+
+ public function read($input)
+ {
+ $xfer = 0;
+ $fname = null;
+ $ftype = 0;
+ $fid = 0;
+ $xfer += $input->readStructBegin($fname);
+ while (true)
+ {
+ $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+ if ($ftype == TType::STOP) {
+ break;
+ }
+ switch ($fid)
+ {
+ case 1:
+ if ($ftype == TType::BOOL) {
+ $xfer += $input->readBool($this->success);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ default:
+ $xfer += $input->skip($ftype);
+ break;
+ }
+ $xfer += $input->readFieldEnd();
+ }
+ $xfer += $input->readStructEnd();
+ return $xfer;
+ }
+
+ public function write($output) {
+ $xfer = 0;
+ $xfer += $output->writeStructBegin('GrantRevokePrivilegeResponse');
+ if ($this->success !== null) {
+ $xfer += $output->writeFieldBegin('success', TType::BOOL, 1);
+ $xfer += $output->writeBool($this->success);
+ $xfer += $output->writeFieldEnd();
+ }
+ $xfer += $output->writeFieldStop();
+ $xfer += $output->writeStructEnd();
+ return $xfer;
+ }
+
+ }
+
+ class Role {
+ static $_TSPEC;
+
+ /**
+ * @var string
+ */
+ public $roleName = null;
+ /**
+ * @var int
+ */
+ public $createTime = null;
+ /**
+ * @var string
+ */
+ public $ownerName = null;
+
+ public function __construct($vals=null) {
+ if (!isset(self::$_TSPEC)) {
+ self::$_TSPEC = array(
+ 1 => array(
+ 'var' => 'roleName',
+ 'type' => TType::STRING,
+ ),
+ 2 => array(
+ 'var' => 'createTime',
+ 'type' => TType::I32,
+ ),
+ 3 => array(
+ 'var' => 'ownerName',
+ 'type' => TType::STRING,
+ ),
+ );
+ }
+ if (is_array($vals)) {
+ if (isset($vals['roleName'])) {
+ $this->roleName = $vals['roleName'];
+ }
+ if (isset($vals['createTime'])) {
+ $this->createTime = $vals['createTime'];
+ }
+ if (isset($vals['ownerName'])) {
+ $this->ownerName = $vals['ownerName'];
+ }
+ }
+ }
+
+ public function getName() {
+ return 'Role';
+ }
+
+ public function read($input)
+ {
+ $xfer = 0;
+ $fname = null;
+ $ftype = 0;
+ $fid = 0;
+ $xfer += $input->readStructBegin($fname);
+ while (true)
+ {
+ $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+ if ($ftype == TType::STOP) {
+ break;
+ }
+ switch ($fid)
+ {
+ case 1:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->roleName);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 2:
+ if ($ftype == TType::I32) {
+ $xfer += $input->readI32($this->createTime);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 3:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->ownerName);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ default:
+ $xfer += $input->skip($ftype);
+ break;
+ }
+ $xfer += $input->readFieldEnd();
+ }
+ $xfer += $input->readStructEnd();
+ return $xfer;
+ }
+
+ public function write($output) {
+ $xfer = 0;
+ $xfer += $output->writeStructBegin('Role');
+ if ($this->roleName !== null) {
+ $xfer += $output->writeFieldBegin('roleName', TType::STRING, 1);
+ $xfer += $output->writeString($this->roleName);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->createTime !== null) {
+ $xfer += $output->writeFieldBegin('createTime', TType::I32, 2);
+ $xfer += $output->writeI32($this->createTime);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->ownerName !== null) {
+ $xfer += $output->writeFieldBegin('ownerName', TType::STRING, 3);
+ $xfer += $output->writeString($this->ownerName);
+ $xfer += $output->writeFieldEnd();
+ }
+ $xfer += $output->writeFieldStop();
+ $xfer += $output->writeStructEnd();
+ return $xfer;
+ }
+
+ }
+
+ class RolePrincipalGrant {
+ static $_TSPEC;
+
+ /**
+ * @var string
+ */
+ public $roleName = null;
+ /**
+ * @var string
+ */
+ public $principalName = null;
+ /**
+ * @var int
+ */
+ public $principalType = null;
+ /**
+ * @var bool
+ */
+ public $grantOption = null;
+ /**
+ * @var int
+ */
+ public $grantTime = null;
+ /**
+ * @var string
+ */
+ public $grantorName = null;
+ /**
+ * @var int
+ */
+ public $grantorPrincipalType = null;
+
+ public function __construct($vals=null) {
+ if (!isset(self::$_TSPEC)) {
+ self::$_TSPEC = array(
+ 1 => array(
+ 'var' => 'roleName',
+ 'type' => TType::STRING,
+ ),
+ 2 => array(
+ 'var' => 'principalName',
+ 'type' => TType::STRING,
+ ),
+ 3 => array(
+ 'var' => 'principalType',
+ 'type' => TType::I32,
+ ),
+ 4 => array(
+ 'var' => 'grantOption',
+ 'type' => TType::BOOL,
+ ),
+ 5 => array(
+ 'var' => 'grantTime',
+ 'type' => TType::I32,
+ ),
+ 6 => array(
+ 'var' => 'grantorName',
+ 'type' => TType::STRING,
+ ),
+ 7 => array(
+ 'var' => 'grantorPrincipalType',
+ 'type' => TType::I32,
+ ),
+ );
+ }
+ if (is_array($vals)) {
+ if (isset($vals['roleName'])) {
+ $this->roleName = $vals['roleName'];
+ }
+ if (isset($vals['principalName'])) {
+ $this->principalName = $vals['principalName'];
+ }
+ if (isset($vals['principalType'])) {
+ $this->principalType = $vals['principalType'];
+ }
+ if (isset($vals['grantOption'])) {
+ $this->grantOption = $vals['grantOption'];
+ }
+ if (isset($vals['grantTime'])) {
+ $this->grantTime = $vals['grantTime'];
+ }
+ if (isset($vals['grantorName'])) {
+ $this->grantorName = $vals['grantorName'];
+ }
+ if (isset($vals['grantorPrincipalType'])) {
+ $this->grantorPrincipalType = $vals['grantorPrincipalType'];
+ }
+ }
+ }
+
+ public function getName() {
+ return 'RolePrincipalGrant';
+ }
+
+ public function read($input)
+ {
+ $xfer = 0;
+ $fname = null;
+ $ftype = 0;
+ $fid = 0;
+ $xfer += $input->readStructBegin($fname);
+ while (true)
+ {
+ $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+ if ($ftype == TType::STOP) {
+ break;
+ }
+ switch ($fid)
+ {
+ case 1:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->roleName);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 2:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->principalName);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 3:
+ if ($ftype == TType::I32) {
+ $xfer += $input->readI32($this->principalType);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 4:
+ if ($ftype == TType::BOOL) {
+ $xfer += $input->readBool($this->grantOption);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 5:
+ if ($ftype == TType::I32) {
+ $xfer += $input->readI32($this->grantTime);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 6:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->grantorName);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 7:
+ if ($ftype == TType::I32) {
+ $xfer += $input->readI32($this->grantorPrincipalType);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ default:
+ $xfer += $input->skip($ftype);
+ break;
+ }
+ $xfer += $input->readFieldEnd();
+ }
+ $xfer += $input->readStructEnd();
+ return $xfer;
+ }
+
+ public function write($output) {
+ $xfer = 0;
+ $xfer += $output->writeStructBegin('RolePrincipalGrant');
+ if ($this->roleName !== null) {
+ $xfer += $output->writeFieldBegin('roleName', TType::STRING, 1);
+ $xfer += $output->writeString($this->roleName);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->principalName !== null) {
+ $xfer += $output->writeFieldBegin('principalName', TType::STRING, 2);
+ $xfer += $output->writeString($this->principalName);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->principalType !== null) {
+ $xfer += $output->writeFieldBegin('principalType', TType::I32, 3);
+ $xfer += $output->writeI32($this->principalType);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->grantOption !== null) {
+ $xfer += $output->writeFieldBegin('grantOption', TType::BOOL, 4);
+ $xfer += $output->writeBool($this->grantOption);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->grantTime !== null) {
+ $xfer += $output->writeFieldBegin('grantTime', TType::I32, 5);
+ $xfer += $output->writeI32($this->grantTime);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->grantorName !== null) {
+ $xfer += $output->writeFieldBegin('grantorName', TType::STRING, 6);
+ $xfer += $output->writeString($this->grantorName);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->grantorPrincipalType !== null) {
+ $xfer += $output->writeFieldBegin('grantorPrincipalType', TType::I32, 7);
+ $xfer += $output->writeI32($this->grantorPrincipalType);
+ $xfer += $output->writeFieldEnd();
+ }
+ $xfer += $output->writeFieldStop();
+ $xfer += $output->writeStructEnd();
+ return $xfer;
+ }
+
+ }
+
+ class GetRoleGrantsForPrincipalRequest {
+ static $_TSPEC;
+
+ /**
+ * @var string
+ */
+ public $principal_name = null;
+ /**
+ * @var int
+ */
+ public $principal_type = null;
+
+ public function __construct($vals=null) {
+ if (!isset(self::$_TSPEC)) {
+ self::$_TSPEC = array(
+ 1 => array(
+ 'var' => 'principal_name',
+ 'type' => TType::STRING,
+ ),
+ 2 => array(
+ 'var' => 'principal_type',
+ 'type' => TType::I32,
+ ),
+ );
+ }
+ if (is_array($vals)) {
+ if (isset($vals['principal_name'])) {
+ $this->principal_name = $vals['principal_name'];
+ }
+ if (isset($vals['principal_type'])) {
+ $this->principal_type = $vals['principal_type'];
+ }
+ }
+ }
+
+ public function getName() {
+ return 'GetRoleGrantsForPrincipalRequest';
+ }
+
+ public function read($input)
+ {
+ $xfer = 0;
+ $fname = null;
+ $ftype = 0;
+ $fid = 0;
+ $xfer += $input->readStructBegin($fname);
+ while (true)
+ {
+ $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+ if ($ftype == TType::STOP) {
+ break;
+ }
+ switch ($fid)
+ {
+ case 1:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->principal_name);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 2:
+ if ($ftype == TType::I32) {
+ $xfer += $input->readI32($this->principal_type);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ default:
+ $xfer += $input->skip($ftype);
+ break;
+ }
+ $xfer += $input->readFieldEnd();
+ }
+ $xfer += $input->readStructEnd();
+ return $xfer;
+ }
+
+ public function write($output) {
+ $xfer = 0;
+ $xfer += $output->writeStru
<TRUNCATED>
[05/50] [abbrv] hive git commit: HIVE-17392: SharedWorkOptimizer
might merge TS operators filtered by not equivalent semijoin operators (Jesus
Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Posted by we...@apache.org.
HIVE-17392: SharedWorkOptimizer might merge TS operators filtered by not equivalent semijoin operators (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/262d8f99
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/262d8f99
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/262d8f99
Branch: refs/heads/hive-14535
Commit: 262d8f992e63ac4aa65e36665fb22748546c511c
Parents: 7567119
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Fri Aug 25 13:25:24 2017 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Fri Aug 25 17:46:15 2017 -0700
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 1 +
.../hive/ql/optimizer/SharedWorkOptimizer.java | 14 +-
.../dynamic_semijoin_reduction_sw.q | 60 +++
.../llap/dynamic_semijoin_reduction_sw.q.out | 518 +++++++++++++++++++
4 files changed, 585 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/262d8f99/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 772113a..37a3757 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -501,6 +501,7 @@ minillaplocal.query.files=acid_globallimit.q,\
dynamic_semijoin_reduction.q,\
dynamic_semijoin_reduction_2.q,\
dynamic_semijoin_reduction_3.q,\
+ dynamic_semijoin_reduction_sw.q,\
dynpart_sort_opt_vectorization.q,\
dynpart_sort_optimization.q,\
dynpart_sort_optimization_acid.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/262d8f99/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
index 01d4f57..37fdb00 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
@@ -443,8 +443,8 @@ public class SharedWorkOptimizer extends Transform {
Set<Operator<?>> ascendants =
findAscendantWorkOperators(pctx, optimizerCache, op);
if (ascendants.contains(tsOp2)) {
- dppsOp1.remove(i);
- i--;
+ // This should not happen, we cannot merge
+ return false;
}
}
}
@@ -454,8 +454,8 @@ public class SharedWorkOptimizer extends Transform {
Set<Operator<?>> ascendants =
findAscendantWorkOperators(pctx, optimizerCache, op);
if (ascendants.contains(tsOp1)) {
- dppsOp2.remove(i);
- i--;
+ // This should not happen, we cannot merge
+ return false;
}
}
}
@@ -464,9 +464,9 @@ public class SharedWorkOptimizer extends Transform {
return false;
}
// Check if DPP branches are equal
+ BitSet bs = new BitSet();
for (int i = 0; i < dppsOp1.size(); i++) {
Operator<?> dppOp1 = dppsOp1.get(i);
- BitSet bs = new BitSet();
for (int j = 0; j < dppsOp2.size(); j++) {
if (!bs.get(j)) {
// If not visited yet
@@ -478,7 +478,7 @@ public class SharedWorkOptimizer extends Transform {
}
}
}
- if (bs.cardinality() == i) {
+ if (bs.cardinality() < i + 1) {
return false;
}
}
@@ -530,7 +530,6 @@ public class SharedWorkOptimizer extends Transform {
if (currentOp1.getChildOperators().size() > 1 ||
currentOp2.getChildOperators().size() > 1) {
// TODO: Support checking multiple child operators to merge further.
- discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, discardableInputOps));
discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, discardableOps));
discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, retainableOps, discardableInputOps));
return new SharedResult(retainableOps, discardableOps, discardableInputOps, dataSize, maxDataSize);
@@ -539,7 +538,6 @@ public class SharedWorkOptimizer extends Transform {
currentOp2 = currentOp2.getChildOperators().get(0);
} else {
// Bail out
- discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, discardableInputOps));
discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, discardableOps));
discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, retainableOps, discardableInputOps));
return new SharedResult(retainableOps, discardableOps, discardableInputOps, dataSize, maxDataSize);
http://git-wip-us.apache.org/repos/asf/hive/blob/262d8f99/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_sw.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_sw.q b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_sw.q
new file mode 100644
index 0000000..cece77d
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_sw.q
@@ -0,0 +1,60 @@
+set hive.compute.query.using.stats=false;
+set hive.mapred.mode=nonstrict;
+set hive.explain.user=false;
+set hive.optimize.ppd=true;
+set hive.ppd.remove.duplicatefilters=true;
+set hive.tez.dynamic.partition.pruning=true;
+set hive.tez.dynamic.semijoin.reduction=true;
+set hive.optimize.metadataonly=false;
+set hive.optimize.index.filter=true;
+set hive.stats.autogather=true;
+set hive.tez.bigtable.minsize.semijoin.reduction=1;
+set hive.tez.min.bloom.filter.entries=1;
+set hive.stats.fetch.column.stats=true;
+
+-- Create Tables
+create table alltypesorc_int ( cint int, cstring string ) stored as ORC;
+create table srcpart_date (key string, value string) partitioned by (ds string ) stored as ORC;
+CREATE TABLE srcpart_small(key1 STRING, value1 STRING) partitioned by (ds1 string) STORED as ORC;
+
+-- Add Partitions
+alter table srcpart_date add partition (ds = "2008-04-08");
+alter table srcpart_date add partition (ds = "2008-04-09");
+
+alter table srcpart_small add partition (ds1 = "2008-04-08");
+alter table srcpart_small add partition (ds1 = "2008-04-09");
+
+-- Load
+insert overwrite table alltypesorc_int select cint, cstring1 from alltypesorc;
+insert overwrite table srcpart_date partition (ds = "2008-04-08" ) select key, value from srcpart where ds = "2008-04-08";
+insert overwrite table srcpart_date partition (ds = "2008-04-09") select key, value from srcpart where ds = "2008-04-09";
+insert overwrite table srcpart_small partition (ds1 = "2008-04-09") select key, value from srcpart where ds = "2008-04-09" limit 20;
+
+set hive.tez.dynamic.semijoin.reduction=false;
+
+analyze table alltypesorc_int compute statistics for columns;
+analyze table srcpart_date compute statistics for columns;
+analyze table srcpart_small compute statistics for columns;
+
+set hive.tez.dynamic.semijoin.reduction=true;
+EXPLAIN
+SELECT count(*)
+FROM (
+ SELECT *
+ FROM (SELECT * FROM srcpart_date WHERE ds = "2008-04-09") `srcpart_date`
+ JOIN (SELECT * FROM srcpart_small WHERE ds1 = "2008-04-08") `srcpart_small`
+ ON (srcpart_date.key = srcpart_small.key1)
+ JOIN alltypesorc_int
+ ON (srcpart_small.key1 = alltypesorc_int.cstring)) a
+JOIN (
+ SELECT *
+ FROM (SELECT * FROM srcpart_date WHERE ds = "2008-04-08") `srcpart_date`
+ JOIN (SELECT * FROM srcpart_small WHERE ds1 = "2008-04-08") `srcpart_small`
+ ON (srcpart_date.key = srcpart_small.key1)
+ JOIN alltypesorc_int
+ ON (srcpart_small.key1 = alltypesorc_int.cstring)) b
+ON ('1' = '1');
+
+drop table srcpart_date;
+drop table srcpart_small;
+drop table alltypesorc_int;
http://git-wip-us.apache.org/repos/asf/hive/blob/262d8f99/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_sw.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_sw.q.out b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_sw.q.out
new file mode 100644
index 0000000..68ea269
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_sw.q.out
@@ -0,0 +1,518 @@
+PREHOOK: query: create table alltypesorc_int ( cint int, cstring string ) stored as ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@alltypesorc_int
+POSTHOOK: query: create table alltypesorc_int ( cint int, cstring string ) stored as ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@alltypesorc_int
+PREHOOK: query: create table srcpart_date (key string, value string) partitioned by (ds string ) stored as ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcpart_date
+POSTHOOK: query: create table srcpart_date (key string, value string) partitioned by (ds string ) stored as ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcpart_date
+PREHOOK: query: CREATE TABLE srcpart_small(key1 STRING, value1 STRING) partitioned by (ds1 string) STORED as ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcpart_small
+POSTHOOK: query: CREATE TABLE srcpart_small(key1 STRING, value1 STRING) partitioned by (ds1 string) STORED as ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcpart_small
+PREHOOK: query: alter table srcpart_date add partition (ds = "2008-04-08")
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@srcpart_date
+POSTHOOK: query: alter table srcpart_date add partition (ds = "2008-04-08")
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@srcpart_date
+POSTHOOK: Output: default@srcpart_date@ds=2008-04-08
+PREHOOK: query: alter table srcpart_date add partition (ds = "2008-04-09")
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@srcpart_date
+POSTHOOK: query: alter table srcpart_date add partition (ds = "2008-04-09")
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@srcpart_date
+POSTHOOK: Output: default@srcpart_date@ds=2008-04-09
+PREHOOK: query: alter table srcpart_small add partition (ds1 = "2008-04-08")
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@srcpart_small
+POSTHOOK: query: alter table srcpart_small add partition (ds1 = "2008-04-08")
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@srcpart_small
+POSTHOOK: Output: default@srcpart_small@ds1=2008-04-08
+PREHOOK: query: alter table srcpart_small add partition (ds1 = "2008-04-09")
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@srcpart_small
+POSTHOOK: query: alter table srcpart_small add partition (ds1 = "2008-04-09")
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@srcpart_small
+POSTHOOK: Output: default@srcpart_small@ds1=2008-04-09
+PREHOOK: query: insert overwrite table alltypesorc_int select cint, cstring1 from alltypesorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: default@alltypesorc_int
+POSTHOOK: query: insert overwrite table alltypesorc_int select cint, cstring1 from alltypesorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: default@alltypesorc_int
+POSTHOOK: Lineage: alltypesorc_int.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ]
+POSTHOOK: Lineage: alltypesorc_int.cstring SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ]
+PREHOOK: query: insert overwrite table srcpart_date partition (ds = "2008-04-08" ) select key, value from srcpart where ds = "2008-04-08"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_date@ds=2008-04-08
+POSTHOOK: query: insert overwrite table srcpart_date partition (ds = "2008-04-08" ) select key, value from srcpart where ds = "2008-04-08"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_date@ds=2008-04-08
+POSTHOOK: Lineage: srcpart_date PARTITION(ds=2008-04-08).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_date PARTITION(ds=2008-04-08).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: insert overwrite table srcpart_date partition (ds = "2008-04-09") select key, value from srcpart where ds = "2008-04-09"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+PREHOOK: Output: default@srcpart_date@ds=2008-04-09
+POSTHOOK: query: insert overwrite table srcpart_date partition (ds = "2008-04-09") select key, value from srcpart where ds = "2008-04-09"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@srcpart_date@ds=2008-04-09
+POSTHOOK: Lineage: srcpart_date PARTITION(ds=2008-04-09).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_date PARTITION(ds=2008-04-09).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: insert overwrite table srcpart_small partition (ds1 = "2008-04-09") select key, value from srcpart where ds = "2008-04-09" limit 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+PREHOOK: Output: default@srcpart_small@ds1=2008-04-09
+POSTHOOK: query: insert overwrite table srcpart_small partition (ds1 = "2008-04-09") select key, value from srcpart where ds = "2008-04-09" limit 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@srcpart_small@ds1=2008-04-09
+POSTHOOK: Lineage: srcpart_small PARTITION(ds1=2008-04-09).key1 SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_small PARTITION(ds1=2008-04-09).value1 SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: analyze table alltypesorc_int compute statistics for columns
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc_int
+PREHOOK: Output: default@alltypesorc_int
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table alltypesorc_int compute statistics for columns
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc_int
+POSTHOOK: Output: default@alltypesorc_int
+#### A masked pattern was here ####
+PREHOOK: query: analyze table srcpart_date compute statistics for columns
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_date
+PREHOOK: Input: default@srcpart_date@ds=2008-04-08
+PREHOOK: Input: default@srcpart_date@ds=2008-04-09
+PREHOOK: Output: default@srcpart_date
+PREHOOK: Output: default@srcpart_date@ds=2008-04-08
+PREHOOK: Output: default@srcpart_date@ds=2008-04-09
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table srcpart_date compute statistics for columns
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_date
+POSTHOOK: Input: default@srcpart_date@ds=2008-04-08
+POSTHOOK: Input: default@srcpart_date@ds=2008-04-09
+POSTHOOK: Output: default@srcpart_date
+POSTHOOK: Output: default@srcpart_date@ds=2008-04-08
+POSTHOOK: Output: default@srcpart_date@ds=2008-04-09
+#### A masked pattern was here ####
+PREHOOK: query: analyze table srcpart_small compute statistics for columns
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart_small
+PREHOOK: Input: default@srcpart_small@ds1=2008-04-08
+PREHOOK: Input: default@srcpart_small@ds1=2008-04-09
+PREHOOK: Output: default@srcpart_small
+PREHOOK: Output: default@srcpart_small@ds1=2008-04-08
+PREHOOK: Output: default@srcpart_small@ds1=2008-04-09
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table srcpart_small compute statistics for columns
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart_small
+POSTHOOK: Input: default@srcpart_small@ds1=2008-04-08
+POSTHOOK: Input: default@srcpart_small@ds1=2008-04-09
+POSTHOOK: Output: default@srcpart_small
+POSTHOOK: Output: default@srcpart_small@ds1=2008-04-08
+POSTHOOK: Output: default@srcpart_small@ds1=2008-04-09
+#### A masked pattern was here ####
+Warning: Shuffle Join MERGEJOIN[93][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 3' is a cross product
+PREHOOK: query: EXPLAIN
+SELECT count(*)
+FROM (
+ SELECT *
+ FROM (SELECT * FROM srcpart_date WHERE ds = "2008-04-09") `srcpart_date`
+ JOIN (SELECT * FROM srcpart_small WHERE ds1 = "2008-04-08") `srcpart_small`
+ ON (srcpart_date.key = srcpart_small.key1)
+ JOIN alltypesorc_int
+ ON (srcpart_small.key1 = alltypesorc_int.cstring)) a
+JOIN (
+ SELECT *
+ FROM (SELECT * FROM srcpart_date WHERE ds = "2008-04-08") `srcpart_date`
+ JOIN (SELECT * FROM srcpart_small WHERE ds1 = "2008-04-08") `srcpart_small`
+ ON (srcpart_date.key = srcpart_small.key1)
+ JOIN alltypesorc_int
+ ON (srcpart_small.key1 = alltypesorc_int.cstring)) b
+ON ('1' = '1')
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT count(*)
+FROM (
+ SELECT *
+ FROM (SELECT * FROM srcpart_date WHERE ds = "2008-04-09") `srcpart_date`
+ JOIN (SELECT * FROM srcpart_small WHERE ds1 = "2008-04-08") `srcpart_small`
+ ON (srcpart_date.key = srcpart_small.key1)
+ JOIN alltypesorc_int
+ ON (srcpart_small.key1 = alltypesorc_int.cstring)) a
+JOIN (
+ SELECT *
+ FROM (SELECT * FROM srcpart_date WHERE ds = "2008-04-08") `srcpart_date`
+ JOIN (SELECT * FROM srcpart_small WHERE ds1 = "2008-04-08") `srcpart_small`
+ ON (srcpart_date.key = srcpart_small.key1)
+ JOIN alltypesorc_int
+ ON (srcpart_small.key1 = alltypesorc_int.cstring)) b
+ON ('1' = '1')
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 1 <- Reducer 7 (BROADCAST_EDGE)
+ Map 10 <- Reducer 5 (BROADCAST_EDGE), Reducer 7 (BROADCAST_EDGE)
+ Map 11 <- Reducer 9 (BROADCAST_EDGE)
+ Map 13 <- Reducer 12 (BROADCAST_EDGE), Reducer 9 (BROADCAST_EDGE)
+ Reducer 12 <- Map 11 (CUSTOM_SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 10 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE)
+ Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE)
+ Reducer 5 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+ Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE)
+ Reducer 8 <- Map 11 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE)
+ Reducer 9 <- Map 6 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_date
+ filterExpr: (key is not null and (key BETWEEN DynamicValue(RS_10_srcpart_small_key1_min) AND DynamicValue(RS_10_srcpart_small_key1_max) and in_bloom_filter(key, DynamicValue(RS_10_srcpart_small_key1_bloom_filter)))) (type: boolean)
+ Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: (key is not null and (key BETWEEN DynamicValue(RS_10_srcpart_small_key1_min) AND DynamicValue(RS_10_srcpart_small_key1_max) and in_bloom_filter(key, DynamicValue(RS_10_srcpart_small_key1_bloom_filter)))) (type: boolean)
+ Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=618)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
+ Execution mode: llap
+ LLAP IO: all inputs
+ Map 10
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc_int
+ filterExpr: (cstring is not null and (cstring BETWEEN DynamicValue(RS_9_srcpart_date_key_min) AND DynamicValue(RS_9_srcpart_date_key_max) and in_bloom_filter(cstring, DynamicValue(RS_9_srcpart_date_key_bloom_filter))) and (cstring BETWEEN DynamicValue(RS_10_srcpart_small_key1_min) AND DynamicValue(RS_10_srcpart_small_key1_max) and in_bloom_filter(cstring, DynamicValue(RS_10_srcpart_small_key1_bloom_filter)))) (type: boolean)
+ Statistics: Num rows: 12288 Data size: 862450 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: (cstring is not null and (cstring BETWEEN DynamicValue(RS_9_srcpart_date_key_min) AND DynamicValue(RS_9_srcpart_date_key_max) and in_bloom_filter(cstring, DynamicValue(RS_9_srcpart_date_key_bloom_filter))) and (cstring BETWEEN DynamicValue(RS_10_srcpart_small_key1_min) AND DynamicValue(RS_10_srcpart_small_key1_max) and in_bloom_filter(cstring, DynamicValue(RS_10_srcpart_small_key1_bloom_filter)))) (type: boolean)
+ Statistics: Num rows: 9174 Data size: 643900 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: cstring (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 9174 Data size: 643900 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 9174 Data size: 643900 Basic stats: COMPLETE Column stats: COMPLETE
+ Execution mode: llap
+ LLAP IO: all inputs
+ Map 11
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_date
+ filterExpr: (key is not null and (key BETWEEN DynamicValue(RS_24_srcpart_small_key1_min) AND DynamicValue(RS_24_srcpart_small_key1_max) and in_bloom_filter(key, DynamicValue(RS_24_srcpart_small_key1_bloom_filter)))) (type: boolean)
+ Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: (key is not null and (key BETWEEN DynamicValue(RS_24_srcpart_small_key1_min) AND DynamicValue(RS_24_srcpart_small_key1_max) and in_bloom_filter(key, DynamicValue(RS_24_srcpart_small_key1_bloom_filter)))) (type: boolean)
+ Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=618)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
+ Execution mode: llap
+ LLAP IO: all inputs
+ Map 13
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc_int
+ filterExpr: (cstring is not null and (cstring BETWEEN DynamicValue(RS_23_srcpart_date_key_min) AND DynamicValue(RS_23_srcpart_date_key_max) and in_bloom_filter(cstring, DynamicValue(RS_23_srcpart_date_key_bloom_filter))) and (cstring BETWEEN DynamicValue(RS_24_srcpart_small_key1_min) AND DynamicValue(RS_24_srcpart_small_key1_max) and in_bloom_filter(cstring, DynamicValue(RS_24_srcpart_small_key1_bloom_filter)))) (type: boolean)
+ Statistics: Num rows: 12288 Data size: 862450 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: (cstring is not null and (cstring BETWEEN DynamicValue(RS_23_srcpart_date_key_min) AND DynamicValue(RS_23_srcpart_date_key_max) and in_bloom_filter(cstring, DynamicValue(RS_23_srcpart_date_key_bloom_filter))) and (cstring BETWEEN DynamicValue(RS_24_srcpart_small_key1_min) AND DynamicValue(RS_24_srcpart_small_key1_max) and in_bloom_filter(cstring, DynamicValue(RS_24_srcpart_small_key1_bloom_filter)))) (type: boolean)
+ Statistics: Num rows: 9174 Data size: 643900 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: cstring (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 9174 Data size: 643900 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 9174 Data size: 643900 Basic stats: COMPLETE Column stats: COMPLETE
+ Execution mode: llap
+ LLAP IO: all inputs
+ Map 6
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_small
+ filterExpr: key1 is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key1 is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key1 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=2)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=2)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
+ Execution mode: llap
+ LLAP IO: all inputs
+ Reducer 12
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=618)
+ mode: final
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 1 to 2
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ Statistics: Num rows: 20182 Data size: 1416580 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 20182 Data size: 1416580 Basic stats: COMPLETE Column stats: NONE
+ Reducer 3
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0
+ 1
+ Statistics: Num rows: 407313124 Data size: 57586148244 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reducer 4
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 5
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=618)
+ mode: final
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
+ Reducer 7
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=2)
+ mode: final
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
+ Reducer 8
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 1 to 2
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ Statistics: Num rows: 20182 Data size: 1416580 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 20182 Data size: 1416580 Basic stats: COMPLETE Column stats: NONE
+ Reducer 9
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=2)
+ mode: final
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: drop table srcpart_date
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@srcpart_date
+PREHOOK: Output: default@srcpart_date
+POSTHOOK: query: drop table srcpart_date
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@srcpart_date
+POSTHOOK: Output: default@srcpart_date
+PREHOOK: query: drop table srcpart_small
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@srcpart_small
+PREHOOK: Output: default@srcpart_small
+POSTHOOK: query: drop table srcpart_small
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@srcpart_small
+POSTHOOK: Output: default@srcpart_small
+PREHOOK: query: drop table alltypesorc_int
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@alltypesorc_int
+PREHOOK: Output: default@alltypesorc_int
+POSTHOOK: query: drop table alltypesorc_int
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@alltypesorc_int
+POSTHOOK: Output: default@alltypesorc_int
[04/50] [abbrv] hive git commit: HIVE-17340 TxnHandler.checkLock() -
reduce number of SQL statements (Eugene Koifman, reviewed by Wei Zheng)
Posted by we...@apache.org.
HIVE-17340 TxnHandler.checkLock() - reduce number of SQL statements (Eugene Koifman, reviewed by Wei Zheng)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/75671197
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/75671197
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/75671197
Branch: refs/heads/hive-14535
Commit: 75671197cfca8d013bc7b6a6346d6318b10ca83c
Parents: 1938eeb
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Fri Aug 25 12:49:17 2017 -0700
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Fri Aug 25 12:49:17 2017 -0700
----------------------------------------------------------------------
.../hadoop/hive/metastore/txn/TxnHandler.java | 81 +++++++++++---------
1 file changed, 46 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/75671197/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
index 71e7c0c..5bfc029 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
@@ -2444,10 +2444,6 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
*
* This is expected to run at READ_COMMITTED.
*
- * Note: this calls acquire() for (extLockId,intLockId) but extLockId is the same and we either take
- * all locks for given extLockId or none. Would be more efficient to update state on all locks
- * at once. Semantics are the same since this is all part of the same txn.
- *
* If there is a concurrent commitTxn/rollbackTxn, those can only remove rows from HIVE_LOCKS.
* If they happen to be for the same txnid, there will be a WW conflict (in MS DB), if different txnid,
* checkLock() will in the worst case keep locks in Waiting state a little longer.
@@ -2654,7 +2650,6 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
// Look at everything in front of this lock to see if it should block
// it or not.
- boolean acquired = false;
for (int i = index - 1; i >= 0; i--) {
// Check if we're operating on the same database, if not, move on
if (!locks[index].db.equals(locks[i].db)) {
@@ -2706,20 +2701,16 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
}
//fall through to ACQUIRE
case ACQUIRE:
- acquire(dbConn, stmt, extLockId, info);
- acquired = true;
break;
case KEEP_LOOKING:
continue;
}
- if (acquired) break; // We've acquired this lock component,
- // so get out of the loop and look at the next component.
+ //if we got here, it means it's ok to acquire 'info' lock
+ break;// so exit the loop and check next lock
}
-
- // If we've arrived here and we have not already acquired, it means there's nothing in the
- // way of the lock, so acquire the lock.
- if (!acquired) acquire(dbConn, stmt, extLockId, info);
}
+ //if here, ther were no locks that blocked any locks in 'locksBeingChecked' - acquire them all
+ acquire(dbConn, stmt, locksBeingChecked);
// We acquired all of the locks, so commit and return acquired.
LOG.debug("Going to commit");
@@ -2733,6 +2724,48 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
}
return response;
}
+ private void acquire(Connection dbConn, Statement stmt, List<LockInfo> locksBeingChecked)
+ throws SQLException, NoSuchLockException, MetaException {
+ if(locksBeingChecked == null || locksBeingChecked.isEmpty()) {
+ return;
+ }
+ long txnId = locksBeingChecked.get(0).txnId;
+ long extLockId = locksBeingChecked.get(0).extLockId;
+ long now = getDbTime(dbConn);
+ String s = "update HIVE_LOCKS set hl_lock_state = '" + LOCK_ACQUIRED + "', " +
+ //if lock is part of txn, heartbeat info is in txn record
+ "hl_last_heartbeat = " + (isValidTxn(txnId) ? 0 : now) +
+ ", hl_acquired_at = " + now + ",HL_BLOCKEDBY_EXT_ID=NULL,HL_BLOCKEDBY_INT_ID=null" +
+ " where hl_lock_ext_id = " + extLockId;
+ LOG.debug("Going to execute update <" + s + ">");
+ int rc = stmt.executeUpdate(s);
+ if (rc < locksBeingChecked.size()) {
+ LOG.debug("Going to rollback acquire(Connection dbConn, Statement stmt, List<LockInfo> locksBeingChecked)");
+ dbConn.rollback();
+ /*select all locks for this ext ID and see which ones are missing*/
+ StringBuilder sb = new StringBuilder("No such lock(s): (" + JavaUtils.lockIdToString(extLockId) + ":");
+ ResultSet rs = stmt.executeQuery("select hl_lock_int_id from HIVE_LOCKS where hl_lock_ext_id = " + extLockId);
+ while(rs.next()) {
+ int intLockId = rs.getInt(1);
+ int idx = 0;
+ for(; idx < locksBeingChecked.size(); idx++) {
+ LockInfo expectedLock = locksBeingChecked.get(idx);
+ if(expectedLock != null && expectedLock.intLockId == intLockId) {
+ locksBeingChecked.set(idx, null);
+ break;
+ }
+ }
+ }
+ for(LockInfo expectedLock : locksBeingChecked) {
+ if(expectedLock != null) {
+ sb.append(expectedLock.intLockId).append(",");
+ }
+ }
+ sb.append(") ").append(JavaUtils.txnIdToString(txnId));
+ close(rs);
+ throw new NoSuchLockException(sb.toString());
+ }
+ }
/**
* the {@link #jumpTable} only deals with LockState/LockType. In some cases it's not
@@ -2775,28 +2808,6 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
LOG.debug("Going to rollback to savepoint");
dbConn.rollback(save);
}
-
- private void acquire(Connection dbConn, Statement stmt, long extLockId, LockInfo lockInfo)
- throws SQLException, NoSuchLockException, MetaException {
- long now = getDbTime(dbConn);
- String s = "update HIVE_LOCKS set hl_lock_state = '" + LOCK_ACQUIRED + "', " +
- //if lock is part of txn, heartbeat info is in txn record
- "hl_last_heartbeat = " + (isValidTxn(lockInfo.txnId) ? 0 : now) +
- ", hl_acquired_at = " + now + ",HL_BLOCKEDBY_EXT_ID=NULL,HL_BLOCKEDBY_INT_ID=null" + " where hl_lock_ext_id = " +
- extLockId + " and hl_lock_int_id = " + lockInfo.intLockId;
- LOG.debug("Going to execute update <" + s + ">");
- int rc = stmt.executeUpdate(s);
- if (rc < 1) {
- LOG.debug("Going to rollback");
- dbConn.rollback();
- throw new NoSuchLockException("No such lock: (" + JavaUtils.lockIdToString(extLockId) + "," +
- + lockInfo.intLockId + ") " + JavaUtils.txnIdToString(lockInfo.txnId));
- }
- // We update the database, but we don't commit because there may be other
- // locks together with this, and we only want to acquire one if we can
- // acquire all.
- }
-
/**
* Heartbeats on the lock table. This commits, so do not enter it with any state.
* Should not be called on a lock that belongs to transaction.
[12/50] [abbrv] hive git commit: HIVE-17375: stddev_samp,
var_samp standard compliance (Zoltan Haindrich, reviewed by Ashutosh Chauhan)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/dd04a92f/ql/src/test/results/clientpositive/vectorization_15.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorization_15.q.out b/ql/src/test/results/clientpositive/vectorization_15.q.out
index 8f0a879..8eff856 100644
--- a/ql/src/test/results/clientpositive/vectorization_15.q.out
+++ b/ql/src/test/results/clientpositive/vectorization_15.q.out
@@ -278,50 +278,50 @@ ORDER BY cfloat, cboolean1, cdouble, cstring1, ctinyint, cint, ctimestamp1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
--48.0 NULL -7196.0 NULL -48 NULL 1969-12-31 16:00:06.337 0.0 NULL -7196.0 -572463.388 33.0 0.0 0.0 -23.0 48 NULL NULL -23 NULL NULL
--51.0 NULL -200.0 NULL -51 NULL 1969-12-31 15:59:55.423 0.0 NULL -200.0 -15910.599999999999 33.0 0.0 0.0 -23.0 51 NULL NULL -23 NULL NULL
--51.0 false NULL 10 -51 1058319346 1969-12-31 16:00:08.451 0.0 -1058319372.28 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 1.05831942E9 -23 1058319372.28 0.0
--51.0 false NULL 10TYIE5S35U6dj3N -51 -469581869 1969-12-31 16:00:08.451 0.0 469581842.72 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 -4.69581792E8 -23 -469581842.72 0.0
--51.0 false NULL 1Lh6Uoq3WhNtOqQHu7WN7U -51 -352637533 1969-12-31 16:00:08.451 0.0 352637506.72 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 -3.52637472E8 -23 -352637506.72 0.0
--51.0 true NULL 04Y1mA17 -51 -114647521 1969-12-31 16:00:08.451 0.0 114647494.72 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 -1.14647472E8 -23 -114647494.72 0.0
--51.0 true NULL 10Wu570aLPO0p02P17FeH -51 405338893 1969-12-31 16:00:08.451 0.0 -405338919.28 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 4.05338944E8 -23 405338919.28 0.0
--51.0 true NULL 3cQp060 -51 -226923315 1969-12-31 16:00:08.451 0.0 226923288.72 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 -2.26923264E8 -23 -226923288.72 0.0
--51.0 true NULL 8EPG0Xi307qd -51 -328662044 1969-12-31 16:00:08.451 0.0 328662017.72 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 -3.28661984E8 -23 -328662017.72 0.0
--51.0 true NULL 8iHtdkJ6d -51 1006818344 1969-12-31 16:00:08.451 0.0 -1006818370.28 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 1.00681843E9 -23 1006818370.28 0.0
--51.0 true NULL QiOcvR0kt6r7f0R7fiPxQTCU -51 266531954 1969-12-31 16:00:08.451 0.0 -266531980.28 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 2.66532E8 -23 266531980.28 0.0
--51.0 true NULL Ybpj38RTTYl7CnJXPNx1g4C -51 -370919370 1969-12-31 16:00:08.451 0.0 370919343.72 NULL NULL 33.0 0.0 0.0 NULL 51 0.0 -3.70919296E8 -23 -370919343.72 0.0
--6.0 NULL -200.0 NULL -6 NULL 1969-12-31 15:59:56.094 0.0 NULL -200.0 -15910.599999999999 3.0 0.0 0.0 -23.0 6 NULL NULL -5 NULL NULL
--62.0 NULL 15601.0 NULL -62 NULL 1969-12-31 15:59:56.527 0.0 NULL 15601.0 1241106.353 33.0 0.0 0.0 -23.0 62 NULL NULL -23 NULL NULL
-11.0 false NULL 10pO8p1LNx4Y 11 271296824 1969-12-31 16:00:02.351 0.0 -271296850.28 NULL NULL 0.0 0.0 0.0 NULL -11 0.0 2.71296832E8 -1 271296850.28 0.0
-11.0 false NULL 1H6wGP 11 -560827082 1969-12-31 16:00:02.351 0.0 560827055.72 NULL NULL 0.0 0.0 0.0 NULL -11 0.0 -5.6082707E8 -1 -560827055.72 0.0
-11.0 false NULL 2a7V63IL7jK3o 11 -325931647 1969-12-31 16:00:02.351 0.0 325931620.72 NULL NULL 0.0 0.0 0.0 NULL -11 0.0 -3.25931648E8 -1 -325931620.72 0.0
-11.0 true NULL 10 11 92365813 1969-12-31 16:00:02.351 0.0 -92365839.28 NULL NULL 0.0 0.0 0.0 NULL -11 0.0 9.2365808E7 -1 92365839.28 0.0
-21.0 NULL 15601.0 NULL 21 NULL 1969-12-31 15:59:56.527 0.0 NULL 15601.0 1241106.353 12.0 0.0 0.0 -23.0 -21 NULL NULL -2 NULL NULL
-32.0 NULL -200.0 NULL 32 NULL 1969-12-31 16:00:02.445 0.0 NULL -200.0 -15910.599999999999 1.0 0.0 0.0 -23.0 -32 NULL NULL -23 NULL NULL
-36.0 NULL -200.0 NULL 36 NULL 1969-12-31 16:00:00.554 0.0 NULL -200.0 -15910.599999999999 33.0 0.0 0.0 -23.0 -36 NULL NULL -23 NULL NULL
-5.0 NULL 15601.0 NULL 5 NULL 1969-12-31 15:59:56.527 0.0 NULL 15601.0 1241106.353 3.0 0.0 0.0 -23.0 -5 NULL NULL -3 NULL NULL
-58.0 NULL 15601.0 NULL 58 NULL 1969-12-31 15:59:56.527 0.0 NULL 15601.0 1241106.353 33.0 0.0 0.0 -23.0 -58 NULL NULL -23 NULL NULL
-8.0 false NULL 10V3pN5r5lI2qWl2lG103 8 -362835731 1969-12-31 16:00:15.892 0.0 362835704.72 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 -3.62835744E8 -7 -362835704.72 0.0
-8.0 false NULL 10c4qt584m5y6uWT 8 -183000142 1969-12-31 16:00:15.892 0.0 183000115.72 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 -1.8300016E8 -7 -183000115.72 0.0
-8.0 false NULL 8GloEukQ0c68JDmnYL53 8 -722873402 1969-12-31 16:00:15.892 0.0 722873375.72 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 -7.2287341E8 -7 -722873375.72 0.0
-8.0 false NULL kA0XH5C5 8 -503903864 1969-12-31 16:00:15.892 0.0 503903837.72 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 -5.03903872E8 -7 -503903837.72 0.0
-8.0 true NULL 100VTM7PEW8GH1uE 8 88129338 1969-12-31 16:00:15.892 0.0 -88129364.28 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 8.8129328E7 -7 88129364.28 0.0
-8.0 true NULL 1062158y 8 -1005155523 1969-12-31 16:00:15.892 0.0 1005155496.72 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 -1.00515552E9 -7 -1005155496.72 0.0
-8.0 true NULL 1063cEnGjSal 8 -624769630 1969-12-31 16:00:15.892 0.0 624769603.72 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 -6.247696E8 -7 -624769603.72 0.0
-8.0 true NULL 4kMasVoB7lX1wc5i64bNk 8 683567667 1969-12-31 16:00:15.892 0.0 -683567693.28 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 6.8356768E8 -7 683567693.28 0.0
-8.0 true NULL XH6I7A417 8 436627202 1969-12-31 16:00:15.892 0.0 -436627228.28 NULL NULL 1.0 0.0 0.0 NULL -8 0.0 4.366272E8 -7 436627228.28 0.0
-NULL false -13008.0 10 NULL -896629175 NULL NULL 896629148.72 -13008.0 -1034825.424 NULL NULL NULL -23.0 NULL 0.0 NULL NULL -896629148.72 0.0
-NULL false -7707.0 10vke853 NULL 424959354 NULL NULL -424959380.28 -7707.0 -613114.971 NULL NULL NULL -23.0 NULL 0.0 NULL NULL 424959380.28 0.0
-NULL false 14771.0 10 NULL 993788576 NULL NULL -993788602.28 14771.0 1175077.363 NULL NULL NULL -23.0 NULL 0.0 NULL NULL 993788602.28 0.0
-NULL false 2459.0 10 NULL -394064473 NULL NULL 394064446.72 2459.0 195620.827 NULL NULL NULL -23.0 NULL 0.0 NULL NULL -394064446.72 0.0
-NULL true -14640.0 101n6n461o NULL -819657767 NULL NULL 819657740.72 -14640.0 -1164655.92 NULL NULL NULL -23.0 NULL 0.0 NULL NULL -819657740.72 0.0
-NULL true -14674.0 105aFDAt30c4rI4U NULL -831072496 NULL NULL 831072469.72 -14674.0 -1167360.722 NULL NULL NULL -23.0 NULL 0.0 NULL NULL -831072469.72 0.0
-NULL true -7238.0 10ljXCFT6fG6Qi3S7414e NULL -256767096 NULL NULL 256767069.72 -7238.0 -575804.614 NULL NULL NULL -23.0 NULL 0.0 NULL NULL -256767069.72 0.0
-NULL true -9489.0 10Yr6 NULL 500274721 NULL NULL -500274747.28 -9489.0 -754878.417 NULL NULL NULL -23.0 NULL 0.0 NULL NULL 500274747.28 0.0
-NULL true 10419.0 10 NULL -721614386 NULL NULL 721614359.72 10419.0 828862.7069999999 NULL NULL NULL -23.0 NULL 0.0 NULL NULL -721614359.72 0.0
-NULL true 14519.0 100xJdkyc NULL 729277608 NULL NULL -729277634.28 14519.0 1155030.007 NULL NULL NULL -23.0 NULL 0.0 NULL NULL 729277634.28 0.0
-NULL true 2261.0 10Hr5oB07Ohu0622u NULL 612000160 NULL NULL -612000186.28 2261.0 179869.33299999998 NULL NULL NULL -23.0 NULL 0.0 NULL NULL 612000186.28 0.0
-NULL true 4516.0 10lL0XD6WP2x64f70N0fHmC1 NULL -974538365 NULL NULL 974538338.72 4516.0 359261.348 NULL NULL NULL -23.0 NULL 0.0 NULL NULL -974538338.72 0.0
-NULL true 4767.0 10dUdwyXp5XwgpkTxLffmv3x NULL 187503456 NULL NULL -187503482.28 4767.0 379229.151 NULL NULL NULL -23.0 NULL 0.0 NULL NULL 187503482.28 0.0
-NULL true 5658.0 10M3eGUsKVonbl70DyoCk25 NULL 486382507 NULL NULL -486382533.28 5658.0 450110.874 NULL NULL NULL -23.0 NULL 0.0 NULL NULL 486382533.28 0.0
-NULL true 8659.0 1047piRsT3c3r134I NULL 487236176 NULL NULL -487236202.28 8659.0 688849.427 NULL NULL NULL -23.0 NULL 0.0 NULL NULL 487236202.28 0.0
+-48.0 NULL -7196.0 NULL -48 NULL 1969-12-31 16:00:06.337 NULL NULL -7196.0 -572463.388 33.0 NULL 0.0 -23.0 48 NULL NULL -23 NULL NULL
+-51.0 NULL -200.0 NULL -51 NULL 1969-12-31 15:59:55.423 NULL NULL -200.0 -15910.599999999999 33.0 NULL 0.0 -23.0 51 NULL NULL -23 NULL NULL
+-51.0 false NULL 10 -51 1058319346 1969-12-31 16:00:08.451 NULL -1058319372.28 NULL NULL 33.0 NULL 0.0 NULL 51 NULL 1.05831942E9 -23 1058319372.28 0.0
+-51.0 false NULL 10TYIE5S35U6dj3N -51 -469581869 1969-12-31 16:00:08.451 NULL 469581842.72 NULL NULL 33.0 NULL 0.0 NULL 51 NULL -4.69581792E8 -23 -469581842.72 0.0
+-51.0 false NULL 1Lh6Uoq3WhNtOqQHu7WN7U -51 -352637533 1969-12-31 16:00:08.451 NULL 352637506.72 NULL NULL 33.0 NULL 0.0 NULL 51 NULL -3.52637472E8 -23 -352637506.72 0.0
+-51.0 true NULL 04Y1mA17 -51 -114647521 1969-12-31 16:00:08.451 NULL 114647494.72 NULL NULL 33.0 NULL 0.0 NULL 51 NULL -1.14647472E8 -23 -114647494.72 0.0
+-51.0 true NULL 10Wu570aLPO0p02P17FeH -51 405338893 1969-12-31 16:00:08.451 NULL -405338919.28 NULL NULL 33.0 NULL 0.0 NULL 51 NULL 4.05338944E8 -23 405338919.28 0.0
+-51.0 true NULL 3cQp060 -51 -226923315 1969-12-31 16:00:08.451 NULL 226923288.72 NULL NULL 33.0 NULL 0.0 NULL 51 NULL -2.26923264E8 -23 -226923288.72 0.0
+-51.0 true NULL 8EPG0Xi307qd -51 -328662044 1969-12-31 16:00:08.451 NULL 328662017.72 NULL NULL 33.0 NULL 0.0 NULL 51 NULL -3.28661984E8 -23 -328662017.72 0.0
+-51.0 true NULL 8iHtdkJ6d -51 1006818344 1969-12-31 16:00:08.451 NULL -1006818370.28 NULL NULL 33.0 NULL 0.0 NULL 51 NULL 1.00681843E9 -23 1006818370.28 0.0
+-51.0 true NULL QiOcvR0kt6r7f0R7fiPxQTCU -51 266531954 1969-12-31 16:00:08.451 NULL -266531980.28 NULL NULL 33.0 NULL 0.0 NULL 51 NULL 2.66532E8 -23 266531980.28 0.0
+-51.0 true NULL Ybpj38RTTYl7CnJXPNx1g4C -51 -370919370 1969-12-31 16:00:08.451 NULL 370919343.72 NULL NULL 33.0 NULL 0.0 NULL 51 NULL -3.70919296E8 -23 -370919343.72 0.0
+-6.0 NULL -200.0 NULL -6 NULL 1969-12-31 15:59:56.094 NULL NULL -200.0 -15910.599999999999 3.0 NULL 0.0 -23.0 6 NULL NULL -5 NULL NULL
+-62.0 NULL 15601.0 NULL -62 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 33.0 NULL 0.0 -23.0 62 NULL NULL -23 NULL NULL
+11.0 false NULL 10pO8p1LNx4Y 11 271296824 1969-12-31 16:00:02.351 NULL -271296850.28 NULL NULL 0.0 NULL 0.0 NULL -11 NULL 2.71296832E8 -1 271296850.28 0.0
+11.0 false NULL 1H6wGP 11 -560827082 1969-12-31 16:00:02.351 NULL 560827055.72 NULL NULL 0.0 NULL 0.0 NULL -11 NULL -5.6082707E8 -1 -560827055.72 0.0
+11.0 false NULL 2a7V63IL7jK3o 11 -325931647 1969-12-31 16:00:02.351 NULL 325931620.72 NULL NULL 0.0 NULL 0.0 NULL -11 NULL -3.25931648E8 -1 -325931620.72 0.0
+11.0 true NULL 10 11 92365813 1969-12-31 16:00:02.351 NULL -92365839.28 NULL NULL 0.0 NULL 0.0 NULL -11 NULL 9.2365808E7 -1 92365839.28 0.0
+21.0 NULL 15601.0 NULL 21 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 12.0 NULL 0.0 -23.0 -21 NULL NULL -2 NULL NULL
+32.0 NULL -200.0 NULL 32 NULL 1969-12-31 16:00:02.445 NULL NULL -200.0 -15910.599999999999 1.0 NULL 0.0 -23.0 -32 NULL NULL -23 NULL NULL
+36.0 NULL -200.0 NULL 36 NULL 1969-12-31 16:00:00.554 NULL NULL -200.0 -15910.599999999999 33.0 NULL 0.0 -23.0 -36 NULL NULL -23 NULL NULL
+5.0 NULL 15601.0 NULL 5 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 3.0 NULL 0.0 -23.0 -5 NULL NULL -3 NULL NULL
+58.0 NULL 15601.0 NULL 58 NULL 1969-12-31 15:59:56.527 NULL NULL 15601.0 1241106.353 33.0 NULL 0.0 -23.0 -58 NULL NULL -23 NULL NULL
+8.0 false NULL 10V3pN5r5lI2qWl2lG103 8 -362835731 1969-12-31 16:00:15.892 NULL 362835704.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -3.62835744E8 -7 -362835704.72 0.0
+8.0 false NULL 10c4qt584m5y6uWT 8 -183000142 1969-12-31 16:00:15.892 NULL 183000115.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -1.8300016E8 -7 -183000115.72 0.0
+8.0 false NULL 8GloEukQ0c68JDmnYL53 8 -722873402 1969-12-31 16:00:15.892 NULL 722873375.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -7.2287341E8 -7 -722873375.72 0.0
+8.0 false NULL kA0XH5C5 8 -503903864 1969-12-31 16:00:15.892 NULL 503903837.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -5.03903872E8 -7 -503903837.72 0.0
+8.0 true NULL 100VTM7PEW8GH1uE 8 88129338 1969-12-31 16:00:15.892 NULL -88129364.28 NULL NULL 1.0 NULL 0.0 NULL -8 NULL 8.8129328E7 -7 88129364.28 0.0
+8.0 true NULL 1062158y 8 -1005155523 1969-12-31 16:00:15.892 NULL 1005155496.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -1.00515552E9 -7 -1005155496.72 0.0
+8.0 true NULL 1063cEnGjSal 8 -624769630 1969-12-31 16:00:15.892 NULL 624769603.72 NULL NULL 1.0 NULL 0.0 NULL -8 NULL -6.247696E8 -7 -624769603.72 0.0
+8.0 true NULL 4kMasVoB7lX1wc5i64bNk 8 683567667 1969-12-31 16:00:15.892 NULL -683567693.28 NULL NULL 1.0 NULL 0.0 NULL -8 NULL 6.8356768E8 -7 683567693.28 0.0
+8.0 true NULL XH6I7A417 8 436627202 1969-12-31 16:00:15.892 NULL -436627228.28 NULL NULL 1.0 NULL 0.0 NULL -8 NULL 4.366272E8 -7 436627228.28 0.0
+NULL false -13008.0 10 NULL -896629175 NULL NULL 896629148.72 -13008.0 -1034825.424 NULL NULL NULL -23.0 NULL NULL NULL NULL -896629148.72 0.0
+NULL false -7707.0 10vke853 NULL 424959354 NULL NULL -424959380.28 -7707.0 -613114.971 NULL NULL NULL -23.0 NULL NULL NULL NULL 424959380.28 0.0
+NULL false 14771.0 10 NULL 993788576 NULL NULL -993788602.28 14771.0 1175077.363 NULL NULL NULL -23.0 NULL NULL NULL NULL 993788602.28 0.0
+NULL false 2459.0 10 NULL -394064473 NULL NULL 394064446.72 2459.0 195620.827 NULL NULL NULL -23.0 NULL NULL NULL NULL -394064446.72 0.0
+NULL true -14640.0 101n6n461o NULL -819657767 NULL NULL 819657740.72 -14640.0 -1164655.92 NULL NULL NULL -23.0 NULL NULL NULL NULL -819657740.72 0.0
+NULL true -14674.0 105aFDAt30c4rI4U NULL -831072496 NULL NULL 831072469.72 -14674.0 -1167360.722 NULL NULL NULL -23.0 NULL NULL NULL NULL -831072469.72 0.0
+NULL true -7238.0 10ljXCFT6fG6Qi3S7414e NULL -256767096 NULL NULL 256767069.72 -7238.0 -575804.614 NULL NULL NULL -23.0 NULL NULL NULL NULL -256767069.72 0.0
+NULL true -9489.0 10Yr6 NULL 500274721 NULL NULL -500274747.28 -9489.0 -754878.417 NULL NULL NULL -23.0 NULL NULL NULL NULL 500274747.28 0.0
+NULL true 10419.0 10 NULL -721614386 NULL NULL 721614359.72 10419.0 828862.7069999999 NULL NULL NULL -23.0 NULL NULL NULL NULL -721614359.72 0.0
+NULL true 14519.0 100xJdkyc NULL 729277608 NULL NULL -729277634.28 14519.0 1155030.007 NULL NULL NULL -23.0 NULL NULL NULL NULL 729277634.28 0.0
+NULL true 2261.0 10Hr5oB07Ohu0622u NULL 612000160 NULL NULL -612000186.28 2261.0 179869.33299999998 NULL NULL NULL -23.0 NULL NULL NULL NULL 612000186.28 0.0
+NULL true 4516.0 10lL0XD6WP2x64f70N0fHmC1 NULL -974538365 NULL NULL 974538338.72 4516.0 359261.348 NULL NULL NULL -23.0 NULL NULL NULL NULL -974538338.72 0.0
+NULL true 4767.0 10dUdwyXp5XwgpkTxLffmv3x NULL 187503456 NULL NULL -187503482.28 4767.0 379229.151 NULL NULL NULL -23.0 NULL NULL NULL NULL 187503482.28 0.0
+NULL true 5658.0 10M3eGUsKVonbl70DyoCk25 NULL 486382507 NULL NULL -486382533.28 5658.0 450110.874 NULL NULL NULL -23.0 NULL NULL NULL NULL 486382533.28 0.0
+NULL true 8659.0 1047piRsT3c3r134I NULL 487236176 NULL NULL -487236202.28 8659.0 688849.427 NULL NULL NULL -23.0 NULL NULL NULL NULL 487236202.28 0.0
http://git-wip-us.apache.org/repos/asf/hive/blob/dd04a92f/ql/src/test/results/clientpositive/vectorization_16.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorization_16.q.out b/ql/src/test/results/clientpositive/vectorization_16.q.out
index 930b476..7af0bbd 100644
--- a/ql/src/test/results/clientpositive/vectorization_16.q.out
+++ b/ql/src/test/results/clientpositive/vectorization_16.q.out
@@ -474,168 +474,168 @@ N6BMOr83ecL NULL 1969-12-31 16:00:08.451 NULL NULL 0 NULL NULL NULL NULL NULL 0.
N6Dh6XreCWb0aA4nmDnFOO NULL 1969-12-31 16:00:02.351 NULL NULL 0 NULL NULL NULL NULL NULL 0.000000 NULL
N8222wByj NULL 1969-12-31 16:00:08.451 NULL NULL 0 NULL NULL NULL NULL NULL 0.000000 NULL
NABd3KhjjaVfcj2Q7SJ46 NULL 1969-12-31 16:00:02.351 NULL NULL 0 NULL NULL NULL NULL NULL 0.000000 NULL
-NULL 15601.0 1969-12-31 15:59:43.919 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:44.07 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:44.179 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:44.394 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:44.477 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:44.568 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:44.571 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:44.708 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:44.782 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:45.816 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:46.114 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:46.82 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:46.953 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:47.134 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:47.406 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:47.511 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:47.616 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:47.975 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:48.052 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:48.299 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:48.429 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:48.552 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:48.679 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:48.943 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:49.331 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:49.896 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:50.345 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:50.66 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:51.104 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:51.265 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:51.413 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:51.596 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:51.637 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:52.076 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:52.311 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:52.326 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:52.357 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:52.587 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:53.038 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:53.583 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:53.584 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:53.635 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:54.024 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:54.116 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:54.334 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:54.342 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:54.454 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:54.583 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:54.994 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:55.411 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:55.847 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:55.989 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:55.998 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:56.068 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:56.338 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:56.806 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:56.858 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:56.913 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:56.97 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:57.215 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:57.261 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:57.28 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:57.386 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:57.524 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:57.678 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:57.729 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:57.932 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:58.134 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:58.279 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:58.343 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 15:59:58.752 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.025 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.054 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.108 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.122 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.123 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.182 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.206 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.326 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.396 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.476 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.523 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.545 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.547 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.648 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.699 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.708 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.741 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.88 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.931 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:00.953 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:01.057 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:01.153 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:01.714 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:02.12 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:02.215 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:02.285 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:02.6 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:02.742 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:02.894 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:02.92 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:02.925 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:03.174 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:03.273 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:03.351 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:03.366 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:03.512 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:03.722 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:04.063 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:04.149 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:04.254 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:04.259 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:04.52 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:04.687 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:04.745 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:04.964 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:05.027 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:05.132 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:05.327 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:05.334 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
+NULL 15601.0 1969-12-31 15:59:43.919 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:44.07 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:44.179 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:44.394 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:44.477 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:44.568 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:44.571 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:44.708 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:44.782 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:45.816 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:46.114 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:46.82 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:46.953 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:47.134 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:47.406 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:47.511 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:47.616 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:47.975 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:48.052 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:48.299 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:48.429 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:48.552 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:48.679 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:48.943 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:49.331 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:49.896 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:50.345 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:50.66 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:51.104 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:51.265 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:51.413 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:51.596 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:51.637 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:52.076 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:52.311 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:52.326 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:52.357 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:52.587 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:53.038 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:53.583 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:53.584 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:53.635 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:54.024 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:54.116 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:54.334 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:54.342 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:54.454 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:54.583 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:54.994 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:55.411 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:55.847 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:55.989 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:55.998 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:56.068 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:56.338 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:56.806 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:56.858 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:56.913 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:56.97 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:57.215 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:57.261 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:57.28 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:57.386 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:57.524 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:57.678 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:57.729 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:57.932 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:58.134 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:58.279 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:58.343 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 15:59:58.752 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.025 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.054 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.108 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.122 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.123 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.182 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.206 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.326 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.396 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.476 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.523 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.545 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.547 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.648 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.699 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.708 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.741 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.88 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.931 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:00.953 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:01.057 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:01.153 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:01.714 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:02.12 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:02.215 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:02.285 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:02.6 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:02.742 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:02.894 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:02.92 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:02.925 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:03.174 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:03.273 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:03.351 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:03.366 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:03.512 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:03.722 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:04.063 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:04.149 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:04.254 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:04.259 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:04.52 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:04.687 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:04.745 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:04.964 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:05.027 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:05.132 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:05.327 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:05.334 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
NULL 15601.0 1969-12-31 16:00:05.617 -9747614.5639 9747614.5639 2 0.0 -0.0 0.0 15601.0 625.8070356964297 -1.439885 0.0
-NULL 15601.0 1969-12-31 16:00:05.83 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:06.051 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:06.692 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:07.844 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:08.176 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:08.252 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:08.368 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:08.607 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:08.868 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:08.948 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:09.357 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:09.473 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:09.582 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:09.697 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:10.045 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:10.132 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:10.173 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:10.259 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:10.649 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:10.738 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:10.898 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:10.957 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:10.983 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:12.205 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:12.498 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:12.848 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:12.853 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:12.948 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:13.029 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:13.183 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:13.503 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:13.801 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:13.955 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:14.452 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:14.565 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:14.733 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:14.747 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:14.903 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:15.39 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:15.805 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
-NULL 15601.0 1969-12-31 16:00:16.076 -9747614.5639 9747614.5639 1 0.0 -0.0 0.0 15601.0 625.8070356964297 -0.719942 0.0
+NULL 15601.0 1969-12-31 16:00:05.83 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:06.051 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:06.692 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:07.844 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:08.176 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:08.252 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:08.368 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:08.607 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:08.868 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:08.948 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:09.357 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:09.473 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:09.582 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:09.697 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:10.045 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:10.132 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:10.173 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:10.259 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:10.649 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:10.738 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:10.898 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:10.957 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:10.983 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:12.205 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:12.498 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:12.848 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:12.853 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:12.948 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:13.029 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:13.183 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:13.503 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:13.801 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:13.955 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:14.452 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:14.565 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:14.733 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:14.747 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:14.903 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:15.39 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:15.805 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
+NULL 15601.0 1969-12-31 16:00:16.076 -9747614.5639 9747614.5639 1 NULL NULL NULL 15601.0 625.8070356964297 -0.719942 NULL
NULL 15601.0 1969-12-31 16:00:16.279 -9747614.5639 9747614.5639 2 0.0 -0.0 0.0 15601.0 625.8070356964297 -1.439885 0.0
Nmt6E360X6dpX58CR2 NULL 1969-12-31 16:00:02.351 NULL NULL 0 NULL NULL NULL NULL NULL 0.000000 NULL
O2U2c43Dx4QtYQ3ynA1CLGI3 NULL 1969-12-31 16:00:15.892 NULL NULL 0 NULL NULL NULL NULL NULL 0.000000 NULL
[18/50] [abbrv] hive git commit: HIVE-17307 Change the metastore to
not use the metrics code in hive/common. This closes #235. (Alan Gates,
reviewed by Vihang Karajgaonkar)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/a6b9cd52/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/metrics/TestMetrics.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/metrics/TestMetrics.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/metrics/TestMetrics.java
new file mode 100644
index 0000000..259a4db
--- /dev/null
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/metrics/TestMetrics.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.metrics;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Timer;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+public class TestMetrics {
+
+ @Test
+ public void jsonReporter() throws Exception {
+ String jsonFile = System.getProperty("java.io.tmpdir") + System.getProperty("file.separator") +
+ "TestMetricsOutput.json";
+ Configuration conf = MetastoreConf.newMetastoreConf();
+ MetastoreConf.setVar(conf, MetastoreConf.ConfVars.METRICS_REPORTERS, "json");
+ MetastoreConf.setVar(conf, MetastoreConf.ConfVars.METRICS_JSON_FILE_LOCATION, jsonFile);
+ MetastoreConf.setTimeVar(conf, MetastoreConf.ConfVars.METRICS_JSON_FILE_INTERVAL, 1,
+ TimeUnit.SECONDS);
+
+ Metrics.initialize(conf);
+
+ final List<String> words = Arrays.asList("mary", "had", "a", "little", "lamb");
+ MetricRegistry registry = Metrics.getRegistry();
+ registry.register("my-gauge", new Gauge<Integer>() {
+
+ @Override
+ public Integer getValue() {
+ return words.size();
+ }
+ });
+
+ Counter counter = Metrics.getOrCreateCounter("my-counter");
+ counter.inc();
+ counter.inc();
+
+ Meter meter = registry.meter("my-meter");
+ meter.mark();
+ Thread.sleep(10);
+ meter.mark();
+
+ Timer timer = Metrics.getOrCreateTimer("my-timer");
+ timer.time(new Callable<Long>() {
+ @Override
+ public Long call() throws Exception {
+ Thread.sleep(100);
+ return 1L;
+ }
+ });
+
+ // Make sure it has a chance to dump it.
+ Thread.sleep(2000);
+
+ FileSystem fs = FileSystem.get(conf);
+ Path path = new Path(jsonFile);
+ Assert.assertTrue(fs.exists(path));
+
+ String json = new String(MetricsTestUtils.getFileData(jsonFile, 200, 10));
+ MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, "my-counter", 2);
+ MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, "my-meter", 2);
+ MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.TIMER, "my-timer", 1);
+ MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.GAUGE, "my-gauge", 5);
+ }
+
+ @Test
+ public void allReporters() throws Exception {
+ String jsonFile = System.getProperty("java.io.tmpdir") + System.getProperty("file.separator") +
+ "TestMetricsOutput.json";
+ Configuration conf = MetastoreConf.newMetastoreConf();
+ MetastoreConf.setVar(conf, MetastoreConf.ConfVars.METRICS_REPORTERS, "json,jmx,console,hadoop");
+ MetastoreConf.setVar(conf, MetastoreConf.ConfVars.METRICS_JSON_FILE_LOCATION, jsonFile);
+
+ Metrics.initialize(conf);
+
+ Assert.assertEquals(4, Metrics.getReporters().size());
+ }
+
+ @Test
+ public void allReportersHiveConfig() throws Exception {
+ String jsonFile = System.getProperty("java.io.tmpdir") + System.getProperty("file.separator") +
+ "TestMetricsOutput.json";
+ Configuration conf = MetastoreConf.newMetastoreConf();
+ conf.set(MetastoreConf.ConfVars.HIVE_CODAHALE_METRICS_REPORTER_CLASSES.hiveName,
+ "org.apache.hadoop.hive.common.metrics.metrics2.JsonFileMetricsReporter," +
+ "org.apache.hadoop.hive.common.metrics.metrics2.JmxMetricsReporter," +
+ "org.apache.hadoop.hive.common.metrics.metrics2.ConsoleMetricsReporter," +
+ "org.apache.hadoop.hive.common.metrics.metrics2.Metrics2Reporter");
+ MetastoreConf.setVar(conf, MetastoreConf.ConfVars.METRICS_JSON_FILE_LOCATION, jsonFile);
+
+ Metrics.initialize(conf);
+
+ Assert.assertEquals(4, Metrics.getReporters().size());
+ }
+
+ @Test
+ public void allReportersOldHiveConfig() throws Exception {
+ String jsonFile = System.getProperty("java.io.tmpdir") + System.getProperty("file.separator") +
+ "TestMetricsOutput.json";
+ Configuration conf = MetastoreConf.newMetastoreConf();
+ conf.set(MetastoreConf.ConfVars.HIVE_METRICS_REPORTER.hiveName,
+ "JSON_FILE,JMX,CONSOLE,HADOOP2");
+ MetastoreConf.setVar(conf, MetastoreConf.ConfVars.METRICS_JSON_FILE_LOCATION, jsonFile);
+
+ Metrics.initialize(conf);
+
+ Assert.assertEquals(4, Metrics.getReporters().size());
+ }
+
+ @Test
+ public void defaults() throws Exception {
+ String jsonFile = System.getProperty("java.io.tmpdir") + System.getProperty("file.separator") +
+ "TestMetricsOutput.json";
+ Configuration conf = MetastoreConf.newMetastoreConf();
+ MetastoreConf.setVar(conf, MetastoreConf.ConfVars.METRICS_JSON_FILE_LOCATION, jsonFile);
+ Metrics.initialize(conf);
+
+ Assert.assertEquals(2, Metrics.getReporters().size());
+ }
+
+ @Before
+ public void shutdownMetrics() {
+ Metrics.shutdown();
+ }
+
+ // Stolen from Hive's MetricsTestUtils. Probably should break it out into it's own class.
+ private static class MetricsTestUtils {
+
+ static final MetricsCategory COUNTER = new MetricsCategory("counters", "count");
+ static final MetricsCategory TIMER = new MetricsCategory("timers", "count");
+ static final MetricsCategory GAUGE = new MetricsCategory("gauges", "value");
+ static final MetricsCategory METER = new MetricsCategory("meters", "count");
+
+ static class MetricsCategory {
+ String category;
+ String metricsHandle;
+ MetricsCategory(String category, String metricsHandle) {
+ this.category = category;
+ this.metricsHandle = metricsHandle;
+ }
+ }
+
+ static void verifyMetricsJson(String json, MetricsCategory category, String metricsName,
+ Object expectedValue) throws Exception {
+ JsonNode jsonNode = getJsonNode(json, category, metricsName);
+ Assert.assertEquals(expectedValue.toString(), jsonNode.asText());
+ }
+
+ static JsonNode getJsonNode(String json, MetricsCategory category, String metricsName) throws Exception {
+ ObjectMapper objectMapper = new ObjectMapper();
+ JsonNode rootNode = objectMapper.readTree(json);
+ JsonNode categoryNode = rootNode.path(category.category);
+ JsonNode metricsNode = categoryNode.path(metricsName);
+ return metricsNode.path(category.metricsHandle);
+ }
+
+ static byte[] getFileData(String path, int timeoutInterval, int tries) throws Exception {
+ File file = new File(path);
+ do {
+ Thread.sleep(timeoutInterval);
+ tries--;
+ } while (tries > 0 && !file.exists());
+ return Files.readAllBytes(Paths.get(path));
+ }
+ }
+}
[42/50] [abbrv] hive git commit: HIVE-14671 : merge master into
hive-14535 (Wei Zheng)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClientCapability.java
----------------------------------------------------------------------
diff --cc standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClientCapability.java
index 0000000,f53bd82..8fc8311
mode 000000,100644..100644
--- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClientCapability.java
+++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClientCapability.java
@@@ -1,0 -1,42 +1,45 @@@
+ /**
+ * Autogenerated by Thrift Compiler (0.9.3)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+ package org.apache.hadoop.hive.metastore.api;
+
+
+ import java.util.Map;
+ import java.util.HashMap;
+ import org.apache.thrift.TEnum;
+
+ public enum ClientCapability implements org.apache.thrift.TEnum {
- TEST_CAPABILITY(1);
++ TEST_CAPABILITY(1),
++ INSERT_ONLY_TABLES(2);
+
+ private final int value;
+
+ private ClientCapability(int value) {
+ this.value = value;
+ }
+
+ /**
+ * Get the integer value of this enum value, as defined in the Thrift IDL.
+ */
+ public int getValue() {
+ return value;
+ }
+
+ /**
+ * Find a the enum type by its integer value, as defined in the Thrift IDL.
+ * @return null if the value is not found.
+ */
+ public static ClientCapability findByValue(int value) {
+ switch (value) {
+ case 1:
+ return TEST_CAPABILITY;
++ case 2:
++ return INSERT_ONLY_TABLES;
+ default:
+ return null;
+ }
+ }
+ }
[35/50] [abbrv] hive git commit: HIVE-17385: incremental repl error
for non-native tables
Posted by we...@apache.org.
HIVE-17385: incremental repl error for non-native tables
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/45f3fac5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/45f3fac5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/45f3fac5
Branch: refs/heads/hive-14535
Commit: 45f3fac554ada3cd4b15a7bf3c9f0047c65fa616
Parents: 601c764
Author: Daniel Dai <da...@hortonworks.com>
Authored: Thu Aug 31 21:08:02 2017 -0700
Committer: Daniel Dai <da...@hortonworks.com>
Committed: Thu Aug 31 21:08:02 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/parse/EximUtil.java | 30 ++++++++++++++++++++
.../hive/ql/parse/ImportSemanticAnalyzer.java | 5 ++++
.../hive/ql/parse/repl/dump/TableExport.java | 7 +----
.../repl/dump/events/CreateTableHandler.java | 8 ++++--
.../ql/parse/repl/dump/io/TableSerializer.java | 7 ++---
5 files changed, 44 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/45f3fac5/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
index 22094c0..40c34bf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.repl.dump.io.DBSerializer;
import org.apache.hadoop.hive.ql.parse.repl.dump.io.JsonWriter;
import org.apache.hadoop.hive.ql.parse.repl.dump.io.ReplicationSpecSerializer;
@@ -385,4 +386,33 @@ public class EximUtil {
};
}
+ /**
+ * Verify if a table should be exported or not
+ */
+ public static Boolean shouldExportTable(ReplicationSpec replicationSpec, Table tableHandle) throws SemanticException {
+ if (replicationSpec == null)
+ {
+ replicationSpec = new ReplicationSpec();
+ }
+
+ if (replicationSpec.isNoop())
+ {
+ return false;
+ }
+
+ if (tableHandle == null)
+ {
+ return false;
+ }
+
+ if (replicationSpec.isInReplicationScope()) {
+ return !(tableHandle == null || tableHandle.isTemporary() || tableHandle.isNonNative());
+ }
+
+ if (tableHandle.isNonNative()) {
+ throw new SemanticException(ErrorMsg.EXIM_FOR_NON_NATIVE.getMsg());
+ }
+
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/45f3fac5/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
index f8d5c8d..7f3460f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
@@ -197,6 +197,11 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(), e);
}
+ if (rv.getTable() == null) {
+ // nothing to do here, silently return.
+ return false;
+ }
+
ReplicationSpec replicationSpec = rv.getReplicationSpec();
if (replicationSpec.isNoop()){
// nothing to do here, silently return.
http://git-wip-us.apache.org/repos/asf/hive/blob/45f3fac5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
index 7533a39..e490633 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
@@ -156,12 +156,7 @@ public class TableExport {
}
private boolean shouldExport() throws SemanticException {
- if (replicationSpec.isInReplicationScope()) {
- return !(tableSpec.tableHandle.isTemporary() || tableSpec.tableHandle.isNonNative());
- } else if (tableSpec.tableHandle.isNonNative()) {
- throw new SemanticException(ErrorMsg.EXIM_FOR_NON_NATIVE.getMsg());
- }
- return true;
+ return EximUtil.shouldExportTable(replicationSpec, tableSpec.tableHandle);
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/45f3fac5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java
index 8737d50..ef6f340 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java
@@ -23,13 +23,12 @@ import org.apache.hadoop.hive.metastore.api.NotificationEvent;
import org.apache.hadoop.hive.metastore.messaging.CreateTableMessage;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.EximUtil;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
-import org.apache.hadoop.hive.ql.parse.repl.DumpType;
-
class CreateTableHandler extends AbstractEventHandler {
CreateTableHandler(NotificationEvent event) {
@@ -48,6 +47,11 @@ class CreateTableHandler extends AbstractEventHandler {
}
Table qlMdTable = new Table(tobj);
+
+ if (!EximUtil.shouldExportTable(withinContext.replicationSpec, qlMdTable)) {
+ return;
+ }
+
if (qlMdTable.isView()) {
withinContext.replicationSpec.setIsMetadataOnly(true);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/45f3fac5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java
index 92408a4..c3a70cc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.parse.EximUtil;
import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.thrift.TException;
@@ -44,7 +45,7 @@ public class TableSerializer implements JsonWriter.Serializer {
@Override
public void writeTo(JsonWriter writer, ReplicationSpec additionalPropertiesProvider)
throws SemanticException, IOException {
- if (cannotReplicateTable(additionalPropertiesProvider)) {
+ if (!EximUtil.shouldExportTable(additionalPropertiesProvider, tableHandle)) {
return;
}
@@ -61,10 +62,6 @@ public class TableSerializer implements JsonWriter.Serializer {
}
}
- private boolean cannotReplicateTable(ReplicationSpec additionalPropertiesProvider) {
- return tableHandle == null || additionalPropertiesProvider.isNoop();
- }
-
private Table addPropertiesToTable(Table table, ReplicationSpec additionalPropertiesProvider)
throws SemanticException, IOException {
if (additionalPropertiesProvider.isInReplicationScope()) {
[36/50] [abbrv] hive git commit: HIVE-17415 - Hit error
"SemanticException View xxx is corresponding to LIMIT,
rather than a SelectOperator." in Hive queries (Deepak Jaiswal,
reviewed by Ashutosh Chauhan)
Posted by we...@apache.org.
HIVE-17415 - Hit error "SemanticException View xxx is corresponding to LIMIT, rather than a SelectOperator." in Hive queries (Deepak Jaiswal, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b82d38aa
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b82d38aa
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b82d38aa
Branch: refs/heads/hive-14535
Commit: b82d38aa38f71a66bee47977c041e7d8a28c8419
Parents: 45f3fac
Author: Deepak Jaiswal <de...@gmail.com>
Authored: Thu Aug 31 21:40:22 2017 -0700
Committer: Vineet Garg <vg...@apache.com>
Committed: Thu Aug 31 21:41:55 2017 -0700
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 2 +
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 8 ++-
.../clientpositive/authorization_view_8.q | 23 +++++++
.../llap/authorization_view_8.q.out | 69 ++++++++++++++++++++
4 files changed, 101 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b82d38aa/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index f452341..6504250 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -87,6 +87,7 @@ minillap.shared.query.files=insert_into1.q,\
minillaplocal.shared.query.files=alter_merge_2_orc.q,\
alter_merge_orc.q,\
alter_merge_stats_orc.q,\
+ authorization_view_8.q,\
auto_join0.q,\
auto_join1.q,\
auto_join21.q,\
@@ -462,6 +463,7 @@ minillaplocal.query.files=\
acid_globallimit.q,\
acid_vectorization_missing_cols.q,\
alter_merge_stats_orc.q,\
+ authorization_view_8.q,\
auto_join30.q,\
auto_join_filters.q,\
auto_join_nulls.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/b82d38aa/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index e8acabe..6f379da 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -92,6 +92,7 @@ import org.apache.hadoop.hive.ql.exec.FunctionInfo;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
import org.apache.hadoop.hive.ql.exec.JoinOperator;
+import org.apache.hadoop.hive.ql.exec.LimitOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.OperatorFactory;
import org.apache.hadoop.hive.ql.exec.RecordReader;
@@ -10547,10 +10548,15 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
// Recurse over the subqueries to fill the subquery part of the plan
for (String alias : qb.getSubqAliases()) {
QBExpr qbexpr = qb.getSubqForAlias(alias);
- Operator operator = genPlan(qb, qbexpr);
+ Operator<?> operator = genPlan(qb, qbexpr);
aliasToOpInfo.put(alias, operator);
if (qb.getViewToTabSchema().containsKey(alias)) {
// we set viewProjectToTableSchema so that we can leverage ColumnPruner.
+ if (operator instanceof LimitOperator) {
+ // If create view has LIMIT operator, this can happen
+ // Fetch parent operator
+ operator = operator.getParentOperators().get(0);
+ }
if (operator instanceof SelectOperator) {
if (this.viewProjectToTableSchema == null) {
this.viewProjectToTableSchema = new LinkedHashMap<>();
http://git-wip-us.apache.org/repos/asf/hive/blob/b82d38aa/ql/src/test/queries/clientpositive/authorization_view_8.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/authorization_view_8.q b/ql/src/test/queries/clientpositive/authorization_view_8.q
new file mode 100644
index 0000000..82fc97b
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/authorization_view_8.q
@@ -0,0 +1,23 @@
+set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.DefaultHiveAuthorizationProvider;
+
+create table my_passwd (
+username string,
+uid int);
+
+insert into my_passwd values
+ ("Deepak", 1),
+ ("Gunther", 2),
+ ("Jason", 3),
+ ("Prasanth", 4),
+ ("Gopal", 5),
+ ("Sergey", 6);
+
+
+set hive.cbo.enable=false;
+create view my_passwd_vw as select * from my_passwd limit 3;
+
+set hive.security.authorization.enabled=true;
+grant select on table my_passwd to user hive_test_user;
+grant select on table my_passwd_vw to user hive_test_user;
+
+select * from my_passwd_vw;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/b82d38aa/ql/src/test/results/clientpositive/llap/authorization_view_8.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/authorization_view_8.q.out b/ql/src/test/results/clientpositive/llap/authorization_view_8.q.out
new file mode 100644
index 0000000..67ddb32
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/authorization_view_8.q.out
@@ -0,0 +1,69 @@
+PREHOOK: query: create table my_passwd (
+username string,
+uid int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@my_passwd
+POSTHOOK: query: create table my_passwd (
+username string,
+uid int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@my_passwd
+PREHOOK: query: insert into my_passwd values
+ ("Deepak", 1),
+ ("Gunther", 2),
+ ("Jason", 3),
+ ("Prasanth", 4),
+ ("Gopal", 5),
+ ("Sergey", 6)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@my_passwd
+POSTHOOK: query: insert into my_passwd values
+ ("Deepak", 1),
+ ("Gunther", 2),
+ ("Jason", 3),
+ ("Prasanth", 4),
+ ("Gopal", 5),
+ ("Sergey", 6)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@my_passwd
+POSTHOOK: Lineage: my_passwd.uid EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: my_passwd.username SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: create view my_passwd_vw as select * from my_passwd limit 3
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@my_passwd
+PREHOOK: Output: database:default
+PREHOOK: Output: default@my_passwd_vw
+POSTHOOK: query: create view my_passwd_vw as select * from my_passwd limit 3
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@my_passwd
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@my_passwd_vw
+POSTHOOK: Lineage: my_passwd_vw.uid SIMPLE [(my_passwd)my_passwd.FieldSchema(name:uid, type:int, comment:null), ]
+POSTHOOK: Lineage: my_passwd_vw.username SIMPLE [(my_passwd)my_passwd.FieldSchema(name:username, type:string, comment:null), ]
+PREHOOK: query: grant select on table my_passwd to user hive_test_user
+PREHOOK: type: GRANT_PRIVILEGE
+PREHOOK: Output: default@my_passwd
+POSTHOOK: query: grant select on table my_passwd to user hive_test_user
+POSTHOOK: type: GRANT_PRIVILEGE
+POSTHOOK: Output: default@my_passwd
+PREHOOK: query: grant select on table my_passwd_vw to user hive_test_user
+PREHOOK: type: GRANT_PRIVILEGE
+PREHOOK: Output: default@my_passwd_vw
+POSTHOOK: query: grant select on table my_passwd_vw to user hive_test_user
+POSTHOOK: type: GRANT_PRIVILEGE
+POSTHOOK: Output: default@my_passwd_vw
+PREHOOK: query: select * from my_passwd_vw
+PREHOOK: type: QUERY
+PREHOOK: Input: default@my_passwd
+PREHOOK: Input: default@my_passwd_vw
+#### A masked pattern was here ####
+POSTHOOK: query: select * from my_passwd_vw
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@my_passwd
+POSTHOOK: Input: default@my_passwd_vw
+#### A masked pattern was here ####
+Deepak 1
+Gunther 2
+Jason 3
[14/50] [abbrv] hive git commit: HIVE-17375: stddev_samp,
var_samp standard compliance (Zoltan Haindrich, reviewed by Ashutosh Chauhan)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/dd04a92f/ql/src/test/results/clientpositive/vectorization_12.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorization_12.q.out b/ql/src/test/results/clientpositive/vectorization_12.q.out
index df3f047..63ea984 100644
--- a/ql/src/test/results/clientpositive/vectorization_12.q.out
+++ b/ql/src/test/results/clientpositive/vectorization_12.q.out
@@ -284,535 +284,535 @@ ORDER BY ctimestamp1, cdouble, cbigint, cstring1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
--1645852809 false DUSKf88a NULL 6764.0 -4.3506048E7 1645852809 1 -1645852809 0.0 6764.0 -6764.0 6764.0 4.3506048E7 -1645858447.15 -1645852809 -1.554726368159204E-4 6764.0 6764.0 4.3512812E7 0.0
--1645852809 false G7Ve8Px6a7J0DafBodF8JMma NULL -1291.0 8303712.0 1645852809 1 -1645852809 0.0 -1291.0 1291.0 -1291.0 -8303712.0 -1645858447.15 -1645852809 -1.554726368159204E-4 -1291.0 -1291.0 -8305003.0 0.0
--1645852809 false K7tGy146ydka NULL -1236.0 7949952.0 1645852809 1 -1645852809 0.0 -1236.0 1236.0 -1236.0 -7949952.0 -1645858447.15 -1645852809 -1.554726368159204E-4 -1236.0 -1236.0 -7951188.0 0.0
--1645852809 false OHG2wWD83Ba NULL 6914.0 -4.4470848E7 1645852809 1 -1645852809 0.0 6914.0 -6914.0 6914.0 4.4470848E7 -1645858447.15 -1645852809 -1.554726368159204E-4 6914.0 6914.0 4.4477762E7 0.0
--1645852809 false S7UM6KgdxTofi6rwXBFa2a NULL 12520.0 -8.052864E7 1645852809 1 -1645852809 0.0 12520.0 -12520.0 12520.0 8.052864E7 -1645858447.15 -1645852809 -1.554726368159204E-4 12520.0 12520.0 8.054116E7 0.0
--1645852809 false eNsh5tYa NULL NULL NULL 1645852809 1 -1645852809 0.0 NULL NULL NULL NULL -1645858447.15 -1645852809 NULL NULL NULL NULL NULL
--1645852809 false iS4P5128HY44wa NULL 3890.0 -2.502048E7 1645852809 1 -1645852809 0.0 3890.0 -3890.0 3890.0 2.502048E7 -1645858447.15 -1645852809 -1.554726368159204E-4 3890.0 3890.0 2.502437E7 0.0
--1645852809 false kro4Xu41bB7hiFa NULL -3277.0 2.1077664E7 1645852809 1 -1645852809 0.0 -3277.0 3277.0 -3277.0 -2.1077664E7 -1645858447.15 -1645852809 -1.554726368159204E-4 -3277.0 -3277.0 -2.1080941E7 0.0
--1645852809 false lJ63qx87BLmdMfa NULL 11619.0 -7.4733408E7 1645852809 1 -1645852809 0.0 11619.0 -11619.0 11619.0 7.4733408E7 -1645858447.15 -1645852809 -1.554726368159204E-4 11619.0 11619.0 7.4745027E7 0.0
--1645852809 true 4gBPJa NULL 13167.0 -8.4690144E7 1645852809 1 -1645852809 0.0 13167.0 -13167.0 13167.0 8.4690144E7 -1645858447.15 -1645852809 -1.554726368159204E-4 13167.0 13167.0 8.4703311E7 0.0
--1645852809 true L057p1HPpJsmA3a NULL -9542.0 6.1374144E7 1645852809 1 -1645852809 0.0 -9542.0 9542.0 -9542.0 -6.1374144E7 -1645858447.15 -1645852809 -1.554726368159204E-4 -9542.0 -9542.0 -6.1383686E7 0.0
--1645852809 true PMoJ1NvQoAm5a NULL 539.0 -3466848.0 1645852809 1 -1645852809 0.0 539.0 -539.0 539.0 3466848.0 -1645858447.15 -1645852809 -1.554726368159204E-4 539.0 539.0 3467387.0 0.0
--1645852809 true Tt484a NULL 754.0 -4849728.0 1645852809 1 -1645852809 0.0 754.0 -754.0 754.0 4849728.0 -1645858447.15 -1645852809 -1.554726368159204E-4 754.0 754.0 4850482.0 0.0
--1645852809 true a NULL -2944.0 1.8935808E7 1645852809 1 -1645852809 0.0 -2944.0 2944.0 -2944.0 -1.8935808E7 -1645858447.15 -1645852809 -1.554726368159204E-4 -2944.0 -2944.0 -1.8938752E7 0.0
--1645852809 true a NULL -5905.0 3.798096E7 1645852809 1 -1645852809 0.0 -5905.0 5905.0 -5905.0 -3.798096E7 -1645858447.15 -1645852809 -1.554726368159204E-4 -5905.0 -5905.0 -3.7986865E7 0.0
--1645852809 true a NULL 4991.0 -3.2102112E7 1645852809 1 -1645852809 0.0 4991.0 -4991.0 4991.0 3.2102112E7 -1645858447.15 -1645852809 -1.554726368159204E-4 4991.0 4991.0 3.2107103E7 0.0
--1645852809 true bBAKio7bAmQq7vIlsc8H14a NULL 1949.0 -1.2535968E7 1645852809 1 -1645852809 0.0 1949.0 -1949.0 1949.0 1.2535968E7 -1645858447.15 -1645852809 -1.554726368159204E-4 1949.0 1949.0 1.2537917E7 0.0
--1645852809 true dun2EEixI701imr3d6a NULL -8352.0 5.3720064E7 1645852809 1 -1645852809 0.0 -8352.0 8352.0 -8352.0 -5.3720064E7 -1645858447.15 -1645852809 -1.554726368159204E-4 -8352.0 -8352.0 -5.3728416E7 0.0
--1645852809 true hnq6hkAfna NULL 5926.0 -3.8116032E7 1645852809 1 -1645852809 0.0 5926.0 -5926.0 5926.0 3.8116032E7 -1645858447.15 -1645852809 -1.554726368159204E-4 5926.0 5926.0 3.8121958E7 0.0
--1887561756 false 5712We1FSa NULL 8801.0 -5.6608032E7 1887561756 1 -1887561756 0.0 8801.0 -8801.0 8801.0 5.6608032E7 -1887567394.15 -1887561756 -1.554726368159204E-4 8801.0 8801.0 5.6616833E7 0.0
--1887561756 false a NULL 3350.0 -2.15472E7 1887561756 1 -1887561756 0.0 3350.0 -3350.0 3350.0 2.15472E7 -1887567394.15 -1887561756 -1.554726368159204E-4 3350.0 3350.0 2.155055E7 0.0
--1887561756 false f3oGa8ByjMs5eo7462S84Aa NULL 4278.0 -2.7516096E7 1887561756 1 -1887561756 0.0 4278.0 -4278.0 4278.0 2.7516096E7 -1887567394.15 -1887561756 -1.554726368159204E-4 4278.0 4278.0 2.7520374E7 0.0
--1887561756 false w62rRn0DnCSWJ1ht6qWa NULL -5638.15 3.62645808E7 1887561756 1 -1887561756 0.0 -5638.15 5638.15 -5638.15 -3.62645808E7 -1887567394.15 -1887561756 -1.554726368159204E-4 -5638.15 -5638.15 -3.6270218949999996E7 0.0
--1887561756 true 055VA1s2XC7q70aD8S0PLpa NULL -12485.0 8.030352E7 1887561756 1 -1887561756 0.0 -12485.0 12485.0 -12485.0 -8.030352E7 -1887567394.15 -1887561756 -1.554726368159204E-4 -12485.0 -12485.0 -8.0316005E7 0.0
--1887561756 true 47x5248dXuiqta NULL -12888.0 8.2895616E7 1887561756 1 -1887561756 0.0 -12888.0 12888.0 -12888.0 -8.2895616E7 -1887567394.15 -1887561756 -1.554726368159204E-4 -12888.0 -12888.0 -8.2908504E7 0.0
--1887561756 true 7C1L24VM7Ya NULL 4122.0 -2.6512704E7 1887561756 1 -1887561756 0.0 4122.0 -4122.0 4122.0 2.6512704E7 -1887567394.15 -1887561756 -1.554726368159204E-4 4122.0 4122.0 2.6516826E7 0.0
--1887561756 true FWCW47mXs2a NULL -6839.0 4.3988448E7 1887561756 1 -1887561756 0.0 -6839.0 6839.0 -6839.0 -4.3988448E7 -1887567394.15 -1887561756 -1.554726368159204E-4 -6839.0 -6839.0 -4.3995287E7 0.0
--1887561756 true LAFo0rFpPj1aW8Js4Scpa NULL 2719.0 -1.7488608E7 1887561756 1 -1887561756 0.0 2719.0 -2719.0 2719.0 1.7488608E7 -1887567394.15 -1887561756 -1.554726368159204E-4 2719.0 2719.0 1.7491327E7 0.0
--1887561756 true hQAra NULL 14460.0 -9.300672E7 1887561756 1 -1887561756 0.0 14460.0 -14460.0 14460.0 9.300672E7 -1887567394.15 -1887561756 -1.554726368159204E-4 14460.0 14460.0 9.302118E7 0.0
-1864027286 true 01I27lE0Ec60Vhk6H72 NULL 4272.0 -2.7477504E7 -1864027286 1 1864027286 0.0 4272.0 -4272.0 4272.0 2.7477504E7 1864021647.85 1864027286 -1.554726368159204E-4 4272.0 4272.0 2.7481776E7 0.0
-1864027286 true 01L3ajd5YosmyM330V3s NULL 3756.0 -2.4158592E7 -1864027286 1 1864027286 0.0 3756.0 -3756.0 3756.0 2.4158592E7 1864021647.85 1864027286 -1.554726368159204E-4 3756.0 3756.0 2.4162348E7 0.0
-1864027286 true 03R4fW3q25Kl NULL -11690.0 7.519008E7 -1864027286 1 1864027286 0.0 -11690.0 11690.0 -11690.0 -7.519008E7 1864021647.85 1864027286 -1.554726368159204E-4 -11690.0 -11690.0 -7.520177E7 0.0
-1864027286 true 03jQEYjRQjm7 NULL -6739.0 4.3345248E7 -1864027286 1 1864027286 0.0 -6739.0 6739.0 -6739.0 -4.3345248E7 1864021647.85 1864027286 -1.554726368159204E-4 -6739.0 -6739.0 -4.3351987E7 0.0
-1864027286 true 067wD7F8YQ8h32jPa NULL -16012.0 1.02989184E8 -1864027286 1 1864027286 0.0 -16012.0 16012.0 -16012.0 -1.02989184E8 1864021647.85 1864027286 -1.554726368159204E-4 -16012.0 -16012.0 -1.03005196E8 0.0
-1864027286 true 08s07Nn26i3mlR5Bl83Ppo8L NULL 474.0 -3048768.0 -1864027286 1 1864027286 0.0 474.0 -474.0 474.0 3048768.0 1864021647.85 1864027286 -1.554726368159204E-4 474.0 474.0 3049242.0 0.0
-1864027286 true 0AP3HERf5Ra NULL 5045.0 -3.244944E7 -1864027286 1 1864027286 0.0 5045.0 -5045.0 5045.0 3.244944E7 1864021647.85 1864027286 -1.554726368159204E-4 5045.0 5045.0 3.2454485E7 0.0
-1864027286 true 0I62LB NULL -5466.0 3.5157312E7 -1864027286 1 1864027286 0.0 -5466.0 5466.0 -5466.0 -3.5157312E7 1864021647.85 1864027286 -1.554726368159204E-4 -5466.0 -5466.0 -3.5162778E7 0.0
-1864027286 true 0RvxJiyole51yN5 NULL -1211.0 7789152.0 -1864027286 1 1864027286 0.0 -1211.0 1211.0 -1211.0 -7789152.0 1864021647.85 1864027286 -1.554726368159204E-4 -1211.0 -1211.0 -7790363.0 0.0
-1864027286 true 0W67K0mT27r22f817281Ocq NULL -5818.0 3.7421376E7 -1864027286 1 1864027286 0.0 -5818.0 5818.0 -5818.0 -3.7421376E7 1864021647.85 1864027286 -1.554726368159204E-4 -5818.0 -5818.0 -3.7427194E7 0.0
-1864027286 true 0ag0Cv NULL -5942.0 3.8218944E7 -1864027286 1 1864027286 0.0 -5942.0 5942.0 -5942.0 -3.8218944E7 1864021647.85 1864027286 -1.554726368159204E-4 -5942.0 -5942.0 -3.8224886E7 0.0
-1864027286 true 0eODhoL30gUMY NULL 2590.0 -1.665888E7 -1864027286 1 1864027286 0.0 2590.0 -2590.0 2590.0 1.665888E7 1864021647.85 1864027286 -1.554726368159204E-4 2590.0 2590.0 1.666147E7 0.0
-1864027286 true 0kywHd7EpIq611b5F8dkKd NULL 14509.0 -9.3321888E7 -1864027286 1 1864027286 0.0 14509.0 -14509.0 14509.0 9.3321888E7 1864021647.85 1864027286 -1.554726368159204E-4 14509.0 14509.0 9.3336397E7 0.0
-1864027286 true 0mrq5CsKD4aq5mt26hUAYN54 NULL 1329.0 -8548128.0 -1864027286 1 1864027286 0.0 1329.0 -1329.0 1329.0 8548128.0 1864021647.85 1864027286 -1.554726368159204E-4 1329.0 1329.0 8549457.0 0.0
-1864027286 true 0oNy2Lac8mgIoM408U8bisc NULL 14705.0 -9.458256E7 -1864027286 1 1864027286 0.0 14705.0 -14705.0 14705.0 9.458256E7 1864021647.85 1864027286 -1.554726368159204E-4 14705.0 14705.0 9.4597265E7 0.0
-1864027286 true 0p3nIvm1c20J2e NULL 2066.0 -1.3288512E7 -1864027286 1 1864027286 0.0 2066.0 -2066.0 2066.0 1.3288512E7 1864021647.85 1864027286 -1.554726368159204E-4 2066.0 2066.0 1.3290578E7 0.0
-1864027286 true 0wyLcN8FuKeK NULL -11456.0 7.3684992E7 -1864027286 1 1864027286 0.0 -11456.0 11456.0 -11456.0 -7.3684992E7 1864021647.85 1864027286 -1.554726368159204E-4 -11456.0 -11456.0 -7.3696448E7 0.0
-1864027286 true 0xsFvigkQf7CEPVyXX78vG7D NULL 4014.0 -2.5818048E7 -1864027286 1 1864027286 0.0 4014.0 -4014.0 4014.0 2.5818048E7 1864021647.85 1864027286 -1.554726368159204E-4 4014.0 4014.0 2.5822062E7 0.0
-1864027286 true 100xJdkyc NULL 14519.0 -9.3386208E7 -1864027286 1 1864027286 0.0 14519.0 -14519.0 14519.0 9.3386208E7 1864021647.85 1864027286 -1.554726368159204E-4 14519.0 14519.0 9.3400727E7 0.0
-1864027286 true 10M3eGUsKVonbl70DyoCk25 NULL 5658.0 -3.6392256E7 -1864027286 1 1864027286 0.0 5658.0 -5658.0 5658.0 3.6392256E7 1864021647.85 1864027286 -1.554726368159204E-4 5658.0 5658.0 3.6397914E7 0.0
-1864027286 true 10lL0XD6WP2x64f70N0fHmC1 NULL 4516.0 -2.9046912E7 -1864027286 1 1864027286 0.0 4516.0 -4516.0 4516.0 2.9046912E7 1864021647.85 1864027286 -1.554726368159204E-4 4516.0 4516.0 2.9051428E7 0.0
-1864027286 true 116MTW7f3P3 NULL -13443.0 8.6465376E7 -1864027286 1 1864027286 0.0 -13443.0 13443.0 -13443.0 -8.6465376E7 1864021647.85 1864027286 -1.554726368159204E-4 -13443.0 -13443.0 -8.6478819E7 0.0
-1864027286 true 11gEw8B737tUg NULL -8278.0 5.3244096E7 -1864027286 1 1864027286 0.0 -8278.0 8278.0 -8278.0 -5.3244096E7 1864021647.85 1864027286 -1.554726368159204E-4 -8278.0 -8278.0 -5.3252374E7 0.0
-1864027286 true 1470P NULL 328.0 -2109696.0 -1864027286 1 1864027286 0.0 328.0 -328.0 328.0 2109696.0 1864021647.85 1864027286 -1.554726368159204E-4 328.0 328.0 2110024.0 0.0
-1864027286 true 16twtB4w2UMSEu3q1L07AMj NULL 2940.0 -1.891008E7 -1864027286 1 1864027286 0.0 2940.0 -2940.0 2940.0 1.891008E7 1864021647.85 1864027286 -1.554726368159204E-4 2940.0 2940.0 1.891302E7 0.0
-1864027286 true 1AV8SL56Iv0rm3vw NULL 9142.0 -5.8801344E7 -1864027286 1 1864027286 0.0 9142.0 -9142.0 9142.0 5.8801344E7 1864021647.85 1864027286 -1.554726368159204E-4 9142.0 9142.0 5.8810486E7 0.0
-1864027286 true 1BQ22Cx70452I4mV1 NULL 10259.0 -6.5985888E7 -1864027286 1 1864027286 0.0 10259.0 -10259.0 10259.0 6.5985888E7 1864021647.85 1864027286 -1.554726368159204E-4 10259.0 10259.0 6.5996147E7 0.0
-1864027286 true 1Ef7Tg NULL 5192.0 -3.3394944E7 -1864027286 1 1864027286 0.0 5192.0 -5192.0 5192.0 3.3394944E7 1864021647.85 1864027286 -1.554726368159204E-4 5192.0 5192.0 3.3400136E7 0.0
-1864027286 true 1K0M0lJ25 NULL 4141.0 -2.6634912E7 -1864027286 1 1864027286 0.0 4141.0 -4141.0 4141.0 2.6634912E7 1864021647.85 1864027286 -1.554726368159204E-4 4141.0 4141.0 2.6639053E7 0.0
-1864027286 true 1KXD04k80RltvQY NULL 1891.0 -1.2162912E7 -1864027286 1 1864027286 0.0 1891.0 -1891.0 1891.0 1.2162912E7 1864021647.85 1864027286 -1.554726368159204E-4 1891.0 1891.0 1.2164803E7 0.0
-1864027286 true 1SkJLW1H NULL -12515.0 8.049648E7 -1864027286 1 1864027286 0.0 -12515.0 12515.0 -12515.0 -8.049648E7 1864021647.85 1864027286 -1.554726368159204E-4 -12515.0 -12515.0 -8.0508995E7 0.0
-1864027286 true 1U0Y0li08r50 NULL -15261.0 9.8158752E7 -1864027286 1 1864027286 0.0 -15261.0 15261.0 -15261.0 -9.8158752E7 1864021647.85 1864027286 -1.554726368159204E-4 -15261.0 -15261.0 -9.8174013E7 0.0
-1864027286 true 1a47CF0K67apXs NULL -7715.0 4.962288E7 -1864027286 1 1864027286 0.0 -7715.0 7715.0 -7715.0 -4.962288E7 1864021647.85 1864027286 -1.554726368159204E-4 -7715.0 -7715.0 -4.9630595E7 0.0
-1864027286 true 1aI03p NULL 9766.0 -6.2814912E7 -1864027286 1 1864027286 0.0 9766.0 -9766.0 9766.0 6.2814912E7 1864021647.85 1864027286 -1.554726368159204E-4 9766.0 9766.0 6.2824678E7 0.0
-1864027286 true 1alMTip5YTi6R3K4Pk8 NULL 2130.0 -1.370016E7 -1864027286 1 1864027286 0.0 2130.0 -2130.0 2130.0 1.370016E7 1864021647.85 1864027286 -1.554726368159204E-4 2130.0 2130.0 1.370229E7 0.0
-1864027286 true 1r3uaJGN7oo7If84Yc NULL 1322.0 -8503104.0 -1864027286 1 1864027286 0.0 1322.0 -1322.0 1322.0 8503104.0 1864021647.85 1864027286 -1.554726368159204E-4 1322.0 1322.0 8504426.0 0.0
-1864027286 true 1t4KWqqqSILisWU5S4md8837 NULL -7101.0 4.5673632E7 -1864027286 1 1864027286 0.0 -7101.0 7101.0 -7101.0 -4.5673632E7 1864021647.85 1864027286 -1.554726368159204E-4 -7101.0 -7101.0 -4.5680733E7 0.0
-1864027286 true 1uerCssknyIB4 NULL 9620.0 -6.187584E7 -1864027286 1 1864027286 0.0 9620.0 -9620.0 9620.0 6.187584E7 1864021647.85 1864027286 -1.554726368159204E-4 9620.0 9620.0 6.188546E7 0.0
-1864027286 true 1wMPbWHES0gcJ4C7438 NULL -10276.0 6.6095232E7 -1864027286 1 1864027286 0.0 -10276.0 10276.0 -10276.0 -6.6095232E7 1864021647.85 1864027286 -1.554726368159204E-4 -10276.0 -10276.0 -6.6105508E7 0.0
-1864027286 true 21I7qFxw2vnAO7N1R1yUMhr0 NULL 15604.0 -1.00364928E8 -1864027286 1 1864027286 0.0 15604.0 -15604.0 15604.0 1.00364928E8 1864021647.85 1864027286 -1.554726368159204E-4 15604.0 15604.0 1.00380532E8 0.0
-1864027286 true 21l7ppi3Q73w7DMg75H1e NULL -447.0 2875104.0 -1864027286 1 1864027286 0.0 -447.0 447.0 -447.0 -2875104.0 1864021647.85 1864027286 -1.554726368159204E-4 -447.0 -447.0 -2875551.0 0.0
-1864027286 true 223qftA0b NULL 15017.0 -9.6589344E7 -1864027286 1 1864027286 0.0 15017.0 -15017.0 15017.0 9.6589344E7 1864021647.85 1864027286 -1.554726368159204E-4 15017.0 15017.0 9.6604361E7 0.0
-1864027286 true 22s17wD60356NWi2m30gkHbm NULL 10267.0 -6.6037344E7 -1864027286 1 1864027286 0.0 10267.0 -10267.0 10267.0 6.6037344E7 1864021647.85 1864027286 -1.554726368159204E-4 10267.0 10267.0 6.6047611E7 0.0
-1864027286 true 24t42K005K7v84Nx820euxD NULL 9362.0 -6.0216384E7 -1864027286 1 1864027286 0.0 9362.0 -9362.0 9362.0 6.0216384E7 1864021647.85 1864027286 -1.554726368159204E-4 9362.0 9362.0 6.0225746E7 0.0
-1864027286 true 25MqX NULL -4221.0 2.7149472E7 -1864027286 1 1864027286 0.0 -4221.0 4221.0 -4221.0 -2.7149472E7 1864021647.85 1864027286 -1.554726368159204E-4 -4221.0 -4221.0 -2.7153693E7 0.0
-1864027286 true 26Mx1k447Tk5 NULL -3888.0 2.5007616E7 -1864027286 1 1864027286 0.0 -3888.0 3888.0 -3888.0 -2.5007616E7 1864021647.85 1864027286 -1.554726368159204E-4 -3888.0 -3888.0 -2.5011504E7 0.0
-1864027286 true 27M4Etiyf304s0aob NULL -5909.0 3.8006688E7 -1864027286 1 1864027286 0.0 -5909.0 5909.0 -5909.0 -3.8006688E7 1864021647.85 1864027286 -1.554726368159204E-4 -5909.0 -5909.0 -3.8012597E7 0.0
-1864027286 true 2ArdYqML3654nUjGJk3 NULL -16379.0 1.05349728E8 -1864027286 1 1864027286 0.0 -16379.0 16379.0 -16379.0 -1.05349728E8 1864021647.85 1864027286 -1.554726368159204E-4 -16379.0 -16379.0 -1.05366107E8 0.0
-1864027286 true 2Fis0xsRWB447Evs6Fa5cH NULL -9721.0 6.2525472E7 -1864027286 1 1864027286 0.0 -9721.0 9721.0 -9721.0 -6.2525472E7 1864021647.85 1864027286 -1.554726368159204E-4 -9721.0 -9721.0 -6.2535193E7 0.0
-1864027286 true 2LTgnBrqS3DAE446015Nc NULL -2942.0 1.8922944E7 -1864027286 1 1864027286 0.0 -2942.0 2942.0 -2942.0 -1.8922944E7 1864021647.85 1864027286 -1.554726368159204E-4 -2942.0 -2942.0 -1.8925886E7 0.0
-1864027286 true 2Q1RY NULL 7887.0 -5.0729184E7 -1864027286 1 1864027286 0.0 7887.0 -7887.0 7887.0 5.0729184E7 1864021647.85 1864027286 -1.554726368159204E-4 7887.0 7887.0 5.0737071E7 0.0
-1864027286 true 2VC0DK60DgLH NULL 10435.0 -6.711792E7 -1864027286 1 1864027286 0.0 10435.0 -10435.0 10435.0 6.711792E7 1864021647.85 1864027286 -1.554726368159204E-4 10435.0 10435.0 6.7128355E7 0.0
-1864027286 true 2c4e2 NULL -11760.0 7.564032E7 -1864027286 1 1864027286 0.0 -11760.0 11760.0 -11760.0 -7.564032E7 1864021647.85 1864027286 -1.554726368159204E-4 -11760.0 -11760.0 -7.565208E7 0.0
-1864027286 true 2cumAMuRN4kC5dJd888m NULL 1603.0 -1.0310496E7 -1864027286 1 1864027286 0.0 1603.0 -1603.0 1603.0 1.0310496E7 1864021647.85 1864027286 -1.554726368159204E-4 1603.0 1603.0 1.0312099E7 0.0
-1864027286 true 2mwT8k NULL -10653.0 6.8520096E7 -1864027286 1 1864027286 0.0 -10653.0 10653.0 -10653.0 -6.8520096E7 1864021647.85 1864027286 -1.554726368159204E-4 -10653.0 -10653.0 -6.8530749E7 0.0
-1864027286 true 2qh6a3is304PThbc NULL 11926.0 -7.6708032E7 -1864027286 1 1864027286 0.0 11926.0 -11926.0 11926.0 7.6708032E7 1864021647.85 1864027286 -1.554726368159204E-4 11926.0 11926.0 7.6719958E7 0.0
-1864027286 true 2uLyD28144vklju213J1mr NULL -5470.0 3.518304E7 -1864027286 1 1864027286 0.0 -5470.0 5470.0 -5470.0 -3.518304E7 1864021647.85 1864027286 -1.554726368159204E-4 -5470.0 -5470.0 -3.518851E7 0.0
-1864027286 true 2y2n4Oh0B5PHX8mAMXq4wId2 NULL -7961.0 5.1205152E7 -1864027286 1 1864027286 0.0 -7961.0 7961.0 -7961.0 -5.1205152E7 1864021647.85 1864027286 -1.554726368159204E-4 -7961.0 -7961.0 -5.1213113E7 0.0
-1864027286 true 316qk10jD0dkAh78 NULL 4257.0 -2.7381024E7 -1864027286 1 1864027286 0.0 4257.0 -4257.0 4257.0 2.7381024E7 1864021647.85 1864027286 -1.554726368159204E-4 4257.0 4257.0 2.7385281E7 0.0
-1864027286 true 3445NVr7c7wfE3Px NULL -15768.0 1.01419776E8 -1864027286 1 1864027286 0.0 -15768.0 15768.0 -15768.0 -1.01419776E8 1864021647.85 1864027286 -1.554726368159204E-4 -15768.0 -15768.0 -1.01435544E8 0.0
-1864027286 true 37EE5NIy NULL -12996.0 8.3590272E7 -1864027286 1 1864027286 0.0 -12996.0 12996.0 -12996.0 -8.3590272E7 1864021647.85 1864027286 -1.554726368159204E-4 -12996.0 -12996.0 -8.3603268E7 0.0
-1864027286 true 3AKRFwBnv2163LyKqSXy NULL -10084.0 6.4860288E7 -1864027286 1 1864027286 0.0 -10084.0 10084.0 -10084.0 -6.4860288E7 1864021647.85 1864027286 -1.554726368159204E-4 -10084.0 -10084.0 -6.4870372E7 0.0
-1864027286 true 3AsYyeNCcv0R7fmt3K1uL NULL 11529.0 -7.4154528E7 -1864027286 1 1864027286 0.0 11529.0 -11529.0 11529.0 7.4154528E7 1864021647.85 1864027286 -1.554726368159204E-4 11529.0 11529.0 7.4166057E7 0.0
-1864027286 true 3B3ubgg3B6a NULL 14468.0 -9.3058176E7 -1864027286 1 1864027286 0.0 14468.0 -14468.0 14468.0 9.3058176E7 1864021647.85 1864027286 -1.554726368159204E-4 14468.0 14468.0 9.3072644E7 0.0
-1864027286 true 3C1y7deXML NULL -4035.0 2.595312E7 -1864027286 1 1864027286 0.0 -4035.0 4035.0 -4035.0 -2.595312E7 1864021647.85 1864027286 -1.554726368159204E-4 -4035.0 -4035.0 -2.5957155E7 0.0
-1864027286 true 3E1qqlB24B NULL 14152.0 -9.1025664E7 -1864027286 1 1864027286 0.0 14152.0 -14152.0 14152.0 9.1025664E7 1864021647.85 1864027286 -1.554726368159204E-4 14152.0 14152.0 9.1039816E7 0.0
-1864027286 true 3T12mSFCYnrAx7EokPLq8002 NULL 5404.0 -3.4758528E7 -1864027286 1 1864027286 0.0 5404.0 -5404.0 5404.0 3.4758528E7 1864021647.85 1864027286 -1.554726368159204E-4 5404.0 5404.0 3.4763932E7 0.0
-1864027286 true 3WsVeqb28VWEEOLI8ail NULL 2563.58 -1.6488946559999999E7 -1864027286 1 1864027286 0.0 2563.58 -2563.58 2563.58 1.6488946559999999E7 1864021647.85 1864027286 -1.554726368159204E-4 2563.58 2563.58 1.6491510139999999E7 0.0
-1864027286 true 3d631tcs1g NULL 10796.0 -6.9439872E7 -1864027286 1 1864027286 0.0 10796.0 -10796.0 10796.0 6.9439872E7 1864021647.85 1864027286 -1.554726368159204E-4 10796.0 10796.0 6.9450668E7 0.0
-1864027286 true 3h01b8LfJ812JV4gwhfT8u NULL 6798.0 -4.3724736E7 -1864027286 1 1864027286 0.0 6798.0 -6798.0 6798.0 4.3724736E7 1864021647.85 1864027286 -1.554726368159204E-4 6798.0 6798.0 4.3731534E7 0.0
-1864027286 true 3kFb68 NULL -11779.0 7.5762528E7 -1864027286 1 1864027286 0.0 -11779.0 11779.0 -11779.0 -7.5762528E7 1864021647.85 1864027286 -1.554726368159204E-4 -11779.0 -11779.0 -7.5774307E7 0.0
-1864027286 true 3q4Mex4ok5Wj6j706Vh NULL -10286.0 6.6159552E7 -1864027286 1 1864027286 0.0 -10286.0 10286.0 -10286.0 -6.6159552E7 1864021647.85 1864027286 -1.554726368159204E-4 -10286.0 -10286.0 -6.6169838E7 0.0
-1864027286 true 3sLC0Y2417i4n6Q5xcMF7 NULL -6106.0 3.9273792E7 -1864027286 1 1864027286 0.0 -6106.0 6106.0 -6106.0 -3.9273792E7 1864021647.85 1864027286 -1.554726368159204E-4 -6106.0 -6106.0 -3.9279898E7 0.0
-1864027286 true 3t3EB NULL 15847.0 -1.01927904E8 -1864027286 1 1864027286 0.0 15847.0 -15847.0 15847.0 1.01927904E8 1864021647.85 1864027286 -1.554726368159204E-4 15847.0 15847.0 1.01943751E8 0.0
-1864027286 true 410L723g40Le351u NULL -11597.0 7.4591904E7 -1864027286 1 1864027286 0.0 -11597.0 11597.0 -11597.0 -7.4591904E7 1864021647.85 1864027286 -1.554726368159204E-4 -11597.0 -11597.0 -7.4603501E7 0.0
-1864027286 true 4186Py40K286Oc NULL 6351.0 -4.0849632E7 -1864027286 1 1864027286 0.0 6351.0 -6351.0 6351.0 4.0849632E7 1864021647.85 1864027286 -1.554726368159204E-4 6351.0 6351.0 4.0855983E7 0.0
-1864027286 true 43d0nGQNH8m6wcT7p0T5Buu NULL -14035.0 9.027312E7 -1864027286 1 1864027286 0.0 -14035.0 14035.0 -14035.0 -9.027312E7 1864021647.85 1864027286 -1.554726368159204E-4 -14035.0 -14035.0 -9.0287155E7 0.0
-1864027286 true 46a8K1 NULL -8764.0 5.6370048E7 -1864027286 1 1864027286 0.0 -8764.0 8764.0 -8764.0 -5.6370048E7 1864021647.85 1864027286 -1.554726368159204E-4 -8764.0 -8764.0 -5.6378812E7 0.0
-1864027286 true 488l506x NULL 8868.0 -5.7038976E7 -1864027286 1 1864027286 0.0 8868.0 -8868.0 8868.0 5.7038976E7 1864021647.85 1864027286 -1.554726368159204E-4 8868.0 8868.0 5.7047844E7 0.0
-1864027286 true 48Dj7hY48w7 NULL 5146.0 -3.3099072E7 -1864027286 1 1864027286 0.0 5146.0 -5146.0 5146.0 3.3099072E7 1864021647.85 1864027286 -1.554726368159204E-4 5146.0 5146.0 3.3104218E7 0.0
-1864027286 true 4BxeN7PLh00qDKq13Nu8eVQ NULL 2336.0 -1.5025152E7 -1864027286 1 1864027286 0.0 2336.0 -2336.0 2336.0 1.5025152E7 1864021647.85 1864027286 -1.554726368159204E-4 2336.0 2336.0 1.5027488E7 0.0
-1864027286 true 4CLH5Pd31NWO NULL 13840.0 -8.901888E7 -1864027286 1 1864027286 0.0 13840.0 -13840.0 13840.0 8.901888E7 1864021647.85 1864027286 -1.554726368159204E-4 13840.0 13840.0 8.903272E7 0.0
-1864027286 true 4D64Q522LOJY7lu4 NULL -6407.0 4.1209824E7 -1864027286 1 1864027286 0.0 -6407.0 6407.0 -6407.0 -4.1209824E7 1864021647.85 1864027286 -1.554726368159204E-4 -6407.0 -6407.0 -4.1216231E7 0.0
-1864027286 true 4F3Tu14b35h26Q7 NULL -4033.0 2.5940256E7 -1864027286 1 1864027286 0.0 -4033.0 4033.0 -4033.0 -2.5940256E7 1864021647.85 1864027286 -1.554726368159204E-4 -4033.0 -4033.0 -2.5944289E7 0.0
-1864027286 true 4Ko41XvrHww1YXrctT NULL 367.0 -2360544.0 -1864027286 1 1864027286 0.0 367.0 -367.0 367.0 2360544.0 1864021647.85 1864027286 -1.554726368159204E-4 367.0 367.0 2360911.0 0.0
-1864027286 true 4O41kg NULL -15027.0 9.6653664E7 -1864027286 1 1864027286 0.0 -15027.0 15027.0 -15027.0 -9.6653664E7 1864021647.85 1864027286 -1.554726368159204E-4 -15027.0 -15027.0 -9.6668691E7 0.0
-1864027286 true 4R0Dk NULL 3617.0 -2.3264544E7 -1864027286 1 1864027286 0.0 3617.0 -3617.0 3617.0 2.3264544E7 1864021647.85 1864027286 -1.554726368159204E-4 3617.0 3617.0 2.3268161E7 0.0
-1864027286 true 4kyK2032wUS2iyU28i NULL 8061.0 -5.1848352E7 -1864027286 1 1864027286 0.0 8061.0 -8061.0 8061.0 5.1848352E7 1864021647.85 1864027286 -1.554726368159204E-4 8061.0 8061.0 5.1856413E7 0.0
-1864027286 true 4srDycbXO8 NULL 4969.0 -3.1960608E7 -1864027286 1 1864027286 0.0 4969.0 -4969.0 4969.0 3.1960608E7 1864021647.85 1864027286 -1.554726368159204E-4 4969.0 4969.0 3.1965577E7 0.0
-1864027286 true 4stOSK0N7i8 NULL -15871.0 1.02082272E8 -1864027286 1 1864027286 0.0 -15871.0 15871.0 -15871.0 -1.02082272E8 1864021647.85 1864027286 -1.554726368159204E-4 -15871.0 -15871.0 -1.02098143E8 0.0
-1864027286 true 4teNUJ1 NULL -13436.0 8.6420352E7 -1864027286 1 1864027286 0.0 -13436.0 13436.0 -13436.0 -8.6420352E7 1864021647.85 1864027286 -1.554726368159204E-4 -13436.0 -13436.0 -8.6433788E7 0.0
-1864027286 true 54yQ6 NULL 7148.0 -4.5975936E7 -1864027286 1 1864027286 0.0 7148.0 -7148.0 7148.0 4.5975936E7 1864021647.85 1864027286 -1.554726368159204E-4 7148.0 7148.0 4.5983084E7 0.0
-1864027286 true 55b1rXQ20u321On2QrDo51K8 NULL -5132.0 3.3009024E7 -1864027286 1 1864027286 0.0 -5132.0 5132.0 -5132.0 -3.3009024E7 1864021647.85 1864027286 -1.554726368159204E-4 -5132.0 -5132.0 -3.3014156E7 0.0
-1864027286 true 55laBDd2J6deffIvr0EknAc NULL 14095.0 -9.065904E7 -1864027286 1 1864027286 0.0 14095.0 -14095.0 14095.0 9.065904E7 1864021647.85 1864027286 -1.554726368159204E-4 14095.0 14095.0 9.0673135E7 0.0
-1864027286 true 563414Ge0cqfJ8v5SaIQ2W3j NULL -7170.0 4.611744E7 -1864027286 1 1864027286 0.0 -7170.0 7170.0 -7170.0 -4.611744E7 1864021647.85 1864027286 -1.554726368159204E-4 -7170.0 -7170.0 -4.612461E7 0.0
-1864027286 true 587FWG5e1NylA0SQD NULL -7788.0 5.0092416E7 -1864027286 1 1864027286 0.0 -7788.0 7788.0 -7788.0 -5.0092416E7 1864021647.85 1864027286 -1.554726368159204E-4 -7788.0 -7788.0 -5.0100204E7 0.0
-1864027286 true 5BFMY8Bb582h6 NULL 4122.0 -2.6512704E7 -1864027286 1 1864027286 0.0 4122.0 -4122.0 4122.0 2.6512704E7 1864021647.85 1864027286 -1.554726368159204E-4 4122.0 4122.0 2.6516826E7 0.0
-1864027286 true 5EOwuCtm184 NULL 6597.0 -4.2431904E7 -1864027286 1 1864027286 0.0 6597.0 -6597.0 6597.0 4.2431904E7 1864021647.85 1864027286 -1.554726368159204E-4 6597.0 6597.0 4.2438501E7 0.0
-1864027286 true 5OcrJ NULL -852.0 5480064.0 -1864027286 1 1864027286 0.0 -852.0 852.0 -852.0 -5480064.0 1864021647.85 1864027286 -1.554726368159204E-4 -852.0 -852.0 -5480916.0 0.0
-1864027286 true 5V14R7pp4m2XvyB3dDDqgxQ0 NULL -6256.0 4.0238592E7 -1864027286 1 1864027286 0.0 -6256.0 6256.0 -6256.0 -4.0238592E7 1864021647.85 1864027286 -1.554726368159204E-4 -6256.0 -6256.0 -4.0244848E7 0.0
-1864027286 true 5Wn74X54OPT5nIbTVM NULL -8790.0 5.653728E7 -1864027286 1 1864027286 0.0 -8790.0 8790.0 -8790.0 -5.653728E7 1864021647.85 1864027286 -1.554726368159204E-4 -8790.0 -8790.0 -5.654607E7 0.0
-1864027286 true 5Xab46Lyo NULL 7598.0 -4.8870336E7 -1864027286 1 1864027286 0.0 7598.0 -7598.0 7598.0 4.8870336E7 1864021647.85 1864027286 -1.554726368159204E-4 7598.0 7598.0 4.8877934E7 0.0
-1864027286 true 5Y503avvhX3gUECL3 NULL 10854.0 -6.9812928E7 -1864027286 1 1864027286 0.0 10854.0 -10854.0 10854.0 6.9812928E7 1864021647.85 1864027286 -1.554726368159204E-4 10854.0 10854.0 6.9823782E7 0.0
-1864027286 true 5eY1KB3 NULL 5204.0 -3.3472128E7 -1864027286 1 1864027286 0.0 5204.0 -5204.0 5204.0 3.3472128E7 1864021647.85 1864027286 -1.554726368159204E-4 5204.0 5204.0 3.3477332E7 0.0
-1864027286 true 5gOeUOB NULL 2506.0 -1.6118592E7 -1864027286 1 1864027286 0.0 2506.0 -2506.0 2506.0 1.6118592E7 1864021647.85 1864027286 -1.554726368159204E-4 2506.0 2506.0 1.6121098E7 0.0
-1864027286 true 5hwHlC8uO8 NULL -294.0 1891008.0 -1864027286 1 1864027286 0.0 -294.0 294.0 -294.0 -1891008.0 1864021647.85 1864027286 -1.554726368159204E-4 -294.0 -294.0 -1891302.0 0.0
-1864027286 true 5lO3R6cjxRdsCi NULL -11252.0 7.2372864E7 -1864027286 1 1864027286 0.0 -11252.0 11252.0 -11252.0 -7.2372864E7 1864021647.85 1864027286 -1.554726368159204E-4 -11252.0 -11252.0 -7.2384116E7 0.0
-1864027286 true 5nXLE NULL -16124.0 1.03709568E8 -1864027286 1 1864027286 0.0 -16124.0 16124.0 -16124.0 -1.03709568E8 1864021647.85 1864027286 -1.554726368159204E-4 -16124.0 -16124.0 -1.03725692E8 0.0
-1864027286 true 5of6ay NULL -9761.0 6.2782752E7 -1864027286 1 1864027286 0.0 -9761.0 9761.0 -9761.0 -6.2782752E7 1864021647.85 1864027286 -1.554726368159204E-4 -9761.0 -9761.0 -6.2792513E7 0.0
-1864027286 true 5rvGhuUle NULL -13956.0 8.9764992E7 -1864027286 1 1864027286 0.0 -13956.0 13956.0 -13956.0 -8.9764992E7 1864021647.85 1864027286 -1.554726368159204E-4 -13956.0 -13956.0 -8.9778948E7 0.0
-1864027286 true 5xaNVvLa NULL 2315.0 -1.489008E7 -1864027286 1 1864027286 0.0 2315.0 -2315.0 2315.0 1.489008E7 1864021647.85 1864027286 -1.554726368159204E-4 2315.0 2315.0 1.4892395E7 0.0
-1864027286 true 5yFe2HK NULL 3396.0 -2.1843072E7 -1864027286 1 1864027286 0.0 3396.0 -3396.0 3396.0 2.1843072E7 1864021647.85 1864027286 -1.554726368159204E-4 3396.0 3396.0 2.1846468E7 0.0
-1864027286 true 60041SoajDs4F2C NULL 12826.0 -8.2496832E7 -1864027286 1 1864027286 0.0 12826.0 -12826.0 12826.0 8.2496832E7 1864021647.85 1864027286 -1.554726368159204E-4 12826.0 12826.0 8.2509658E7 0.0
-1864027286 true 60M56qKrd2j NULL -15205.0 9.779856E7 -1864027286 1 1864027286 0.0 -15205.0 15205.0 -15205.0 -9.779856E7 1864021647.85 1864027286 -1.554726368159204E-4 -15205.0 -15205.0 -9.7813765E7 0.0
-1864027286 true 60Ydc418lOl284ss63 NULL 3316.0 -2.1328512E7 -1864027286 1 1864027286 0.0 3316.0 -3316.0 3316.0 2.1328512E7 1864021647.85 1864027286 -1.554726368159204E-4 3316.0 3316.0 2.1331828E7 0.0
-1864027286 true 61fdP5u NULL 4143.0 -2.6647776E7 -1864027286 1 1864027286 0.0 4143.0 -4143.0 4143.0 2.6647776E7 1864021647.85 1864027286 -1.554726368159204E-4 4143.0 4143.0 2.6651919E7 0.0
-1864027286 true 61gE6oOT4E0G83 NULL -3714.0 2.3888448E7 -1864027286 1 1864027286 0.0 -3714.0 3714.0 -3714.0 -2.3888448E7 1864021647.85 1864027286 -1.554726368159204E-4 -3714.0 -3714.0 -2.3892162E7 0.0
-1864027286 true 63L57061J754YaaV NULL -15253.0 9.8107296E7 -1864027286 1 1864027286 0.0 -15253.0 15253.0 -15253.0 -9.8107296E7 1864021647.85 1864027286 -1.554726368159204E-4 -15253.0 -15253.0 -9.8122549E7 0.0
-1864027286 true 6648LI57SdO7 NULL 8854.0 -5.6948928E7 -1864027286 1 1864027286 0.0 8854.0 -8854.0 8854.0 5.6948928E7 1864021647.85 1864027286 -1.554726368159204E-4 8854.0 8854.0 5.6957782E7 0.0
-1864027286 true 686HHW45wojg5OCxqdn NULL -3320.0 2.135424E7 -1864027286 1 1864027286 0.0 -3320.0 3320.0 -3320.0 -2.135424E7 1864021647.85 1864027286 -1.554726368159204E-4 -3320.0 -3320.0 -2.135756E7 0.0
-1864027286 true 6D47xA0FaDfy4h NULL 3100.0 -1.99392E7 -1864027286 1 1864027286 0.0 3100.0 -3100.0 3100.0 1.99392E7 1864021647.85 1864027286 -1.554726368159204E-4 3100.0 3100.0 1.99423E7 0.0
-1864027286 true 6D8pQ38Wn NULL -16140.0 1.0381248E8 -1864027286 1 1864027286 0.0 -16140.0 16140.0 -16140.0 -1.0381248E8 1864021647.85 1864027286 -1.554726368159204E-4 -16140.0 -16140.0 -1.0382862E8 0.0
-1864027286 true 6E5g66uV1fm6 NULL -9886.0 6.3586752E7 -1864027286 1 1864027286 0.0 -9886.0 9886.0 -9886.0 -6.3586752E7 1864021647.85 1864027286 -1.554726368159204E-4 -9886.0 -9886.0 -6.3596638E7 0.0
-1864027286 true 6H463iHBu1HNq3oBr1ehE NULL -13152.0 8.4593664E7 -1864027286 1 1864027286 0.0 -13152.0 13152.0 -13152.0 -8.4593664E7 1864021647.85 1864027286 -1.554726368159204E-4 -13152.0 -13152.0 -8.4606816E7 0.0
-1864027286 true 6J2wyLGv NULL 6441.0 -4.1428512E7 -1864027286 1 1864027286 0.0 6441.0 -6441.0 6441.0 4.1428512E7 1864021647.85 1864027286 -1.554726368159204E-4 6441.0 6441.0 4.1434953E7 0.0
-1864027286 true 6P5hI87IBw5BwP4T36lkB2 NULL -1388.0 8927616.0 -1864027286 1 1864027286 0.0 -1388.0 1388.0 -1388.0 -8927616.0 1864021647.85 1864027286 -1.554726368159204E-4 -1388.0 -1388.0 -8929004.0 0.0
-1864027286 true 6Qb7hMltqN0MY0xRf8 NULL 8243.0 -5.3018976E7 -1864027286 1 1864027286 0.0 8243.0 -8243.0 8243.0 5.3018976E7 1864021647.85 1864027286 -1.554726368159204E-4 8243.0 8243.0 5.3027219E7 0.0
-1864027286 true 6XR3D100e NULL -13345.0 8.583504E7 -1864027286 1 1864027286 0.0 -13345.0 13345.0 -13345.0 -8.583504E7 1864021647.85 1864027286 -1.554726368159204E-4 -13345.0 -13345.0 -8.5848385E7 0.0
-1864027286 true 6Xh62epM8Akab NULL -7786.0 5.0079552E7 -1864027286 1 1864027286 0.0 -7786.0 7786.0 -7786.0 -5.0079552E7 1864021647.85 1864027286 -1.554726368159204E-4 -7786.0 -7786.0 -5.0087338E7 0.0
-1864027286 true 6bO0XXrj NULL 11248.0 -7.2347136E7 -1864027286 1 1864027286 0.0 11248.0 -11248.0 11248.0 7.2347136E7 1864021647.85 1864027286 -1.554726368159204E-4 11248.0 11248.0 7.2358384E7 0.0
-1864027286 true 6c6b1XPMiEw5 NULL -8731.0 5.6157792E7 -1864027286 1 1864027286 0.0 -8731.0 8731.0 -8731.0 -5.6157792E7 1864021647.85 1864027286 -1.554726368159204E-4 -8731.0 -8731.0 -5.6166523E7 0.0
-1864027286 true 6gYlws NULL -11061.0 7.1144352E7 -1864027286 1 1864027286 0.0 -11061.0 11061.0 -11061.0 -7.1144352E7 1864021647.85 1864027286 -1.554726368159204E-4 -11061.0 -11061.0 -7.1155413E7 0.0
-1864027286 true 6nhFMfJ6 NULL 109.0 -701088.0 -1864027286 1 1864027286 0.0 109.0 -109.0 109.0 701088.0 1864021647.85 1864027286 -1.554726368159204E-4 109.0 109.0 701197.0 0.0
-1864027286 true 720r2q1xoXc3Kcf3 NULL -8554.0 5.5019328E7 -1864027286 1 1864027286 0.0 -8554.0 8554.0 -8554.0 -5.5019328E7 1864021647.85 1864027286 -1.554726368159204E-4 -8554.0 -8554.0 -5.5027882E7 0.0
-1864027286 true 7258G5fYVY NULL 13206.0 -8.4940992E7 -1864027286 1 1864027286 0.0 13206.0 -13206.0 13206.0 8.4940992E7 1864021647.85 1864027286 -1.554726368159204E-4 13206.0 13206.0 8.4954198E7 0.0
-1864027286 true 74iV6r7bnrdp03E4uW NULL -6917.0 4.4490144E7 -1864027286 1 1864027286 0.0 -6917.0 6917.0 -6917.0 -4.4490144E7 1864021647.85 1864027286 -1.554726368159204E-4 -6917.0 -6917.0 -4.4497061E7 0.0
-1864027286 true 74shmoR1 NULL -13746.0 8.8414272E7 -1864027286 1 1864027286 0.0 -13746.0 13746.0 -13746.0 -8.8414272E7 1864021647.85 1864027286 -1.554726368159204E-4 -13746.0 -13746.0 -8.8428018E7 0.0
-1864027286 true 764u1WA24hRh3rs NULL -2120.0 1.363584E7 -1864027286 1 1864027286 0.0 -2120.0 2120.0 -2120.0 -1.363584E7 1864021647.85 1864027286 -1.554726368159204E-4 -2120.0 -2120.0 -1.363796E7 0.0
-1864027286 true 7716wo8bn1 NULL -6978.0 4.4882496E7 -1864027286 1 1864027286 0.0 -6978.0 6978.0 -6978.0 -4.4882496E7 1864021647.85 1864027286 -1.554726368159204E-4 -6978.0 -6978.0 -4.4889474E7 0.0
-1864027286 true 7JDt8xM8G778vdBUA1 NULL -16092.0 1.03503744E8 -1864027286 1 1864027286 0.0 -16092.0 16092.0 -16092.0 -1.03503744E8 1864021647.85 1864027286 -1.554726368159204E-4 -16092.0 -16092.0 -1.03519836E8 0.0
-1864027286 true 7MHXQ0V71I NULL -5564.0 3.5787648E7 -1864027286 1 1864027286 0.0 -5564.0 5564.0 -5564.0 -3.5787648E7 1864021647.85 1864027286 -1.554726368159204E-4 -5564.0 -5564.0 -3.5793212E7 0.0
-1864027286 true 7PE3Nv5LTl NULL 6206.0 -3.9916992E7 -1864027286 1 1864027286 0.0 6206.0 -6206.0 6206.0 3.9916992E7 1864021647.85 1864027286 -1.554726368159204E-4 6206.0 6206.0 3.9923198E7 0.0
-1864027286 true 7Spfb6Q8pJBNWi3T NULL 6897.0 -4.4361504E7 -1864027286 1 1864027286 0.0 6897.0 -6897.0 6897.0 4.4361504E7 1864021647.85 1864027286 -1.554726368159204E-4 6897.0 6897.0 4.4368401E7 0.0
-1864027286 true 7XhwAvjDFx87 NULL -7033.0 4.5236256E7 -1864027286 1 1864027286 0.0 -7033.0 7033.0 -7033.0 -4.5236256E7 1864021647.85 1864027286 -1.554726368159204E-4 -7033.0 -7033.0 -4.5243289E7 0.0
-1864027286 true 7afdC4616LFIHN NULL -2179.0 1.4015328E7 -1864027286 1 1864027286 0.0 -2179.0 2179.0 -2179.0 -1.4015328E7 1864021647.85 1864027286 -1.554726368159204E-4 -2179.0 -2179.0 -1.4017507E7 0.0
-1864027286 true 7dqm3Oc6um NULL 5543.0 -3.5652576E7 -1864027286 1 1864027286 0.0 5543.0 -5543.0 5543.0 3.5652576E7 1864021647.85 1864027286 -1.554726368159204E-4 5543.0 5543.0 3.5658119E7 0.0
-1864027286 true 7gGmkmKO80vxDN4 NULL -3322.0 2.1367104E7 -1864027286 1 1864027286 0.0 -3322.0 3322.0 -3322.0 -2.1367104E7 1864021647.85 1864027286 -1.554726368159204E-4 -3322.0 -3322.0 -2.1370426E7 0.0
-1864027286 true 7ois1q60TPT4ckv5 NULL 1803.0 -1.1596896E7 -1864027286 1 1864027286 0.0 1803.0 -1803.0 1803.0 1.1596896E7 1864021647.85 1864027286 -1.554726368159204E-4 1803.0 1803.0 1.1598699E7 0.0
-1864027286 true 7sA426CHy4 NULL 3822.0 -2.4583104E7 -1864027286 1 1864027286 0.0 3822.0 -3822.0 3822.0 2.4583104E7 1864021647.85 1864027286 -1.554726368159204E-4 3822.0 3822.0 2.4586926E7 0.0
-1864027286 true 7smvc50Lf0Vc75l0Aw1 NULL 15538.0 -9.9940416E7 -1864027286 1 1864027286 0.0 15538.0 -15538.0 15538.0 9.9940416E7 1864021647.85 1864027286 -1.554726368159204E-4 15538.0 15538.0 9.9955954E7 0.0
-1864027286 true 7t7tL288aFIHcovPB8 NULL 8982.0 -5.7772224E7 -1864027286 1 1864027286 0.0 8982.0 -8982.0 8982.0 5.7772224E7 1864021647.85 1864027286 -1.554726368159204E-4 8982.0 8982.0 5.7781206E7 0.0
-1864027286 true 7u351EK474IcTOFW NULL -13653.0 8.7816096E7 -1864027286 1 1864027286 0.0 -13653.0 13653.0 -13653.0 -8.7816096E7 1864021647.85 1864027286 -1.554726368159204E-4 -13653.0 -13653.0 -8.7829749E7 0.0
-1864027286 true 7v3bUgTi6IBDVdvyb6sU NULL 14124.0 -9.0845568E7 -1864027286 1 1864027286 0.0 14124.0 -14124.0 14124.0 9.0845568E7 1864021647.85 1864027286 -1.554726368159204E-4 14124.0 14124.0 9.0859692E7 0.0
-1864027286 true 7xINFn3pugc8IOw4GWi7nR NULL -4854.0 3.1220928E7 -1864027286 1 1864027286 0.0 -4854.0 4854.0 -4854.0 -3.1220928E7 1864021647.85 1864027286 -1.554726368159204E-4 -4854.0 -4854.0 -3.1225782E7 0.0
-1864027286 true 81TewRpuYX3 NULL -7310.0 4.701792E7 -1864027286 1 1864027286 0.0 -7310.0 7310.0 -7310.0 -4.701792E7 1864021647.85 1864027286 -1.554726368159204E-4 -7310.0 -7310.0 -4.702523E7 0.0
-1864027286 true 83bn3y1 NULL -4638.0 2.9831616E7 -1864027286 1 1864027286 0.0 -4638.0 4638.0 -4638.0 -2.9831616E7 1864021647.85 1864027286 -1.554726368159204E-4 -4638.0 -4638.0 -2.9836254E7 0.0
-1864027286 true 840ng7eC1Ap8bgNEgSAVnwas NULL 5625.0 -3.618E7 -1864027286 1 1864027286 0.0 5625.0 -5625.0 5625.0 3.618E7 1864021647.85 1864027286 -1.554726368159204E-4 5625.0 5625.0 3.6185625E7 0.0
-1864027286 true 84TvhtF NULL 352.0 -2264064.0 -1864027286 1 1864027286 0.0 352.0 -352.0 352.0 2264064.0 1864021647.85 1864027286 -1.554726368159204E-4 352.0 352.0 2264416.0 0.0
-1864027286 true 87y8G77XofAGWgM115XGM NULL -16026.0 1.03079232E8 -1864027286 1 1864027286 0.0 -16026.0 16026.0 -16026.0 -1.03079232E8 1864021647.85 1864027286 -1.554726368159204E-4 -16026.0 -16026.0 -1.03095258E8 0.0
-1864027286 true 88SB8 NULL -6209.0 3.9936288E7 -1864027286 1 1864027286 0.0 -6209.0 6209.0 -6209.0 -3.9936288E7 1864021647.85 1864027286 -1.554726368159204E-4 -6209.0 -6209.0 -3.9942497E7 0.0
-1864027286 true 8B7U2E2o5byWd3KV7i NULL -11273.0 7.2507936E7 -1864027286 1 1864027286 0.0 -11273.0 11273.0 -11273.0 -7.2507936E7 1864021647.85 1864027286 -1.554726368159204E-4 -11273.0 -11273.0 -7.2519209E7 0.0
-1864027286 true 8IcQ0DU NULL 13107.0 -8.4304224E7 -1864027286 1 1864027286 0.0 13107.0 -13107.0 13107.0 8.4304224E7 1864021647.85 1864027286 -1.554726368159204E-4 13107.0 13107.0 8.4317331E7 0.0
-1864027286 true 8M42dX6x214GLI NULL 7956.0 -5.1172992E7 -1864027286 1 1864027286 0.0 7956.0 -7956.0 7956.0 5.1172992E7 1864021647.85 1864027286 -1.554726368159204E-4 7956.0 7956.0 5.1180948E7 0.0
-1864027286 true 8M8BPR10t2W0ypOh8 NULL -11817.0 7.6006944E7 -1864027286 1 1864027286 0.0 -11817.0 11817.0 -11817.0 -7.6006944E7 1864021647.85 1864027286 -1.554726368159204E-4 -11817.0 -11817.0 -7.6018761E7 0.0
-1864027286 true 8Qr143GYBM NULL 12819.0 -8.2451808E7 -1864027286 1 1864027286 0.0 12819.0 -12819.0 12819.0 8.2451808E7 1864021647.85 1864027286 -1.554726368159204E-4 12819.0 12819.0 8.2464627E7 0.0
-1864027286 true 8SGc8Ly1WTgwV1 NULL -6099.0 3.9228768E7 -1864027286 1 1864027286 0.0 -6099.0 6099.0 -6099.0 -3.9228768E7 1864021647.85 1864027286 -1.554726368159204E-4 -6099.0 -6099.0 -3.9234867E7 0.0
-1864027286 true 8W3527304W1WeGNo0q12l NULL 8804.0 -5.6627328E7 -1864027286 1 1864027286 0.0 8804.0 -8804.0 8804.0 5.6627328E7 1864021647.85 1864027286 -1.554726368159204E-4 8804.0 8804.0 5.6636132E7 0.0
-1864027286 true 8Xmc82JogMCeiE5 NULL 11982.0 -7.7068224E7 -1864027286 1 1864027286 0.0 11982.0 -11982.0 11982.0 7.7068224E7 1864021647.85 1864027286 -1.554726368159204E-4 11982.0 11982.0 7.7080206E7 0.0
-1864027286 true 8b1rapGl7vy44odt4jFI NULL 13561.0 -8.7224352E7 -1864027286 1 1864027286 0.0 13561.0 -13561.0 13561.0 8.7224352E7 1864021647.85 1864027286 -1.554726368159204E-4 13561.0 13561.0 8.7237913E7 0.0
-1864027286 true 8fjJStK8D7bsF7P3d65118S NULL 11040.0 -7.100928E7 -1864027286 1 1864027286 0.0 11040.0 -11040.0 11040.0 7.100928E7 1864021647.85 1864027286 -1.554726368159204E-4 11040.0 11040.0 7.102032E7 0.0
-1864027286 true 8hMHl64qhfWSdC NULL -8814.0 5.6691648E7 -1864027286 1 1864027286 0.0 -8814.0 8814.0 -8814.0 -5.6691648E7 1864021647.85 1864027286 -1.554726368159204E-4 -8814.0 -8814.0 -5.6700462E7 0.0
-1864027286 true 8lAl0YbpyMmPgI NULL -14696.0 9.4524672E7 -1864027286 1 1864027286 0.0 -14696.0 14696.0 -14696.0 -9.4524672E7 1864021647.85 1864027286 -1.554726368159204E-4 -14696.0 -14696.0 -9.4539368E7 0.0
-1864027286 true 8n431HuJF6X2x46Rt NULL -5513.0 3.5459616E7 -1864027286 1 1864027286 0.0 -5513.0 5513.0 -5513.0 -3.5459616E7 1864021647.85 1864027286 -1.554726368159204E-4 -5513.0 -5513.0 -3.5465129E7 0.0
-1864027286 true 8pbggxc NULL -3914.0 2.5174848E7 -1864027286 1 1864027286 0.0 -3914.0 3914.0 -3914.0 -2.5174848E7 1864021647.85 1864027286 -1.554726368159204E-4 -3914.0 -3914.0 -2.5178762E7 0.0
-1864027286 true 8r2TI3Svqra1Jc253gAYR3 NULL 15879.0 -1.02133728E8 -1864027286 1 1864027286 0.0 15879.0 -15879.0 15879.0 1.02133728E8 1864021647.85 1864027286 -1.554726368159204E-4 15879.0 15879.0 1.02149607E8 0.0
-1864027286 true 8r5uX85x2Pn7g3gJ0 NULL -3005.0 1.932816E7 -1864027286 1 1864027286 0.0 -3005.0 3005.0 -3005.0 -1.932816E7 1864021647.85 1864027286 -1.554726368159204E-4 -3005.0 -3005.0 -1.9331165E7 0.0
-1864027286 true 8tL4e4XE8jF2YLJ8l NULL 15061.0 -9.6872352E7 -1864027286 1 1864027286 0.0 15061.0 -15061.0 15061.0 9.6872352E7 1864021647.85 1864027286 -1.554726368159204E-4 15061.0 15061.0 9.6887413E7 0.0
-1864027286 true 8v0iU4C NULL -5891.0 3.7890912E7 -1864027286 1 1864027286 0.0 -5891.0 5891.0 -5891.0 -3.7890912E7 1864021647.85 1864027286 -1.554726368159204E-4 -5891.0 -5891.0 -3.7896803E7 0.0
-1864027286 true A2REERChgbC5c4 NULL 11056.0 -7.1112192E7 -1864027286 1 1864027286 0.0 11056.0 -11056.0 11056.0 7.1112192E7 1864021647.85 1864027286 -1.554726368159204E-4 11056.0 11056.0 7.1123248E7 0.0
-1864027286 true AFv66x72c72hjHPYqV0y4Qi NULL 14099.0 -9.0684768E7 -1864027286 1 1864027286 0.0 14099.0 -14099.0 14099.0 9.0684768E7 1864021647.85 1864027286 -1.554726368159204E-4 14099.0 14099.0 9.0698867E7 0.0
-1864027286 true AGYktyr3k0GMQx7bWp NULL -12990.0 8.355168E7 -1864027286 1 1864027286 0.0 -12990.0 12990.0 -12990.0 -8.355168E7 1864021647.85 1864027286 -1.554726368159204E-4 -12990.0 -12990.0 -8.356467E7 0.0
-1864027286 true AS86Ghu6q7 NULL 10681.0 -6.8700192E7 -1864027286 1 1864027286 0.0 10681.0 -10681.0 10681.0 6.8700192E7 1864021647.85 1864027286 -1.554726368159204E-4 10681.0 10681.0 6.8710873E7 0.0
-1864027286 true Ag7jo42O8LQxbFwe6TK NULL 570.0 -3666240.0 -1864027286 1 1864027286 0.0 570.0 -570.0 570.0 3666240.0 1864021647.85 1864027286 -1.554726368159204E-4 570.0 570.0 3666810.0 0.0
-1864027286 true B0q1K7dlcKAC46176yc83 NULL -12313.0 7.9197216E7 -1864027286 1 1864027286 0.0 -12313.0 12313.0 -12313.0 -7.9197216E7 1864021647.85 1864027286 -1.554726368159204E-4 -12313.0 -12313.0 -7.9209529E7 0.0
-1864027286 true BH3PJ6Nf5T0Tg NULL -5400.0 3.47328E7 -1864027286 1 1864027286 0.0 -5400.0 5400.0 -5400.0 -3.47328E7 1864021647.85 1864027286 -1.554726368159204E-4 -5400.0 -5400.0 -3.47382E7 0.0
-1864027286 true BPm3v8Y4 NULL 3151.0 -2.0267232E7 -1864027286 1 1864027286 0.0 3151.0 -3151.0 3151.0 2.0267232E7 1864021647.85 1864027286 -1.554726368159204E-4 3151.0 3151.0 2.0270383E7 0.0
-1864027286 true BS8FR NULL 12619.0 -8.1165408E7 -1864027286 1 1864027286 0.0 12619.0 -12619.0 12619.0 8.1165408E7 1864021647.85 1864027286 -1.554726368159204E-4 12619.0 12619.0 8.1178027E7 0.0
-1864027286 true Bbow1DFvD65Sx6 NULL 7182.0 -4.6194624E7 -1864027286 1 1864027286 0.0 7182.0 -7182.0 7182.0 4.6194624E7 1864021647.85 1864027286 -1.554726368159204E-4 7182.0 7182.0 4.6201806E7 0.0
-1864027286 true BfDk1WlFIoug NULL 4220.0 -2.714304E7 -1864027286 1 1864027286 0.0 4220.0 -4220.0 4220.0 2.714304E7 1864021647.85 1864027286 -1.554726368159204E-4 4220.0 4220.0 2.714726E7 0.0
-1864027286 true Bl1vfIc3iDf8iM7S1p8o2 NULL -15895.0 1.0223664E8 -1864027286 1 1864027286 0.0 -15895.0 15895.0 -15895.0 -1.0223664E8 1864021647.85 1864027286 -1.554726368159204E-4 -15895.0 -15895.0 -1.02252535E8 0.0
-1864027286 true Bug1pfMQCEHkV6M1O4u NULL 9784.0 -6.2930688E7 -1864027286 1 1864027286 0.0 9784.0 -9784.0 9784.0 6.2930688E7 1864021647.85 1864027286 -1.554726368159204E-4 9784.0 9784.0 6.2940472E7 0.0
-1864027286 true C043G NULL -13678.0 8.7976896E7 -1864027286 1 1864027286 0.0 -13678.0 13678.0 -13678.0 -8.7976896E7 1864021647.85 1864027286 -1.554726368159204E-4 -13678.0 -13678.0 -8.7990574E7 0.0
-1864027286 true C1KV2I0wL8wk7C6371 NULL 2776.0 -1.7855232E7 -1864027286 1 1864027286 0.0 2776.0 -2776.0 2776.0 1.7855232E7 1864021647.85 1864027286 -1.554726368159204E-4 2776.0 2776.0 1.7858008E7 0.0
-1864027286 true C2HD3c8PSr8q NULL -9328.0 5.9997696E7 -1864027286 1 1864027286 0.0 -9328.0 9328.0 -9328.0 -5.9997696E7 1864021647.85 1864027286 -1.554726368159204E-4 -9328.0 -9328.0 -6.0007024E7 0.0
-1864027286 true CHP5367P06dFMPWw23eQ NULL -15760.0 1.0136832E8 -1864027286 1 1864027286 0.0 -15760.0 15760.0 -15760.0 -1.0136832E8 1864021647.85 1864027286 -1.554726368159204E-4 -15760.0 -15760.0 -1.0138408E8 0.0
-1864027286 true Cq7458Q8iJtn4aq8I3E NULL -6900.0 4.43808E7 -1864027286 1 1864027286 0.0 -6900.0 6900.0 -6900.0 -4.43808E7 1864021647.85 1864027286 -1.554726368159204E-4 -6900.0 -6900.0 -4.43877E7 0.0
-1864027286 true CwKybtG8352074kNi8cV6qSN NULL -15279.0 9.8274528E7 -1864027286 1 1864027286 0.0 -15279.0 15279.0 -15279.0 -9.8274528E7 1864021647.85 1864027286 -1.554726368159204E-4 -15279.0 -15279.0 -9.8289807E7 0.0
-1864027286 true Cxv2002dg27NL7053ily2CE NULL 9882.0 -6.3561024E7 -1864027286 1 1864027286 0.0 9882.0 -9882.0 9882.0 6.3561024E7 1864021647.85 1864027286 -1.554726368159204E-4 9882.0 9882.0 6.3570906E7 0.0
-1864027286 true D3rrf4BKs5TE NULL 10659.0 -6.8558688E7 -1864027286 1 1864027286 0.0 10659.0 -10659.0 10659.0 6.8558688E7 1864021647.85 1864027286 -1.554726368159204E-4 10659.0 10659.0 6.8569347E7 0.0
-1864027286 true D4tl3Bm NULL 7231.0 -4.6509792E7 -1864027286 1 1864027286 0.0 7231.0 -7231.0 7231.0 4.6509792E7 1864021647.85 1864027286 -1.554726368159204E-4 7231.0 7231.0 4.6517023E7 0.0
-1864027286 true D7d5u8c2q2td7F8wwQSn2Tab NULL -2785.0 1.791312E7 -1864027286 1 1864027286 0.0 -2785.0 2785.0 -2785.0 -1.791312E7 1864021647.85 1864027286 -1.554726368159204E-4 -2785.0 -2785.0 -1.7915905E7 0.0
-1864027286 true D8uSK63TOFY064bwF NULL -13470.0 8.663904E7 -1864027286 1 1864027286 0.0 -13470.0 13470.0 -13470.0 -8.663904E7 1864021647.85 1864027286 -1.554726368159204E-4 -13470.0 -13470.0 -8.665251E7 0.0
-1864027286 true Dy70nFW20WY NULL -4606.0 2.9625792E7 -1864027286 1 1864027286 0.0 -4606.0 4606.0 -4606.0 -2.9625792E7 1864021647.85 1864027286 -1.554726368159204E-4 -4606.0 -4606.0 -2.9630398E7 0.0
-1864027286 true DyDe58BA NULL -8620.0 5.544384E7 -1864027286 1 1864027286 0.0 -8620.0 8620.0 -8620.0 -5.544384E7 1864021647.85 1864027286 -1.554726368159204E-4 -8620.0 -8620.0 -5.545246E7 0.0
-1864027286 true E7T18u2ir5LfC5yywht NULL 5005.0 -3.219216E7 -1864027286 1 1864027286 0.0 5005.0 -5005.0 5005.0 3.219216E7 1864021647.85 1864027286 -1.554726368159204E-4 5005.0 5005.0 3.2197165E7 0.0
-1864027286 true E82GlbIr2v62H5d248gn662 NULL 15492.0 -9.9644544E7 -1864027286 1 1864027286 0.0 15492.0 -15492.0 15492.0 9.9644544E7 1864021647.85 1864027286 -1.554726368159204E-4 15492.0 15492.0 9.9660036E7 0.0
-1864027286 true EbLh7DAd NULL -682.0 4386624.0 -1864027286 1 1864027286 0.0 -682.0 682.0 -682.0 -4386624.0 1864021647.85 1864027286 -1.554726368159204E-4 -682.0 -682.0 -4387306.0 0.0
-1864027286 true Eq4NvWHH4Qb NULL -1911.0 1.2291552E7 -1864027286 1 1864027286 0.0 -1911.0 1911.0 -1911.0 -1.2291552E7 1864021647.85 1864027286 -1.554726368159204E-4 -1911.0 -1911.0 -1.2293463E7 0.0
-1864027286 true F4e1XPV2Hwg7a3d3x530818 NULL 14688.0 -9.4473216E7 -1864027286 1 1864027286 0.0 14688.0 -14688.0 14688.0 9.4473216E7 1864021647.85 1864027286 -1.554726368159204E-4 14688.0 14688.0 9.4487904E7 0.0
-1864027286 true F5n0SfL8CT53dFr51vvW0S3 NULL 4432.0 -2.8506624E7 -1864027286 1 1864027286 0.0 4432.0 -4432.0 4432.0 2.8506624E7 1864021647.85 1864027286 -1.554726368159204E-4 4432.0 4432.0 2.8511056E7 0.0
-1864027286 true F88n72F NULL -15666.0 1.00763712E8 -1864027286 1 1864027286 0.0 -15666.0 15666.0 -15666.0 -1.00763712E8 1864021647.85 1864027286 -1.554726368159204E-4 -15666.0 -15666.0 -1.00779378E8 0.0
-1864027286 true FpcR5Ph NULL -10241.0 6.5870112E7 -1864027286 1 1864027286 0.0 -10241.0 10241.0 -10241.0 -6.5870112E7 1864021647.85 1864027286 -1.554726368159204E-4 -10241.0 -10241.0 -6.5880353E7 0.0
-1864027286 true FpsIohh60Bho67Fb7f NULL -5732.0 3.6868224E7 -1864027286 1 1864027286 0.0 -5732.0 5732.0 -5732.0 -3.6868224E7 1864021647.85 1864027286 -1.554726368159204E-4 -5732.0 -5732.0 -3.6873956E7 0.0
-1864027286 true Fq87rJI5RvYG3 NULL -15729.0 1.01168928E8 -1864027286 1 1864027286 0.0 -15729.0 15729.0 -15729.0 -1.01168928E8 1864021647.85 1864027286 -1.554726368159204E-4 -15729.0 -15729.0 -1.01184657E8 0.0
-1864027286 true G3gsRF NULL 12814.0 -8.2419648E7 -1864027286 1 1864027286 0.0 12814.0 -12814.0 12814.0 8.2419648E7 1864021647.85 1864027286 -1.554726368159204E-4 12814.0 12814.0 8.2432462E7 0.0
-1864027286 true G54It40daSr8MF NULL -10301.0 6.6256032E7 -1864027286 1 1864027286 0.0 -10301.0 10301.0 -10301.0 -6.6256032E7 1864021647.85 1864027286 -1.554726368159204E-4 -10301.0 -10301.0 -6.6266333E7 0.0
-1864027286 true G8N7338fFG NULL -1298.0 8348736.0 -1864027286 1 1864027286 0.0 -1298.0 1298.0 -1298.0 -8348736.0 1864021647.85 1864027286 -1.554726368159204E-4 -1298.0 -1298.0 -8350034.0 0.0
-1864027286 true GP1Kc84XR7Vk10384m7S2J NULL -9375.0 6.03E7 -1864027286 1 1864027286 0.0 -9375.0 9375.0 -9375.0 -6.03E7 1864021647.85 1864027286 -1.554726368159204E-4 -9375.0 -9375.0 -6.0309375E7 0.0
-1864027286 true GPntPwnx0 NULL -14438.0 9.2865216E7 -1864027286 1 1864027286 0.0 -14438.0 14438.0 -14438.0 -9.2865216E7 1864021647.85 1864027286 -1.554726368159204E-4 -14438.0 -14438.0 -9.2879654E7 0.0
-1864027286 true GvcXQ8626I6NBGQm4w NULL -10742.0 6.9092544E7 -1864027286 1 1864027286 0.0 -10742.0 10742.0 -10742.0 -6.9092544E7 1864021647.85 1864027286 -1.554726368159204E-4 -10742.0 -10742.0 -6.9103286E7 0.0
-1864027286 true H1V38u NULL -809.0 5203488.0 -1864027286 1 1864027286 0.0 -809.0 809.0 -809.0 -5203488.0 1864021647.85 1864027286 -1.554726368159204E-4 -809.0 -809.0 -5204297.0 0.0
-1864027286 true H8P4VX62803V NULL 8752.0 -5.6292864E7 -1864027286 1 1864027286 0.0 8752.0 -8752.0 8752.0 5.6292864E7 1864021647.85 1864027286 -1.554726368159204E-4 8752.0 8752.0 5.6301616E7 0.0
-1864027286 true HcPXG7EhIs11eU4iYK5G NULL 11908.0 -7.6592256E7 -1864027286 1 1864027286 0.0 11908.0 -11908.0 11908.0 7.6592256E7 1864021647.85 1864027286 -1.554726368159204E-4 11908.0 11908.0 7.6604164E7 0.0
-1864027286 true Hh8Q8yObmEPI017 NULL -8485.0 5.457552E7 -1864027286 1 1864027286 0.0 -8485.0 8485.0 -8485.0 -5.457552E7 1864021647.85 1864027286 -1.554726368159204E-4 -8485.0 -8485.0 -5.4584005E7 0.0
-1864027286 true HmBi32XWTjC3dd7stD0GY NULL -212.0 1363584.0 -1864027286 1 1864027286 0.0 -212.0 212.0 -212.0 -1363584.0 1864021647.85 1864027286 -1.554726368159204E-4 -212.0 -212.0 -1363796.0 0.0
-1864027286 true HuetF38A4rj7w2 NULL -9710.0 6.245472E7 -1864027286 1 1864027286 0.0 -9710.0 9710.0 -9710.0 -6.245472E7 1864021647.85 1864027286 -1.554726368159204E-4 -9710.0 -9710.0 -6.246443E7 0.0
-1864027286 true I3F7N7s7M NULL 16011.0 -1.02982752E8 -1864027286 1 1864027286 0.0 16011.0 -16011.0 16011.0 1.02982752E8 1864021647.85 1864027286 -1.554726368159204E-4 16011.0 16011.0 1.02998763E8 0.0
-1864027286 true IA46V76LhS4etye16E NULL 2402.0 -1.5449664E7 -1864027286 1 1864027286 0.0 2402.0 -2402.0 2402.0 1.5449664E7 1864021647.85 1864027286 -1.554726368159204E-4 2402.0 2402.0 1.5452066E7 0.0
-1864027286 true IFW3AU8X61t86CljEALEgrr NULL 11329.0 -7.2868128E7 -1864027286 1 1864027286 0.0 11329.0 -11329.0 11329.0 7.2868128E7 1864021647.85 1864027286 -1.554726368159204E-4 11329.0 11329.0 7.2879457E7 0.0
-1864027286 true IL6Ct0hm2 NULL -12970.0 8.342304E7 -1864027286 1 1864027286 0.0 -12970.0 12970.0 -12970.0 -8.342304E7 1864021647.85 1864027286 -1.554726368159204E-4 -12970.0 -12970.0 -8.343601E7 0.0
-1864027286 true ILCAW28PE NULL 5674.0 -3.6495168E7 -1864027286 1 1864027286 0.0 5674.0 -5674.0 5674.0 3.6495168E7 1864021647.85 1864027286 -1.554726368159204E-4 5674.0 5674.0 3.6500842E7 0.0
-1864027286 true INxp2d10SKEd75iE4A7Yq2vc NULL 5492.0 -3.5324544E7 -1864027286 1 1864027286 0.0 5492.0 -5492.0 5492.0 3.5324544E7 1864021647.85 1864027286 -1.554726368159204E-4 5492.0 5492.0 3.5330036E7 0.0
-1864027286 true Io7Mj0g8fwd7L8b4Di NULL 1575.0 -1.01304E7 -1864027286 1 1864027286 0.0 1575.0 -1575.0 1575.0 1.01304E7 1864021647.85 1864027286 -1.554726368159204E-4 1575.0 1575.0 1.0131975E7 0.0
-1864027286 true Is4ogkJ64Sqcqf NULL -13815.0 8.885808E7 -1864027286 1 1864027286 0.0 -13815.0 13815.0 -13815.0 -8.885808E7 1864021647.85 1864027286 -1.554726368159204E-4 -13815.0 -13815.0 -8.8871895E7 0.0
-1864027286 true Iw8wY NULL -668.0 4296576.0 -1864027286 1 1864027286 0.0 -668.0 668.0 -668.0 -4296576.0 1864021647.85 1864027286 -1.554726368159204E-4 -668.0 -668.0 -4297244.0 0.0
-1864027286 true J2El2C63y31dNp4rx NULL -4190.0 2.695008E7 -1864027286 1 1864027286 0.0 -4190.0 4190.0 -4190.0 -2.695008E7 1864021647.85 1864027286 -1.554726368159204E-4 -4190.0 -4190.0 -2.695427E7 0.0
-1864027286 true J34ijU3243 NULL -7672.0 4.9346304E7 -1864027286 1 1864027286 0.0 -7672.0 7672.0 -7672.0 -4.9346304E7 1864021647.85 1864027286 -1.554726368159204E-4 -7672.0 -7672.0 -4.9353976E7 0.0
-1864027286 true J54mWKFYUD081SIe NULL -12288.0 7.9036416E7 -1864027286 1 1864027286 0.0 -12288.0 12288.0 -12288.0 -7.9036416E7 1864021647.85 1864027286 -1.554726368159204E-4 -12288.0 -12288.0 -7.9048704E7 0.0
-1864027286 true J6fBeMaj7b6M8 NULL -16221.0 1.04333472E8 -1864027286 1 1864027286 0.0 -16221.0 16221.0 -16221.0 -1.04333472E8 1864021647.85 1864027286 -1.554726368159204E-4 -16221.0 -16221.0 -1.04349693E8 0.0
-1864027286 true JRN4nLo30dv0bRtsrJa NULL -4319.0 2.7779808E7 -1864027286 1 1864027286 0.0 -4319.0 4319.0 -4319.0 -2.7779808E7 1864021647.85 1864027286 -1.554726368159204E-4 -4319.0 -4319.0 -2.7784127E7 0.0
-1864027286 true Jh7KP0 NULL 13878.0 -8.9263296E7 -1864027286 1 1864027286 0.0 13878.0 -13878.0 13878.0 8.9263296E7 1864021647.85 1864027286 -1.554726368159204E-4 13878.0 13878.0 8.9277174E7 0.0
-1864027286 true Jy4CAuL25v4JrHsIdj3d4q2M NULL -11781.0 7.5775392E7 -1864027286 1 1864027286 0.0 -11781.0 11781.0 -11781.0 -7.5775392E7 1864021647.85 1864027286 -1.554726368159204E-4 -11781.0 -11781.0 -7.5787173E7 0.0
-1864027286 true K26B60qNA761SuYdXKhu NULL 15278.0 -9.8268096E7 -1864027286 1 1864027286 0.0 15278.0 -15278.0 15278.0 9.8268096E7 1864021647.85 1864027286 -1.554726368159204E-4 15278.0 15278.0 9.8283374E7 0.0
-1864027286 true K54bM1PBEyv85M7J6G NULL 5277.0 -3.3941664E7 -1864027286 1 1864027286 0.0 5277.0 -5277.0 5277.0 3.3941664E7 1864021647.85 1864027286 -1.554726368159204E-4 5277.0 5277.0 3.3946941E7 0.0
-1864027286 true KA2M874c7v83T NULL -7352.0 4.7288064E7 -1864027286 1 1864027286 0.0 -7352.0 7352.0 -7352.0 -4.7288064E7 1864021647.85 1864027286 -1.554726368159204E-4 -7352.0 -7352.0 -4.7295416E7 0.0
-1864027286 true KBV5WE6y76le NULL 10683.0 -6.8713056E7 -1864027286 1 1864027286 0.0 10683.0 -10683.0 10683.0 6.8713056E7 1864021647.85 1864027286 -1.554726368159204E-4 10683.0 10683.0 6.8723739E7 0.0
-1864027286 true Kc1lPGJx6JXTcDsck00 NULL 2803.0 -1.8028896E7 -1864027286 1 1864027286 0.0 2803.0 -2803.0 2803.0 1.8028896E7 1864021647.85 1864027286 -1.554726368159204E-4 2803.0 2803.0 1.8031699E7 0.0
-1864027286 true KlP8GX12PxC4giG475 NULL -8630.0 5.550816E7 -1864027286 1 1864027286 0.0 -8630.0 8630.0 -8630.0 -5.550816E7 1864021647.85 1864027286 -1.554726368159204E-4 -8630.0 -8630.0 -5.551679E7 0.0
-1864027286 true KwqjKvxg17Ro85YEQYKl NULL -4971.0 3.1973472E7 -1864027286 1 1864027286 0.0 -4971.0 4971.0 -4971.0 -3.1973472E7 1864021647.85 1864027286 -1.554726368159204E-4 -4971.0 -4971.0 -3.1978443E7 0.0
-1864027286 true L28vl NULL 2438.0 -1.5681216E7 -1864027286 1 1864027286 0.0 2438.0 -2438.0 2438.0 1.5681216E7 1864021647.85 1864027286 -1.554726368159204E-4 2438.0 2438.0 1.5683654E7 0.0
-1864027286 true L4WQG81b36T NULL 1970.0 -1.267104E7 -1864027286 1 1864027286 0.0 1970.0 -1970.0 1970.0 1.267104E7 1864021647.85 1864027286 -1.554726368159204E-4 1970.0 1970.0 1.267301E7 0.0
-1864027286 true L577vXI27E4kGm NULL -11345.0 7.297104E7 -1864027286 1 1864027286 0.0 -11345.0 11345.0 -11345.0 -7.297104E7 1864021647.85 1864027286 -1.554726368159204E-4 -11345.0 -11345.0 -7.2982385E7 0.0
-1864027286 true L5X4732Ib1Vj5ev NULL 8542.0 -5.4942144E7 -1864027286 1 1864027286 0.0 8542.0 -8542.0 8542.0 5.4942144E7 1864021647.85 1864027286 -1.554726368159204E-4 8542.0 8542.0 5.4950686E7 0.0
-1864027286 true LCUh4H7E8RT8opWRW8m NULL -4593.0 2.9542176E7 -1864027286 1 1864027286 0.0 -4593.0 4593.0 -4593.0 -2.9542176E7 1864021647.85 1864027286 -1.554726368159204E-4 -4593.0 -4593.0 -2.9546769E7 0.0
-1864027286 true LHtKPAbAXa4QGM2y NULL -2847.0 1.8311904E7 -1864027286 1 1864027286 0.0 -2847.0 2847.0 -2847.0 -1.8311904E7 1864021647.85 1864027286 -1.554726368159204E-4 -2847.0 -2847.0 -1.8314751E7 0.0
-1864027286 true LOeiVy1yE NULL -11326.0 7.2848832E7 -1864027286 1 1864027286 0.0 -11326.0 11326.0 -11326.0 -7.2848832E7 1864021647.85 1864027286 -1.554726368159204E-4 -11326.0 -11326.0 -7.2860158E7 0.0
-1864027286 true LSt435WAB5OKB NULL -7333.0 4.7165856E7 -1864027286 1 1864027286 0.0 -7333.0 7333.0 -7333.0 -4.7165856E7 1864021647.85 1864027286 -1.554726368159204E-4 -7333.0 -7333.0 -4.7173189E7 0.0
-1864027286 true M0kjTU3N2L5P NULL 368.0 -2366976.0 -1864027286 1 1864027286 0.0 368.0 -368.0 368.0 2366976.0 1864021647.85 1864027286 -1.554726368159204E-4 368.0 368.0 2367344.0 0.0
-1864027286 true M7J5a5vG8s3 NULL 1338.0 -8606016.0 -1864027286 1 1864027286 0.0 1338.0 -1338.0 1338.0 8606016.0 1864021647.85 1864027286 -1.554726368159204E-4 1338.0 1338.0 8607354.0 0.0
-1864027286 true MFaMcxlV NULL -9039.0 5.8138848E7 -1864027286 1 1864027286 0.0 -9039.0 9039.0 -9039.0 -5.8138848E7 1864021647.85 1864027286 -1.554726368159204E-4 -9039.0 -9039.0 -5.8147887E7 0.0
-1864027286 true MGsGfU7253gN2Hnt2W NULL -5679.0 3.6527328E7 -1864027286 1 1864027286 0.0 -5679.0 5679.0 -5679.0 -3.6527328E7 1864021647.85 1864027286 -1.554726368159204E-4 -5679.0 -5679.0 -3.6533007E7 0.0
-1864027286 true MUg2eGVMxLEn2JlY3stOYR NULL -741.0 4766112.0 -1864027286 1 1864027286 0.0 -741.0 741.0 -741.0 -4766112.0 1864021647.85 1864027286 -1.554726368159204E-4 -741.0 -741.0 -4766853.0 0.0
-1864027286 true Mr3q8uV NULL 354.0 -2276928.0 -1864027286 1 1864027286 0.0 354.0 -354.0 354.0 2276928.0 1864021647.85 1864027286 -1.554726368159204E-4 354.0 354.0 2277282.0 0.0
-1864027286 true N2TL0cw5gA4VFFI6xo NULL 1554.0 -9995328.0 -1864027286 1 1864027286 0.0 1554.0 -1554.0 1554.0 9995328.0 1864021647.85 1864027286 -1.554726368159204E-4 1554.0 1554.0 9996882.0 0.0
-1864027286 true N5yMwlmd8beg7N2jPn NULL 1684.0 -1.0831488E7 -1864027286 1 1864027286 0.0 1684.0 -1684.0 1684.0 1.0831488E7 1864021647.85 1864027286 -1.554726368159204E-4 1684.0 1684.0 1.0833172E7 0.0
-1864027286 true N6G5QssB8L7DoJW6BSSGFUFI NULL -5296.0 3.4063872E7 -1864027286 1 1864027286 0.0 -5296.0 5296.0 -5296.0 -3.4063872E7 1864021647.85 1864027286 -1.554726368159204E-4 -5296.0 -5296.0 -3.4069168E7 0.0
-1864027286 true N7L608vFx24p0uNVwJr2o6G NULL -5536.0 3.5607552E7 -1864027286 1 1864027286 0.0 -5536.0 5536.0 -5536.0 -3.5607552E7 1864021647.85 1864027286 -1.554726368159204E-4 -5536.0 -5536.0 -3.5613088E7 0.0
-1864027286 true NEK1MY7NTS36Ov4FI7xQx NULL -10682.0 6.8706624E7 -1864027286 1 1864027286 0.0 -10682.0 10682.0 -10682.0 -6.8706624E7 1864021647.85 1864027286 -1.554726368159204E-4 -10682.0 -10682.0 -6.8717306E7 0.0
-1864027286 true NdtQ8j30gg2U5O NULL -8369.0 5.3829408E7 -1864027286 1 1864027286 0.0 -8369.0 8369.0 -8369.0 -5.3829408E7 1864021647.85 1864027286 -1.554726368159204E-4 -8369.0 -8369.0 -5.3837777E7 0.0
-1864027286 true O1Rlpc2lK3YRjAQu34gE2UK5 NULL -6216.0 3.9981312E7 -1864027286 1 1864027286 0.0 -6216.0 6216.0 -6216.0 -3.9981312E7 1864021647.85 1864027286 -1.554726368159204E-4 -6216.0 -6216.0 -3.9987528E7 0.0
-1864027286 true O6o7xl47446MR NULL 7031.0 -4.5223392E7 -1864027286 1 1864027286 0.0 7031.0 -7031.0 7031.0 4.5223392E7 1864021647.85 1864027286 -1.554726368159204E-4 7031.0 7031.0 4.5230423E7 0.0
-1864027286 true ODLrXI8882q8LS8 NULL 10782.0 -6.9349824E7 -1864027286 1 1864027286 0.0 10782.0 -10782.0 10782.0 6.9349824E7 1864021647.85 1864027286 -1.554726368159204E-4 10782.0 10782.0 6.9360606E7 0.0
-1864027286 true OIj6IQ7c4U NULL 8233.0 -5.2954656E7 -1864027286 1 1864027286 0.0 8233.0 -8233.0 8233.0 5.2954656E7 1864021647.85 1864027286 -1.554726368159204E-4 8233.0 8233.0 5.2962889E7 0.0
-1864027286 true OKlMC73w40s4852R75 NULL 12464.0 -8.0168448E7 -1864027286 1 1864027286 0.0 12464.0 -12464.0 12464.0 8.0168448E7 1864021647.85 1864027286 -1.554726368159204E-4 12464.0 12464.0 8.0180912E7 0.0
-1864027286 true Ocv25R6uD751tb7f2 NULL -3657.0 2.3521824E7 -1864027286 1 1864027286 0.0 -3657.0 3657.0 -3657.0 -2.3521824E7 1864021647.85 1864027286 -1.554726368159204E-4 -3657.0 -3657.0 -2.3525481E7 0.0
-1864027286 true Oqh7OlT63e0RO74or NULL 13600.0 -8.74752E7 -1864027286 1 1864027286 0.0 13600.0 -13600.0 13600.0 8.74752E7 1864021647.85 1864027286 -1.554726368159204E-4 13600.0 13600.0 8.74888E7 0.0
-1864027286 true P3484jw0Gpff2VgoSdALY NULL 7872.0 -5.0632704E7 -1864027286 1 1864027286 0.0 7872.0 -7872.0 7872.0 5.0632704E7 1864021647.85 1864027286 -1.554726368159204E-4 7872.0 7872.0 5.0640576E7 0.0
-1864027286 true P35JtWWC5M42H7cTpwJN NULL -12207.0 7.8515424E7 -1864027286 1 1864027286 0.0 -12207.0 12207.0 -12207.0 -7.8515424E7 1864021647.85 1864027286 -1.554726368159204E-4 -12207.0 -12207.0 -7.8527631E7 0.0
-1864027286 true P35q3 NULL -14317.0 9.2086944E7 -1864027286 1 1864027286 0.0 -14317.0 14317.0 -14317.0 -9.2086944E7 1864021647.85 1864027286 -1.554726368159204E-4 -14317.0 -14317.0 -9.2101261E7 0.0
-1864027286 true P3T4PNGG1QqCpM NULL -7577.0 4.8735264E7 -1864027286 1 1864027286 0.0 -7577.0 7577.0 -7577.0 -4.8735264E7 1864021647.85 1864027286 -1.554726368159204E-4 -7577.0 -7577.0 -4.8742841E7 0.0
-1864027286 true P5iS0 NULL -4168.0 2.6808576E7 -1864027286 1 1864027286 0.0 -4168.0 4168.0 -4168.0 -2.6808576E7 1864021647.85 1864027286 -1.554726368159204E-4 -4168.0 -4168.0 -2.6812744E7 0.0
-1864027286 true P61xNCa0H NULL 10775.0 -6.93048E7 -1864027286 1 1864027286 0.0 10775.0 -10775.0 10775.0 6.93048E7 1864021647.85 1864027286 -1.554726368159204E-4 10775.0 10775.0 6.9315575E7 0.0
-1864027286 true P8NPOlehc210j8c781 NULL 12949.0 -8.3287968E7 -1864027286 1 1864027286 0.0 12949.0 -12949.0 12949.0 8.3287968E7 1864021647.85 1864027286 -1.554726368159204E-4 12949.0 12949.0 8.3300917E7 0.0
-1864027286 true PC25sHxt4J NULL 9052.0 -5.8222464E7 -1864027286 1 1864027286 0.0 9052.0 -9052.0 9052.0 5.8222464E7 1864021647.85 1864027286 -1.554726368159204E-4 9052.0 9052.0 5.8231516E7 0.0
-1864027286 true PQ71uI1bCFcvHK7 NULL -13872.0 8.9224704E7 -1864027286 1 1864027286 0.0 -13872.0 13872.0 -13872.0 -8.9224704E7 1864021647.85 1864027286 -1.554726368159204E-4 -13872.0 -13872.0 -8.9238576E7 0.0
-1864027286 true PlOxor04p5cvVl NULL 5064.0 -3.2571648E7 -1864027286 1 1864027286 0.0 5064.0 -5064.0 5064.0 3.2571648E7 1864021647.85 1864027286 -1.554726368159204E-4 5064.0 5064.0 3.2576712E7 0.0
-1864027286 true Po4rrk NULL 3442.0 -2.2138944E7 -1864027286 1 1864027286 0.0 3442.0 -3442.0 3442.0 2.2138944E7 1864021647.85 1864027286 -1.554726368159204E-4 3442.0 3442.0 2.2142386E7 0.0
-1864027286 true PovkPN NULL 5312.0 -3.4166784E7 -1864027286 1 1864027286 0.0 5312.0 -5312.0 5312.0 3.4166784E7 1864021647.85 1864027286 -1.554726368159204E-4 5312.0 5312.0 3.4172096E7 0.0
-1864027286 true PxgAPl26H6hsU47TPD NULL -12794.0 8.2291008E7 -1864027286 1 1864027286 0.0 -12794.0 12794.0 -12794.0 -8.2291008E7 1864021647.85 1864027286 -1.554726368159204E-4 -12794.0 -12794.0 -8.2303802E7 0.0
-1864027286 true PyQ4Q7MF23J4AtYu6W NULL 2327.0 -1.4967264E7 -1864027286 1 1864027286 0.0 2327.0 -2327.0 2327.0 1.4967264E7 1864021647.85 1864027286 -1.554726368159204E-4 2327.0 2327.0 1.4969591E7 0.0
-1864027286 true QAgnk2L5bnLH580a143KUc NULL 12738.0 -8.1930816E7 -1864027286 1 1864027286 0.0 12738.0 -12738.0 12738.0 8.1930816E7 1864021647.85 1864027286 -1.554726368159204E-4 12738.0 12738.0 8.1943554E7 0.0
-1864027286 true QEF7UG67MDaTK504bNrF NULL 15217.0 -9.7875744E7 -1864027286 1 1864027286 0.0 15217.0 -15217.0 15217.0 9.7875744E7 1864021647.85 1864027286 -1.554726368159204E-4 15217.0 15217.0 9.7890961E7 0.0
-1864027286 true QJxfy45 NULL 12427.0 -7.9930464E7 -1864027286 1 1864027286 0.0 12427.0 -12427.0 12427.0 7.9930464E7 1864021647.85 1864027286 -1.554726368159204E-4 12427.0 12427.0 7.9942891E7 0.0
-1864027286 true QN3Ru4uhSNA62bgc4HI35 NULL -12165.0 7.824528E7 -1864027286 1 1864027286 0.0 -12165.0 12165.0 -12165.0 -7.824528E7 1864021647.85 1864027286 -1.554726368159204E-4 -12165.0 -12165.0 -7.8257445E7 0.0
-1864027286 true QOt28D6Ov NULL -8010.0 5.152032E7 -1864027286 1 1864027286 0.0 -8010.0 8010.0 -8010.0 -5.152032E7 1864021647.85 1864027286 -1.554726368159204E-4 -8010.0 -8010.0 -5.152833E7 0.0
-1864027286 true QWfu6dR4Na2g5 NULL -9974.0 6.4152768E7 -1864027286 1 1864027286 0.0 -9974.0 9974.0 -9974.0 -6.4152768E7 1864021647.85 1864027286 -1.554726368159204E-4 -9974.0 -9974.0 -6.4162742E7 0.0
-1864027286 true Qa8XbKYNym5Se NULL 2442.0 -1.5706944E7 -1864027286 1 1864027286 0.0 2442.0 -2442.0 2442.0 1.5706944E7 1864021647.85 1864027286 -1.554726368159204E-4 2442.0 2442.0 1.5709386E7 0.0
-1864027286 true R03eo03Ntqej0VDQbL3 NULL -1976.0 1.2709632E7 -1864027286 1 1864027286 0.0 -1976.0 1976.0 -1976.0 -1.2709632E7 1864021647.85 1864027286 -1.554726368159204E-4 -1976.0 -1976.0 -1.2711608E7 0.0
-1864027286 true R04RF7qkQ8Gn1PPd33pU6 NULL 6637.0 -4.2689184E7 -1864027286 1 1864027286 0.0 6637.0 -6637.0 6637.0 4.2689184E7 1864021647.85 1864027286 -1.554726368159204E-4 6637.0 6637.0 4.2695821E7 0.0
-1864027286 true R0hA3Hq2VsjnFh NULL 9931.0 -6.3876192E7 -1864027286 1 1864027286 0.0 9931.0 -9931.0 9931.0 6.3876192E7 1864021647.85 1864027286 -1.554726368159204E-4 9931.0 9931.0 6.3886123E7 0.0
-1864027286 true R1VmJ10Ie NULL 14947.0 -9.6139104E7 -1864027286 1 1864027286 0.0 14947.0 -14947.0 14947.0 9.6139104E7 1864021647.85 1864027286 -1.554726368159204E-4 14947.0 14947.0 9.6154051E7 0.0
-1864027286 true R61IdER NULL 1321.0 -8496672.0 -1864027286 1 1864027286 0.0 1321.0 -1321.0 1321.0 8496672.0 1864021647.85 1864027286 -1.554726368159204E-4 1321.0 1321.0 8497993.0 0.0
-1864027286 true R6xXNwfbk NULL -2129.0 1.3693728E7 -1864027286 1 1864027286 0.0 -2129.0 2129.0 -2129.0 -1.3693728E7 1864021647.85 1864027286 -1.554726368159204E-4 -2129.0 -2129.0 -1.3695857E7 0.0
-1864027286 true RAUe5p NULL 2686.0 -1.7276352E7 -1864027286 1 1864027286 0.0 2686.0 -2686.0 2686.0 1.7276352E7 1864021647.85 1864027286 -1.554726368159204E-4 2686.0 2686.0 1.7279038E7 0.0
-1864027286 true RBtE7gkmLOh22A4 NULL 9614.0 -6.1837248E7 -1864027286 1 1864027286 0.0 9614.0 -9614.0 9614.0 6.1837248E7 1864021647.85 1864027286 -1.554726368159204E-4 9614.0 9614.0 6.1846862E7 0.0
-1864027286 true RBvPK67 NULL 8146.0 -5.2395072E7 -1864027286 1 1864027286 0.0 8146.0 -8146.0 8146.0 5.2395072E7 1864021647.85 1864027286 -1.554726368159204E-4 8146.0 8146.0 5.2403218E7 0.0
-1864027286 true RDLOWd758CODQgBBA8hd172 NULL 423.0 -2720736.0 -1864027286 1 1864027286 0.0 423.0 -423.0 423.0 2720736.0 1864021647.85 1864027286 -1.554726368159204E-4 423.0 423.0 2721159.0 0.0
-1864027286 true RW6K24 NULL -9580.0 6.161856E7 -1864027286 1 1864027286 0.0 -9580.0 9580.0 -9580.0 -6.161856E7 1864021647.85 1864027286 -1.554726368159204E-4 -9580.0 -9580.0 -6.162814E7 0.0
-1864027286 true Ru7fjpH4C0YOXs6E NULL 6474.0 -4.1640768E7 -1864027286 1 1864027286 0.0 6474.0 -6474.0 6474.0 4.1640768E7 1864021647.85 1864027286 -1.554726368159204E-4 6474.0 6474.0 4.1647242E7 0.0
-1864027286 true S2I2nIEii3X5 NULL -1207.0 7763424.0 -1864027286 1 1864027286 0.0 -1207.0 1207.0 -1207.0 -7763424.0 1864021647.85 1864027286 -1.554726368159204E-4 -1207.0 -1207.0 -7764631.0 0.0
-1864027286 true S45s3B0rSCbDkMx3Q NULL 2852.0 -1.8344064E7 -1864027286 1 1864027286 0.0 2852.0 -2852.0 2852.0 1.8344064E7 1864021647.85 1864027286 -1.554726368159204E-4 2852.0 2852.0 1.8346916E7 0.0
-1864027286 true Se4jyihvl80uOdFD NULL 15076.0 -9.6968832E7 -1864027286 1 1864027286 0.0 15076.0 -15076.0 15076.0 9.6968832E7 1864021647.85 1864027286 -1.554726368159204E-4 15076.0 15076.0 9.6983908E7 0.0
-1864027286 true T2o8XRFAL0HC4ikDQnfoCymw NULL 1535.0 -9873120.0 -1864027286 1 1864027286 0.0 1535.0 -1535.0 1535.0 9873120.0 1864021647.85 1864027286 -1.554726368159204E-4 1535.0 1535.0 9874655.0 0.0
-1864027286 true TBbxkMGlYD17B7d76b7x3 NULL 13786.0 -8.8671552E7 -1864027286 1 1864027286 0.0 13786.0 -13786.0 13786.0 8.8671552E7 1864021647.85 1864027286 -1.554726368159204E-4 13786.0 13786.0 8.8685338E7 0.0
-1864027286 true TT4CHN NULL -6060.0 3.897792E7 -1864027286 1 1864027286 0.0 -6060.0 6060.0 -6060.0 -3.897792E7 1864021647.85 1864027286 -1.554726368159204E-4 -6060.0 -6060.0 -3.898398E7 0.0
-1864027286 true ToOQ4YhGHo NULL 14146.0 -9.0987072E7 -1864027286 1 1864027286 0.0 14146.0 -14146.0 14146.0 9.0987072E7 1864021647.85 1864027286 -1.554726368159204E-4 14146.0 14146.0 9.1001218E7 0.0
-1864027286 true U4MrN4CKBl84 NULL 15895.0 -1.0223664E8 -1864027286 1 1864027286 0.0 15895.0 -15895.0 15895.0 1.0223664E8 1864021647.85 1864027286 -1.554726368159204E-4 15895.0 15895.0 1.02252535E8 0.0
-1864027286 true UR83Iqx405t0jOOhF NULL 12605.0 -8.107536E7 -1864027286 1 1864027286 0.0 12605.0 -12605.0 12605.0 8.107536E7 1864021647.85 1864027286 -1.554726368159204E-4 12605.0 12605.0 8.1087965E7 0.0
-1864027286 true Uj28ubp026RCw NULL -5469.0 3.5176608E7 -1864027286 1 1864027286 0.0 -5469.0 5469.0 -5469.0 -3.5176608E7 1864021647.85 1864027286 -1.554726368159204E-4 -5469.0 -5469.0 -3.5182077E7 0.0
-1864027286 true Usb4N NULL -9174.0 5.9007168E7 -1864027286 1 1864027286 0.0 -9174.0 9174.0 -9174.0 -5.9007168E7 1864021647.85 1864027286 -1.554726368159204E-4 -9174.0 -9174.0 -5.9016342E7 0.0
-1864027286 true VMlhJes4CVgyK7uFOX NULL -10868.0 6.9902976E7 -1864027286 1 1864027286 0.0 -10868.0 10868.0 -10868.0 -6.9902976E7 1864021647.85 1864027286 -1.554726368159204E-4 -10868.0 -10868.0 -6.9913844E7 0.0
-1864027286 true Vb8ub0i0Maa NULL -9883.0 6.3567456E7 -1864027286 1 1864027286 0.0 -9883.0 9883.0 -9883.0 -6.3567456E7 1864021647.85 1864027286 -1.554726368159204E-4 -9883.0 -9883.0 -6.3577339E7 0.0
-1864027286 true W2mhptJ NULL 8246.0 -5.3038272E7 -1864027286 1 1864027286 0.0 8246.0 -8246.0 8246.0 5.3038272E7 1864021647.85 1864027286 -1.554726368159204E-4 8246.0 8246.0 5.3046518E7 0.0
-1864027286 true W4GLKnA2Nwk0HJ NULL 9528.0 -6.1284096E7 -1864027286 1 1864027286 0.0 9528.0 -9528.0 9528.0 6.1284096E7 1864021647.85 1864027286 -1.554726368159204E-4 9528.0 9528.0 6.1293624E7 0.0
-1864027286 true W772E0x NULL 7864.0 -5.0581248E7 -1864027286 1 1864027286 0.0 7864.0 -7864.0 7864.0 5.0581248E7 1864021647.85 1864027286 -1.554726368159204E-4 7864.0 7864.0 5.0589112E7 0.0
-1864027286 true WL65H3J NULL -13307.0 8.5590624E7 -1864027286 1 1864027286 0.0 -13307.0 13307.0 -13307.0 -8.5590624E7 1864021647.85 1864027286 -1.554726368159204E-4 -13307.0 -13307.0 -8.5603931E7 0.0
-1864027286 true WQk67I0Gk NULL 2489.0 -1.6009248E7 -1864027286 1 1864027286 0.0 2489.0 -2489.0 2489.0 1.6009248E7 1864021647.85 1864027286 -1.554726368159204E-4 2489.0 2489.0 1.6011737E7 0.0
-1864027286 true WU7g0T0a15w2v5t NULL -9418.0 6.0576576E7 -1864027286 1 1864027286 0.0 -9418.0 9418.0 -9418.0 -6.0576576E7 1864021647.85 1864027286 -1.554726368159204E-4 -9418.0 -9418.0 -6.0585994E7 0.0
-1864027286 true WWo570W28lhx415 NULL 6392.0 -4.1113344E7 -1864027286 1 1864027286 0.0 6392.0 -6392.0 6392.0 4.1113344E7 1864021647.85 1864027286 -1.554726368159204E-4 6392.0 6392.0 4.1119736E7 0.0
-1864027286 true WhgF327bC NULL -4837.0 3.1111584E7 -1864027286 1 1864027286 0.0 -4837.0 4837.0 -4837.0 -3.1111584E7 1864021647.85 1864027286 -1.554726368159204E-4 -4837.0 -4837.0 -3.1116421E7 0.0
-1864027286 true X18ccPrLl NULL -10096.0 6.4937472E7 -1864027286 1 1864027286 0.0 -10096.0 10096.0 -10096.0 -6.4937472E7 1864021647.85 1864027286 -1.554726368159204E-4 -10096.0 -10096.0 -6.4947568E7 0.0
-1864027286 true X6155iP NULL 4774.0 -3.0706368E7 -1864027286 1 1864027286 0.0 4774.0 -4774.0 4774.0 3.0706368E7 1864021647.85 1864027286 -1.554726368159204E-4 4774.0 4774.0 3.0711142E7 0.0
-1864027286 true X75olERkL08uR NULL 12481.0 -8.0277792E7 -1864027286 1 1864027286 0.0 12481.0 -12481.0 12481.0 8.0277792E7 1864021647.85 1864027286 -1.554726368159204E-4 12481.0 12481.0 8.0290273E7 0.0
-1864027286 true XP2cjyx NULL -9367.0 6.0248544E7 -1864027286 1 1864027286 0.0 -9367.0 9367.0 -9367.0 -6.0248544E7 1864021647.85 1864027286 -1.554726368159204E-4 -9367.0 -9367.0 -6.0257911E7 0.0
-1864027286 true Xvyjl2vcUcxY4 NULL -14086.0 9.0601152E7 -1864027286 1 1864027286 0.0 -14086.0 14086.0 -14086.0 -9.0601152E7 1864021647.85 1864027286 -1.554726368159204E-4 -14086.0 -14086.0 -9.0615238E7 0.0
-1864027286 true Y2C704h6OUXJQ3 NULL -13177.0 8.4754464E7 -1864027286 1 1864027286 0.0 -13177.0 13177.0 -13177.0 -8.4754464E7 1864021647.85 1864027286 -1.554726368159204E-4 -13177.0 -13177.0 -8.4767641E7 0.0
-1864027286 true Y4JQvk NULL 10557.0 -6.7902624E7 -1864027286 1 1864027286 0.0 10557.0 -10557.0 10557.0 6.7902624E7 1864021647.85 1864027286 -1.554726368159204E-4 10557.0 10557.0 6.7913181E7 0.0
-1864027286 true YtN1m7B NULL -3416.0 2.1971712E7 -1864027286 1 1864027286 0.0 -3416.0 3416.0 -3416.0 -2.1971712E7 1864021647.85 1864027286 -1.554726368159204E-4 -3416.0 -3416.0 -2.1975128E7 0.0
-1864027286 true a NULL 12004.0 -7.7209728E7 -1864027286 1 1864027286 0.0 12004.0 -12004.0 12004.0 7.7209728E7 1864021647.85 1864027286 -1.554726368159204E-4 12004.0 12004.0 7.7221732E7 0.0
-1864027286 true a0YMQr03O NULL 10671.0 -6.8635872E7 -1864027286 1 1864027286 0.0 10671.0 -10671.0 10671.0 6.8635872E7 1864021647.85 1864027286 -1.554726368159204E-4 10671.0 10671.0 6.8646543E7 0.0
-1864027286 true a0mdHI0HtSL0o8 NULL 8163.0 -5.2504416E7 -1864027286 1 1864027286 0.0 8163.0 -8163.0 8163.0 5.2504416E7 1864021647.85 1864027286 -1.554726368159204E-4 8163.0 8163.0 5.2512579E7 0.0
-1864027286 true a250165354I3O4fw42l7DG NULL 14108.0 -9.0742656E7 -1864027286 1 1864027286 0.0 14108.0 -14108.0 14108.0 9.0742656E7 1864021647.85 1864027286 -1.554726368159204E-4 14108.0 14108.0 9.0756764E7 0.0
-1864027286 true a4PMyxYPeTA0Js14lFCV3f NULL -3746.0 2.4094272E7 -1864027286 1 1864027286 0.0 -3746.0 3746.0 -3746.0 -2.4094272E7 1864021647.85 1864027286 -1.554726368159204E-4 -3746.0 -3746.0 -2.4098018E7 0.0
-1864027286 true aDNmF88FfTwOx7u NULL -8251.0 5.3070432E7 -1864027286 1 1864027286 0.0 -8251.0 8251.0 -8251.0 -5.3070432E7 1864021647.85 1864027286 -1.554726368159204E-4 -8251.0 -8251.0 -5.3078683E7 0.0
-1864027286 true aH38aH4ob NULL 12197.0 -7.8451104E7 -1864027286 1 1864027286 0.0 12197.0 -12197.0 12197.0 7.8451104E7 1864021647.85 1864027286 -1.554726368159204E-4 12197.0 12197.0 7.8463301E7 0.0
-1864027286 true aT5XuK NULL -10736.0 6.9053952E7 -1864027286 1 1864027286 0.0 -10736.0 10736.0 -10736.0 -6.9053952E7 1864021647.85 1864027286 -1.554726368159204E-4 -10736.0 -10736.0 -6.9064688E7 0.0
-1864027286 true ap7PY4878sX8F6YUn6Wh1Vg4 NULL -3684.0 2.3695488E7 -1864027286 1 1864027286 0.0 -3684.0 3684.0 -3684.0 -2.3695488E7 1864021647.85 1864027286 -1.554726368159204E-4 -3684.0 -3684.0 -2.3699172E7 0.0
-1864027286 true axu5k1BMtA6Ki0 NULL -1227.0 7892064.0 -1864027286 1 1864027286 0.0 -1227.0 1227.0 -1227.0 -7892064.0 1864021647.85 1864027286 -1.554726368159204E-4 -1227.0 -1227.0 -7893291.0 0.0
-1864027286 true b NULL 10938.0 -7.0353216E7 -1864027286 1 1864027286 0.0 10938.0 -10938.0 10938.0 7.0353216E7 1864021647.85 1864027286 -1.554726368159204E-4 10938.0 10938.0 7.0364154E7 0.0
-1864027286 true b NULL 13839.0 -8.9012448E7 -1864027286 1 1864027286 0.0 13839.0 -13839.0 13839.0 8.9012448E7 1864021647.85 1864027286 -1.554726368159204E-4 13839.0 13839.0 8.9026287E7 0.0
-1864027286 true b2Mvom63qTp4o NULL -14355.0 9.233136E7 -1864027286 1 1864027286 0.0 -14355.0 14355.0 -14355.0 -9.233136E7 1864021647.85 1864027286 -1.554726368159204E-4 -14355.0 -14355.0 -9.2345715E7 0.0
-1864027286 true b565l4rv1444T25Gv0 NULL 9517.0 -6.1213344E7 -1864027286 1 1864027286 0.0 9517.0 -9517.0 9517.0 6.1213344E7 1864021647.85 1864027286 -1.554726368159204E-4 9517.0 9517.0 6.1222861E7 0.0
-1864027286 true bFmH03DgwC5s88 NULL 3956.0 -2.5444992E7 -1864027286 1 1864027286 0.0 3956.0 -3956.0 3956.0 2.5444992E7 1864021647.85 1864027286 -1.554726368159204E-4 3956.0 3956.0 2.5448948E7 0.0
-1864027286 true bVvdKDfUwoKNMosc2esLYVe NULL -10016.0 6.4422912E7 -1864027286 1 1864027286 0.0 -10016.0 10016.0 -10016.0 -6.4422912E7 1864021647.85 1864027286 -1.554726368159204E-4 -10016.0 -10016.0 -6.4432928E7 0.0
-1864027286 true bvoO6VwRmH6181mdOm87Do NULL 10144.0 -6.5246208E7 -1864027286 1 1864027286 0.0 10144.0 -10144.0 10144.0 6.5246208E7 1864021647.85 1864027286 -1.554726368159204E-4 10144.0 10144.0 6.5256352E7 0.0
-1864027286 true c7VDm103iwF1c7M NULL -14542.0 9.3534144E7 -1864027286 1 1864027286 0.0 -14542.0 14542.0 -14542.0 -9.3534144E7 1864021647.85 1864027286 -1.554726368159204E-4 -14542.0 -14542.0 -9.3548686E7 0.0
-1864027286 true cM0xm3h8463l57s NULL 1253.0 -8059296.0 -1864027286 1 1864027286 0.0 1253.0 -1253.0 1253.0 8059296.0 1864021647.85 1864027286 -1.554726368159204E-4 1253.0 1253.0 8060549.0 0.0
-1864027286 true cwEvSRx2cuarX7I21UGe NULL -1434.0 9223488.0 -1864027286 1 1864027286 0.0 -1434.0 1434.0 -1434.0 -9223488.0 1864021647.85 1864027286 -1.554726368159204E-4 -1434.0 -1434.0 -9224922.0 0.0
-1864027286 true d2A5U2557V347stTcy5bb NULL -13334.0 8.5764288E7 -1864027286 1 1864027286 0.0 -13334.0 13334.0 -13334.0 -8.5764288E7 1864021647.85 1864027286 -1.554726368159204E-4 -13334.0 -13334.0 -8.5777622E7 0.0
-1864027286 true d4YeS73lyC6l NULL -16168.0 1.03992576E8 -1864027286 1 1864027286 0.0 -16168.0 16168.0 -16168.0 -1.03992576E8 1864021647.85 1864027286 -1.554726368159204E-4 -16168.0 -16168.0 -1.04008744E8 0.0
-1864027286 true d77tW1Y01AT7U NULL -15267.0 9.8197344E7 -1864027286 1 1864027286 0.0 -15267.0 15267.0 -15267.0 -9.8197344E7 1864021647.85 1864027286 -1.554726368159204E-4 -15267.0 -15267.0 -9.8212611E7 0.0
-1864027286 true dGF1yf NULL 3426.0 -2.2036032E7 -1864027286 1 1864027286 0.0 3426.0 -3426.0 3426.0 2.2036032E7 1864021647.85 1864027286 -1.554726368159204E-4 3426.0 3426.0 2.2039458E7 0.0
-1864027286 true dIw0j NULL 9774.0 -6.2866368E7 -1864027286 1 1864027286 0.0 9774.0 -9774.0 9774.0 6.2866368E7 1864021647.85 1864027286 -1.554726368159204E-4 9774.0 9774.0 6.2876142E7 0.0
-1864027286 true dPkN74F7 NULL 8373.0 -5.3855136E7 -1864027286 1 1864027286 0.0 8373.0 -8373.0 8373.0 5.3855136E7 1864021647.85 1864027286 -1.554726368159204E-4 8373.0 8373.0 5.3863509E7 0.0
-1864027286 true dQsIgL NULL 2624.0 -1.6877568E7 -1864027286 1 1864027286 0.0 2624.0 -2624.0 2624.0 1.6877568E7 1864021647.85 1864027286 -1.554726368159204E-4 2624.0 2624.0 1.6880192E7 0.0
-1864027286 true dV86D7yr0I62C NULL -13617.0 8.7584544E7 -1864027286 1 1864027286 0.0 -13617.0 13617.0 -13617.0 -8.7584544E7 1864021647.85 1864027286 -1.554726368159204E-4 -13617.0 -13617.0 -8.7598161E7 0.0
-1864027286 true dqSh2nXp NULL 15296.0 -9.8383872E7 -1864027286 1 1864027286 0.0 15296.0 -15296.0 15296.0 9.8383872E7 1864021647.85 1864027286 -1.554726368159204E-4 15296.0 15296.0 9.8399168E7 0.0
-1864027286 true e2tRWV1I2oE NULL -12310.0 7.917792E7 -1864027286 1 1864027286 0.0 -12310.0 12310.0 -12310.0 -7.917792E7 1864021647.85 1864027286 -1.554726368159204E-4 -12310.0 -12310.0 -7.919023E7 0.0
-1864027286 true e4rLBwDgWm1S4fl264fmpC NULL 9962.0 -6.4075584E7 -1864027286 1 1864027286 0.0 9962.0 -9962.0 9962.0 6.4075584E7 1864021647.85 1864027286 -1.554726368159204E-4 9962.0 9962.0 6.4085546E7 0.0
-1864027286 true e6SAAy5o0so6LM30k NULL -548.0 3524736.0 -1864027286 1 1864027286 0.0 -548.0 548.0 -548.0 -3524736.0 1864021647.85 1864027286 -1.554726368159204E-4 -548.0 -548.0 -3525284.0 0.0
-1864027286 true eHxtaCo643hV3BIi2Le35Eq NULL 9814.0 -6.3123648E7 -1864027286 1 1864027286 0.0 9814.0 -9814.0 9814.0 6.3123648E7 1864021647.85 1864027286 -1.554726368159204E-4 9814.0 9814.0 6.3133462E7 0.0
-1864027286 true eWq33N3Xk6 NULL -11596.0 7.4585472E7 -1864027286 1 1864027286 0.0 -11596.0 11596.0 -11596.0 -7.4585472E7 1864021647.85 1864027286 -1.554726368159204E-4 -11596.0 -11596.0 -7.4597068E7 0.0
-1864027286 true eeLpfP6O NULL -828.0 5325696.0 -1864027286 1 1864027286 0.0 -828.0 828.0 -828.0 -5325696.0 1864021647.85 1864027286 -1.554726368159204E-4 -828.0 -828.0 -5326524.0 0.0
-1864027286 true f12qhlvH NULL -3544.0 2.2795008E7 -1864027286 1 1864027286 0.0 -3544.0 3544.0 -3544.0 -2.2795008E7 1864021647.85 1864027286 -1.554726368159204E-4 -3544.0 -3544.0 -2.2798552E7 0.0
-1864027286 true f1b7368iTH NULL 11837.0 -7.6135584E7 -1864027286 1 1864027286 0.0 11837.0 -11837.0 11837.0 7.6135584E7 1864021647.85 1864027286 -1.554726368159204E-4 11837.0 11837.0 7.6147421E7 0.0
-1864027286 true f6B6I2d7180wveu1BG63b NULL 4178.0 -2.6872896E7 -1864027286 1 1864027286 0.0 4178.0 -4178.0 4178.0 2.6872896E7 1864021647.85 1864027286 -1.554726368159204E-4 4178.0 4178.0 2.6877074E7 0.0
-1864027286 true f8e16sE7qHnJFq8IjXe6uSE NULL -9408.0 6.0512256E7 -1864027286 1 1864027286 0.0 -9408.0 9408.0 -9408.0 -6.0512256E7 1864021647.85 1864027286 -1.554726368159204E-4 -9408.0 -9408.0 -6.0521664E7 0.0
-1864027286 true fJWe8p2jkqws5d04a5lSvLH NULL -14942.0 9.6106944E7 -1864027286 1 1864027286 0.0 -14942.0 14942.0 -14942.0 -9.6106944E7 1864021647.85 1864027286 -1.554726368159204E-4 -14942.0 -14942.0 -9.6121886E7 0.0
-1864027286 true far4S170PC NULL 13691.0 -8.8060512E7 -1864027286 1 1864027286 0.0 13691.0 -13691.0 13691.0 8.8060512E7 1864021647.85 1864027286 -1.554726368159204E-4 13691.0 13691.0 8.8074203E7 0.0
-1864027286 true g0C6gENIKCKayurchl7pjs2 NULL 12201.0 -7.8476832E7 -1864027286 1 1864027286 0.0 12201.0 -12201.0 12201.0 7.8476832E7 1864021647.85 1864027286 -1.554726368159204E-4 12201.0 12201.0 7.8489033E7 0.0
-1864027286 true gLGK7D0V NULL 11865.0 -7.631568E7 -1864027286 1 1864027286 0.0 11865.0 -11865.0 11865.0 7.631568E7 1864021647.85 1864027286 -1.554726368159204E-4 11865.0 11865.0 7.6327545E7 0.0
-1864027286 true gls8SspE NULL 231.0 -1485792.0 -1864027286 1 1864027286 0.0 231.0 -231.0 231.0 1485792.0 1864021647.85 1864027286 -1.554726368159204E-4 231.0 231.0 1486023.0 0.0
-1864027286 true gppEomS0ce2G6k6 NULL 4577.0 -2.9439264E7 -1864027286 1 1864027286 0.0 4577.0 -4577.0 4577.0 2.9439264E7 1864021647.85 1864027286 -1.554726368159204E-4 4577.0 4577.0 2.9443841E7 0.0
-1864027286 true hA4lNb NULL 8634.0 -5.5533888E7 -1864027286 1 1864027286 0.0 8634.0 -8634.0 8634.0 5.5533888E7 1864021647.85 1864027286 -1.554726368159204E-4 8634.0 8634.0 5.5542522E7 0.0
-1864027286 true iDlPQmQC7RSxNA NULL -16004.0 1.02937728E8 -1864027286 1 1864027286 0.0 -16004.0 16004.0 -16004.0 -1.02937728E8 1864021647.85 1864027286 -1.554726368159204E-4 -16004.0 -16004.0 -1.02953732E8 0.0
-1864027286 true iF1fQ7gn0qgpH7HKS5N3 NULL -4561.0 2.9336352E7 -1864027286 1 1864027286 0.0 -4561.0 4561.0 -4561.0 -2.9336352E7 1864021647.85 1864027286 -1.554726368159204E-4 -4561.0 -4561.0 -2.9340913E7 0.0
-1864027286 true iG1K1q1 NULL -8530.0 5.486496E7 -1864027286 1 1864027286 0.0 -8530.0 8530.0 -8530.0 -5.486496E7 1864021647.85 1864027286 -1.554726368159204E-4 -8530.0 -8530.0 -5.487349E7 0.0
-1864027286 true iP2ABL NULL -8162.0 5.2497984E7 -1864027286 1 1864027286 0.0 -8162.0 8162.0 -8162.0 -5.2497984E7 1864021647.85 1864027286 -1.554726368159204E-4 -8162.0 -8162.0 -5.2506146E7 0.0
-1864027286 true iUAMMN23Vq5jREr832nxXn NULL 4149.0 -2.6686368E7 -1864027286 1 1864027286 0.0 4149.0 -4149.0 4149.0 2.6686368E7 1864021647.85 1864027286 -1.554726368159204E-4 4149.0 4149.0 2.6690517E7 0.0
-1864027286 true ihlorJE62ik1WuKfS NULL -8390.0 5.396448E7 -1864027286 1 1864027286 0.0 -8390.0 8390.0 -8390.0 -5.396448E7 1864021647.85 1864027286 -1.554726368159204E-4 -8390.0 -8390.0 -5.397287E7 0.0
-1864027286 true ii6d0V0 NULL 12732.0 -8.1892224E7 -1864027286 1 1864027286 0.0 12732.0 -12732.0 12732.0 8.1892224E7 1864021647.85 1864027286 -1.554726368159204E-4 12732.0 12732.0 8.1904956E7 0.0
-1864027286 true iuSQEi3rpt2ctxK08ut3 NULL -12574.0 8.0875968E7 -1864027286 1 1864027286 0.0 -12574.0 12574.0 -12574.0 -8.0875968E7 1864021647.85 1864027286 -1.554726368159204E-4 -12574.0 -12574.0 -8.0888542E7 0.0
-1864027286 true j8fJ4l2w4F8fI51 NULL -7691.0 4.9468512E7 -1864027286 1 1864027286 0.0 -7691.0 7691.0 -7691.0 -4.9468512E7 1864021647.85 1864027286 -1.554726368159204E-4 -7691.0 -7691.0 -4.9476203E7 0.0
-1864027286 true jLX0SrR6OP NULL -12264.0 7.8882048E7 -1864027286 1 1864027286 0.0 -12264.0 12264.0 -12264.0 -7.8882048E7 1864021647.85 1864027286 -1.554726368159204E-4 -12264.0 -12264.0 -7.8894312E7 0.0
-1864027286 true jSUVVR NULL -7375.0 4.7436E7 -1864027286 1 1864027286 0.0 -7375.0 7375.0 -7375.0 -4.7436E7 1864021647.85 1864027286 -1.554726368159204E-4 -7375.0 -7375.0 -4.7443375E7 0.0
-1864027286 true jc3G2mefLm8mpl8tua3b3 NULL 236.0 -1517952.0 -1864027286 1 1864027286 0.0 236.0 -236.0 236.0 1517952.0 1864021647.85 1864027286 -1.554726368159204E-4 236.0 236.0 1518188.0 0.0
-1864027286 true jcS1NU2R06MX2 NULL 14177.0 -9.1186464E7 -1864027286 1 1864027286 0.0 14177.0 -14177.0 14177.0 9.1186464E7 1864021647.85 1864027286 -1.554726368159204E-4 14177.0 14177.0 9.1200641E7 0.0
-1864027286 true jjc503pMQskjqb8T3tCL0 NULL -12883.0 8.2863456E7 -1864027286 1 1864027286 0.0 -12883.0 12883.0 -12883.0 -8.2863456E7 1864021647.85 1864027286 -1.554726368159204E-4 -12883.0 -12883.0 -8.2876339E7 0.0
-1864027286 true k1VX0eFh56x3ErERaS2y55B NULL 14909.0 -9.5894688E7 -1864027286 1 1864027286 0.0 14909.0 -14909.0 14909.0 9.5894688E7 1864021647.85 1864027286 -1.554726368159204E-4 14909.0 14909.0 9.5909597E7 0.0
-1864027286 true k7RL0DH3Dj4218Jd NULL 14863.0 -9.5598816E7 -1864027286 1 1864027286 0.0 14863.0 -14863.0 14863.0 9.5598816E7 1864021647.85 1864027286 -1.554726368159204E-4 14863.0 14863.0 9.5613679E7 0.0
-1864027286 true k8184H NULL 6645.0 -4.274064E7 -1864027286 1 1864027286 0.0 6645.0 -6645.0 6645.0 4.274064E7 1864021647.85 1864027286 -1.554726368159204E-4 6645.0 6645.0 4.2747285E7 0.0
-1864027286 true kPpivtTi0S43BIo NULL 6581.0 -4.2328992E7 -1864027286 1 1864027286 0.0 6581.0 -6581.0 6581.0 4.2328992E7 1864021647.85 1864027286 -1.554726368159204E-4 6581.0 6581.0 4.2335573E7 0.0
-1864027286 true kRa26RQDv3Sk NULL -13118.0 8.4374976E7 -1864027286 1 1864027286 0.0 -13118.0 13118.0 -13118.0 -8.4374976E7 1864021647.85 1864027286 -1.554726368159204E-4 -13118.0 -13118.0 -8.4388094E7 0.0
-1864027286 true kcA1Sw5 NULL 6182.0 -3.9762624E7 -1864027286 1 1864027286 0.0 6182.0 -6182.0 6182.0 3.9762624E7 1864021647.85 1864027286 -1.554726368159204E-4 6182.0 6182.0 3.9768806E7 0.0
-1864027286 true kwgr1l8iVOT NULL -6410.0 4.122912E7 -1864027286 1 1864027286 0.0 -6410.0 6410.0 -6410.0 -4.122912E7 1864021647.85 1864027286 -1.554726368159204E-4 -6410.0 -6410.0 -4.123553E7 0.0
-1864027286 true l20qY NULL 8919.0 -5.7367008E7 -1864027286 1 1864027286 0.0 8919.0 -8919.0 8919.0 5.7367008E7 1864021647.85 1864027286 -1.554726368159204E-4 8919.0 8919.0 5.7375927E7 0.0
-1864027286 true l3j1vwt6TY65u7m NULL 11499.0 -7.3961568E7 -1864027286 1 1864027286 0.0 11499.0 -11499.0 11499.0 7.3961568E7 1864021647.85 1864027286 -1.554726368159204E-4 11499.0 11499.0 7.3973067E7 0.0
-1864027286 true l4iq01SNoFl7kABN NULL 15311.0 -9.8480352E7 -1864027286 1 1864027286 0.0 15311.0 -15311.0 15311.0 9.8480352E7 1864021647.85 1864027286 -1.554726368159204E-4 15311.0 15311.0 9.8495663E7 0.0
-1864027286 true lEXXcvYRGqGd31V5R7paYE5 NULL 1225.0 -7879200.0 -1864027286 1 1864027286 0.0 1225.0 -1225.0 1225.0 7879200.0 1864021647.85 1864027286 -1.554726368159204E-4 1225.0 1225.0 7880425.0 0.0
-1864027286 true lP7HUebhIc6T NULL 8196.0 -5.2716672E7 -1864027286 1 1864027286 0.0 8196.0 -8196.0 8196.0 5.2716672E7 1864021647.85 1864027286 -1.554726368159204E-4 8196.0 8196.0 5.2724868E7 0.0
-1864027286 true lVXCI385cbcEk NULL -607.0 3904224.0 -1864027286 1 1864027286 0.0 -607.0 607.0 -607.0 -3904224.0 1864021647.85 1864027286 -1.554726368159204E-4 -607.0 -607.0 -3904831.0 0.0
-1864027286 true lm60Wii25 NULL 9304.0 -5.9843328E7 -1864027286 1 1864027286 0.0 9304.0 -9304.0 9304.0 5.9843328E7 1864021647.85 1864027286 -1.554726368159204E-4 9304.0 9304.0 5.9852632E7 0.0
-1864027286 true lxQp116 NULL -5638.15 3.62645808E7 -1864027286 1 1864027286 0.0 -5638.15 5638.15 -5638.15 -3.62645808E7 1864021647.85 1864027286 -1.554726368159204E-4 -5638.15 -5638.15 -3.6270218949999996E7 0.0
-1864027286 true m2482tQ NULL 4049.0 -2.6043168E7 -1864027286 1 1864027286 0.0 4049.0 -4049.0 4049.0 2.6043168E7 1864021647.85 1864027286
<TRUNCATED>
[16/50] [abbrv] hive git commit: HIVE-17276: Check max shuffle size
when converting to dynamically partitioned hash join (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by we...@apache.org.
HIVE-17276: Check max shuffle size when converting to dynamically partitioned hash join (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b67d52c2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b67d52c2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b67d52c2
Branch: refs/heads/hive-14535
Commit: b67d52c294c1c2db21ed89c86486c946aa5d3ca4
Parents: dd04a92
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Tue Aug 29 10:36:29 2017 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Wed Aug 30 08:29:33 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 7 +-
.../hive/ql/optimizer/ConvertJoinMapJoin.java | 55 +++++--
.../queries/clientpositive/join_max_hashtable.q | 13 ++
.../llap/join_max_hashtable.q.out | 154 +++++++++++++++++++
4 files changed, 217 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b67d52c2/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 0d8d7ae..e4b09a2 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1435,9 +1435,14 @@ public class HiveConf extends Configuration {
HIVECONVERTJOINMAXENTRIESHASHTABLE("hive.auto.convert.join.hashtable.max.entries", 40000000L,
"If hive.auto.convert.join.noconditionaltask is off, this parameter does not take affect. \n" +
- "However, if it is on, and the predicated number of entries in hashtable for a given join \n" +
+ "However, if it is on, and the predicted number of entries in hashtable for a given join \n" +
"input is larger than this number, the join will not be converted to a mapjoin. \n" +
"The value \"-1\" means no limit."),
+ HIVECONVERTJOINMAXSHUFFLESIZE("hive.auto.convert.join.shuffle.max.size", 10000000L,
+ "If hive.auto.convert.join.noconditionaltask is off, this parameter does not take affect. \n" +
+ "However, if it is on, and the predicted size of the larger input for a given join is greater \n" +
+ "than this number, the join will not be converted to a dynamically partitioned hash join. \n" +
+ "The value \"-1\" means no limit."),
HIVEHASHTABLEKEYCOUNTADJUSTMENT("hive.hashtable.key.count.adjustment", 1.0f,
"Adjustment to mapjoin hashtable size derived from table and column statistics; the estimate" +
" of the number of keys is divided by this value. If the value is 0, statistics are not used" +
http://git-wip-us.apache.org/repos/asf/hive/blob/b67d52c2/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
index 21d0053..a2414f3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
@@ -579,12 +579,12 @@ public class ConvertJoinMapJoin implements NodeProcessor {
* for Dynamic Hash Join conversion consideration
* @param skipJoinTypeChecks whether to skip join type checking
* @param maxSize size threshold for Map Join conversion
- * @param checkHashTableEntries whether to check threshold for distinct keys in hash table for Map Join
+ * @param checkMapJoinThresholds whether to check thresholds to convert to Map Join
* @return returns big table position or -1 if it cannot be determined
* @throws SemanticException
*/
public int getMapJoinConversionPos(JoinOperator joinOp, OptimizeTezProcContext context,
- int buckets, boolean skipJoinTypeChecks, long maxSize, boolean checkHashTableEntries)
+ int buckets, boolean skipJoinTypeChecks, long maxSize, boolean checkMapJoinThresholds)
throws SemanticException {
if (!skipJoinTypeChecks) {
/*
@@ -634,6 +634,9 @@ public class ConvertJoinMapJoin implements NodeProcessor {
// total size of the inputs
long totalSize = 0;
+ // convert to DPHJ
+ boolean convertDPHJ = false;
+
for (int pos = 0; pos < joinOp.getParentOperators().size(); pos++) {
Operator<? extends OperatorDesc> parentOp = joinOp.getParentOperators().get(pos);
@@ -693,19 +696,19 @@ public class ConvertJoinMapJoin implements NodeProcessor {
// We are replacing the current big table with a new one, thus
// we need to count the current one as a map table then.
totalSize += bigInputStat.getDataSize();
- // Check if number of distinct keys is larger than given max
- // number of entries for HashMap. If it is, we do not convert.
- if (checkHashTableEntries && !checkNumberOfEntriesForHashTable(joinOp, bigTablePosition, context)) {
- return -1;
+ // Check if number of distinct keys is greater than given max number of entries
+ // for HashMap
+ if (checkMapJoinThresholds && !checkNumberOfEntriesForHashTable(joinOp, bigTablePosition, context)) {
+ convertDPHJ = true;
}
} else if (!selectedBigTable) {
// This is not the first table and we are not using it as big table,
// in fact, we're adding this table as a map table
totalSize += inputSize;
- // Check if number of distinct keys is larger than given max
- // number of entries for HashMap. If it is, we do not convert.
- if (checkHashTableEntries && !checkNumberOfEntriesForHashTable(joinOp, pos, context)) {
- return -1;
+ // Check if number of distinct keys is greater than given max number of entries
+ // for HashMap
+ if (checkMapJoinThresholds && !checkNumberOfEntriesForHashTable(joinOp, pos, context)) {
+ convertDPHJ = true;
}
}
@@ -723,6 +726,13 @@ public class ConvertJoinMapJoin implements NodeProcessor {
}
+ // Check if size of data to shuffle (larger table) is less than given max size
+ if (checkMapJoinThresholds && convertDPHJ
+ && checkShuffleSizeForLargeTable(joinOp, bigTablePosition, context)) {
+ LOG.debug("Conditions to convert to MapJoin are not met");
+ return -1;
+ }
+
// We store the total memory that this MapJoin is going to use,
// which is calculated as totalSize/buckets, with totalSize
// equal to sum of small tables size.
@@ -1087,13 +1097,36 @@ public class ConvertJoinMapJoin implements NodeProcessor {
if (estimation > max) {
// Estimation larger than max
LOG.debug("Number of different entries for HashTable is greater than the max; "
- + "we do not converting to MapJoin");
+ + "we do not convert to MapJoin");
return false;
}
// We can proceed with the conversion
return true;
}
+ /* Returns true if it passes the test, false otherwise. */
+ private boolean checkShuffleSizeForLargeTable(JoinOperator joinOp, int position,
+ OptimizeTezProcContext context) {
+ long max = HiveConf.getLongVar(context.parseContext.getConf(),
+ HiveConf.ConfVars.HIVECONVERTJOINMAXSHUFFLESIZE);
+ if (max < 1) {
+ // Max is disabled, we can safely return true
+ return true;
+ }
+ // Evaluate
+ ReduceSinkOperator rsOp = (ReduceSinkOperator) joinOp.getParentOperators().get(position);
+ Statistics inputStats = rsOp.getStatistics();
+ long inputSize = inputStats.getDataSize();
+ LOG.debug("Estimated size for input {}: {}; Max size for DPHJ conversion: {}",
+ position, inputSize, max);
+ if (inputSize > max) {
+ LOG.debug("Size of input is greater than the max; "
+ + "we do not convert to DPHJ");
+ return false;
+ }
+ return true;
+ }
+
private static long estimateNDV(long numRows, List<ColStatistics> columnStats) {
// If there is a single column, return the number of distinct values
if (columnStats.size() == 1) {
http://git-wip-us.apache.org/repos/asf/hive/blob/b67d52c2/ql/src/test/queries/clientpositive/join_max_hashtable.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/join_max_hashtable.q b/ql/src/test/queries/clientpositive/join_max_hashtable.q
index 9c30a0d..8d0ccb7 100644
--- a/ql/src/test/queries/clientpositive/join_max_hashtable.q
+++ b/ql/src/test/queries/clientpositive/join_max_hashtable.q
@@ -1,6 +1,7 @@
set hive.auto.convert.join=true;
set hive.optimize.dynamic.partition.hashjoin=true;
set hive.auto.convert.join.hashtable.max.entries=500;
+set hive.auto.convert.join.shuffle.max.size=100000;
-- CONVERT
EXPLAIN
@@ -35,3 +36,15 @@ FROM src x JOIN src y ON (x.key = y.key);
EXPLAIN
SELECT x.key, x.value
FROM src x JOIN src y ON (x.key = y.key AND x.value = y.value);
+
+set hive.auto.convert.join.shuffle.max.size=80000;
+
+-- CONVERT
+EXPLAIN
+SELECT x.key, x.value
+FROM src x JOIN src y ON (x.key = y.key);
+
+-- CONVERT
+EXPLAIN
+SELECT x.key, x.value
+FROM src x JOIN src y ON (x.key = y.key AND x.value = y.value);
http://git-wip-us.apache.org/repos/asf/hive/blob/b67d52c2/ql/src/test/results/clientpositive/llap/join_max_hashtable.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/join_max_hashtable.q.out b/ql/src/test/results/clientpositive/llap/join_max_hashtable.q.out
index 63f5d28..ef1a6f3 100644
--- a/ql/src/test/results/clientpositive/llap/join_max_hashtable.q.out
+++ b/ql/src/test/results/clientpositive/llap/join_max_hashtable.q.out
@@ -498,3 +498,157 @@ STAGE PLANS:
Processor Tree:
ListSink
+PREHOOK: query: EXPLAIN
+SELECT x.key, x.value
+FROM src x JOIN src y ON (x.key = y.key)
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT x.key, x.value
+FROM src x JOIN src y ON (x.key = y.key)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 1 <- Map 2 (BROADCAST_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: x
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1
+ input vertices:
+ 1 Map 2
+ Statistics: Num rows: 809 Data size: 144002 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 809 Data size: 144002 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: llap
+ LLAP IO: no inputs
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: y
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Execution mode: llap
+ LLAP IO: no inputs
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: EXPLAIN
+SELECT x.key, x.value
+FROM src x JOIN src y ON (x.key = y.key AND x.value = y.value)
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT x.key, x.value
+FROM src x JOIN src y ON (x.key = y.key AND x.value = y.value)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 1 <- Map 2 (BROADCAST_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: x
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: (key is not null and value is not null) (type: boolean)
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string), _col1 (type: string)
+ 1 _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1
+ input vertices:
+ 1 Map 2
+ Statistics: Num rows: 809 Data size: 144002 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 809 Data size: 144002 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: llap
+ LLAP IO: no inputs
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: y
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: (key is not null and value is not null) (type: boolean)
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Execution mode: llap
+ LLAP IO: no inputs
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
[08/50] [abbrv] hive git commit: HIVE-17205 - add functional support
for unbucketed tables (Eugene Koifman, reviewed by Wei Zheng)
Posted by we...@apache.org.
HIVE-17205 - add functional support for unbucketed tables (Eugene Koifman, reviewed by Wei Zheng)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6be50b76
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6be50b76
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6be50b76
Branch: refs/heads/hive-14535
Commit: 6be50b76be5956b3c52ed6024fd7b4a3dee65fb6
Parents: 262d8f9
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Fri Aug 25 20:14:57 2017 -0700
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Fri Aug 25 20:15:26 2017 -0700
----------------------------------------------------------------------
.../streaming/AbstractRecordWriter.java | 44 +-
.../hive/hcatalog/streaming/HiveEndPoint.java | 4 +-
.../apache/hive/hcatalog/streaming/package.html | 21 +-
.../hive/hcatalog/streaming/TestStreaming.java | 86 +-
.../hive/metastore/TestHiveMetaStore.java | 8 +-
.../apache/hadoop/hive/ql/TestAcidOnTez.java | 367 +++-
.../hive/ql/TestAcidOnTezWithSplitUpdate.java | 28 -
.../test/resources/testconfiguration.properties | 4 +-
.../TransactionalValidationListener.java | 42 +-
.../org/apache/hadoop/hive/ql/ErrorMsg.java | 4 +-
.../hadoop/hive/ql/exec/FileSinkOperator.java | 22 +-
.../hadoop/hive/ql/io/orc/OrcInputFormat.java | 60 +-
.../hive/ql/io/orc/OrcRawRecordMerger.java | 225 +-
.../hadoop/hive/ql/io/orc/OrcRecordUpdater.java | 14 +-
.../apache/hadoop/hive/ql/io/orc/OrcSplit.java | 10 +-
.../io/orc/VectorizedOrcAcidRowBatchReader.java | 26 +-
.../optimizer/SortedDynPartitionOptimizer.java | 1 +
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 6 -
.../hive/ql/txn/compactor/CompactorMR.java | 3 +-
.../apache/hadoop/hive/ql/TestTxnCommands.java | 108 +-
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 12 +-
.../hadoop/hive/ql/TestTxnCommandsBase.java | 162 ++
.../apache/hadoop/hive/ql/TestTxnNoBuckets.java | 297 +++
.../hive/ql/io/orc/TestInputOutputFormat.java | 28 +-
.../hive/ql/io/orc/TestOrcRawRecordMerger.java | 86 +-
.../queries/clientnegative/create_not_acid.q | 2 +-
.../queries/clientpositive/acid_no_buckets.q | 210 ++
.../clientnegative/create_not_acid.q.out | 4 +-
.../clientnegative/delete_non_acid_table.q.out | 2 +-
.../clientnegative/update_non_acid_table.q.out | 2 +-
.../clientpositive/llap/acid_no_buckets.q.out | 1976 ++++++++++++++++++
31 files changed, 3534 insertions(+), 330 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/AbstractRecordWriter.java
----------------------------------------------------------------------
diff --git a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/AbstractRecordWriter.java b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/AbstractRecordWriter.java
index e409e75..4ec10ad 100644
--- a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/AbstractRecordWriter.java
+++ b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/AbstractRecordWriter.java
@@ -46,6 +46,7 @@ import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Properties;
@@ -54,19 +55,23 @@ import java.util.Properties;
public abstract class AbstractRecordWriter implements RecordWriter {
static final private Logger LOG = LoggerFactory.getLogger(AbstractRecordWriter.class.getName());
- final HiveConf conf;
- final HiveEndPoint endPoint;
+ private final HiveConf conf;
+ private final HiveEndPoint endPoint;
final Table tbl;
- final IMetaStoreClient msClient;
- protected final List<Integer> bucketIds;
- ArrayList<RecordUpdater> updaters = null;
+ private final IMetaStoreClient msClient;
+ final List<Integer> bucketIds;
+ private ArrayList<RecordUpdater> updaters = null;
- public final int totalBuckets;
+ private final int totalBuckets;
+ /**
+ * Indicates whether target table is bucketed
+ */
+ private final boolean isBucketed;
private final Path partitionPath;
- final AcidOutputFormat<?,?> outf;
+ private final AcidOutputFormat<?,?> outf;
private Object[] bucketFieldData; // Pre-allocated in constructor. Updated on each write.
private Long curBatchMinTxnId;
private Long curBatchMaxTxnId;
@@ -109,16 +114,22 @@ public abstract class AbstractRecordWriter implements RecordWriter {
this.tbl = twp.tbl;
this.partitionPath = twp.partitionPath;
}
- this.totalBuckets = tbl.getSd().getNumBuckets();
- if (totalBuckets <= 0) {
- throw new StreamingException("Cannot stream to table that has not been bucketed : "
- + endPoint);
+ this.isBucketed = tbl.getSd().getNumBuckets() > 0;
+ /**
+ * For unbucketed tables we have exactly 1 RecrodUpdater for each AbstractRecordWriter which
+ * ends up writing to a file bucket_000000
+ * See also {@link #getBucket(Object)}
+ */
+ this.totalBuckets = isBucketed ? tbl.getSd().getNumBuckets() : 1;
+ if(isBucketed) {
+ this.bucketIds = getBucketColIDs(tbl.getSd().getBucketCols(), tbl.getSd().getCols());
+ this.bucketFieldData = new Object[bucketIds.size()];
+ }
+ else {
+ bucketIds = Collections.emptyList();
}
- this.bucketIds = getBucketColIDs(tbl.getSd().getBucketCols(), tbl.getSd().getCols());
- this.bucketFieldData = new Object[bucketIds.size()];
String outFormatName = this.tbl.getSd().getOutputFormat();
outf = (AcidOutputFormat<?, ?>) ReflectionUtils.newInstance(JavaUtils.loadClass(outFormatName), conf);
- bucketFieldData = new Object[bucketIds.size()];
} catch(InterruptedException e) {
throw new StreamingException(endPoint2.toString(), e);
} catch (MetaException | NoSuchObjectException e) {
@@ -169,6 +180,9 @@ public abstract class AbstractRecordWriter implements RecordWriter {
// returns the bucket number to which the record belongs to
protected int getBucket(Object row) throws SerializationError {
+ if(!isBucketed) {
+ return 0;
+ }
ObjectInspector[] inspectors = getBucketObjectInspectors();
Object[] bucketFields = getBucketFields(row);
return ObjectInspectorUtils.getBucketNumber(bucketFields, inspectors, totalBuckets);
@@ -204,7 +218,7 @@ public abstract class AbstractRecordWriter implements RecordWriter {
curBatchMaxTxnId = maxTxnID;
updaters = new ArrayList<RecordUpdater>(totalBuckets);
for (int bucket = 0; bucket < totalBuckets; bucket++) {
- updaters.add(bucket, null);
+ updaters.add(bucket, null);//so that get(i) returns null rather than ArrayOutOfBounds
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
----------------------------------------------------------------------
diff --git a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
index 81f6155..28c98bd 100644
--- a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
+++ b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
@@ -20,6 +20,8 @@ package org.apache.hive.hcatalog.streaming;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hive.metastore.api.DataOperationType;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.cli.CliSessionState;
@@ -338,7 +340,7 @@ public class HiveEndPoint {
// 1 - check if TBLPROPERTIES ('transactional'='true') is set on table
Map<String, String> params = t.getParameters();
if (params != null) {
- String transactionalProp = params.get("transactional");
+ String transactionalProp = params.get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
if (transactionalProp == null || !transactionalProp.equalsIgnoreCase("true")) {
LOG.error("'transactional' property is not set on Table " + endPoint);
throw new InvalidTable(endPoint.database, endPoint.table, "\'transactional\' property" +
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/package.html
----------------------------------------------------------------------
diff --git a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/package.html b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/package.html
index ed4d307..a879b97 100644
--- a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/package.html
+++ b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/package.html
@@ -30,7 +30,7 @@ partition. Once data is committed it becomes immediately visible to
all Hive queries initiated subsequently.</p>
<p>
-This API is intended for streaming clients such as Flume and Storm,
+This API is intended for streaming clients such as NiFi, Flume and Storm,
which continuously generate data. Streaming support is built on top of
ACID based insert/update support in Hive.</p>
@@ -56,10 +56,7 @@ A few things are currently required to use streaming.
<ol>
<li> Currently, only ORC storage format is supported. So
'<b>stored as orc</b>' must be specified during table creation.</li>
- <li> The hive table must be bucketed, but not sorted. So something like
- '<b>clustered by (<i>colName</i>) into <i>10</i> buckets</b>' must
- be specified during table creation. The number of buckets
- is ideally the same as the number of streaming writers.</li>
+ <li> The hive table may be bucketed but must not be sorted. </li>
<li> User of the client streaming process must have the necessary
permissions to write to the table or partition and create partitions in
the table.</li>
@@ -67,7 +64,6 @@ A few things are currently required to use streaming.
<ol>
<li><b>hive.input.format =
org.apache.hadoop.hive.ql.io.HiveInputFormat</b></li>
- <li><b>hive.vectorized.execution.enabled = false</b></li>
</ol></li>
The above client settings are a temporary requirement and the intention is to
drop the need for them in the near future.
@@ -165,8 +161,21 @@ additional implementations of the <b>RecordWriter</b> interface.
- Delimited text input.</li>
<li> <a href="StrictJsonWriter.html"><b>StrictJsonWriter</b></a>
- JSON text input.</li>
+ <li> <a href="StrictRegexWriter.html"><b>StrictRegexWriter</b></a>
+ - text input with regex.</li>
</ul></p>
+<h2>Performance, Concurrency, Etc.</h2>
+<p>
+ Each StreamingConnection is writing data at the rate the underlying
+ FileSystem can accept it. If that is not sufficient, multiple StreamingConnection objects can
+ be created concurrently.
+</p>
+<p>
+ Each StreamingConnection can have at most 1 outstanding TransactionBatch and each TransactionBatch
+ may have at most 2 threads operaing on it.
+ See <a href="TransactionBatch.html"><b>TransactionBatch</b></a>
+</p>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
----------------------------------------------------------------------
diff --git a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
index f3ef92b..49520ef 100644
--- a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
+++ b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
@@ -33,6 +33,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -52,6 +53,7 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse;
import org.apache.hadoop.hive.metastore.api.LockState;
import org.apache.hadoop.hive.metastore.api.LockType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.ShowLocksRequest;
@@ -65,6 +67,7 @@ import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.io.AcidUtils;
+import org.apache.hadoop.hive.ql.io.BucketCodec;
import org.apache.hadoop.hive.ql.io.IOConstants;
import org.apache.hadoop.hive.ql.io.orc.OrcFile;
import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
@@ -74,6 +77,7 @@ import org.apache.hadoop.hive.ql.io.orc.RecordReader;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.txn.AcidHouseKeeperService;
+import org.apache.hadoop.hive.ql.txn.compactor.Worker;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
@@ -193,7 +197,6 @@ public class TestStreaming {
conf = new HiveConf(this.getClass());
conf.set("fs.raw.impl", RawFileSystem.class.getName());
- conf.set("hive.enforce.bucketing", "true");
conf
.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
"org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory");
@@ -339,6 +342,83 @@ public class TestStreaming {
}
}
+ /**
+ * Test that streaming can write to unbucketed table.
+ */
+ @Test
+ public void testNoBuckets() throws Exception {
+ queryTable(driver, "drop table if exists default.streamingnobuckets");
+ //todo: why does it need transactional_properties?
+ queryTable(driver, "create table default.streamingnobuckets (a string, b string) stored as orc TBLPROPERTIES('transactional'='true', 'transactional_properties'='default')");
+ queryTable(driver, "insert into default.streamingnobuckets values('foo','bar')");
+ List<String> rs = queryTable(driver, "select * from default.streamingnobuckets");
+ Assert.assertEquals(1, rs.size());
+ Assert.assertEquals("foo\tbar", rs.get(0));
+ HiveEndPoint endPt = new HiveEndPoint(metaStoreURI, "default", "streamingnobuckets", null);
+ String[] colNames1 = new String[] { "a", "b" };
+ StreamingConnection connection = endPt.newConnection(false, "UT_" + Thread.currentThread().getName());
+ DelimitedInputWriter wr = new DelimitedInputWriter(colNames1,",", endPt, connection);
+
+ TransactionBatch txnBatch = connection.fetchTransactionBatch(2, wr);
+ txnBatch.beginNextTransaction();
+ txnBatch.write("a1,b2".getBytes());
+ txnBatch.write("a3,b4".getBytes());
+ txnBatch.commit();
+ txnBatch.beginNextTransaction();
+ txnBatch.write("a5,b6".getBytes());
+ txnBatch.write("a7,b8".getBytes());
+ txnBatch.commit();
+ txnBatch.close();
+
+ Assert.assertEquals("", 0, BucketCodec.determineVersion(536870912).decodeWriterId(536870912));
+ rs = queryTable(driver,"select ROW__ID, a, b, INPUT__FILE__NAME from default.streamingnobuckets order by ROW__ID");
+
+ Assert.assertTrue(rs.get(0), rs.get(0).startsWith("{\"transactionid\":16,\"bucketid\":536870912,\"rowid\":0}\tfoo\tbar"));
+ Assert.assertTrue(rs.get(0), rs.get(0).endsWith("streamingnobuckets/delta_0000016_0000016_0000/bucket_00000"));
+ Assert.assertTrue(rs.get(1), rs.get(1).startsWith("{\"transactionid\":18,\"bucketid\":536870912,\"rowid\":0}\ta1\tb2"));
+ Assert.assertTrue(rs.get(1), rs.get(1).endsWith("streamingnobuckets/delta_0000018_0000019/bucket_00000"));
+ Assert.assertTrue(rs.get(2), rs.get(2).startsWith("{\"transactionid\":18,\"bucketid\":536870912,\"rowid\":1}\ta3\tb4"));
+ Assert.assertTrue(rs.get(2), rs.get(2).endsWith("streamingnobuckets/delta_0000018_0000019/bucket_00000"));
+ Assert.assertTrue(rs.get(3), rs.get(3).startsWith("{\"transactionid\":19,\"bucketid\":536870912,\"rowid\":0}\ta5\tb6"));
+ Assert.assertTrue(rs.get(3), rs.get(3).endsWith("streamingnobuckets/delta_0000018_0000019/bucket_00000"));
+ Assert.assertTrue(rs.get(4), rs.get(4).startsWith("{\"transactionid\":19,\"bucketid\":536870912,\"rowid\":1}\ta7\tb8"));
+ Assert.assertTrue(rs.get(4), rs.get(4).endsWith("streamingnobuckets/delta_0000018_0000019/bucket_00000"));
+
+ queryTable(driver, "update default.streamingnobuckets set a=0, b=0 where a='a7'");
+ queryTable(driver, "delete from default.streamingnobuckets where a='a1'");
+ rs = queryTable(driver, "select a, b from default.streamingnobuckets order by a, b");
+ int row = 0;
+ Assert.assertEquals("at row=" + row, "0\t0", rs.get(row++));
+ Assert.assertEquals("at row=" + row, "a3\tb4", rs.get(row++));
+ Assert.assertEquals("at row=" + row, "a5\tb6", rs.get(row++));
+ Assert.assertEquals("at row=" + row, "foo\tbar", rs.get(row++));
+
+ queryTable(driver, "alter table default.streamingnobuckets compact 'major'");
+ runWorker(conf);
+ rs = queryTable(driver,"select ROW__ID, a, b, INPUT__FILE__NAME from default.streamingnobuckets order by ROW__ID");
+
+ Assert.assertTrue(rs.get(0), rs.get(0).startsWith("{\"transactionid\":16,\"bucketid\":536870912,\"rowid\":0}\tfoo\tbar"));
+ Assert.assertTrue(rs.get(0), rs.get(0).endsWith("streamingnobuckets/base_0000022/bucket_00000"));
+ Assert.assertTrue(rs.get(1), rs.get(1).startsWith("{\"transactionid\":18,\"bucketid\":536870912,\"rowid\":1}\ta3\tb4"));
+ Assert.assertTrue(rs.get(1), rs.get(1).endsWith("streamingnobuckets/base_0000022/bucket_00000"));
+ Assert.assertTrue(rs.get(2), rs.get(2).startsWith("{\"transactionid\":19,\"bucketid\":536870912,\"rowid\":0}\ta5\tb6"));
+ Assert.assertTrue(rs.get(2), rs.get(2).endsWith("streamingnobuckets/base_0000022/bucket_00000"));
+ Assert.assertTrue(rs.get(3), rs.get(3).startsWith("{\"transactionid\":21,\"bucketid\":536870912,\"rowid\":0}\t0\t0"));
+ Assert.assertTrue(rs.get(3), rs.get(3).endsWith("streamingnobuckets/base_0000022/bucket_00000"));
+ }
+
+ /**
+ * this is a clone from TestTxnStatement2....
+ */
+ public static void runWorker(HiveConf hiveConf) throws MetaException {
+ AtomicBoolean stop = new AtomicBoolean(true);
+ Worker t = new Worker();
+ t.setThreadId((int) t.getId());
+ t.setHiveConf(hiveConf);
+ AtomicBoolean looped = new AtomicBoolean();
+ t.init(stop, looped);
+ t.run();
+ }
// stream data into streaming table with N buckets, then copy the data into another bucketed table
// check if bucketing in both was done in the same way
@@ -453,8 +533,8 @@ public class TestStreaming {
}
/**
- * @deprecated use {@link #checkDataWritten2(Path, long, long, int, String, String...)} - there is
- * little value in using InputFormat directly
+ * @deprecated use {@link #checkDataWritten2(Path, long, long, int, String, boolean, String...)} -
+ * there is little value in using InputFormat directly
*/
private void checkDataWritten(Path partitionPath, long minTxn, long maxTxn, int buckets, int numExpectedFiles,
String... records) throws Exception {
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
index 8bd23cc..50e5274 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
@@ -3018,7 +3018,7 @@ public abstract class TestHiveMetaStore extends TestCase {
Table t = createTable(dbName, tblName, owner, params, null, sd, 0);
Assert.assertTrue("Expected exception", false);
} catch (MetaException e) {
- Assert.assertEquals("The table must be bucketed and stored using an ACID compliant format (such as ORC)", e.getMessage());
+ Assert.assertEquals("The table must be stored using an ACID compliant format (such as ORC)", e.getMessage());
}
// Fail - "transactional" is set to true, and the table is bucketed, but doesn't use ORC
@@ -3031,7 +3031,7 @@ public abstract class TestHiveMetaStore extends TestCase {
Table t = createTable(dbName, tblName, owner, params, null, sd, 0);
Assert.assertTrue("Expected exception", false);
} catch (MetaException e) {
- Assert.assertEquals("The table must be bucketed and stored using an ACID compliant format (such as ORC)", e.getMessage());
+ Assert.assertEquals("The table must be stored using an ACID compliant format (such as ORC)", e.getMessage());
}
// Succeed - "transactional" is set to true, and the table is bucketed, and uses ORC
@@ -3064,13 +3064,14 @@ public abstract class TestHiveMetaStore extends TestCase {
tblName += "1";
params.clear();
sd.unsetBucketCols();
+ sd.setInputFormat("org.apache.hadoop.mapred.FileInputFormat");
t = createTable(dbName, tblName, owner, params, null, sd, 0);
params.put("transactional", "true");
t.setParameters(params);
client.alter_table(dbName, tblName, t);
Assert.assertTrue("Expected exception", false);
} catch (MetaException e) {
- Assert.assertEquals("The table must be bucketed and stored using an ACID compliant format (such as ORC)", e.getMessage());
+ Assert.assertEquals("The table must be stored using an ACID compliant format (such as ORC)", e.getMessage());
}
// Succeed - trying to set "transactional" to "true", and satisfies bucketing and Input/OutputFormat requirement
@@ -3078,6 +3079,7 @@ public abstract class TestHiveMetaStore extends TestCase {
params.clear();
sd.setNumBuckets(1);
sd.setBucketCols(bucketCols);
+ sd.setInputFormat("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat");
t = createTable(dbName, tblName, owner, params, null, sd, 0);
params.put("transactional", "true");
t.setParameters(params);
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
index 2bf9871..d0b5cf6 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
@@ -22,7 +22,10 @@ import java.io.File;
import java.util.ArrayList;
import java.util.List;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.ShowCompactRequest;
@@ -30,23 +33,26 @@ import org.apache.hadoop.hive.metastore.api.ShowCompactResponse;
import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
import org.apache.hadoop.hive.metastore.txn.TxnStore;
import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.apache.hadoop.hive.ql.io.BucketCodec;
import org.apache.hadoop.hive.ql.io.HiveInputFormat;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.ql.txn.compactor.Cleaner;
-import org.apache.hadoop.hive.ql.txn.compactor.Worker;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This class resides in itests to facilitate running query using Tez engine, since the jars are
* fully loaded here, which is not the case if it stays in ql.
*/
public class TestAcidOnTez {
+ static final private Logger LOG = LoggerFactory.getLogger(TestAcidOnTez.class);
private static final String TEST_DATA_DIR = new File(System.getProperty("java.io.tmpdir") +
File.separator + TestAcidOnTez.class.getCanonicalName()
+ "-" + System.currentTimeMillis()
@@ -61,8 +67,10 @@ public class TestAcidOnTez {
private static enum Table {
ACIDTBL("acidTbl"),
ACIDTBLPART("acidTblPart"),
+ ACIDNOBUCKET("acidNoBucket"),
NONACIDORCTBL("nonAcidOrcTbl"),
- NONACIDPART("nonAcidPart");
+ NONACIDPART("nonAcidPart"),
+ NONACIDNONBUCKET("nonAcidNonBucket");
private final String name;
@Override
@@ -159,6 +167,359 @@ public class TestAcidOnTez {
testJoin("tez", "MapJoin");
}
+ /**
+ * Tests non acid to acid conversion where starting table has non-standard layout, i.e.
+ * where "original" files are not immediate children of the partition dir
+ */
+ @Test
+ public void testNonStandardConversion01() throws Exception {
+ HiveConf confForTez = new HiveConf(hiveConf); // make a clone of existing hive conf
+ setupTez(confForTez);
+ //CTAS with non-ACID target table
+ runStatementOnDriver("create table " + Table.NONACIDNONBUCKET + " stored as ORC TBLPROPERTIES('transactional'='false') as " +
+ "select a, b from " + Table.ACIDTBL + " where a <= 5 union all select a, b from " + Table.NONACIDORCTBL + " where a >= 5", confForTez);
+
+ List<String> rs = runStatementOnDriver("select a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by a, b, INPUT__FILE__NAME");
+ String expected0[][] = {
+ {"1\t2", "/1/000000_0"},
+ {"3\t4", "/1/000000_0"},
+ {"5\t6", "/1/000000_0"},
+ {"5\t6", "/2/000000_0"},
+ {"7\t8", "/2/000000_0"},
+ {"9\t10", "/2/000000_0"},
+ };
+ Assert.assertEquals("Unexpected row count after ctas", expected0.length, rs.size());
+ //verify data and layout
+ for(int i = 0; i < expected0.length; i++) {
+ Assert.assertTrue("Actual line " + i + " bc: " + rs.get(i), rs.get(i).startsWith(expected0[i][0]));
+ Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), rs.get(i).endsWith(expected0[i][1]));
+ }
+ //make the table ACID
+ runStatementOnDriver("alter table " + Table.NONACIDNONBUCKET + " SET TBLPROPERTIES ('transactional'='true')");
+
+ rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by ROW__ID");
+ LOG.warn("after ctas:");
+ for (String s : rs) {
+ LOG.warn(s);
+ }
+ Assert.assertEquals(0, BucketCodec.determineVersion(536870912).decodeWriterId(536870912));
+ /*
+ * Expected result 0th entry i the RecordIdentifier + data. 1st entry file before compact*/
+ String expected[][] = {
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}\t1\t2", "/1/000000_0"},
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}\t3\t4", "/1/000000_0"},
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":2}\t5\t6", "/1/000000_0"},
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t9\t10", "/2/000000_0"},
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}\t7\t8", "/2/000000_0"},
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":5}\t5\t6", "/2/000000_0"},
+ };
+ Assert.assertEquals("Unexpected row count after ctas", expected.length, rs.size());
+ //verify data and layout
+ for(int i = 0; i < expected.length; i++) {
+ Assert.assertTrue("Actual line " + i + " bc: " + rs.get(i), rs.get(i).startsWith(expected[i][0]));
+ Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), rs.get(i).endsWith(expected[i][1]));
+ }
+ //perform some Update/Delete
+ runStatementOnDriver("update " + Table.NONACIDNONBUCKET + " set a = 70, b = 80 where a = 7");
+ runStatementOnDriver("delete from " + Table.NONACIDNONBUCKET + " where a = 5");
+ rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by ROW__ID");
+ LOG.warn("after update/delete:");
+ for (String s : rs) {
+ LOG.warn(s);
+ }
+ String[][] expected2 = {
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}\t1\t2", "/1/000000_0"},
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}\t3\t4", "/1/000000_0"},
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t9\t10", "/2/000000_0"},
+ {"{\"transactionid\":21,\"bucketid\":536870912,\"rowid\":0}\t70\t80", "delta_0000021_0000021_0000/bucket_00000"}
+ };
+ Assert.assertEquals("Unexpected row count after update", expected2.length, rs.size());
+ //verify data and layout
+ for(int i = 0; i < expected2.length; i++) {
+ Assert.assertTrue("Actual line " + i + " bc: " + rs.get(i), rs.get(i).startsWith(expected2[i][0]));
+ Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), rs.get(i).endsWith(expected2[i][1]));
+ }
+ //now make sure delete deltas are present
+ FileSystem fs = FileSystem.get(hiveConf);
+ FileStatus[] status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+ (Table.NONACIDNONBUCKET).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+ String[] expectedDelDelta = {"delete_delta_0000021_0000021_0000", "delete_delta_0000022_0000022_0000"};
+ for(FileStatus stat : status) {
+ for(int i = 0; i < expectedDelDelta.length; i++) {
+ if(expectedDelDelta[i] != null && stat.getPath().toString().endsWith(expectedDelDelta[i])) {
+ expectedDelDelta[i] = null;
+ }
+ }
+ }
+ for(int i = 0; i < expectedDelDelta.length; i++) {
+ Assert.assertNull("at " + i + " " + expectedDelDelta[i] + " not found on disk", expectedDelDelta[i]);
+ }
+ //run Minor compaction
+ runStatementOnDriver("alter table " + Table.NONACIDNONBUCKET + " compact 'minor'");
+ TestTxnCommands2.runWorker(hiveConf);
+ rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by ROW__ID");
+ LOG.warn("after compact minor:");
+ for (String s : rs) {
+ LOG.warn(s);
+ }
+ Assert.assertEquals("Unexpected row count after update", expected2.length, rs.size());
+ //verify the data is the same
+ for(int i = 0; i < expected2.length; i++) {
+ Assert.assertTrue("Actual line " + i + " bc: " + rs.get(i), rs.get(i).startsWith(expected2[i][0]));
+ //todo: since HIVE-16669 is not done, Minor compact compacts insert delta as well - it should not
+ //Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), rs.get(i).endsWith(expected2[i][1]));
+ }
+ //check we have right delete delta files after minor compaction
+ status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+ (Table.NONACIDNONBUCKET).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+ String[] expectedDelDelta2 = {"delete_delta_0000021_0000021_0000", "delete_delta_0000022_0000022_0000", "delete_delta_0000021_0000022"};
+ for(FileStatus stat : status) {
+ for(int i = 0; i < expectedDelDelta2.length; i++) {
+ if(expectedDelDelta2[i] != null && stat.getPath().toString().endsWith(expectedDelDelta2[i])) {
+ expectedDelDelta2[i] = null;
+ break;
+ }
+ }
+ }
+ for(int i = 0; i < expectedDelDelta2.length; i++) {
+ Assert.assertNull("at " + i + " " + expectedDelDelta2[i] + " not found on disk", expectedDelDelta2[i]);
+ }
+ //run Major compaction
+ runStatementOnDriver("alter table " + Table.NONACIDNONBUCKET + " compact 'major'");
+ TestTxnCommands2.runWorker(hiveConf);
+ rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by ROW__ID");
+ LOG.warn("after compact major:");
+ for (String s : rs) {
+ LOG.warn(s);
+ }
+ Assert.assertEquals("Unexpected row count after major compact", expected2.length, rs.size());
+ for(int i = 0; i < expected2.length; i++) {
+ Assert.assertTrue("Actual line " + i + " bc: " + rs.get(i), rs.get(i).startsWith(expected2[i][0]));
+ //everything is now in base/
+ Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), rs.get(i).endsWith("base_0000022/bucket_00000"));
+ }
+ }
+ /**
+ * Tests non acid to acid conversion where starting table has non-standard layout, i.e.
+ * where "original" files are not immediate children of the partition dir - partitioned table
+ *
+ * How to do this? CTAS is the only way to create data files which are not immediate children
+ * of the partition dir. CTAS/Union/Tez doesn't support partition tables. The only way is to copy
+ * data files in directly.
+ */
+ @Ignore("HIVE-17214")
+ @Test
+ public void testNonStandardConversion02() throws Exception {
+ HiveConf confForTez = new HiveConf(hiveConf); // make a clone of existing hive conf
+ confForTez.setBoolean("mapred.input.dir.recursive", true);
+ setupTez(confForTez);
+ runStatementOnDriver("create table " + Table.NONACIDNONBUCKET + " stored as ORC " +
+ "TBLPROPERTIES('transactional'='false') as " +
+ "select a, b from " + Table.ACIDTBL + " where a <= 3 union all " +
+ "select a, b from " + Table.NONACIDORCTBL + " where a >= 7 " +
+ "union all select a, b from " + Table.ACIDTBL + " where a = 5", confForTez);
+
+ List<String> rs = runStatementOnDriver("select a, b, INPUT__FILE__NAME from " +
+ Table.NONACIDNONBUCKET + " order by a, b");
+ String expected0[][] = {
+ {"1\t2", "/1/000000_0"},
+ {"3\t4", "/1/000000_0"},
+ {"5\t6", "/3/000000_0"},
+ {"7\t8", "/2/000000_0"},
+ {"9\t10", "/2/000000_0"},
+ };
+ Assert.assertEquals("Unexpected row count after ctas", expected0.length, rs.size());
+ //verify data and layout
+ for(int i = 0; i < expected0.length; i++) {
+ Assert.assertTrue("Actual line " + i + " bc: " + rs.get(i), rs.get(i).startsWith(expected0[i][0]));
+ Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), rs.get(i).endsWith(expected0[i][1]));
+ }
+ FileSystem fs = FileSystem.get(hiveConf);
+ FileStatus[] status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+ (Table.NONACIDNONBUCKET).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+ //ensure there is partition dir
+ runStatementOnDriver("insert into " + Table.NONACIDPART + " partition (p=1) values (100,110)");
+ //creates more files in that partition
+ for(FileStatus stat : status) {
+ int limit = 5;
+ Path p = stat.getPath();//dirs 1/, 2/, 3/
+ Path to = new Path(TEST_WAREHOUSE_DIR + "/" + Table.NONACIDPART+ "/p=1/" + p.getName());
+ while(limit-- > 0 && !fs.rename(p, to)) {
+ Thread.sleep(200);
+ }
+ if(limit <= 0) {
+ throw new IllegalStateException("Could not rename " + p + " to " + to);
+ }
+ }
+ /*
+ This is what we expect on disk
+ ekoifman:warehouse ekoifman$ tree nonacidpart/
+ nonacidpart/
+ └── p=1
+ ├── 000000_0
+ ├── 1
+ │  └── 000000_0
+ ├── 2
+ │  └── 000000_0
+ └── 3
+ └── 000000_0
+
+4 directories, 4 files
+ **/
+ //make the table ACID
+ runStatementOnDriver("alter table " + Table.NONACIDPART + " SET TBLPROPERTIES ('transactional'='true')");
+ rs = runStatementOnDriver("select ROW__ID, a, b, p, INPUT__FILE__NAME from " + Table.NONACIDPART + " order by ROW__ID");
+ LOG.warn("after acid conversion:");
+ for (String s : rs) {
+ LOG.warn(s);
+ }
+ String[][] expected = {
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}\t100\t110\t1", "nonacidpart/p=1/000000_0"},
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t1", "nonacidpart/p=1/1/000000_0"},
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":2}\t3\t4\t1", "nonacidpart/p=1/1/000000_0"},
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t9\t10\t1", "nonacidpart/p=1/2/000000_0"},
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}\t7\t8\t1", "nonacidpart/p=1/2/000000_0"},
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":5}\t5\t6\t1", "nonacidpart/p=1/3/000000_0"}
+ };
+ Assert.assertEquals("Wrong row count", expected.length, rs.size());
+ //verify data and layout
+ for(int i = 0; i < expected.length; i++) {
+ Assert.assertTrue("Actual line " + i + " bc: " + rs.get(i), rs.get(i).startsWith(expected[i][0]));
+ Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), rs.get(i).endsWith(expected[i][1]));
+ }
+
+ //run Major compaction
+ runStatementOnDriver("alter table " + Table.NONACIDPART + " partition (p=1) compact 'major'");
+ TestTxnCommands2.runWorker(hiveConf);
+ rs = runStatementOnDriver("select ROW__ID, a, b, p, INPUT__FILE__NAME from " + Table.NONACIDPART + " order by ROW__ID");
+ LOG.warn("after major compaction:");
+ for (String s : rs) {
+ LOG.warn(s);
+ }
+ //verify data and layout
+ for(int i = 0; i < expected.length; i++) {
+ Assert.assertTrue("Actual line " + i + " ac: " + rs.get(i), rs.get(i).startsWith(expected[i][0]));
+ Assert.assertTrue("Actual line(file) " + i + " ac: " +
+ rs.get(i), rs.get(i).endsWith("nonacidpart/p=1/base_-9223372036854775808/bucket_00000"));
+ }
+
+ }
+ /**
+ * CTAS + Tez + Union creates a non-standard layout in table dir
+ * Each leg of the union places data into a subdir of the table/partition. Subdirs are named 1/, 2/, etc
+ * The way this currently works is that CTAS creates an Acid table but the insert statement writes
+ * the data in non-acid layout. Then on read, it's treated like an non-acid to acid conversion.
+ * Longer term CTAS should create acid layout from the get-go.
+ */
+ @Test
+ public void testCtasTezUnion() throws Exception {
+ HiveConf confForTez = new HiveConf(hiveConf); // make a clone of existing hive conf
+ setupTez(confForTez);
+ //CTAS with ACID target table
+ runStatementOnDriver("create table " + Table.ACIDNOBUCKET + " stored as ORC TBLPROPERTIES('transactional'='true') as " +
+ "select a, b from " + Table.ACIDTBL + " where a <= 5 union all select a, b from " + Table.NONACIDORCTBL + " where a >= 5", confForTez);
+ List<String> rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.ACIDNOBUCKET + " order by ROW__ID");
+ LOG.warn("after ctas:");
+ for (String s : rs) {
+ LOG.warn(s);
+ }
+ Assert.assertEquals(0, BucketCodec.determineVersion(536870912).decodeWriterId(536870912));
+ /*
+ * Expected result 0th entry i the RecordIdentifier + data. 1st entry file before compact*/
+ String expected[][] = {
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}\t1\t2", "/1/000000_0"},
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}\t3\t4", "/1/000000_0"},
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":2}\t5\t6", "/1/000000_0"},
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t9\t10", "/2/000000_0"},
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}\t7\t8", "/2/000000_0"},
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":5}\t5\t6", "/2/000000_0"},
+ };
+ Assert.assertEquals("Unexpected row count after ctas", expected.length, rs.size());
+ //verify data and layout
+ for(int i = 0; i < expected.length; i++) {
+ Assert.assertTrue("Actual line " + i + " bc: " + rs.get(i), rs.get(i).startsWith(expected[i][0]));
+ Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), rs.get(i).endsWith(expected[i][1]));
+ }
+ //perform some Update/Delete
+ runStatementOnDriver("update " + Table.ACIDNOBUCKET + " set a = 70, b = 80 where a = 7");
+ runStatementOnDriver("delete from " + Table.ACIDNOBUCKET + " where a = 5");
+ rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.ACIDNOBUCKET + " order by ROW__ID");
+ LOG.warn("after update/delete:");
+ for (String s : rs) {
+ LOG.warn(s);
+ }
+ String[][] expected2 = {
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}\t1\t2", "/1/000000_0"},
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}\t3\t4", "/1/000000_0"},
+ {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t9\t10", "/2/000000_0"},
+ {"{\"transactionid\":19,\"bucketid\":536870912,\"rowid\":0}\t70\t80", "delta_0000019_0000019_0000/bucket_00000"}
+ };
+ Assert.assertEquals("Unexpected row count after update", expected2.length, rs.size());
+ //verify data and layout
+ for(int i = 0; i < expected2.length; i++) {
+ Assert.assertTrue("Actual line " + i + " bc: " + rs.get(i), rs.get(i).startsWith(expected2[i][0]));
+ Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), rs.get(i).endsWith(expected2[i][1]));
+ }
+ //now make sure delete deltas are present
+ FileSystem fs = FileSystem.get(hiveConf);
+ FileStatus[] status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+ (Table.ACIDNOBUCKET).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+ String[] expectedDelDelta = {"delete_delta_0000019_0000019_0000", "delete_delta_0000020_0000020_0000"};
+ for(FileStatus stat : status) {
+ for(int i = 0; i < expectedDelDelta.length; i++) {
+ if(expectedDelDelta[i] != null && stat.getPath().toString().endsWith(expectedDelDelta[i])) {
+ expectedDelDelta[i] = null;
+ }
+ }
+ }
+ for(int i = 0; i < expectedDelDelta.length; i++) {
+ Assert.assertNull("at " + i + " " + expectedDelDelta[i] + " not found on disk", expectedDelDelta[i]);
+ }
+ //run Minor compaction
+ runStatementOnDriver("alter table " + Table.ACIDNOBUCKET + " compact 'minor'");
+ TestTxnCommands2.runWorker(hiveConf);
+ rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.ACIDNOBUCKET + " order by ROW__ID");
+ LOG.warn("after compact minor:");
+ for (String s : rs) {
+ LOG.warn(s);
+ }
+ Assert.assertEquals("Unexpected row count after update", expected2.length, rs.size());
+ //verify the data is the same
+ for(int i = 0; i < expected2.length; i++) {
+ Assert.assertTrue("Actual line " + i + " bc: " + rs.get(i), rs.get(i).startsWith(expected2[i][0]));
+ //todo: since HIVE-16669 is not done, Minor compact compacts insert delta as well - it should not
+ //Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), rs.get(i).endsWith(expected2[i][1]));
+ }
+ //check we have right delete delta files after minor compaction
+ status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+ (Table.ACIDNOBUCKET).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+ String[] expectedDelDelta2 = { "delete_delta_0000019_0000019_0000", "delete_delta_0000020_0000020_0000", "delete_delta_0000019_0000020"};
+ for(FileStatus stat : status) {
+ for(int i = 0; i < expectedDelDelta2.length; i++) {
+ if(expectedDelDelta2[i] != null && stat.getPath().toString().endsWith(expectedDelDelta2[i])) {
+ expectedDelDelta2[i] = null;
+ break;
+ }
+ }
+ }
+ for(int i = 0; i < expectedDelDelta2.length; i++) {
+ Assert.assertNull("at " + i + " " + expectedDelDelta2[i] + " not found on disk", expectedDelDelta2[i]);
+ }
+ //run Major compaction
+ runStatementOnDriver("alter table " + Table.ACIDNOBUCKET + " compact 'major'");
+ TestTxnCommands2.runWorker(hiveConf);
+ rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.ACIDNOBUCKET + " order by ROW__ID");
+ LOG.warn("after compact major:");
+ for (String s : rs) {
+ LOG.warn(s);
+ }
+ Assert.assertEquals("Unexpected row count after major compact", expected2.length, rs.size());
+ for(int i = 0; i < expected2.length; i++) {
+ Assert.assertTrue("Actual line " + i + " bc: " + rs.get(i), rs.get(i).startsWith(expected2[i][0]));
+ //everything is now in base/
+ Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), rs.get(i).endsWith("base_0000020/bucket_00000"));
+ }
+ }
// Ideally test like this should be a qfile test. However, the explain output from qfile is always
// slightly different depending on where the test is run, specifically due to file size estimation
private void testJoin(String engine, String joinType) throws Exception {
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTezWithSplitUpdate.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTezWithSplitUpdate.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTezWithSplitUpdate.java
deleted file mode 100644
index 3dacf08..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTezWithSplitUpdate.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql;
-
-/**
- * Same as parent class but covers Acid 2.0 tables
- */
-public class TestAcidOnTezWithSplitUpdate extends TestAcidOnTez {
- @Override
- String getTblProperties() {
- return "TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default')";
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 37a3757..fa6a2aa 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -456,7 +456,9 @@ minillap.query.files=acid_bucket_pruning.q,\
llap_stats.q,\
multi_count_distinct_null.q
-minillaplocal.query.files=acid_globallimit.q,\
+minillaplocal.query.files=\
+ acid_no_buckets.q, \
+ acid_globallimit.q,\
acid_vectorization_missing_cols.q,\
alter_merge_stats_orc.q,\
auto_join30.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java b/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
index 023d703..3a3d184 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
@@ -98,14 +98,26 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
// that will use it down below.
}
}
+ Table oldTable = context.getOldTable();
+ String oldTransactionalValue = null;
+ String oldTransactionalPropertiesValue = null;
+ for (String key : oldTable.getParameters().keySet()) {
+ if (hive_metastoreConstants.TABLE_IS_TRANSACTIONAL.equalsIgnoreCase(key)) {
+ oldTransactionalValue = oldTable.getParameters().get(key);
+ }
+ if (hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES.equalsIgnoreCase(key)) {
+ oldTransactionalPropertiesValue = oldTable.getParameters().get(key);
+ }
+ }
+
if (transactionalValuePresent) {
//normalize prop name
parameters.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, transactionalValue);
}
- if ("true".equalsIgnoreCase(transactionalValue)) {
+ if ("true".equalsIgnoreCase(transactionalValue) && !"true".equalsIgnoreCase(oldTransactionalValue)) {
+ //only need to check conformance if alter table enabled aicd
if (!conformToAcid(newTable)) {
- throw new MetaException("The table must be bucketed and stored using an ACID compliant" +
- " format (such as ORC)");
+ throw new MetaException("The table must be stored using an ACID compliant format (such as ORC)");
}
if (newTable.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) {
@@ -115,17 +127,6 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
hasValidTransactionalValue = true;
}
- Table oldTable = context.getOldTable();
- String oldTransactionalValue = null;
- String oldTransactionalPropertiesValue = null;
- for (String key : oldTable.getParameters().keySet()) {
- if (hive_metastoreConstants.TABLE_IS_TRANSACTIONAL.equalsIgnoreCase(key)) {
- oldTransactionalValue = oldTable.getParameters().get(key);
- }
- if (hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES.equalsIgnoreCase(key)) {
- oldTransactionalPropertiesValue = oldTable.getParameters().get(key);
- }
- }
if (oldTransactionalValue == null ? transactionalValue == null
@@ -195,8 +196,7 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
if ("true".equalsIgnoreCase(transactionalValue)) {
if (!conformToAcid(newTable)) {
- throw new MetaException("The table must be bucketed and stored using an ACID compliant" +
- " format (such as ORC)");
+ throw new MetaException("The table must be stored using an ACID compliant format (such as ORC)");
}
if (newTable.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) {
@@ -214,14 +214,12 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
throw new MetaException("'transactional' property of TBLPROPERTIES may only have value 'true'");
}
- // Check if table is bucketed and InputFormatClass/OutputFormatClass should implement
- // AcidInputFormat/AcidOutputFormat
+ /**
+ * Check that InputFormatClass/OutputFormatClass should implement
+ * AcidInputFormat/AcidOutputFormat
+ */
private boolean conformToAcid(Table table) throws MetaException {
StorageDescriptor sd = table.getSd();
- if (sd.getBucketColsSize() < 1) {
- return false;
- }
-
try {
Class inputFormatClass = Class.forName(sd.getInputFormat());
Class outputFormatClass = Class.forName(sd.getOutputFormat());
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
index 9c9d4e7..b3ef916 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
@@ -418,8 +418,8 @@ public enum ErrorMsg {
" does not support these operations."),
VALUES_TABLE_CONSTRUCTOR_NOT_SUPPORTED(10296,
"Values clause with table constructor not yet supported"),
- ACID_OP_ON_NONACID_TABLE(10297, "Attempt to do update or delete on table {0} that does not use " +
- "an AcidOutputFormat or is not bucketed", true),
+ ACID_OP_ON_NONACID_TABLE(10297, "Attempt to do update or delete on table {0} that is " +
+ "not transactional", true),
ACID_NO_SORTED_BUCKETS(10298, "ACID insert, update, delete not supported on tables that are " +
"sorted, table {0}", true),
ALTER_TABLE_TYPE_PARTIAL_PARTITION_SPEC_NO_SUPPORTED(10299,
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index 25ad1e9..bc265eb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConfUtil;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.CompilationOpContext;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.io.AcidUtils;
@@ -36,6 +37,7 @@ import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
import org.apache.hadoop.hive.ql.io.HiveKey;
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
import org.apache.hadoop.hive.ql.io.HivePartitioner;
+import org.apache.hadoop.hive.ql.io.RecordIdentifier;
import org.apache.hadoop.hive.ql.io.RecordUpdater;
import org.apache.hadoop.hive.ql.io.StatsProvidingRecordWriter;
import org.apache.hadoop.hive.ql.io.StreamingOutputFormat;
@@ -285,6 +287,7 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
private transient int numFiles;
protected transient boolean multiFileSpray;
protected transient final Map<Integer, Integer> bucketMap = new HashMap<Integer, Integer>();
+ private transient boolean isBucketed = false;
private transient ObjectInspector[] partitionObjectInspectors;
protected transient HivePartitioner<HiveKey, Object> prtner;
@@ -345,6 +348,7 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
isNativeTable = !conf.getTableInfo().isNonNative();
isTemporary = conf.isTemporary();
multiFileSpray = conf.isMultiFileSpray();
+ this.isBucketed = hconf.getInt(hive_metastoreConstants.BUCKET_COUNT, 0) > 0;
totalFiles = conf.getTotalFiles();
numFiles = conf.getNumFiles();
dpCtx = conf.getDynPartCtx();
@@ -791,9 +795,23 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
* Hive.copyFiles() will make one of them bucket_N_copy_M in the final location. The
* reset of acid (read path) doesn't know how to handle copy_N files except for 'original'
* files (HIVE-16177)*/
+ int writerId = -1;
+ if(!isBucketed) {
+ assert !multiFileSpray;
+ assert writerOffset == 0;
+ /**For un-bucketed tables, Deletes with ROW__IDs with different 'bucketNum' values can
+ * be written to the same bucketN file.
+ * N in this case is writerId and there is no relationship
+ * between the file name and any property of the data in it. Inserts will be written
+ * to bucketN file such that all {@link RecordIdentifier#getBucketProperty()} indeed
+ * contain writerId=N.
+ * Since taskId is unique (at least per statementId and thus
+ * per [delete_]delta_x_y_stmtId/) there will not be any copy_N files.*/
+ writerId = Integer.parseInt(Utilities.getTaskIdFromFilename(taskId));
+ }
fpaths.updaters[writerOffset] = HiveFileFormatUtils.getAcidRecordUpdater(
- jc, conf.getTableInfo(), bucketNum, conf, fpaths.outPaths[writerOffset],
- rowInspector, reporter, 0);
+ jc, conf.getTableInfo(), writerId >= 0 ? writerId : bucketNum, conf,
+ fpaths.outPaths[writerOffset], rowInspector, reporter, 0);
if (LOG.isDebugEnabled()) {
LOG.debug("Created updater for bucket number " + bucketNum + " using file " +
fpaths.outPaths[writerOffset]);
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
index 751fca8..69a9f9f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
@@ -964,6 +964,9 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
private final boolean allowSyntheticFileIds;
private final boolean isDefaultFs;
+ /**
+ * @param dir - root of partition dir
+ */
public BISplitStrategy(Context context, FileSystem fs, Path dir,
List<HdfsFileStatusWithId> fileStatuses, boolean isOriginal, List<DeltaMetaData> deltas,
boolean[] covered, boolean allowSyntheticFileIds, boolean isDefaultFs) {
@@ -996,7 +999,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
}
OrcSplit orcSplit = new OrcSplit(fileStatus.getPath(), fileKey, entry.getKey(),
entry.getValue().getLength(), entry.getValue().getHosts(), null, isOriginal, true,
- deltas, -1, logicalLen);
+ deltas, -1, logicalLen, dir);
splits.add(orcSplit);
}
}
@@ -1017,18 +1020,16 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
* ACID split strategy is used when there is no base directory (when transactions are enabled).
*/
static class ACIDSplitStrategy implements SplitStrategy<OrcSplit> {
- private Path dir;
+ Path dir;
private List<DeltaMetaData> deltas;
- private boolean[] covered;
- private int numBuckets;
private AcidOperationalProperties acidOperationalProperties;
-
+ /**
+ * @param dir root of partition dir
+ */
ACIDSplitStrategy(Path dir, int numBuckets, List<DeltaMetaData> deltas, boolean[] covered,
AcidOperationalProperties acidOperationalProperties) {
this.dir = dir;
- this.numBuckets = numBuckets;
this.deltas = deltas;
- this.covered = covered;
this.acidOperationalProperties = acidOperationalProperties;
}
@@ -1234,6 +1235,8 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
private final UserGroupInformation ugi;
private final boolean allowSyntheticFileIds;
private SchemaEvolution evolution;
+ //this is the root of the partition in which the 'file' is located
+ private final Path rootDir;
public SplitGenerator(SplitInfo splitInfo, UserGroupInformation ugi,
boolean allowSyntheticFileIds, boolean isDefaultFs) throws IOException {
@@ -1250,6 +1253,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
this.isOriginal = splitInfo.isOriginal;
this.deltas = splitInfo.deltas;
this.hasBase = splitInfo.hasBase;
+ this.rootDir = splitInfo.dir;
this.projColsUncompressedSize = -1;
this.deltaSplits = splitInfo.getSplits();
this.allowSyntheticFileIds = allowSyntheticFileIds;
@@ -1361,7 +1365,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
fileKey = new SyntheticFileId(file);
}
return new OrcSplit(file.getPath(), fileKey, offset, length, hosts,
- orcTail, isOriginal, hasBase, deltas, scaledProjSize, fileLen);
+ orcTail, isOriginal, hasBase, deltas, scaledProjSize, fileLen, rootDir);
}
private static final class OffsetAndLength { // Java cruft; pair of long.
@@ -1641,7 +1645,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
pathFutures.add(ecs.submit(fileGenerator));
}
- boolean isTransactionalTableScan =
+ boolean isTransactionalTableScan =//this never seems to be set correctly
HiveConf.getBoolVar(conf, ConfVars.HIVE_TRANSACTIONAL_TABLE_SCAN);
boolean isSchemaEvolution = HiveConf.getBoolVar(conf, ConfVars.HIVE_SCHEMA_EVOLUTION);
TypeDescription readerSchema =
@@ -1932,16 +1936,17 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
final OrcSplit split = (OrcSplit) inputSplit;
final Path path = split.getPath();
-
Path root;
if (split.hasBase()) {
if (split.isOriginal()) {
- root = path.getParent();
+ root = split.getRootDir();
} else {
root = path.getParent().getParent();
+ assert root.equals(split.getRootDir()) : "root mismatch: baseDir=" + split.getRootDir() +
+ " path.p.p=" + root;
}
- } else {//here path is a delta/ but above it's a partition/
- root = path;
+ } else {
+ throw new IllegalStateException("Split w/o base: " + path);
}
// Retrieve the acidOperationalProperties for the table, initialized in HiveInputFormat.
@@ -2037,21 +2042,6 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
}
};
}
- private static Path findOriginalBucket(FileSystem fs,
- Path directory,
- int bucket) throws IOException {
- for(FileStatus stat: fs.listStatus(directory)) {
- if(stat.getLen() <= 0) {
- continue;
- }
- AcidOutputFormat.Options bucketInfo =
- AcidUtils.parseBaseOrDeltaBucketFilename(stat.getPath(), fs.getConf());
- if(bucketInfo.getBucketId() == bucket) {
- return stat.getPath();
- }
- }
- throw new IllegalArgumentException("Can't find bucket " + bucket + " in " + directory);
- }
static Reader.Options createOptionsForReader(Configuration conf) {
/**
@@ -2275,20 +2265,22 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
) throws IOException {
Reader reader = null;
boolean isOriginal = false;
- if (baseDirectory != null) {
- Path bucketFile;
+ if (baseDirectory != null) {//this is NULL for minor compaction
+ Path bucketFile = null;
if (baseDirectory.getName().startsWith(AcidUtils.BASE_PREFIX)) {
bucketFile = AcidUtils.createBucketFile(baseDirectory, bucket);
} else {
+ /**we don't know which file to start reading -
+ * {@link OrcRawRecordMerger.OriginalReaderPairToCompact} does*/
isOriginal = true;
- bucketFile = findOriginalBucket(baseDirectory.getFileSystem(conf),
- baseDirectory, bucket);
}
- reader = OrcFile.createReader(bucketFile, OrcFile.readerOptions(conf));
+ if(bucketFile != null) {
+ reader = OrcFile.createReader(bucketFile, OrcFile.readerOptions(conf));
+ }
}
OrcRawRecordMerger.Options mergerOptions = new OrcRawRecordMerger.Options()
.isCompacting(true)
- .rootPath(baseDirectory);
+ .rootPath(baseDirectory).isMajorCompaction(baseDirectory != null);
return new OrcRawRecordMerger(conf, collapseEvents, reader, isOriginal,
bucket, validTxnList, new Reader.Options(), deltaDirectory, mergerOptions);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
index 97c4e3d..cbbb4c4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
@@ -22,6 +22,8 @@ import java.util.List;
import java.util.Map;
import java.util.TreeMap;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.io.AcidOutputFormat;
import org.apache.hadoop.hive.ql.io.BucketCodec;
@@ -29,7 +31,6 @@ import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.orc.OrcUtils;
import org.apache.orc.StripeInformation;
import org.apache.orc.TypeDescription;
-import org.apache.orc.impl.AcidStats;
import org.apache.orc.impl.OrcAcidUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -68,7 +69,6 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
private final RecordIdentifier maxKey;
// an extra value so that we can return it while reading ahead
private OrcStruct extraValue;
-
/**
* A RecordIdentifier extended with the current transaction id. This is the
* key of our merge sort with the originalTransaction, bucket, and rowId
@@ -294,9 +294,15 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
* Running multiple Insert statements on the same partition (of non acid table) creates files
* like so: 00000_0, 00000_0_copy1, 00000_0_copy2, etc. So the OriginalReaderPair must treat all
* of these files as part of a single logical bucket file.
+ *
+ * Also, for unbucketed (non acid) tables, there are no guarantees where data files may be placed.
+ * For example, CTAS+Tez+Union creates subdirs 1/, 2/, etc for each leg of the Union. Thus the
+ * data file need not be an immediate child of partition dir. All files for a given writerId are
+ * treated as one logical unit to assign {@link RecordIdentifier}s to them consistently.
*
* For Compaction, where each split includes the whole bucket, this means reading over all the
* files in order to assign ROW__ID.rowid in one sequence for the entire logical bucket.
+ * For unbucketed tables, a Compaction split is all files written by a given writerId.
*
* For a read after the table is marked transactional but before it's rewritten into a base/
* by compaction, each of the original files may be split into many pieces. For each split we
@@ -305,7 +311,7 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
* split of the original file and used to filter rows from all the deltas. The ROW__ID.rowid for
* the rows of the 'original' file of course, must be assigned from the beginning of logical
* bucket. The last split of the logical bucket, i.e. the split that has the end of last file,
- * should include all insert events from deltas.
+ * should include all insert events from deltas (last sentence is obsolete for Acid 2: HIVE-17320)
*/
private static abstract class OriginalReaderPair implements ReaderPair {
OrcStruct nextRecord;
@@ -407,18 +413,18 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
RecordIdentifier newMaxKey = maxKey;
recordReader = reader.rowsOptions(options);
/**
- * Logically each bucket consists of 0000_0, 0000_0_copy_1... 0000_0_copyN. etc We don't
- * know N a priori so if this is true, then the current split is from 0000_0_copyN file.
+ * Logically each bucket consists of 0000_0, 0000_0_copy_1... 0000_0_copy_N. etc We don't
+ * know N a priori so if this is true, then the current split is from 0000_0_copy_N file.
* It's needed to correctly set maxKey. In particular, set maxKey==null if this split
* is the tail of the last file for this logical bucket to include all deltas written after
- * non-acid to acid table conversion.
+ * non-acid to acid table conversion (todo: HIVE-17320).
+ * Also, see comments at {@link OriginalReaderPair} about unbucketed tables.
*/
- boolean isLastFileForThisBucket = false;
+ boolean isLastFileForThisBucket = true;
boolean haveSeenCurrentFile = false;
long rowIdOffsetTmp = 0;
- if (mergerOptions.getCopyIndex() > 0) {
+ {
//the split is from something other than the 1st file of the logical bucket - compute offset
-
AcidUtils.Directory directoryState = AcidUtils.getAcidState(mergerOptions.getRootPath(),
conf, validTxnList, false, true);
for (HadoopShims.HdfsFileStatusWithId f : directoryState.getOriginalFiles()) {
@@ -467,23 +473,6 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
maxKey.setRowId(maxKey.getRowId() + rowIdOffset);
}
}
- } else {
- rowIdOffset = 0;
- isLastFileForThisBucket = true;
- AcidUtils.Directory directoryState = AcidUtils.getAcidState(mergerOptions.getRootPath(),
- conf, validTxnList, false, true);
- int numFilesInBucket = 0;
- for (HadoopShims.HdfsFileStatusWithId f : directoryState.getOriginalFiles()) {
- AcidOutputFormat.Options bucketOptions =
- AcidUtils.parseBaseOrDeltaBucketFilename(f.getFileStatus().getPath(), conf);
- if (bucketOptions.getBucketId() == bucketId) {
- numFilesInBucket++;
- if (numFilesInBucket > 1) {
- isLastFileForThisBucket = false;
- break;
- }
- }
- }
}
if (!isLastFileForThisBucket && maxKey == null) {
/*
@@ -651,6 +640,12 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
}
/**
* Find the key range for original bucket files.
+ * For unbucketed tables the insert event data is still written to bucket_N file except that
+ * N is just a writer ID - it still matches {@link RecordIdentifier#getBucketProperty()}. For
+ * 'original' files (ubucketed) the same applies. A file 000000_0 encodes a taskId/wirterId and
+ * at read time we synthesize {@link RecordIdentifier#getBucketProperty()} to match the file name
+ * and so the same bucketProperty is used here to create minKey/maxKey, i.e. these keys are valid
+ * to filter data from delete_delta files even for unbucketed tables.
* @param reader the reader
* @param bucket the bucket number we are reading
* @param options the options for reading with
@@ -740,7 +735,6 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
*/
static Reader.Options createEventOptions(Reader.Options options) {
Reader.Options result = options.clone();
- //result.range(options.getOffset(), Long.MAX_VALUE);WTF?
result.include(options.getInclude());
// slide the column names down by 6 for the name array
@@ -755,11 +749,17 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
return result;
}
+ /**
+ * {@link OrcRawRecordMerger} Acid reader is used slightly differently in various contexts.
+ * This makes the "context" explicit.
+ */
static class Options {
private int copyIndex = 0;
private boolean isCompacting = false;
private Path bucketPath;
private Path rootPath;
+ private boolean isMajorCompaction = false;
+ private boolean isDeleteReader = false;
Options copyIndex(int copyIndex) {
assert copyIndex >= 0;
this.copyIndex = copyIndex;
@@ -767,6 +767,7 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
}
Options isCompacting(boolean isCompacting) {
this.isCompacting = isCompacting;
+ assert !isDeleteReader;
return this;
}
Options bucketPath(Path bucketPath) {
@@ -777,6 +778,16 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
this.rootPath = rootPath;
return this;
}
+ Options isMajorCompaction(boolean isMajor) {
+ this.isMajorCompaction = isMajor;
+ assert !isDeleteReader;
+ return this;
+ }
+ Options isDeleteReader(boolean isDeleteReader) {
+ this.isDeleteReader = isDeleteReader;
+ assert !isCompacting;
+ return this;
+ }
/**
* 0 means it's the original file, without {@link Utilities#COPY_KEYWORD} suffix
*/
@@ -788,7 +799,6 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
}
/**
* Full path to the data file
- * @return
*/
Path getBucketPath() {
return bucketPath;
@@ -797,6 +807,22 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
* Partition folder (Table folder if not partitioned)
*/
Path getRootPath() { return rootPath; }
+ /**
+ * @return true if major compaction, false if minor
+ */
+ boolean isMajorCompaction() {
+ return isMajorCompaction && isCompacting;
+ }
+ boolean isMinorCompaction() {
+ return !isMajorCompaction && isCompacting;
+ }
+ /**
+ * true if this is only processing delete deltas to load in-memory table for
+ * vectorized reader
+ */
+ boolean isDeleteReader() {
+ return isDeleteReader;
+ }
}
/**
* Create a reader that merge sorts the ACID events together.
@@ -820,6 +846,39 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
this.offset = options.getOffset();
this.length = options.getLength();
this.validTxnList = validTxnList;
+ /**
+ * @since Hive 3.0
+ * With split update (HIVE-14035) we have base/, delta/ and delete_delta/ - the latter only
+ * has Delete events and the others only have Insert events. Thus {@link #baseReader} is
+ * a split of a file in base/ or delta/.
+ *
+ * For Compaction, each split (for now) is a logical bucket, i.e. all files from base/ + delta(s)/
+ * for a given bucket ID and delete_delta(s)/
+ *
+ * For bucketed tables, the data files are named bucket_N and all rows in this file are such
+ * that {@link org.apache.hadoop.hive.ql.io.BucketCodec#decodeWriterId(int)} of
+ * {@link RecordIdentifier#getBucketProperty()} is N. This is currently true for all types of
+ * files but may not be true for for delete_delta/ files in the future.
+ *
+ * For un-bucketed tables, the system is designed so that it works when there is no relationship
+ * between delete_delta file name (bucket_N) and the value of {@link RecordIdentifier#getBucketProperty()}.
+ * (Later we this maybe optimized to take advantage of situations where it is known that
+ * bucket_N matches bucketProperty().) This implies that for a given {@link baseReader} all
+ * files in delete_delta/ have to be opened ({@link ReaderPair} created). Insert events are
+ * still written such that N in file name (writerId) matches what's in bucketProperty().
+ *
+ * Compactor for un-bucketed tables works exactly the same as for bucketed ones though it
+ * should be optimized (see HIVE-17206). In particular, each split is a set of files
+ * created by a writer with the same writerId, i.e. all bucket_N files across base/ &
+ * deleta/ for the same N. Unlike bucketed tables, there is no relationship between
+ * any values in user columns to file name.
+ * The maximum N is determined by the number of writers the system chose for the the "largest"
+ * write into a given partition.
+ *
+ * In both cases, Compactor should be changed so that Minor compaction is run very often and
+ * only compacts delete_delta/. Major compaction can do what it does now.
+ */
+ boolean isBucketed = conf.getInt(hive_metastoreConstants.BUCKET_COUNT, 0) > 0;
TypeDescription typeDescr =
OrcInputFormat.getDesiredRowTypeDescr(conf, true, Integer.MAX_VALUE);
@@ -829,16 +888,26 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
// modify the options to reflect the event instead of the base row
Reader.Options eventOptions = createEventOptions(options);
- if (reader == null) {
+ if((mergerOptions.isCompacting() && mergerOptions.isMinorCompaction()) ||
+ mergerOptions.isDeleteReader()) {
+ //for minor compaction, there is no progress report and we don't filter deltas
baseReader = null;
minKey = maxKey = null;
+ assert reader == null : "unexpected input reader during minor compaction: " +
+ mergerOptions.getRootPath();
} else {
KeyInterval keyInterval;
- // find the min/max based on the offset and length (and more for 'original')
- if (isOriginal) {
- keyInterval = discoverOriginalKeyBounds(reader, bucket, options, conf);
+ if (mergerOptions.isCompacting()) {
+ assert mergerOptions.isMajorCompaction();
+ //compaction doesn't filter deltas but *may* have a reader for 'base'
+ keyInterval = new KeyInterval(null, null);
} else {
- keyInterval = discoverKeyBounds(reader, options);
+ // find the min/max based on the offset and length (and more for 'original')
+ if (isOriginal) {
+ keyInterval = discoverOriginalKeyBounds(reader, bucket, options, conf);
+ } else {
+ keyInterval = discoverKeyBounds(reader, options);
+ }
}
LOG.info("min key = " + keyInterval.getMinKey() + ", max key = " + keyInterval.getMaxKey());
// use the min/max instead of the byte range
@@ -849,8 +918,7 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
if(mergerOptions.isCompacting()) {
pair = new OriginalReaderPairToCompact(key, bucket, options, mergerOptions,
conf, validTxnList);
- }
- else {
+ } else {
pair = new OriginalReaderPairToRead(key, reader, bucket, keyInterval.getMinKey(),
keyInterval.getMaxKey(), options, mergerOptions, conf, validTxnList);
}
@@ -868,35 +936,31 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
baseReader = pair.getRecordReader();
}
- // we always want to read all of the deltas
- eventOptions.range(0, Long.MAX_VALUE);
if (deltaDirectory != null) {
+ /*whatever SARG maybe applicable to base it's not applicable to delete_delta since it has no
+ * user columns
+ * HIVE-17320: we should compute a SARG to push down min/max key to delete_delta*/
+ Reader.Options deltaEventOptions = eventOptions.clone()
+ .searchArgument(null, null).range(0, Long.MAX_VALUE);
for(Path delta: deltaDirectory) {
if(!mergerOptions.isCompacting() && !AcidUtils.isDeleteDelta(delta)) {
//all inserts should be in baseReader for normal read so this should always be delete delta if not compacting
throw new IllegalStateException(delta + " is not delete delta and is not compacting.");
}
ReaderKey key = new ReaderKey();
- Path deltaFile = AcidUtils.createBucketFile(delta, bucket);
AcidUtils.ParsedDelta deltaDir = AcidUtils.parsedDelta(delta);
- FileSystem fs = deltaFile.getFileSystem(conf);
- long length = OrcAcidUtils.getLastFlushLength(fs, deltaFile);
- if (length != -1 && fs.exists(deltaFile)) {
- Reader deltaReader = OrcFile.createReader(deltaFile,
- OrcFile.readerOptions(conf).maxLength(length));
- Reader.Options deltaEventOptions = null;
- if(eventOptions.getSearchArgument() != null) {
- // Turn off the sarg before pushing it to delta. We never want to push a sarg to a delta as
- // it can produce wrong results (if the latest valid version of the record is filtered out by
- // the sarg) or ArrayOutOfBounds errors (when the sarg is applied to a delete record)
- // unless the delta only has insert events
- AcidStats acidStats = OrcAcidUtils.parseAcidStats(deltaReader);
- if(acidStats.deletes > 0 || acidStats.updates > 0) {
- deltaEventOptions = eventOptions.clone().searchArgument(null, null);
- }
+ for (Path deltaFile : getDeltaFiles(delta, bucket, conf, mergerOptions, isBucketed)) {
+ FileSystem fs = deltaFile.getFileSystem(conf);
+ if(!fs.exists(deltaFile)) {
+ continue;
}
+ /* side files are only created by streaming ingest. If this is a compaction, we may
+ * have an insert delta/ here with side files there because the original writer died.*/
+ long length = AcidUtils.getLogicalLength(fs, fs.getFileStatus(deltaFile));
+ assert length >= 0;
+ Reader deltaReader = OrcFile.createReader(deltaFile, OrcFile.readerOptions(conf).maxLength(length));
ReaderPairAcid deltaPair = new ReaderPairAcid(key, deltaReader, minKey, maxKey,
- deltaEventOptions != null ? deltaEventOptions : eventOptions, deltaDir.getStatementId());
+ deltaEventOptions, deltaDir.getStatementId());
if (deltaPair.nextRecord() != null) {
readers.put(key, deltaPair);
}
@@ -921,6 +985,59 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
}
}
+ /**
+ * This determines the set of {@link ReaderPairAcid} to create for a given delta/.
+ * For unbucketed tables {@code bucket} can be thought of as a write tranche.
+ */
+ static Path[] getDeltaFiles(Path deltaDirectory, int bucket, Configuration conf,
+ Options mergerOptions, boolean isBucketed) throws IOException {
+ if(isBucketed) {
+ /**
+ * for bucketed tables (for now) we always trust that the N in bucketN file name means that
+ * all records have {@link RecordIdentifier#getBucketProperty()} encoding bucketId = N. This
+ * means that a delete event in bucketN can only modify an insert in another bucketN file for
+ * the same N. (Down the road we may trust it only in certain delta dirs)
+ *
+ * Compactor takes all types of deltas for a given bucket. For regular read, any file that
+ * contains (only) insert events is treated as base and only
+ * delete_delta/ are treated as deltas.
+ */
+ assert (!mergerOptions.isCompacting &&
+ deltaDirectory.getName().startsWith(AcidUtils.DELETE_DELTA_PREFIX)
+ ) || mergerOptions.isCompacting : "Unexpected delta: " + deltaDirectory;
+ Path deltaFile = AcidUtils.createBucketFile(deltaDirectory, bucket);
+ return new Path[]{deltaFile};
+ }
+ /**
+ * For unbucketed tables insert events are also stored in bucketN files but here N is
+ * the writer ID. We can trust that N matches info in {@link RecordIdentifier#getBucketProperty()}
+ * delta_x_y but it's not required since we can't trust N for delete_delta_x_x/bucketN.
+ * Thus we always have to take all files in a delete_delta.
+ * For regular read, any file that has (only) insert events is treated as base so
+ * {@link deltaDirectory} can only be delete_delta and so we take all files in it.
+ * For compacting, every split contains base/bN + delta(s)/bN + delete_delta(s){all buckets} for
+ * a given N.
+ */
+ if(deltaDirectory.getName().startsWith(AcidUtils.DELETE_DELTA_PREFIX)) {
+ //it's not wrong to take all delete events for bucketed tables but it's more efficient
+ //to only take those that belong to the 'bucket' assuming we trust the file name
+ //un-bucketed table - get all files
+ FileSystem fs = deltaDirectory.getFileSystem(conf);
+ FileStatus[] dataFiles = fs.listStatus(deltaDirectory, AcidUtils.bucketFileFilter);
+ Path[] deltaFiles = new Path[dataFiles.length];
+ int i = 0;
+ for (FileStatus stat : dataFiles) {
+ deltaFiles[i++] = stat.getPath();
+ }//todo: need a test where we actually have more than 1 file
+ return deltaFiles;
+ }
+ //if here it must be delta_x_y - insert events only, so we must be compacting
+ assert mergerOptions.isCompacting() : "Expected to be called as part of compaction";
+ Path deltaFile = AcidUtils.createBucketFile(deltaDirectory, bucket);
+ return new Path[] {deltaFile};
+
+ }
+
@VisibleForTesting
RecordIdentifier getMinKey() {
return minKey;
http://git-wip-us.apache.org/repos/asf/hive/blob/6be50b76/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
index 429960b..1e19a91 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
@@ -243,7 +243,8 @@ public class OrcRecordUpdater implements RecordUpdater {
}
if (options.getMinimumTransactionId() != options.getMaximumTransactionId()
&& !options.isWritingBase()){
- flushLengths = fs.create(OrcAcidUtils.getSideFile(this.path), true, 8,
+ //throw if file already exists as that should never happen
+ flushLengths = fs.create(OrcAcidUtils.getSideFile(this.path), false, 8,
options.getReporter());
flushLengths.writeLong(0);
OrcInputFormat.SHIMS.hflush(flushLengths);
@@ -349,6 +350,12 @@ public class OrcRecordUpdater implements RecordUpdater {
return newInspector;
}
}
+
+ /**
+ * The INSERT event always uses {@link #bucket} that this {@link RecordUpdater} was created with
+ * thus even for unbucketed tables, the N in bucket_N file name matches writerId/bucketId even for
+ * late split
+ */
private void addSimpleEvent(int operation, long currentTransaction, long rowId, Object row)
throws IOException {
this.operation.set(operation);
@@ -394,6 +401,11 @@ public class OrcRecordUpdater implements RecordUpdater {
Integer currentBucket = null;
if (operation == DELETE_OPERATION || operation == UPDATE_OPERATION) {
+ /**
+ * make sure bucketProperty in the delete event is from the {@link row} rather than whatever
+ * {@link this#bucket} is. For bucketed tables, the 2 must agree on bucketId encoded in it
+ * not for necessarily the whole value. For unbucketed tables there is no relationship.
+ */
currentBucket = setBucket(bucketInspector.get(
recIdInspector.getStructFieldData(rowValue, bucketField)), operation);
// Initialize a deleteEventWriter if not yet done. (Lazy initialization)
[23/50] [abbrv] hive git commit: HIVE-17100 : Improve HS2 operation
logs for REPL commands (Sankar Hariappan, reviewed by Anishek Agarwal,
Thejas Nair)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
index f4cc70f..569bc45 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
@@ -1240,14 +1240,14 @@ uint32_t ThriftHiveMetastore_get_databases_result::read(::apache::thrift::protoc
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->success.clear();
- uint32_t _size861;
- ::apache::thrift::protocol::TType _etype864;
- xfer += iprot->readListBegin(_etype864, _size861);
- this->success.resize(_size861);
- uint32_t _i865;
- for (_i865 = 0; _i865 < _size861; ++_i865)
+ uint32_t _size865;
+ ::apache::thrift::protocol::TType _etype868;
+ xfer += iprot->readListBegin(_etype868, _size865);
+ this->success.resize(_size865);
+ uint32_t _i869;
+ for (_i869 = 0; _i869 < _size865; ++_i869)
{
- xfer += iprot->readString(this->success[_i865]);
+ xfer += iprot->readString(this->success[_i869]);
}
xfer += iprot->readListEnd();
}
@@ -1286,10 +1286,10 @@ uint32_t ThriftHiveMetastore_get_databases_result::write(::apache::thrift::proto
xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->success.size()));
- std::vector<std::string> ::const_iterator _iter866;
- for (_iter866 = this->success.begin(); _iter866 != this->success.end(); ++_iter866)
+ std::vector<std::string> ::const_iterator _iter870;
+ for (_iter870 = this->success.begin(); _iter870 != this->success.end(); ++_iter870)
{
- xfer += oprot->writeString((*_iter866));
+ xfer += oprot->writeString((*_iter870));
}
xfer += oprot->writeListEnd();
}
@@ -1334,14 +1334,14 @@ uint32_t ThriftHiveMetastore_get_databases_presult::read(::apache::thrift::proto
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
(*(this->success)).clear();
- uint32_t _size867;
- ::apache::thrift::protocol::TType _etype870;
- xfer += iprot->readListBegin(_etype870, _size867);
- (*(this->success)).resize(_size867);
- uint32_t _i871;
- for (_i871 = 0; _i871 < _size867; ++_i871)
+ uint32_t _size871;
+ ::apache::thrift::protocol::TType _etype874;
+ xfer += iprot->readListBegin(_etype874, _size871);
+ (*(this->success)).resize(_size871);
+ uint32_t _i875;
+ for (_i875 = 0; _i875 < _size871; ++_i875)
{
- xfer += iprot->readString((*(this->success))[_i871]);
+ xfer += iprot->readString((*(this->success))[_i875]);
}
xfer += iprot->readListEnd();
}
@@ -1458,14 +1458,14 @@ uint32_t ThriftHiveMetastore_get_all_databases_result::read(::apache::thrift::pr
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->success.clear();
- uint32_t _size872;
- ::apache::thrift::protocol::TType _etype875;
- xfer += iprot->readListBegin(_etype875, _size872);
- this->success.resize(_size872);
- uint32_t _i876;
- for (_i876 = 0; _i876 < _size872; ++_i876)
+ uint32_t _size876;
+ ::apache::thrift::protocol::TType _etype879;
+ xfer += iprot->readListBegin(_etype879, _size876);
+ this->success.resize(_size876);
+ uint32_t _i880;
+ for (_i880 = 0; _i880 < _size876; ++_i880)
{
- xfer += iprot->readString(this->success[_i876]);
+ xfer += iprot->readString(this->success[_i880]);
}
xfer += iprot->readListEnd();
}
@@ -1504,10 +1504,10 @@ uint32_t ThriftHiveMetastore_get_all_databases_result::write(::apache::thrift::p
xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->success.size()));
- std::vector<std::string> ::const_iterator _iter877;
- for (_iter877 = this->success.begin(); _iter877 != this->success.end(); ++_iter877)
+ std::vector<std::string> ::const_iterator _iter881;
+ for (_iter881 = this->success.begin(); _iter881 != this->success.end(); ++_iter881)
{
- xfer += oprot->writeString((*_iter877));
+ xfer += oprot->writeString((*_iter881));
}
xfer += oprot->writeListEnd();
}
@@ -1552,14 +1552,14 @@ uint32_t ThriftHiveMetastore_get_all_databases_presult::read(::apache::thrift::p
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
(*(this->success)).clear();
- uint32_t _size878;
- ::apache::thrift::protocol::TType _etype881;
- xfer += iprot->readListBegin(_etype881, _size878);
- (*(this->success)).resize(_size878);
- uint32_t _i882;
- for (_i882 = 0; _i882 < _size878; ++_i882)
+ uint32_t _size882;
+ ::apache::thrift::protocol::TType _etype885;
+ xfer += iprot->readListBegin(_etype885, _size882);
+ (*(this->success)).resize(_size882);
+ uint32_t _i886;
+ for (_i886 = 0; _i886 < _size882; ++_i886)
{
- xfer += iprot->readString((*(this->success))[_i882]);
+ xfer += iprot->readString((*(this->success))[_i886]);
}
xfer += iprot->readListEnd();
}
@@ -2621,17 +2621,17 @@ uint32_t ThriftHiveMetastore_get_type_all_result::read(::apache::thrift::protoco
if (ftype == ::apache::thrift::protocol::T_MAP) {
{
this->success.clear();
- uint32_t _size883;
- ::apache::thrift::protocol::TType _ktype884;
- ::apache::thrift::protocol::TType _vtype885;
- xfer += iprot->readMapBegin(_ktype884, _vtype885, _size883);
- uint32_t _i887;
- for (_i887 = 0; _i887 < _size883; ++_i887)
+ uint32_t _size887;
+ ::apache::thrift::protocol::TType _ktype888;
+ ::apache::thrift::protocol::TType _vtype889;
+ xfer += iprot->readMapBegin(_ktype888, _vtype889, _size887);
+ uint32_t _i891;
+ for (_i891 = 0; _i891 < _size887; ++_i891)
{
- std::string _key888;
- xfer += iprot->readString(_key888);
- Type& _val889 = this->success[_key888];
- xfer += _val889.read(iprot);
+ std::string _key892;
+ xfer += iprot->readString(_key892);
+ Type& _val893 = this->success[_key892];
+ xfer += _val893.read(iprot);
}
xfer += iprot->readMapEnd();
}
@@ -2670,11 +2670,11 @@ uint32_t ThriftHiveMetastore_get_type_all_result::write(::apache::thrift::protoc
xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_MAP, 0);
{
xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(this->success.size()));
- std::map<std::string, Type> ::const_iterator _iter890;
- for (_iter890 = this->success.begin(); _iter890 != this->success.end(); ++_iter890)
+ std::map<std::string, Type> ::const_iterator _iter894;
+ for (_iter894 = this->success.begin(); _iter894 != this->success.end(); ++_iter894)
{
- xfer += oprot->writeString(_iter890->first);
- xfer += _iter890->second.write(oprot);
+ xfer += oprot->writeString(_iter894->first);
+ xfer += _iter894->second.write(oprot);
}
xfer += oprot->writeMapEnd();
}
@@ -2719,17 +2719,17 @@ uint32_t ThriftHiveMetastore_get_type_all_presult::read(::apache::thrift::protoc
if (ftype == ::apache::thrift::protocol::T_MAP) {
{
(*(this->success)).clear();
- uint32_t _size891;
- ::apache::thrift::protocol::TType _ktype892;
- ::apache::thrift::protocol::TType _vtype893;
- xfer += iprot->readMapBegin(_ktype892, _vtype893, _size891);
- uint32_t _i895;
- for (_i895 = 0; _i895 < _size891; ++_i895)
+ uint32_t _size895;
+ ::apache::thrift::protocol::TType _ktype896;
+ ::apache::thrift::protocol::TType _vtype897;
+ xfer += iprot->readMapBegin(_ktype896, _vtype897, _size895);
+ uint32_t _i899;
+ for (_i899 = 0; _i899 < _size895; ++_i899)
{
- std::string _key896;
- xfer += iprot->readString(_key896);
- Type& _val897 = (*(this->success))[_key896];
- xfer += _val897.read(iprot);
+ std::string _key900;
+ xfer += iprot->readString(_key900);
+ Type& _val901 = (*(this->success))[_key900];
+ xfer += _val901.read(iprot);
}
xfer += iprot->readMapEnd();
}
@@ -2883,14 +2883,14 @@ uint32_t ThriftHiveMetastore_get_fields_result::read(::apache::thrift::protocol:
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->success.clear();
- uint32_t _size898;
- ::apache::thrift::protocol::TType _etype901;
- xfer += iprot->readListBegin(_etype901, _size898);
- this->success.resize(_size898);
- uint32_t _i902;
- for (_i902 = 0; _i902 < _size898; ++_i902)
+ uint32_t _size902;
+ ::apache::thrift::protocol::TType _etype905;
+ xfer += iprot->readListBegin(_etype905, _size902);
+ this->success.resize(_size902);
+ uint32_t _i906;
+ for (_i906 = 0; _i906 < _size902; ++_i906)
{
- xfer += this->success[_i902].read(iprot);
+ xfer += this->success[_i906].read(iprot);
}
xfer += iprot->readListEnd();
}
@@ -2945,10 +2945,10 @@ uint32_t ThriftHiveMetastore_get_fields_result::write(::apache::thrift::protocol
xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(this->success.size()));
- std::vector<FieldSchema> ::const_iterator _iter903;
- for (_iter903 = this->success.begin(); _iter903 != this->success.end(); ++_iter903)
+ std::vector<FieldSchema> ::const_iterator _iter907;
+ for (_iter907 = this->success.begin(); _iter907 != this->success.end(); ++_iter907)
{
- xfer += (*_iter903).write(oprot);
+ xfer += (*_iter907).write(oprot);
}
xfer += oprot->writeListEnd();
}
@@ -3001,14 +3001,14 @@ uint32_t ThriftHiveMetastore_get_fields_presult::read(::apache::thrift::protocol
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
(*(this->success)).clear();
- uint32_t _size904;
- ::apache::thrift::protocol::TType _etype907;
- xfer += iprot->readListBegin(_etype907, _size904);
- (*(this->success)).resize(_size904);
- uint32_t _i908;
- for (_i908 = 0; _i908 < _size904; ++_i908)
+ uint32_t _size908;
+ ::apache::thrift::protocol::TType _etype911;
+ xfer += iprot->readListBegin(_etype911, _size908);
+ (*(this->success)).resize(_size908);
+ uint32_t _i912;
+ for (_i912 = 0; _i912 < _size908; ++_i912)
{
- xfer += (*(this->success))[_i908].read(iprot);
+ xfer += (*(this->success))[_i912].read(iprot);
}
xfer += iprot->readListEnd();
}
@@ -3194,14 +3194,14 @@ uint32_t ThriftHiveMetastore_get_fields_with_environment_context_result::read(::
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->success.clear();
- uint32_t _size909;
- ::apache::thrift::protocol::TType _etype912;
- xfer += iprot->readListBegin(_etype912, _size909);
- this->success.resize(_size909);
- uint32_t _i913;
- for (_i913 = 0; _i913 < _size909; ++_i913)
+ uint32_t _size913;
+ ::apache::thrift::protocol::TType _etype916;
+ xfer += iprot->readListBegin(_etype916, _size913);
+ this->success.resize(_size913);
+ uint32_t _i917;
+ for (_i917 = 0; _i917 < _size913; ++_i917)
{
- xfer += this->success[_i913].read(iprot);
+ xfer += this->success[_i917].read(iprot);
}
xfer += iprot->readListEnd();
}
@@ -3256,10 +3256,10 @@ uint32_t ThriftHiveMetastore_get_fields_with_environment_context_result::write(:
xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(this->success.size()));
- std::vector<FieldSchema> ::const_iterator _iter914;
- for (_iter914 = this->success.begin(); _iter914 != this->success.end(); ++_iter914)
+ std::vector<FieldSchema> ::const_iterator _iter918;
+ for (_iter918 = this->success.begin(); _iter918 != this->success.end(); ++_iter918)
{
- xfer += (*_iter914).write(oprot);
+ xfer += (*_iter918).write(oprot);
}
xfer += oprot->writeListEnd();
}
@@ -3312,14 +3312,14 @@ uint32_t ThriftHiveMetastore_get_fields_with_environment_context_presult::read(:
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
(*(this->success)).clear();
- uint32_t _size915;
- ::apache::thrift::protocol::TType _etype918;
- xfer += iprot->readListBegin(_etype918, _size915);
- (*(this->success)).resize(_size915);
- uint32_t _i919;
- for (_i919 = 0; _i919 < _size915; ++_i919)
+ uint32_t _size919;
+ ::apache::thrift::protocol::TType _etype922;
+ xfer += iprot->readListBegin(_etype922, _size919);
+ (*(this->success)).resize(_size919);
+ uint32_t _i923;
+ for (_i923 = 0; _i923 < _size919; ++_i923)
{
- xfer += (*(this->success))[_i919].read(iprot);
+ xfer += (*(this->success))[_i923].read(iprot);
}
xfer += iprot->readListEnd();
}
@@ -3489,14 +3489,14 @@ uint32_t ThriftHiveMetastore_get_schema_result::read(::apache::thrift::protocol:
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->success.clear();
- uint32_t _size920;
- ::apache::thrift::protocol::TType _etype923;
- xfer += iprot->readListBegin(_etype923, _size920);
- this->success.resize(_size920);
- uint32_t _i924;
- for (_i924 = 0; _i924 < _size920; ++_i924)
+ uint32_t _size924;
+ ::apache::thrift::protocol::TType _etype927;
+ xfer += iprot->readListBegin(_etype927, _size924);
+ this->success.resize(_size924);
+ uint32_t _i928;
+ for (_i928 = 0; _i928 < _size924; ++_i928)
{
- xfer += this->success[_i924].read(iprot);
+ xfer += this->success[_i928].read(iprot);
}
xfer += iprot->readListEnd();
}
@@ -3551,10 +3551,10 @@ uint32_t ThriftHiveMetastore_get_schema_result::write(::apache::thrift::protocol
xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(this->success.size()));
- std::vector<FieldSchema> ::const_iterator _iter925;
- for (_iter925 = this->success.begin(); _iter925 != this->success.end(); ++_iter925)
+ std::vector<FieldSchema> ::const_iterator _iter929;
+ for (_iter929 = this->success.begin(); _iter929 != this->success.end(); ++_iter929)
{
- xfer += (*_iter925).write(oprot);
+ xfer += (*_iter929).write(oprot);
}
xfer += oprot->writeListEnd();
}
@@ -3607,14 +3607,14 @@ uint32_t ThriftHiveMetastore_get_schema_presult::read(::apache::thrift::protocol
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
(*(this->success)).clear();
- uint32_t _size926;
- ::apache::thrift::protocol::TType _etype929;
- xfer += iprot->readListBegin(_etype929, _size926);
- (*(this->success)).resize(_size926);
- uint32_t _i930;
- for (_i930 = 0; _i930 < _size926; ++_i930)
+ uint32_t _size930;
+ ::apache::thrift::protocol::TType _etype933;
+ xfer += iprot->readListBegin(_etype933, _size930);
+ (*(this->success)).resize(_size930);
+ uint32_t _i934;
+ for (_i934 = 0; _i934 < _size930; ++_i934)
{
- xfer += (*(this->success))[_i930].read(iprot);
+ xfer += (*(this->success))[_i934].read(iprot);
}
xfer += iprot->readListEnd();
}
@@ -3800,14 +3800,14 @@ uint32_t ThriftHiveMetastore_get_schema_with_environment_context_result::read(::
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->success.clear();
- uint32_t _size931;
- ::apache::thrift::protocol::TType _etype934;
- xfer += iprot->readListBegin(_etype934, _size931);
- this->success.resize(_size931);
- uint32_t _i935;
- for (_i935 = 0; _i935 < _size931; ++_i935)
+ uint32_t _size935;
+ ::apache::thrift::protocol::TType _etype938;
+ xfer += iprot->readListBegin(_etype938, _size935);
+ this->success.resize(_size935);
+ uint32_t _i939;
+ for (_i939 = 0; _i939 < _size935; ++_i939)
{
- xfer += this->success[_i935].read(iprot);
+ xfer += this->success[_i939].read(iprot);
}
xfer += iprot->readListEnd();
}
@@ -3862,10 +3862,10 @@ uint32_t ThriftHiveMetastore_get_schema_with_environment_context_result::write(:
xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(this->success.size()));
- std::vector<FieldSchema> ::const_iterator _iter936;
- for (_iter936 = this->success.begin(); _iter936 != this->success.end(); ++_iter936)
+ std::vector<FieldSchema> ::const_iterator _iter940;
+ for (_iter940 = this->success.begin(); _iter940 != this->success.end(); ++_iter940)
{
- xfer += (*_iter936).write(oprot);
+ xfer += (*_iter940).write(oprot);
}
xfer += oprot->writeListEnd();
}
@@ -3918,14 +3918,14 @@ uint32_t ThriftHiveMetastore_get_schema_with_environment_context_presult::read(:
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
(*(this->success)).clear();
- uint32_t _size937;
- ::apache::thrift::protocol::TType _etype940;
- xfer += iprot->readListBegin(_etype940, _size937);
- (*(this->success)).resize(_size937);
- uint32_t _i941;
- for (_i941 = 0; _i941 < _size937; ++_i941)
+ uint32_t _size941;
+ ::apache::thrift::protocol::TType _etype944;
+ xfer += iprot->readListBegin(_etype944, _size941);
+ (*(this->success)).resize(_size941);
+ uint32_t _i945;
+ for (_i945 = 0; _i945 < _size941; ++_i945)
{
- xfer += (*(this->success))[_i941].read(iprot);
+ xfer += (*(this->success))[_i945].read(iprot);
}
xfer += iprot->readListEnd();
}
@@ -4518,14 +4518,14 @@ uint32_t ThriftHiveMetastore_create_table_with_constraints_args::read(::apache::
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->primaryKeys.clear();
- uint32_t _size942;
- ::apache::thrift::protocol::TType _etype945;
- xfer += iprot->readListBegin(_etype945, _size942);
- this->primaryKeys.resize(_size942);
- uint32_t _i946;
- for (_i946 = 0; _i946 < _size942; ++_i946)
+ uint32_t _size946;
+ ::apache::thrift::protocol::TType _etype949;
+ xfer += iprot->readListBegin(_etype949, _size946);
+ this->primaryKeys.resize(_size946);
+ uint32_t _i950;
+ for (_i950 = 0; _i950 < _size946; ++_i950)
{
- xfer += this->primaryKeys[_i946].read(iprot);
+ xfer += this->primaryKeys[_i950].read(iprot);
}
xfer += iprot->readListEnd();
}
@@ -4538,14 +4538,14 @@ uint32_t ThriftHiveMetastore_create_table_with_constraints_args::read(::apache::
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->foreignKeys.clear();
- uint32_t _size947;
- ::apache::thrift::protocol::TType _etype950;
- xfer += iprot->readListBegin(_etype950, _size947);
- this->foreignKeys.resize(_size947);
- uint32_t _i951;
- for (_i951 = 0; _i951 < _size947; ++_i951)
+ uint32_t _size951;
+ ::apache::thrift::protocol::TType _etype954;
+ xfer += iprot->readListBegin(_etype954, _size951);
+ this->foreignKeys.resize(_size951);
+ uint32_t _i955;
+ for (_i955 = 0; _i955 < _size951; ++_i955)
{
- xfer += this->foreignKeys[_i951].read(iprot);
+ xfer += this->foreignKeys[_i955].read(iprot);
}
xfer += iprot->readListEnd();
}
@@ -4558,14 +4558,14 @@ uint32_t ThriftHiveMetastore_create_table_with_constraints_args::read(::apache::
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->uniqueConstraints.clear();
- uint32_t _size952;
- ::apache::thrift::protocol::TType _etype955;
- xfer += iprot->readListBegin(_etype955, _size952);
- this->uniqueConstraints.resize(_size952);
- uint32_t _i956;
- for (_i956 = 0; _i956 < _size952; ++_i956)
+ uint32_t _size956;
+ ::apache::thrift::protocol::TType _etype959;
+ xfer += iprot->readListBegin(_etype959, _size956);
+ this->uniqueConstraints.resize(_size956);
+ uint32_t _i960;
+ for (_i960 = 0; _i960 < _size956; ++_i960)
{
- xfer += this->uniqueConstraints[_i956].read(iprot);
+ xfer += this->uniqueConstraints[_i960].read(iprot);
}
xfer += iprot->readListEnd();
}
@@ -4578,14 +4578,14 @@ uint32_t ThriftHiveMetastore_create_table_with_constraints_args::read(::apache::
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->notNullConstraints.clear();
- uint32_t _size957;
- ::apache::thrift::protocol::TType _etype960;
- xfer += iprot->readListBegin(_etype960, _size957);
- this->notNullConstraints.resize(_size957);
- uint32_t _i961;
- for (_i961 = 0; _i961 < _size957; ++_i961)
+ uint32_t _size961;
+ ::apache::thrift::protocol::TType _etype964;
+ xfer += iprot->readListBegin(_etype964, _size961);
+ this->notNullConstraints.resize(_size961);
+ uint32_t _i965;
+ for (_i965 = 0; _i965 < _size961; ++_i965)
{
- xfer += this->notNullConstraints[_i961].read(iprot);
+ xfer += this->notNullConstraints[_i965].read(iprot);
}
xfer += iprot->readListEnd();
}
@@ -4618,10 +4618,10 @@ uint32_t ThriftHiveMetastore_create_table_with_constraints_args::write(::apache:
xfer += oprot->writeFieldBegin("primaryKeys", ::apache::thrift::protocol::T_LIST, 2);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(this->primaryKeys.size()));
- std::vector<SQLPrimaryKey> ::const_iterator _iter962;
- for (_iter962 = this->primaryKeys.begin(); _iter962 != this->primaryKeys.end(); ++_iter962)
+ std::vector<SQLPrimaryKey> ::const_iterator _iter966;
+ for (_iter966 = this->primaryKeys.begin(); _iter966 != this->primaryKeys.end(); ++_iter966)
{
- xfer += (*_iter962).write(oprot);
+ xfer += (*_iter966).write(oprot);
}
xfer += oprot->writeListEnd();
}
@@ -4630,10 +4630,10 @@ uint32_t ThriftHiveMetastore_create_table_with_constraints_args::write(::apache:
xfer += oprot->writeFieldBegin("foreignKeys", ::apache::thrift::protocol::T_LIST, 3);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(this->foreignKeys.size()));
- std::vector<SQLForeignKey> ::const_iterator _iter963;
- for (_iter963 = this->foreignKeys.begin(); _iter963 != this->foreignKeys.end(); ++_iter963)
+ std::vector<SQLForeignKey> ::const_iterator _iter967;
+ for (_iter967 = this->foreignKeys.begin(); _iter967 != this->foreignKeys.end(); ++_iter967)
{
- xfer += (*_iter963).write(oprot);
+ xfer += (*_iter967).write(oprot);
}
xfer += oprot->writeListEnd();
}
@@ -4642,10 +4642,10 @@ uint32_t ThriftHiveMetastore_create_table_with_constraints_args::write(::apache:
xfer += oprot->writeFieldBegin("uniqueConstraints", ::apache::thrift::protocol::T_LIST, 4);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(this->uniqueConstraints.size()));
- std::vector<SQLUniqueConstraint> ::const_iterator _iter964;
- for (_iter964 = this->uniqueConstraints.begin(); _iter964 != this->uniqueConstraints.end(); ++_iter964)
+ std::vector<SQLUniqueConstraint> ::const_iterator _iter968;
+ for (_iter968 = this->uniqueConstraints.begin(); _iter968 != this->uniqueConstraints.end(); ++_iter968)
{
- xfer += (*_iter964).write(oprot);
+ xfer += (*_iter968).write(oprot);
}
xfer += oprot->writeListEnd();
}
@@ -4654,10 +4654,10 @@ uint32_t ThriftHiveMetastore_create_table_with_constraints_args::write(::apache:
xfer += oprot->writeFieldBegin("notNullConstraints", ::apache::thrift::protocol::T_LIST, 5);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(this->notNullConstraints.size()));
- std::vector<SQLNotNullConstraint> ::const_iterator _iter965;
- for (_iter965 = this->notNullConstraints.begin(); _iter965 != this->notNullConstraints.end(); ++_iter965)
+ std::vector<SQLNotNullConstraint> ::const_iterator _iter969;
+ for (_iter969 = this->notNullConstraints.begin(); _iter969 != this->notNullConstraints.end(); ++_iter969)
{
- xfer += (*_iter965).write(oprot);
+ xfer += (*_iter969).write(oprot);
}
xfer += oprot->writeListEnd();
}
@@ -4685,10 +4685,10 @@ uint32_t ThriftHiveMetastore_create_table_with_constraints_pargs::write(::apache
xfer += oprot->writeFieldBegin("primaryKeys", ::apache::thrift::protocol::T_LIST, 2);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>((*(this->primaryKeys)).size()));
- std::vector<SQLPrimaryKey> ::const_iterator _iter966;
- for (_iter966 = (*(this->primaryKeys)).begin(); _iter966 != (*(this->primaryKeys)).end(); ++_iter966)
+ std::vector<SQLPrimaryKey> ::const_iterator _iter970;
+ for (_iter970 = (*(this->primaryKeys)).begin(); _iter970 != (*(this->primaryKeys)).end(); ++_iter970)
{
- xfer += (*_iter966).write(oprot);
+ xfer += (*_iter970).write(oprot);
}
xfer += oprot->writeListEnd();
}
@@ -4697,10 +4697,10 @@ uint32_t ThriftHiveMetastore_create_table_with_constraints_pargs::write(::apache
xfer += oprot->writeFieldBegin("foreignKeys", ::apache::thrift::protocol::T_LIST, 3);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>((*(this->foreignKeys)).size()));
- std::vector<SQLForeignKey> ::const_iterator _iter967;
- for (_iter967 = (*(this->foreignKeys)).begin(); _iter967 != (*(this->foreignKeys)).end(); ++_iter967)
+ std::vector<SQLForeignKey> ::const_iterator _iter971;
+ for (_iter971 = (*(this->foreignKeys)).begin(); _iter971 != (*(this->foreignKeys)).end(); ++_iter971)
{
- xfer += (*_iter967).write(oprot);
+ xfer += (*_iter971).write(oprot);
}
xfer += oprot->writeListEnd();
}
@@ -4709,10 +4709,10 @@ uint32_t ThriftHiveMetastore_create_table_with_constraints_pargs::write(::apache
xfer += oprot->writeFieldBegin("uniqueConstraints", ::apache::thrift::protocol::T_LIST, 4);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>((*(this->uniqueConstraints)).size()));
- std::vector<SQLUniqueConstraint> ::const_iterator _iter968;
- for (_iter968 = (*(this->uniqueConstraints)).begin(); _iter968 != (*(this->uniqueConstraints)).end(); ++_iter968)
+ std::vector<SQLUniqueConstraint> ::const_iterator _iter972;
+ for (_iter972 = (*(this->uniqueConstraints)).begin(); _iter972 != (*(this->uniqueConstraints)).end(); ++_iter972)
{
- xfer += (*_iter968).write(oprot);
+ xfer += (*_iter972).write(oprot);
}
xfer += oprot->writeListEnd();
}
@@ -4721,10 +4721,10 @@ uint32_t ThriftHiveMetastore_create_table_with_constraints_pargs::write(::apache
xfer += oprot->writeFieldBegin("notNullConstraints", ::apache::thrift::protocol::T_LIST, 5);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>((*(this->notNullConstraints)).size()));
- std::vector<SQLNotNullConstraint> ::const_iterator _iter969;
- for (_iter969 = (*(this->notNullConstraints)).begin(); _iter969 != (*(this->notNullConstraints)).end(); ++_iter969)
+ std::vector<SQLNotNullConstraint> ::const_iterator _iter973;
+ for (_iter973 = (*(this->notNullConstraints)).begin(); _iter973 != (*(this->notNullConstraints)).end(); ++_iter973)
{
- xfer += (*_iter969).write(oprot);
+ xfer += (*_iter973).write(oprot);
}
xfer += oprot->writeListEnd();
}
@@ -6478,14 +6478,14 @@ uint32_t ThriftHiveMetastore_truncate_table_args::read(::apache::thrift::protoco
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->partNames.clear();
- uint32_t _size970;
- ::apache::thrift::protocol::TType _etype973;
- xfer += iprot->readListBegin(_etype973, _size970);
- this->partNames.resize(_size970);
- uint32_t _i974;
- for (_i974 = 0; _i974 < _size970; ++_i974)
+ uint32_t _size974;
+ ::apache::thrift::protocol::TType _etype977;
+ xfer += iprot->readListBegin(_etype977, _size974);
+ this->partNames.resize(_size974);
+ uint32_t _i978;
+ for (_i978 = 0; _i978 < _size974; ++_i978)
{
- xfer += iprot->readString(this->partNames[_i974]);
+ xfer += iprot->readString(this->partNames[_i978]);
}
xfer += iprot->readListEnd();
}
@@ -6522,10 +6522,10 @@ uint32_t ThriftHiveMetastore_truncate_table_args::write(::apache::thrift::protoc
xfer += oprot->writeFieldBegin("partNames", ::apache::thrift::protocol::T_LIST, 3);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->partNames.size()));
- std::vector<std::string> ::const_iterator _iter975;
- for (_iter975 = this->partNames.begin(); _iter975 != this->partNames.end(); ++_iter975)
+ std::vector<std::string> ::const_iterator _iter979;
+ for (_iter979 = this->partNames.begin(); _iter979 != this->partNames.end(); ++_iter979)
{
- xfer += oprot->writeString((*_iter975));
+ xfer += oprot->writeString((*_iter979));
}
xfer += oprot->writeListEnd();
}
@@ -6557,10 +6557,10 @@ uint32_t ThriftHiveMetastore_truncate_table_pargs::write(::apache::thrift::proto
xfer += oprot->writeFieldBegin("partNames", ::apache::thrift::protocol::T_LIST, 3);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>((*(this->partNames)).size()));
- std::vector<std::string> ::const_iterator _iter976;
- for (_iter976 = (*(this->partNames)).begin(); _iter976 != (*(this->partNames)).end(); ++_iter976)
+ std::vector<std::string> ::const_iterator _iter980;
+ for (_iter980 = (*(this->partNames)).begin(); _iter980 != (*(this->partNames)).end(); ++_iter980)
{
- xfer += oprot->writeString((*_iter976));
+ xfer += oprot->writeString((*_iter980));
}
xfer += oprot->writeListEnd();
}
@@ -6804,14 +6804,14 @@ uint32_t ThriftHiveMetastore_get_tables_result::read(::apache::thrift::protocol:
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->success.clear();
- uint32_t _size977;
- ::apache::thrift::protocol::TType _etype980;
- xfer += iprot->readListBegin(_etype980, _size977);
- this->success.resize(_size977);
- uint32_t _i981;
- for (_i981 = 0; _i981 < _size977; ++_i981)
+ uint32_t _size981;
+ ::apache::thrift::protocol::TType _etype984;
+ xfer += iprot->readListBegin(_etype984, _size981);
+ this->success.resize(_size981);
+ uint32_t _i985;
+ for (_i985 = 0; _i985 < _size981; ++_i985)
{
- xfer += iprot->readString(this->success[_i981]);
+ xfer += iprot->readString(this->success[_i985]);
}
xfer += iprot->readListEnd();
}
@@ -6850,10 +6850,10 @@ uint32_t ThriftHiveMetastore_get_tables_result::write(::apache::thrift::protocol
xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->success.size()));
- std::vector<std::string> ::const_iterator _iter982;
- for (_iter982 = this->success.begin(); _iter982 != this->success.end(); ++_iter982)
+ std::vector<std::string> ::const_iterator _iter986;
+ for (_iter986 = this->success.begin(); _iter986 != this->success.end(); ++_iter986)
{
- xfer += oprot->writeString((*_iter982));
+ xfer += oprot->writeString((*_iter986));
}
xfer += oprot->writeListEnd();
}
@@ -6898,14 +6898,14 @@ uint32_t ThriftHiveMetastore_get_tables_presult::read(::apache::thrift::protocol
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
(*(this->success)).clear();
- uint32_t _size983;
- ::apache::thrift::protocol::TType _etype986;
- xfer += iprot->readListBegin(_etype986, _size983);
- (*(this->success)).resize(_size983);
- uint32_t _i987;
- for (_i987 = 0; _i987 < _size983; ++_i987)
+ uint32_t _size987;
+ ::apache::thrift::protocol::TType _etype990;
+ xfer += iprot->readListBegin(_etype990, _size987);
+ (*(this->success)).resize(_size987);
+ uint32_t _i991;
+ for (_i991 = 0; _i991 < _size987; ++_i991)
{
- xfer += iprot->readString((*(this->success))[_i987]);
+ xfer += iprot->readString((*(this->success))[_i991]);
}
xfer += iprot->readListEnd();
}
@@ -7075,14 +7075,14 @@ uint32_t ThriftHiveMetastore_get_tables_by_type_result::read(::apache::thrift::p
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->success.clear();
- uint32_t _size988;
- ::apache::thrift::protocol::TType _etype991;
- xfer += iprot->readListBegin(_etype991, _size988);
- this->success.resize(_size988);
- uint32_t _i992;
- for (_i992 = 0; _i992 < _size988; ++_i992)
+ uint32_t _size992;
+ ::apache::thrift::protocol::TType _etype995;
+ xfer += iprot->readListBegin(_etype995, _size992);
+ this->success.resize(_size992);
+ uint32_t _i996;
+ for (_i996 = 0; _i996 < _size992; ++_i996)
{
- xfer += iprot->readString(this->success[_i992]);
+ xfer += iprot->readString(this->success[_i996]);
}
xfer += iprot->readListEnd();
}
@@ -7121,10 +7121,10 @@ uint32_t ThriftHiveMetastore_get_tables_by_type_result::write(::apache::thrift::
xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->success.size()));
- std::vector<std::string> ::const_iterator _iter993;
- for (_iter993 = this->success.begin(); _iter993 != this->success.end(); ++_iter993)
+ std::vector<std::string> ::const_iterator _iter997;
+ for (_iter997 = this->success.begin(); _iter997 != this->success.end(); ++_iter997)
{
- xfer += oprot->writeString((*_iter993));
+ xfer += oprot->writeString((*_iter997));
}
xfer += oprot->writeListEnd();
}
@@ -7169,14 +7169,14 @@ uint32_t ThriftHiveMetastore_get_tables_by_type_presult::read(::apache::thrift::
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
(*(this->success)).clear();
- uint32_t _size994;
- ::apache::thrift::protocol::TType _etype997;
- xfer += iprot->readListBegin(_etype997, _size994);
- (*(this->success)).resize(_size994);
- uint32_t _i998;
- for (_i998 = 0; _i998 < _size994; ++_i998)
+ uint32_t _size998;
+ ::apache::thrift::protocol::TType _etype1001;
+ xfer += iprot->readListBegin(_etype1001, _size998);
+ (*(this->success)).resize(_size998);
+ uint32_t _i1002;
+ for (_i1002 = 0; _i1002 < _size998; ++_i1002)
{
- xfer += iprot->readString((*(this->success))[_i998]);
+ xfer += iprot->readString((*(this->success))[_i1002]);
}
xfer += iprot->readListEnd();
}
@@ -7251,14 +7251,14 @@ uint32_t ThriftHiveMetastore_get_table_meta_args::read(::apache::thrift::protoco
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->tbl_types.clear();
- uint32_t _size999;
- ::apache::thrift::protocol::TType _etype1002;
- xfer += iprot->readListBegin(_etype1002, _size999);
- this->tbl_types.resize(_size999);
- uint32_t _i1003;
- for (_i1003 = 0; _i1003 < _size999; ++_i1003)
+ uint32_t _size1003;
+ ::apache::thrift::protocol::TType _etype1006;
+ xfer += iprot->readListBegin(_etype1006, _size1003);
+ this->tbl_types.resize(_size1003);
+ uint32_t _i1007;
+ for (_i1007 = 0; _i1007 < _size1003; ++_i1007)
{
- xfer += iprot->readString(this->tbl_types[_i1003]);
+ xfer += iprot->readString(this->tbl_types[_i1007]);
}
xfer += iprot->readListEnd();
}
@@ -7295,10 +7295,10 @@ uint32_t ThriftHiveMetastore_get_table_meta_args::write(::apache::thrift::protoc
xfer += oprot->writeFieldBegin("tbl_types", ::apache::thrift::protocol::T_LIST, 3);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->tbl_types.size()));
- std::vector<std::string> ::const_iterator _iter1004;
- for (_iter1004 = this->tbl_types.begin(); _iter1004 != this->tbl_types.end(); ++_iter1004)
+ std::vector<std::string> ::const_iterator _iter1008;
+ for (_iter1008 = this->tbl_types.begin(); _iter1008 != this->tbl_types.end(); ++_iter1008)
{
- xfer += oprot->writeString((*_iter1004));
+ xfer += oprot->writeString((*_iter1008));
}
xfer += oprot->writeListEnd();
}
@@ -7330,10 +7330,10 @@ uint32_t ThriftHiveMetastore_get_table_meta_pargs::write(::apache::thrift::proto
xfer += oprot->writeFieldBegin("tbl_types", ::apache::thrift::protocol::T_LIST, 3);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>((*(this->tbl_types)).size()));
- std::vector<std::string> ::const_iterator _iter1005;
- for (_iter1005 = (*(this->tbl_types)).begin(); _iter1005 != (*(this->tbl_types)).end(); ++_iter1005)
+ std::vector<std::string> ::const_iterator _iter1009;
+ for (_iter1009 = (*(this->tbl_types)).begin(); _iter1009 != (*(this->tbl_types)).end(); ++_iter1009)
{
- xfer += oprot->writeString((*_iter1005));
+ xfer += oprot->writeString((*_iter1009));
}
xfer += oprot->writeListEnd();
}
@@ -7374,14 +7374,14 @@ uint32_t ThriftHiveMetastore_get_table_meta_result::read(::apache::thrift::proto
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->success.clear();
- uint32_t _size1006;
- ::apache::thrift::protocol::TType _etype1009;
- xfer += iprot->readListBegin(_etype1009, _size1006);
- this->success.resize(_size1006);
- uint32_t _i1010;
- for (_i1010 = 0; _i1010 < _size1006; ++_i1010)
+ uint32_t _size1010;
+ ::apache::thrift::protocol::TType _etype1013;
+ xfer += iprot->readListBegin(_etype1013, _size1010);
+ this->success.resize(_size1010);
+ uint32_t _i1014;
+ for (_i1014 = 0; _i1014 < _size1010; ++_i1014)
{
- xfer += this->success[_i1010].read(iprot);
+ xfer += this->success[_i1014].read(iprot);
}
xfer += iprot->readListEnd();
}
@@ -7420,10 +7420,10 @@ uint32_t ThriftHiveMetastore_get_table_meta_result::write(::apache::thrift::prot
xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(this->success.size()));
- std::vector<TableMeta> ::const_iterator _iter1011;
- for (_iter1011 = this->success.begin(); _iter1011 != this->success.end(); ++_iter1011)
+ std::vector<TableMeta> ::const_iterator _iter1015;
+ for (_iter1015 = this->success.begin(); _iter1015 != this->success.end(); ++_iter1015)
{
- xfer += (*_iter1011).write(oprot);
+ xfer += (*_iter1015).write(oprot);
}
xfer += oprot->writeListEnd();
}
@@ -7468,14 +7468,14 @@ uint32_t ThriftHiveMetastore_get_table_meta_presult::read(::apache::thrift::prot
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
(*(this->success)).clear();
- uint32_t _size1012;
- ::apache::thrift::protocol::TType _etype1015;
- xfer += iprot->readListBegin(_etype1015, _size1012);
- (*(this->success)).resize(_size1012);
- uint32_t _i1016;
- for (_i1016 = 0; _i1016 < _size1012; ++_i1016)
+ uint32_t _size1016;
+ ::apache::thrift::protocol::TType _etype1019;
+ xfer += iprot->readListBegin(_etype1019, _size1016);
+ (*(this->success)).resize(_size1016);
+ uint32_t _i1020;
+ for (_i1020 = 0; _i1020 < _size1016; ++_i1020)
{
- xfer += (*(this->success))[_i1016].read(iprot);
+ xfer += (*(this->success))[_i1020].read(iprot);
}
xfer += iprot->readListEnd();
}
@@ -7613,14 +7613,14 @@ uint32_t ThriftHiveMetastore_get_all_tables_result::read(::apache::thrift::proto
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->success.clear();
- uint32_t _size1017;
- ::apache::thrift::protocol::TType _etype1020;
- xfer += iprot->readListBegin(_etype1020, _size1017);
- this->success.resize(_size1017);
- uint32_t _i1021;
- for (_i1021 = 0; _i1021 < _size1017; ++_i1021)
+ uint32_t _size1021;
+ ::apache::thrift::protocol::TType _etype1024;
+ xfer += iprot->readListBegin(_etype1024, _size1021);
+ this->success.resize(_size1021);
+ uint32_t _i1025;
+ for (_i1025 = 0; _i1025 < _size1021; ++_i1025)
{
- xfer += iprot->readString(this->success[_i1021]);
+ xfer += iprot->readString(this->success[_i1025]);
}
xfer += iprot->readListEnd();
}
@@ -7659,10 +7659,10 @@ uint32_t ThriftHiveMetastore_get_all_tables_result::write(::apache::thrift::prot
xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->success.size()));
- std::vector<std::string> ::const_iterator _iter1022;
- for (_iter1022 = this->success.begin(); _iter1022 != this->success.end(); ++_iter1022)
+ std::vector<std::string> ::const_iterator _iter1026;
+ for (_iter1026 = this->success.begin(); _iter1026 != this->success.end(); ++_iter1026)
{
- xfer += oprot->writeString((*_iter1022));
+ xfer += oprot->writeString((*_iter1026));
}
xfer += oprot->writeListEnd();
}
@@ -7707,14 +7707,14 @@ uint32_t ThriftHiveMetastore_get_all_tables_presult::read(::apache::thrift::prot
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
(*(this->success)).clear();
- uint32_t _size1023;
- ::apache::thrift::protocol::TType _etype1026;
- xfer += iprot->readListBegin(_etype1026, _size1023);
- (*(this->success)).resize(_size1023);
- uint32_t _i1027;
- for (_i1027 = 0; _i1027 < _size1023; ++_i1027)
+ uint32_t _size1027;
+ ::apache::thrift::protocol::TType _etype1030;
+ xfer += iprot->readListBegin(_etype1030, _size1027);
+ (*(this->success)).resize(_size1027);
+ uint32_t _i1031;
+ for (_i1031 = 0; _i1031 < _size1027; ++_i1031)
{
- xfer += iprot->readString((*(this->success))[_i1027]);
+ xfer += iprot->readString((*(this->success))[_i1031]);
}
xfer += iprot->readListEnd();
}
@@ -8024,14 +8024,14 @@ uint32_t ThriftHiveMetastore_get_table_objects_by_name_args::read(::apache::thri
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->tbl_names.clear();
- uint32_t _size1028;
- ::apache::thrift::protocol::TType _etype1031;
- xfer += iprot->readListBegin(_etype1031, _size1028);
- this->tbl_names.resize(_size1028);
- uint32_t _i1032;
- for (_i1032 = 0; _i1032 < _size1028; ++_i1032)
+ uint32_t _size1032;
+ ::apache::thrift::protocol::TType _etype1035;
+ xfer += iprot->readListBegin(_etype1035, _size1032);
+ this->tbl_names.resize(_size1032);
+ uint32_t _i1036;
+ for (_i1036 = 0; _i1036 < _size1032; ++_i1036)
{
- xfer += iprot->readString(this->tbl_names[_i1032]);
+ xfer += iprot->readString(this->tbl_names[_i1036]);
}
xfer += iprot->readListEnd();
}
@@ -8064,10 +8064,10 @@ uint32_t ThriftHiveMetastore_get_table_objects_by_name_args::write(::apache::thr
xfer += oprot->writeFieldBegin("tbl_names", ::apache::thrift::protocol::T_LIST, 2);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->tbl_names.size()));
- std::vector<std::string> ::const_iterator _iter1033;
- for (_iter1033 = this->tbl_names.begin(); _iter1033 != this->tbl_names.end(); ++_iter1033)
+ std::vector<std::string> ::const_iterator _iter1037;
+ for (_iter1037 = this->tbl_names.begin(); _iter1037 != this->tbl_names.end(); ++_iter1037)
{
- xfer += oprot->writeString((*_iter1033));
+ xfer += oprot->writeString((*_iter1037));
}
xfer += oprot->writeListEnd();
}
@@ -8095,10 +8095,10 @@ uint32_t ThriftHiveMetastore_get_table_objects_by_name_pargs::write(::apache::th
xfer += oprot->writeFieldBegin("tbl_names", ::apache::thrift::protocol::T_LIST, 2);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>((*(this->tbl_names)).size()));
- std::vector<std::string> ::const_iterator _iter1034;
- for (_iter1034 = (*(this->tbl_names)).begin(); _iter1034 != (*(this->tbl_names)).end(); ++_iter1034)
+ std::vector<std::string> ::const_iterator _iter1038;
+ for (_iter1038 = (*(this->tbl_names)).begin(); _iter1038 != (*(this->tbl_names)).end(); ++_iter1038)
{
- xfer += oprot->writeString((*_iter1034));
+ xfer += oprot->writeString((*_iter1038));
}
xfer += oprot->writeListEnd();
}
@@ -8139,14 +8139,14 @@ uint32_t ThriftHiveMetastore_get_table_objects_by_name_result::read(::apache::th
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->success.clear();
- uint32_t _size1035;
- ::apache::thrift::protocol::TType _etype1038;
- xfer += iprot->readListBegin(_etype1038, _size1035);
- this->success.resize(_size1035);
- uint32_t _i1039;
- for (_i1039 = 0; _i1039 < _size1035; ++_i1039)
+ uint32_t _size1039;
+ ::apache::thrift::protocol::TType _etype1042;
+ xfer += iprot->readListBegin(_etype1042, _size1039);
+ this->success.resize(_size1039);
+ uint32_t _i1043;
+ for (_i1043 = 0; _i1043 < _size1039; ++_i1043)
{
- xfer += this->success[_i1039].read(iprot);
+ xfer += this->success[_i1043].read(iprot);
}
xfer += iprot->readListEnd();
}
@@ -8177,10 +8177,10 @@ uint32_t ThriftHiveMetastore_get_table_objects_by_name_result::write(::apache::t
xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(this->success.size()));
- std::vector<Table> ::const_iterator _iter1040;
- for (_iter1040 = this->success.begin(); _iter1040 != this->success.end(); ++_iter1040)
+ std::vector<Table> ::const_iterator _iter1044;
+ for (_iter1044 = this->success.begin(); _iter1044 != this->success.end(); ++_iter1044)
{
- xfer += (*_iter1040).write(oprot);
+ xfer += (*_iter1044).write(oprot);
}
xfer += oprot->writeListEnd();
}
@@ -8221,14 +8221,14 @@ uint32_t ThriftHiveMetastore_get_table_objects_by_name_presult::read(::apache::t
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
(*(this->success)).clear();
- uint32_t _size1041;
- ::apache::thrift::protocol::TType _etype1044;
- xfer += iprot->readListBegin(_etype1044, _size1041);
- (*(this->success)).resize(_size1041);
- uint32_t _i1045;
- for (_i1045 = 0; _i1045 < _size1041; ++_i1045)
+ uint32_t _size1045;
+ ::apache::thrift::protocol::TType _etype1048;
+ xfer += iprot->readListBegin(_etype1048, _size1045);
+ (*(this->success)).resize(_size1045);
+ uint32_t _i1049;
+ for (_i1049 = 0; _i1049 < _size1045; ++_i1049)
{
- xfer += (*(this->success))[_i1045].read(iprot);
+ xfer += (*(this->success))[_i1049].read(iprot);
}
xfer += iprot->readListEnd();
}
@@ -8864,14 +8864,14 @@ uint32_t ThriftHiveMetastore_get_table_names_by_filter_result::read(::apache::th
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->success.clear();
- uint32_t _size1046;
- ::apache::thrift::protocol::TType _etype1049;
- xfer += iprot->readListBegin(_etype1049, _size1046);
- this->success.resize(_size1046);
- uint32_t _i1050;
- for (_i1050 = 0; _i1050 < _size1046; ++_i1050)
+ uint32_t _size1050;
+ ::apache::thrift::protocol::TType _etype1053;
+ xfer += iprot->readListBegin(_etype1053, _size1050);
+ this->success.resize(_size1050);
+ uint32_t _i1054;
+ for (_i1054 = 0; _i1054 < _size1050; ++_i1054)
{
- xfer += iprot->readString(this->success[_i1050]);
+ xfer += iprot->readString(this->success[_i1054]);
}
xfer += iprot->readListEnd();
}
@@ -8926,10 +8926,10 @@ uint32_t ThriftHiveMetastore_get_table_names_by_filter_result::write(::apache::t
xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->success.size()));
- std::vector<std::string> ::const_iterator _iter1051;
- for (_iter1051 = this->success.begin(); _iter1051 != this->success.end(); ++_iter1051)
+ std::vector<std::string> ::const_iterator _iter1055;
+ for (_iter1055 = this->success.begin(); _iter1055 != this->success.end(); ++_iter1055)
{
- xfer += oprot->writeString((*_iter1051));
+ xfer += oprot->writeString((*_iter1055));
}
xfer += oprot->writeListEnd();
}
@@ -8982,14 +8982,14 @@ uint32_t ThriftHiveMetastore_get_table_names_by_filter_presult::read(::apache::t
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
(*(this->success)).clear();
- uint32_t _size1052;
- ::apache::thrift::protocol::TType _etype1055;
- xfer += iprot->readListBegin(_etype1055, _size1052);
- (*(this->success)).resize(_size1052);
- uint32_t _i1056;
- for (_i1056 = 0; _i1056 < _size1052; ++_i1056)
+ uint32_t _size1056;
+ ::apache::thrift::protocol::TType _etype1059;
+ xfer += iprot->readListBegin(_etype1059, _size1056);
+ (*(this->success)).resize(_size1056);
+ uint32_t _i1060;
+ for (_i1060 = 0; _i1060 < _size1056; ++_i1060)
{
- xfer += iprot->readString((*(this->success))[_i1056]);
+ xfer += iprot->readString((*(this->success))[_i1060]);
}
xfer += iprot->readListEnd();
}
@@ -10323,14 +10323,14 @@ uint32_t ThriftHiveMetastore_add_partitions_args::read(::apache::thrift::protoco
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->new_parts.clear();
- uint32_t _size1057;
- ::apache::thrift::protocol::TType _etype1060;
- xfer += iprot->readListBegin(_etype1060, _size1057);
- this->new_parts.resize(_size1057);
- uint32_t _i1061;
- for (_i1061 = 0; _i1061 < _size1057; ++_i1061)
+ uint32_t _size1061;
+ ::apache::thrift::protocol::TType _etype1064;
+ xfer += iprot->readListBegin(_etype1064, _size1061);
+ this->new_parts.resize(_size1061);
+ uint32_t _i1065;
+ for (_i1065 = 0; _i1065 < _size1061; ++_i1065)
{
- xfer += this->new_parts[_i1061].read(iprot);
+ xfer += this->new_parts[_i1065].read(iprot);
}
xfer += iprot->readListEnd();
}
@@ -10359,10 +10359,10 @@ uint32_t ThriftHiveMetastore_add_partitions_args::write(::apache::thrift::protoc
xfer += oprot->writeFieldBegin("new_parts", ::apache::thrift::protocol::T_LIST, 1);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(this->new_parts.size()));
- std::vector<Partition> ::const_iterator _iter1062;
- for (_iter1062 = this->new_parts.begin(); _iter1062 != this->new_parts.end(); ++_iter1062)
+ std::vector<Partition> ::const_iterator _iter1066;
+ for (_iter1066 = this->new_parts.begin(); _iter1066 != this->new_parts.end(); ++_iter1066)
{
- xfer += (*_iter1062).write(oprot);
+ xfer += (*_iter1066).write(oprot);
}
xfer += oprot->writeListEnd();
}
@@ -10386,10 +10386,10 @@ uint32_t ThriftHiveMetastore_add_partitions_pargs::write(::apache::thrift::proto
xfer += oprot->writeFieldBegin("new_parts", ::apache::thrift::protocol::T_LIST, 1);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>((*(this->new_parts)).size()));
- std::vector<Partition> ::const_iterator _iter1063;
- for (_iter1063 = (*(this->new_parts)).begin(); _iter1063 != (*(this->new_parts)).end(); ++_iter1063)
+ std::vector<Partition> ::const_iterator _iter1067;
+ for (_iter1067 = (*(this->new_parts)).begin(); _iter1067 != (*(this->new_parts)).end(); ++_iter1067)
{
- xfer += (*_iter1063).write(oprot);
+ xfer += (*_iter1067).write(oprot);
}
xfer += oprot->writeListEnd();
}
@@ -10598,14 +10598,14 @@ uint32_t ThriftHiveMetastore_add_partitions_pspec_args::read(::apache::thrift::p
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->new_parts.clear();
- uint32_t _size1064;
- ::apache::thrift::protocol::TType _etype1067;
- xfer += iprot->readListBegin(_etype1067, _size1064);
- this->new_parts.resize(_size1064);
- uint32_t _i1068;
- for (_i1068 = 0; _i1068 < _size1064; ++_i1068)
+ uint32_t _size1068;
+ ::apache::thrift::protocol::TType _etype1071;
+ xfer += iprot->readListBegin(_etype1071, _size1068);
+ this->new_parts.resize(_size1068);
+ uint32_t _i1072;
+ for (_i1072 = 0; _i1072 < _size1068; ++_i1072)
{
- xfer += this->new_parts[_i1068].read(iprot);
+ xfer += this->new_parts[_i1072].read(iprot);
}
xfer += iprot->readListEnd();
}
@@ -10634,10 +10634,10 @@ uint32_t ThriftHiveMetastore_add_partitions_pspec_args::write(::apache::thrift::
xfer += oprot->writeFieldBegin("new_parts", ::apache::thrift::protocol::T_LIST, 1);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(this->new_parts.size()));
- std::vector<PartitionSpec> ::const_iterator _iter1069;
- for (_iter1069 = this->new_parts.begin(); _iter1069 != this->new_parts.end(); ++_iter1069)
+ std::vector<PartitionSpec> ::const_iterator _iter1073;
+ for (_iter1073 = this->new_parts.begin(); _iter1073 != this->new_parts.end(); ++_iter1073)
{
- xfer += (*_iter1069).write(oprot);
+ xfer += (*_iter1073).write(oprot);
}
xfer += oprot->writeListEnd();
}
@@ -10661,10 +10661,10 @@ uint32_t ThriftHiveMetastore_add_partitions_pspec_pargs::write(::apache::thrift:
xfer += oprot->writeFieldBegin("new_parts", ::apache::thrift::protocol::T_LIST, 1);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>((*(this->new_parts)).size()));
- std::vector<PartitionSpec> ::const_iterator _iter1070;
- for (_iter1070 = (*(this->new_parts)).begin(); _iter1070 != (*(this->new_parts)).end(); ++_iter1070)
+ std::vector<PartitionSpec> ::const_iterator _iter1074;
+ for (_iter1074 = (*(this->new_parts)).begin(); _iter1074 != (*(this->new_parts)).end(); ++_iter1074)
{
- xfer += (*_iter1070).write(oprot);
+ xfer += (*_iter1074).write(oprot);
}
xfer += oprot->writeListEnd();
}
@@ -10889,14 +10889,14 @@ uint32_t ThriftHiveMetastore_append_partition_args::read(::apache::thrift::proto
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->part_vals.clear();
- uint32_t _size1071;
- ::apache::thrift::protocol::TType _etype1074;
- xfer += iprot->readListBegin(_etype1074, _size1071);
- this->part_vals.resize(_size1071);
- uint32_t _i1075;
- for (_i1075 = 0; _i1075 < _size1071; ++_i1075)
+ uint32_t _size1075;
+ ::apache::thrift::protocol::TType _etype1078;
+ xfer += iprot->readListBegin(_etype1078, _size1075);
+ this->part_vals.resize(_size1075);
+ uint32_t _i1079;
+ for (_i1079 = 0; _i1079 < _size1075; ++_i1079)
{
- xfer += iprot->readString(this->part_vals[_i1075]);
+ xfer += iprot->readString(this->part_vals[_i1079]);
}
xfer += iprot->readListEnd();
}
@@ -10933,10 +10933,10 @@ uint32_t ThriftHiveMetastore_append_partition_args::write(::apache::thrift::prot
xfer += oprot->writeFieldBegin("part_vals", ::apache::thrift::protocol::T_LIST, 3);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->part_vals.size()));
- std::vector<std::string> ::const_iterator _iter1076;
- for (_iter1076 = this->part_vals.begin(); _iter1076 != this->part_vals.end(); ++_iter1076)
+ std::vector<std::string> ::const_iterator _iter1080;
+ for (_iter1080 = this->part_vals.begin(); _iter1080 != this->part_vals.end(); ++_iter1080)
{
- xfer += oprot->writeString((*_iter1076));
+ xfer += oprot->writeString((*_iter1080));
}
xfer += oprot->writeListEnd();
}
@@ -10968,10 +10968,10 @@ uint32_t ThriftHiveMetastore_append_partition_pargs::write(::apache::thrift::pro
xfer += oprot->writeFieldBegin("part_vals", ::apache::thrift::protocol::T_LIST, 3);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>((*(this->part_vals)).size()));
- std::vector<std::string> ::const_iterator _iter1077;
- for (_iter1077 = (*(this->part_vals)).begin(); _iter1077 != (*(this->part_vals)).end(); ++_iter1077)
+ std::vector<std::string> ::const_iterator _iter1081;
+ for (_iter1081 = (*(this->part_vals)).begin(); _iter1081 != (*(this->part_vals)).end(); ++_iter1081)
{
- xfer += oprot->writeString((*_iter1077));
+ xfer += oprot->writeString((*_iter1081));
}
xfer += oprot->writeListEnd();
}
@@ -11443,14 +11443,14 @@ uint32_t ThriftHiveMetastore_append_partition_with_environment_context_args::rea
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->part_vals.clear();
- uint32_t _size1078;
- ::apache::thrift::protocol::TType _etype1081;
- xfer += iprot->readListBegin(_etype1081, _size1078);
- this->part_vals.resize(_size1078);
- uint32_t _i1082;
- for (_i1082 = 0; _i1082 < _size1078; ++_i1082)
+ uint32_t _size1082;
+ ::apache::thrift::protocol::TType _etype1085;
+ xfer += iprot->readListBegin(_etype1085, _size1082);
+ this->part_vals.resize(_size1082);
+ uint32_t _i1086;
+ for (_i1086 = 0; _i1086 < _size1082; ++_i1086)
{
- xfer += iprot->readString(this->part_vals[_i1082]);
+ xfer += iprot->readString(this->part_vals[_i1086]);
}
xfer += iprot->readListEnd();
}
@@ -11495,10 +11495,10 @@ uint32_t ThriftHiveMetastore_append_partition_with_environment_context_args::wri
xfer += oprot->writeFieldBegin("part_vals", ::apache::thrift::protocol::T_LIST, 3);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->part_vals.size()));
- std::vector<std::string> ::const_iterator _iter1083;
- for (_iter1083 = this->part_vals.begin(); _iter1083 != this->part_vals.end(); ++_iter1083)
+ std::vector<std::string> ::const_iterator _iter1087;
+ for (_iter1087 = this->part_vals.begin(); _iter1087 != this->part_vals.end(); ++_iter1087)
{
- xfer += oprot->writeString((*_iter1083));
+ xfer += oprot->writeString((*_iter1087));
}
xfer += oprot->writeListEnd();
}
@@ -11534,10 +11534,10 @@ uint32_t ThriftHiveMetastore_append_partition_with_environment_context_pargs::wr
xfer += oprot->writeFieldBegin("part_vals", ::apache::thrift::protocol::T_LIST, 3);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>((*(this->part_vals)).size()));
- std::vector<std::string> ::const_iterator _iter1084;
- for (_iter1084 = (*(this->part_vals)).begin(); _iter1084 != (*(this->part_vals)).end(); ++_iter1084)
+ std::vector<std::string> ::const_iterator _iter1088;
+ for (_iter1088 = (*(this->part_vals)).begin(); _iter1088 != (*(this->part_vals)).end(); ++_iter1088)
{
- xfer += oprot->writeString((*_iter1084));
+ xfer += oprot->writeString((*_iter1088));
}
xfer += oprot->writeListEnd();
}
@@ -12340,14 +12340,14 @@ uint32_t ThriftHiveMetastore_drop_partition_args::read(::apache::thrift::protoco
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->part_vals.clear();
- uint32_t _size1085;
- ::apache::thrift::protocol::TType _etype1088;
- xfer += iprot->readListBegin(_etype1088, _size1085);
- this->part_vals.resize(_size1085);
- uint32_t _i1089;
- for (_i1089 = 0; _i1089 < _size1085; ++_i1089)
+ uint32_t _size1089;
+ ::apache::thrift::protocol::TType _etype1092;
+ xfer += iprot->readListBegin(_etype1092, _size1089);
+ this->part_vals.resize(_size1089);
+ uint32_t _i1093;
+ for (_i1093 = 0; _i1093 < _size1089; ++_i1093)
{
- xfer += iprot->readString(this->part_vals[_i1089]);
+ xfer += iprot->readString(this->part_vals[_i1093]);
}
xfer += iprot->readListEnd();
}
@@ -12392,10 +12392,10 @@ uint32_t ThriftHiveMetastore_drop_partition_args::write(::apache::thrift::protoc
xfer += oprot->writeFieldBegin("part_vals", ::apache::thrift::protocol::T_LIST, 3);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->part_vals.size()));
- std::vector<std::string> ::const_iterator _iter1090;
- for (_iter1090 = this->part_vals.begin(); _iter1090 != this->part_vals.end(); ++_iter1090)
+ std::vector<std::string> ::const_iterator _iter1094;
+ for (_iter1094 = this->part_vals.begin(); _iter1094 != this->part_vals.end(); ++_iter1094)
{
- xfer += oprot->writeString((*_iter1090));
+ xfer += oprot->writeString((*_iter1094));
}
xfer += oprot->writeListEnd();
}
@@ -12431,10 +12431,10 @@ uint32_t ThriftHiveMetastore_drop_partition_pargs::write(::apache::thrift::proto
xfer += oprot->writeFieldBegin("part_vals", ::apache::thrift::protocol::T_LIST, 3);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>((*(this->part_vals)).size()));
- std::vector<std::string> ::const_iterator _iter1091;
- for (_iter1091 = (*(this->part_vals)).begin(); _iter1091 != (*(this->part_vals)).end(); ++_iter1091)
+ std::vector<std::string> ::const_iterator _iter1095;
+ for (_iter1095 = (*(this->part_vals)).begin(); _iter1095 != (*(this->part_vals)).end(); ++_iter1095)
{
- xfer += oprot->writeString((*_iter1091));
+ xfer += oprot->writeString((*_iter1095));
}
xfer += oprot->writeListEnd();
}
@@ -12643,14 +12643,14 @@ uint32_t ThriftHiveMetastore_drop_partition_with_environment_context_args::read(
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->part_vals.clear();
- uint32_t _size1092;
- ::apache::thrift::protocol::TType _etype1095;
- xfer += iprot->readListBegin(_etype1095, _size1092);
- this->part_vals.resize(_size1092);
- uint32_t _i1096;
- for (_i1096 = 0; _i1096 < _size1092; ++_i1096)
+ uint32_t _size1096;
+ ::apache::thrift::protocol::TType _etype1099;
+ xfer += iprot->readListBegin(_etype1099, _size1096);
+ this->part_vals.resize(_size1096);
+ uint32_t _i1100;
+ for (_i1100 = 0; _i1100 < _size1096; ++_i1100)
{
- xfer += iprot->readString(this->part_vals[_i1096]);
+ xfer += iprot->readString(this->part_vals[_i1100]);
}
xfer += iprot->readListEnd();
}
@@ -12703,10 +12703,10 @@ uint32_t ThriftHiveMetastore_drop_partition_with_environment_context_args::write
xfer += oprot->writeFieldBegin("part_vals", ::apache::thrift::protocol::T_LIST, 3);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->part_vals.size()));
- std::vector<std::string> ::const_iterator _iter1097;
- for (_iter1097 = this->part_vals.begin(); _iter1097 != this->part_vals.end(); ++_iter1097)
+ std::vector<std::string> ::const_iterator _iter1101;
+ for (_iter1101 = this->part_vals.begin(); _iter1101 != this->part_vals.end(); ++_iter1101)
{
- xfer += oprot->writeString((*_iter1097));
+ xfer += oprot->writeString((*_iter1101));
}
xfer += oprot->writeListEnd();
}
@@ -12746,10 +12746,10 @@ uint32_t ThriftHiveMetastore_drop_partition_with_environment_context_pargs::writ
xfer += oprot->writeFieldBegin("part_vals", ::apache::thrift::protocol::T_LIST, 3);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>((*(this->part_vals)).size()));
- std::vector<std::string> ::const_iterator _iter1098;
- for (_iter1098 = (*(this->part_vals)).begin(); _iter1098 != (*(this->part_vals)).end(); ++_iter1098)
+ std::vector<std::string> ::const_iterator _iter1102;
+ for (_iter1102 = (*(this->part_vals)).begin(); _iter1102 != (*(this->part_vals)).end(); ++_iter1102)
{
- xfer += oprot->writeString((*_iter1098));
+ xfer += oprot->writeString((*_iter1102));
}
xfer += oprot->writeListEnd();
}
@@ -13755,14 +13755,14 @@ uint32_t ThriftHiveMetastore_get_partition_args::read(::apache::thrift::protocol
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->part_vals.clear();
- uint32_t _size1099;
- ::apache::thrift::protocol::TType _etype1102;
- xfer += iprot->readListBegin(_etype1102, _size1099);
- this->part_vals.resize(_size1099);
- uint32_t _i1103;
- for (_i1103 = 0; _i1103 < _size1099; ++_i1103)
+ uint32_t _size1103;
+ ::apache::thrift::protocol::TType _etype1106;
+ xfer += iprot->readListBegin(_etype1106, _size1103);
+ this->part_vals.resize(_size1103);
+ uint32_t _i1107;
+ for (_i1107 = 0; _i1107 < _size1103; ++_i1107)
{
- xfer += iprot->readString(this->part_vals[_i1103]);
+ xfer += iprot->readString(this->part_vals[_i1107]);
}
xfer += iprot->readListEnd();
}
@@ -13799,10 +13799,10 @@ uint32_t ThriftHiveMetastore_get_partition_args::write(::apache::thrift::protoco
xfer += oprot->writeFieldBegin("part_vals", ::apache::thrift::protocol::T_LIST, 3);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->part_vals.size()));
- std::vector<std::string> ::const_iterator _iter1104;
- for (_iter1104 = this->part_vals.begin(); _iter1104 != this->part_vals.end(); ++_iter1104)
+ std::vector<std::string> ::const_iterator _iter1108;
+ for (_iter1108 = this->part_vals.begin(); _iter1108 != this->part_vals.end(); ++_iter1108)
{
- xfer += oprot->writeString((*_iter1104));
+ xfer += oprot->writeString((*_iter1108));
}
xfer += oprot->writeListEnd();
}
@@ -13834,10 +13834,10 @@ uint32_t ThriftHiveMetastore_get_partition_pargs::write(::apache::thrift::protoc
xfer += oprot->writeFieldBegin("part_vals", ::apache::thrift::protocol::T_LIST, 3);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>((*(this->part_vals)).size()));
- std::vector<std::string> ::const_iterator _iter1105;
- for (_iter1105 = (*(this->part_vals)).begin(); _iter1105 != (*(this->part_vals)).end(); ++_iter1105)
+ std::vector<std::string> ::const_iterator _iter1109;
+ for (_iter1109 = (*(this->part_vals)).begin(); _iter1109 != (*(this->part_vals)).end(); ++_iter1109)
{
- xfer += oprot->writeString((*_iter1105));
+ xfer += oprot->writeString((*_iter1109));
}
xfer += oprot->writeListEnd();
}
@@ -14026,17 +14026,17 @@ uint32_t ThriftHiveMetastore_exchange_partition_args::read(::apache::thrift::pro
if (ftype == ::apache::thrift::protocol::T_MAP) {
{
this->partitionSpecs.clear();
- uint32_t _size1106;
- ::apache::thrift::protocol::TType _ktype1107;
- ::apache::thrift::protocol::TType _vtype1108;
- xfer += iprot->readMapBegin(_ktype1107, _vtype1108, _size1106);
- uint32_t _i1110;
- for (_i1110 = 0; _i1110 < _size1106; ++_i1110)
+ uint32_t _size1110;
+ ::apache::thrift::protocol::TType _ktype1111;
+ ::apache::thrift::protocol::TType _vtype1112;
+ xfer += iprot->readMapBegin(_ktype1111, _vtype1112, _size1110);
+ uint32_t _i1114;
+ for (_i1114 = 0; _i1114 < _size1110; ++_i1114)
{
- std::string _key1111;
- xfer += iprot->readString(_key1111);
- std::string& _val1112 = this->partitionSpecs[_key1111];
- xfer += iprot->readString(_val1112);
+ std::string _key1115;
+ xfer += iprot->readString(_key1115);
+ std::string& _val1116 = this->partitionSpecs[_key1115];
+ xfer += iprot->readString(_val1116);
}
xfer += iprot->readMapEnd();
}
@@ -14097,11 +14097,11 @@ uint32_t ThriftHiveMetastore_exchange_partition_args::write(::apache::thrift::pr
xfer += oprot->writeFieldBegin("partitionSpecs", ::apache::thrift::protocol::T_MAP, 1);
{
xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->partitionSpecs.size()));
- std::map<std::string, std::string> ::const_iterator _iter1113;
- for (_iter1113 = this->partitionSpecs.begin(); _iter1113 != this->partitionSpecs.end(); ++_iter1113)
+ std::map<std::string, std::string> ::const_iterator _iter1117;
+ for (_iter1117 = this->partitionSpecs.begin(); _iter1117 != this->partitionSpecs.end(); ++_iter1117)
{
- xfer += oprot->writeString(_iter1113->first);
- xfer += oprot->writeString(_iter1113->second);
+ xfer += oprot->writeString(_iter1117->first);
+ xfer += oprot->writeString(_iter1117->second);
}
xfer += oprot->writeMapEnd();
}
@@ -14141,11 +14141,11 @@ uint32_t ThriftHiveMetastore_exchange_partition_pargs::write(::apache::thrift::p
xfer += oprot->writeFieldBegin("partitionSpecs", ::apache::thrift::protocol::T_MAP, 1);
{
xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_STRING, static_cast<uint32_t>((*(this->partitionSpecs)).size()));
- std::map<std::string, std::string> ::const_iterator _iter1114;
- for (_iter1114 = (*(this->partitionSpecs)).begin(); _iter1114 != (*(this->partitionSpecs)).end(); ++_iter1114)
+ std::map<std::string, std::string> ::const_iterator _iter1118;
+ for (_iter1118 = (*(this->partitionSpecs)).begin(); _iter1118 != (*(this->partitionSpecs)).end(); ++_iter1118)
{
- xfer += oprot->writeString(_iter1114->first);
- xfer += oprot->writeString(_iter1114->second);
+ xfer += oprot->writeString(_iter1118->first);
+ xfer += oprot->writeString(_iter1118->second);
}
xfer += oprot->writeMapEnd();
}
@@ -14390,17 +14390,17 @@ uint32_t ThriftHiveMetastore_exchange_partitions_args::read(::apache::thrift::pr
if (ftype == ::apache::thrift::protocol::T_MAP) {
{
this->partitionSpecs.clear();
- uint32_t _size1115;
- ::apache::thrift::protocol::TType _ktype1116;
- ::apache::thrift::protocol::TType _vtype1117;
- xfer += iprot->readMapBegin(_ktype1116, _vtype1117, _size1115);
- uint32_t _i1119;
- for (_i1119 = 0; _i1119 < _size1115; ++_i1119)
+ uint32_t _size1119;
+ ::apache::thrift::protocol::TType _ktype1120;
+ ::apache::thrift::protocol::TType _vtype1121;
+ xfer += iprot->readMapBegin(_ktype1120, _vtype1121, _size1119);
+ uint32_t _i1123;
+ for (_i1123 = 0; _i1123 < _size1119; ++_i1123)
{
- std::string _key1120;
- xfer += iprot->readString(_key1120);
- std::string& _val1121 = this->partitionSpecs[_key1120];
- xfer += iprot->readString(_val1121);
+ std::string _key1124;
+ xfer += iprot->readString(_key1124);
+ std::string& _val1125 = this->partitionSpecs[_key1124];
+ xfer += iprot->readString(_val1125);
}
xfer += iprot->readMapEnd();
}
@@ -14461,11 +14461,11 @@ uint32_t ThriftHiveMetastore_exchange_partitions_args::write(::apache::thrift::p
xfer += oprot->writeFieldBegin("partitionSpecs", ::apache::thrift::protocol::T_MAP, 1);
{
xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->partitionSpecs.size()));
- std::map<std::string, std::string> ::const_iterator _iter1122;
- for (_iter1122 = this->partitionSpecs.begin(); _iter1122 != this->partitionSpecs.end(); ++_iter1122)
+ std::map<std::string, std::string> ::const_iterator _iter1126;
+ for (_iter1126 = this->partitionSpecs.begin(); _iter1126 != this->partitionSpecs.end(); ++_iter1126)
{
- xfer += oprot->writeString(_iter1122->first);
- xfer += oprot->writeString(_iter1122->second);
+ xfer += oprot->writeString(_iter1126->first);
+ xfer += oprot->writeString(_iter1126->second);
}
xfer += oprot->writeMapEnd();
}
@@ -14505,11 +14505,11 @@ uint32_t ThriftHiveMetastore_exchange_partitions_pargs::write(::apache::thrift::
xfer += oprot->writeFieldBegin("partitionSpecs", ::apache::thrift::protocol::T_MAP, 1);
{
xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_STRING, static_cast<uint32_t>((*(this->partitionSpecs)).size()));
- std::map<std::string, std::string> ::const_iterator _iter1123;
- for (_iter1123 = (*(this->partitionSpecs)).begin(); _iter1123 != (*(this->partitionSpecs)).end(); ++_iter1123)
+ std::map<std::string, std::string> ::const_iterator _iter1127;
+ for (_iter1127 = (*(this->partitionSpecs)).begin(); _iter1127 != (*(this->partitionSpecs)).end(); ++_iter1127)
{
- xfer += oprot->writeString(_iter1123->first);
- xfer += oprot->writeString(_iter1123->second);
+ xfer += oprot->writeString(_iter1127->first);
+ xfer += oprot->writeString(_iter1127->second);
}
xfer += oprot->writeMapEnd();
}
@@ -14566,14 +14566,14 @@ uint32_t ThriftHiveMetastore_exchange_partitions_result::read(::apache::thrift::
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->success.clear();
- uint32_t _size1124;
- ::apache::thrift::protocol::TType _etype1127;
- xfer += iprot->readListBegin(_etype1127, _size1124);
- this->success.resize(_size1124);
- uint32_t _i1128;
- for (_i1128 = 0; _i1128 < _size1124; ++_i1128)
+ uint32_t _size1128;
+ ::apache::thrift::protocol::TType _etype1131;
+ xfer += iprot->readListBegin(_etype1131, _size1128);
+ this->success.resize(_size1128);
+ uint32_t _i1132;
+ for (_i1132 = 0; _i1132 < _size1128; ++_i1132)
{
- xfer += this->success[_i1128].read(iprot);
+ xfer += this->success[_i1132].read(iprot);
}
xfer += iprot->readListEnd();
}
@@ -14636,10 +14636,10 @@ uint32_t ThriftHiveMetastore_exchange_partitions_result::write(::apache::thrift:
xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(this->success.size()));
- std::vector<Partition> ::const_iterator _iter1129;
- for (_iter1129 = this->success.begin(); _iter1129 != this->success.end(); ++_iter1129)
+ std::vector<Partition> ::const_iterator _iter1133;
+ for (_iter1133 = this->success.begin(); _iter1133 != this->success.end(); ++_iter1133)
{
- xfer += (*_iter1129).write(oprot);
+ xfer += (*_iter1133).write(oprot);
}
xfer += oprot->writeListEnd();
}
@@ -14696,14 +14696,14 @@ uint32_t ThriftHiveMetastore_exchange_partitions_presult::read(::apache::thrift:
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
(*(this->success)).clear();
- uint32_t _size1130;
- ::apache::thrift::protocol::TType _etype1133;
- xfer += iprot->readListBegin(_etype1133, _size1130);
- (*(this->success)).resize(_size1130);
- uint32_t _i1134;
- for (_i1134 = 0; _i1134 < _size1130; ++_i1134)
+ uint32_t _size1134;
+ ::apache::thrift::protocol::TType _etype1137;
+ xfer += iprot->readListBegin(_etype1137, _size1134);
+ (*(this->success)).resize(_size1134);
+ uint32_t _i1138;
+ for (_i1138 = 0; _i1138 < _size1134; ++_i1138)
{
- xfer += (*(this->success))[_i1134].read(iprot);
+ xfer += (*(this->success))[_i1138].read(iprot);
}
xfer += iprot->readListEnd();
}
@@ -14802,14 +14802,14 @@ uint32_t ThriftHiveMetastore_get_partition_with_auth_args::read(::apache::thrift
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->part_vals.clear();
- uint32_t _size1135;
- ::apache::thrift::protocol::TType _etype1138;
- xfer += iprot->readListBegin(_etype1138, _size1135);
- this->part_vals.resize(_size1135);
- uint32_t _i1139;
- for (_i1139 = 0; _i1139 < _size1135; ++_i1139)
+ uint32_t _size1139;
+ ::apache::thrift::protocol::TType _etype1142;
+ xfer += iprot->readListBegin(_etype1142, _size1139);
+ this->part_vals.resize(_size1139);
+ uint32_t _i1143;
+ for (_i1143 = 0; _i1143 < _size1139; ++_i1143)
{
- xfer += iprot->readString(this->part_vals[_i1139]);
+ xfer += iprot->readString(this->part_vals[_i1143]);
}
xfer += iprot->readListEnd();
}
@@ -14830,14 +14830,14 @@ uint32_t ThriftHiveMetastore_get_partition_with_auth_args::read(::apache::thrift
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->group_names.clear();
- uint32_t _size1140;
- ::apache::thrift::protocol::TType _etype1143;
- xfer += iprot->readListBegin(_etype1143, _size1140);
- this->group_names.resize(_size1140);
- uint32_t _i1144;
- for (_i1144 = 0; _i1144 < _size1140; ++_i1144)
+ uint32_t _size1144;
+ ::apache::thrift::protocol::TType _etype1147;
+ xfer += iprot->readListBegin(_etype1147, _size1144);
+ this->group_names.resize(_size1144);
+ uint32_t _i1148;
+ for (_i1148 = 0; _i1148 < _size1144; ++_i1148)
{
- xfer += iprot->readString(this->group_names[_i1144]);
+ xfer += iprot->readString(this->group_names[_i1148]);
}
xfer += iprot->readListEnd();
}
@@ -14874,10 +14874,10 @@ uint32_t ThriftHiveMetastore_get_partition_with_auth_args::write(::apache::thrif
xfer += oprot->writeFieldBegin("part_vals", ::apache::thrift::protocol::T_LIST, 3);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->part_vals.size()));
- std::vector<std::string> ::const_iterator _iter1145;
- for (_iter1145 = this->part_vals.begin(); _iter1145 != this->part_vals.end(); ++_iter1145)
+ std::vector<std::string> ::const_iterator _iter1149;
+ for (_iter1149 = this->part_vals.begin(); _iter1149 != this->part_vals.end(); ++_iter1149)
{
- xfer += oprot->writeString((*_iter1145));
+ xfer += oprot->writeString((*_iter1149));
}
xfer += oprot->writeListEnd();
}
@@ -14890,10 +14890,10 @@ uint32_t ThriftHiveMetastore_get_partition_with_auth_args::write(::apache::thrif
xfer += oprot->writeFieldBegin("group_names", ::apache::thrift::protocol::T_LIST, 5);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->group_names.size()));
- std::vector<std::string> ::const_iterator _iter1146;
- for (_iter1146 = this->group_names.begin(); _iter1146 != this->group_names.end(); ++_iter1146)
+ std::vector<std::string> ::const_iterator _iter1150;
+ for (_iter1150 = this->group_names.begin(); _iter1150 != this->group_names.end(); ++_iter1150)
{
- xfer += oprot->writeString((*_iter1146));
+ xfer += oprot->writeString((*_iter1150));
}
xfer += oprot->writeListEnd();
}
@@ -14925,10 +14925,10 @@ uint32_t ThriftHiveMetastore_get_partition_with_auth_pargs::write(::apache::thri
xfer += oprot->writeFieldBegin("part_vals", ::apache::thrift::protocol::T_LIST, 3);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>((*(this->part_vals)).size()));
- std::vector<std::string> ::const_iterator _iter1147;
- for (_iter1147 = (*(this->part_vals)).begin(); _iter1147 != (*(this->part_vals)).end(); ++_iter1147)
+ std::vector<std::string> ::const_iterator _iter1151;
+ for (_iter1151 = (*(this->part_vals)).begin(); _iter1151 != (*(this->part_vals)).end(); ++_iter1151)
{
- xfer += oprot->writeString((*_iter1147));
+ xfer += oprot->writeString((*_iter1151));
}
xfer += oprot->writeListEnd();
}
@@ -14941,10 +14941,10 @@ uint32_t ThriftHiveMetastore_get_partition_with_auth_pargs::write(::apache::thri
xfer += oprot->writeFieldBegin("group_names", ::apache::thrift::protocol::T_LIST, 5);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>((*(this->group_names)).size()));
- std::vector<std::string> ::const_iterator _iter1148;
- for (_iter1148 = (*(this->group_names)).begin(); _iter1148 != (*(this->group_names)).end(); ++_iter1148)
+ std::vector<std::string> ::const_iterator _iter1152;
+ for (_iter1152 = (*(this->group_names)).begin(); _iter1152 != (*(this->group_names)).end(); ++_iter1152)
{
- xfer += oprot->writeString((*_iter1148));
+ xfer += oprot->writeString((*_iter1152));
}
xfer += oprot->writeListEnd();
}
@@ -15503,14 +15503,14 @@ uint32_t ThriftHiveMetastore_get_partitions_result::read(::apache::thrift::proto
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->success.clear();
- uint32_t _size1149;
- ::apache::thrift::protocol::TType _etype1152;
- xfer += iprot->readListBegin(_etype1152, _size1149);
- this->success.resize(_size1149);
- uint32_t _i1153;
- for (_i1153 = 0; _i1153 < _size1149; ++_i1153)
+ uint32_t
<TRUNCATED>
[50/50] [abbrv] hive git commit: HIVE-14671 : merge master into
hive-14535 (Wei Zheng)
Posted by we...@apache.org.
HIVE-14671 : merge master into hive-14535 (Wei Zheng)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/42a38577
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/42a38577
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/42a38577
Branch: refs/heads/hive-14535
Commit: 42a38577bc9d242bed040c503b0d1d46f74e0301
Parents: 81853c1 b82d38a
Author: Wei Zheng <we...@apache.org>
Authored: Wed Sep 6 21:24:02 2017 -0700
Committer: Wei Zheng <we...@apache.org>
Committed: Wed Sep 6 21:24:02 2017 -0700
----------------------------------------------------------------------
.gitignore | 1 +
LICENSE | 2 +-
.../accumulo/AccumuloConnectionParameters.java | 57 +-
.../hive/accumulo/AccumuloStorageHandler.java | 77 +-
.../hive/accumulo/HiveAccumuloHelper.java | 396 +-
.../org/apache/hadoop/hive/accumulo/Utils.java | 3 +-
.../mr/HiveAccumuloTableInputFormat.java | 244 +-
.../mr/HiveAccumuloTableOutputFormat.java | 84 +-
.../serde/CompositeAccumuloRowIdFactory.java | 3 +-
.../serde/DefaultAccumuloRowIdFactory.java | 5 +-
.../accumulo/TestAccumuloStorageHandler.java | 6 +-
.../hive/accumulo/TestHiveAccumuloHelper.java | 184 +-
.../mr/TestHiveAccumuloTableInputFormat.java | 37 +-
.../mr/TestHiveAccumuloTableOutputFormat.java | 29 +-
.../positive/accumulo_predicate_pushdown.q.out | 3 +-
.../results/positive/accumulo_queries.q.out | 2 +-
.../java/org/apache/hive/beeline/BeeLine.java | 4 +-
.../org/apache/hive/beeline/BeeLineOpts.java | 9 +
.../java/org/apache/hive/beeline/Commands.java | 32 +-
.../org/apache/hive/beeline/HiveSchemaTool.java | 58 +-
.../src/java/org/apache/hive/beeline/Rows.java | 21 +-
beeline/src/main/resources/BeeLine.properties | 1 +
.../apache/hive/beeline/TestBufferedRows.java | 2 +-
.../org/apache/hive/beeline/TestCommands.java | 29 +-
.../hive/beeline/TestIncrementalRows.java | 136 +
.../TestIncrementalRowsWithNormalization.java | 90 -
.../hive/beeline/TestTableOutputFormat.java | 2 +-
bin/ext/hbaseimport.sh | 27 -
bin/ext/hbaseschematool.sh | 27 -
bin/ext/metastore.sh | 2 +-
binary-package-licenses/org.json-LICENSE | 22 -
.../org/apache/hadoop/hive/cli/CliDriver.java | 10 +-
common/pom.xml | 8 +-
.../apache/hadoop/hive/common/FileUtils.java | 80 +-
.../org/apache/hadoop/hive/common/LogUtils.java | 41 +
.../hadoop/hive/common/jsonexplain/Vertex.java | 2 +-
.../common/jsonexplain/tez/TezJsonParser.java | 2 +
.../common/metrics/common/MetricsConstant.java | 14 -
.../common/ndv/NumDistinctValueEstimator.java | 51 +
.../ndv/NumDistinctValueEstimatorFactory.java | 74 +
.../hadoop/hive/common/ndv/fm/FMSketch.java | 359 +
.../hive/common/ndv/fm/FMSketchUtils.java | 132 +
.../hive/common/ndv/hll/HLLConstants.java | 929 +
.../hive/common/ndv/hll/HLLDenseRegister.java | 192 +
.../hadoop/hive/common/ndv/hll/HLLRegister.java | 50 +
.../hive/common/ndv/hll/HLLSparseRegister.java | 260 +
.../hadoop/hive/common/ndv/hll/HyperLogLog.java | 634 +
.../hive/common/ndv/hll/HyperLogLogUtils.java | 392 +
.../hadoop/hive/common/type/TimestampTZ.java | 122 +-
.../hive/common/type/TimestampTZUtil.java | 150 +
.../org/apache/hadoop/hive/conf/Constants.java | 2 +
.../org/apache/hadoop/hive/conf/HiveConf.java | 259 +-
.../apache/hadoop/hive/conf/HiveConfUtil.java | 20 +-
.../hive/common/util/HiveStringUtils.java | 92 +-
.../java/org/apache/hive/http/HttpServer.java | 1 +
.../java/org/apache/hive/http/StackServlet.java | 2 +-
.../src/main/resources/hive-log4j2.properties | 8 +-
.../hadoop/hive/common/TestFileUtils.java | 17 +-
.../ndv/fm/TestFMSketchSerialization.java | 98 +
.../hive/common/ndv/hll/TestHLLNoBias.java | 114 +
.../common/ndv/hll/TestHLLSerialization.java | 267 +
.../hive/common/ndv/hll/TestHyperLogLog.java | 227 +
.../common/ndv/hll/TestHyperLogLogDense.java | 82 +
.../common/ndv/hll/TestHyperLogLogSparse.java | 81 +
.../common/ndv/hll/TestSparseEncodeHash.java | 56 +
.../hive/common/type/TestTimestampTZ.java | 63 +-
.../apache/hadoop/hive/conf/TestHiveConf.java | 11 +
.../hive/conf/TestHiveConfRestrictList.java | 12 +-
.../hadoop/hive/conf/TestHiveConfUtil.java | 68 +
.../hive/common/util/TestHiveStringUtils.java | 51 +
.../clientpositive/udf_row_sequence.q.out | 6 +-
data/conf/hive-log4j2.properties | 11 +-
data/conf/hive-site.xml | 4 +
data/conf/llap/hive-site.xml | 9 +
data/conf/spark/standalone/hive-site.xml | 17 +-
data/conf/spark/yarn-client/hive-site.xml | 10 +
data/conf/tez/hive-site.xml | 10 +
data/files/having_data.txt | 8 +
data/files/impala_int96_timestamp.parq | Bin 241 -> 0 bytes
data/scripts/input20_script.py | 18 +-
dev-support/checkstyle_YETUS-484.sh | 406 +
dev-support/findbugs_YETUS-471.sh | 488 +
dev-support/hive-personality.sh | 100 +
dev-support/maven_YETUS-506.sh | 687 +
dev-support/smart-apply-patch.sh | 18 +
dev-support/test-patch.sh | 18 +
dev-support/yetus-wrapper.sh | 185 +
druid-handler/pom.xml | 4 +
.../hadoop/hive/druid/DruidStorageHandler.java | 189 +-
.../hive/druid/DruidStorageHandlerUtils.java | 378 +-
.../hadoop/hive/druid/io/DruidOutputFormat.java | 21 +-
.../hive/druid/security/DruidKerberosUtil.java | 126 +
.../hive/druid/security/KerberosHttpClient.java | 163 +
.../druid/security/ResponseCookieHandler.java | 92 +
.../RetryIfUnauthorizedResponseHandler.java | 104 +
.../druid/security/RetryResponseHolder.java | 46 +
.../serde/DruidGroupByQueryRecordReader.java | 44 +-
.../hadoop/hive/druid/serde/DruidSerDe.java | 42 +-
.../hive/druid/serde/DruidSerDeUtils.java | 26 +-
.../hadoop/hive/druid/TestDruidSerDe.java | 50 +-
.../hive/druid/TestDruidStorageHandler.java | 384 +-
.../hive/ql/io/TestDruidRecordWriter.java | 6 +-
errata.txt | 1 +
.../results/positive/hbase_ppd_key_range.q.out | 3 +-
.../test/results/positive/hbase_queries.q.out | 2 +-
.../test/results/positive/ppd_key_ranges.q.out | 34 +-
.../cli/SemanticAnalysis/HCatAuthUtil.java | 3 +-
.../mapreduce/FileRecordWriterContainer.java | 7 +-
.../hcatalog/mapreduce/HCatBaseInputFormat.java | 3 +
.../mapreduce/HCatBaseOutputFormat.java | 15 +-
.../hive/hcatalog/mapreduce/SpecialCases.java | 27 +
.../apache/hive/hcatalog/cli/TestPermsGrp.java | 4 +-
.../mapreduce/TestHCatOutputFormat.java | 43 +-
.../mapreduce/TestHCatPartitionPublish.java | 6 +-
.../apache/hive/hcatalog/pig/HCatLoader.java | 2 +-
.../pig/TestHCatLoaderComplexSchema.java | 3 -
.../hcatalog/pig/TestHCatLoaderEncryption.java | 9 +-
.../hive/hcatalog/pig/TestHCatStorerMulti.java | 10 +-
.../hcatalog/pig/TestParquetHCatLoader.java | 49 -
.../hcatalog/pig/TestParquetHCatStorer.java | 167 -
.../listener/DbNotificationListener.java | 101 +-
.../MetaStoreEventListenerConstants.java | 33 -
.../hcatalog/messaging/AddPartitionMessage.java | 2 +
.../messaging/AlterPartitionMessage.java | 2 +
.../hcatalog/messaging/AlterTableMessage.java | 1 +
.../hcatalog/messaging/CreateTableMessage.java | 2 +
.../messaging/DropPartitionMessage.java | 1 +
.../hcatalog/messaging/DropTableMessage.java | 1 +
.../hive/hcatalog/messaging/InsertMessage.java | 1 +
.../hive/hcatalog/messaging/MessageFactory.java | 12 +
.../messaging/json/JSONAddPartitionMessage.java | 15 +-
.../json/JSONAlterPartitionMessage.java | 19 +-
.../messaging/json/JSONAlterTableMessage.java | 18 +-
.../messaging/json/JSONCreateTableMessage.java | 16 +-
.../json/JSONDropPartitionMessage.java | 15 +-
.../messaging/json/JSONDropTableMessage.java | 11 +-
.../messaging/json/JSONInsertMessage.java | 14 +-
.../messaging/json/JSONMessageFactory.java | 28 +-
.../listener/TestNotificationListener.java | 7 +
hcatalog/src/test/e2e/hcatalog/tests/hcat.conf | 2 +-
.../streaming/AbstractRecordWriter.java | 44 +-
.../hive/hcatalog/streaming/HiveEndPoint.java | 4 +-
.../mutate/worker/BucketIdResolverImpl.java | 6 +-
.../mutate/worker/MutatorCoordinator.java | 7 +-
.../streaming/mutate/worker/MutatorImpl.java | 16 +-
.../apache/hive/hcatalog/streaming/package.html | 21 +-
.../hive/hcatalog/streaming/TestStreaming.java | 229 +-
.../streaming/mutate/StreamingAssert.java | 46 +-
.../streaming/mutate/TestMutations.java | 56 +-
.../mutate/worker/TestBucketIdResolverImpl.java | 6 +-
.../mutate/worker/TestMutatorImpl.java | 2 +-
.../hive/hcatalog/api/TestHCatClient.java | 118 +-
.../api/TestHCatClientNotification.java | 5 +
.../listener/DummyRawStoreFailEvent.java | 27 +-
.../listener/TestDbNotificationListener.java | 19 +-
.../src/test/queries/clientpositive/having.q | 12 +
...rt_overwrite_dynamic_partitions_merge_move.q | 37 +
...rt_overwrite_dynamic_partitions_merge_only.q | 37 +
...ert_overwrite_dynamic_partitions_move_only.q | 37 +
.../test/results/clientpositive/having.q.out | 59 +
.../insert_blobstore_to_blobstore.q.out | 6 +
.../insert_empty_into_blobstore.q.out | 3 +
.../clientpositive/insert_into_table.q.out | 2 +
...verwrite_dynamic_partitions_merge_move.q.out | 138 +
...verwrite_dynamic_partitions_merge_only.q.out | 128 +
...overwrite_dynamic_partitions_move_only.q.out | 158 +
.../clientpositive/insert_overwrite_table.q.out | 2 +
.../clientpositive/orc_format_nonpart.q.out | 6 +
.../clientpositive/orc_format_part.q.out | 16 +
.../orc_nonstd_partitions_loc.q.out | 24 +
.../clientpositive/rcfile_format_nonpart.q.out | 6 +
.../clientpositive/rcfile_format_part.q.out | 16 +
.../rcfile_nonstd_partitions_loc.q.out | 24 +
.../clientpositive/zero_rows_blobstore.q.out | 2 +
itests/hive-jmh/pom.xml | 6 +
.../benchmark/storage/ColumnarStorageBench.java | 3 +-
.../vectorization/mapjoin/AbstractMapJoin.java | 179 +
.../mapjoin/MapJoinMultiKeyBench.java | 313 +
.../mapjoin/MapJoinMultiKeyBenchBase.java | 68 +
.../mapjoin/MapJoinOneLongKeyBench.java | 313 +
.../mapjoin/MapJoinOneLongKeyBenchBase.java | 66 +
.../mapjoin/MapJoinOneStringKeyBench.java | 313 +
.../mapjoin/MapJoinOneStringKeyBenchBase.java | 66 +
.../hive/minikdc/TestHs2HooksWithMiniKdc.java | 28 +-
.../security/TestHadoopAuthBridge23.java | 416 +
.../hive/thrift/TestHadoopAuthBridge23.java | 417 -
itests/hive-unit/pom.xml | 6 +
.../apache/hadoop/hive/hooks/TestHs2Hooks.java | 27 +-
.../AbstractTestAuthorizationApiAuthorizer.java | 4 +-
.../hadoop/hive/metastore/TestFilterHooks.java | 4 +-
.../hive/metastore/TestHiveMetaStore.java | 144 +-
.../metastore/TestHiveMetaStoreStatsMerge.java | 199 -
...TestHiveMetaStoreWithEnvironmentContext.java | 4 +-
.../metastore/TestMetaStoreAuthorization.java | 4 +-
.../TestMetaStoreEndFunctionListener.java | 5 +-
.../metastore/TestMetaStoreEventListener.java | 72 +-
.../TestMetaStoreEventListenerOnlyOnCommit.java | 4 +-
.../metastore/TestMetaStoreInitListener.java | 5 +-
.../metastore/TestMetaStoreListenersError.java | 6 +-
.../hive/metastore/TestMetaStoreMetrics.java | 82 +-
.../hive/metastore/TestRemoteHiveMetaStore.java | 4 +-
.../TestRemoteHiveMetaStoreIpAddress.java | 4 +-
.../hive/metastore/TestReplChangeManager.java | 37 +-
.../hive/metastore/TestRetryingHMSHandler.java | 4 +-
.../metastore/hbase/HBaseIntegrationTests.java | 112 -
.../TestHBaseAggrStatsCacheIntegration.java | 691 -
.../hive/metastore/hbase/TestHBaseImport.java | 775 -
.../hbase/TestHBaseMetastoreMetrics.java | 130 -
.../metastore/hbase/TestHBaseMetastoreSql.java | 223 -
.../metastore/hbase/TestHBaseSchemaTool.java | 582 -
.../metastore/hbase/TestHBaseSchemaTool2.java | 59 -
.../hbase/TestHBaseStoreIntegration.java | 1801 -
.../hbase/TestStorageDescriptorSharing.java | 191 -
.../apache/hadoop/hive/ql/TestAcidOnTez.java | 367 +-
.../hive/ql/TestAcidOnTezWithSplitUpdate.java | 28 -
...estDDLWithRemoteMetastoreSecondNamenode.java | 4 +-
.../hadoop/hive/ql/TestLocationQueries.java | 2 +-
.../hadoop/hive/ql/parse/TestExportImport.java | 100 +
.../hive/ql/parse/TestReplicationScenarios.java | 2772 +-
...TestReplicationScenariosAcrossInstances.java | 91 +-
.../hadoop/hive/ql/parse/WarehouseInstance.java | 89 +-
.../security/StorageBasedMetastoreTestBase.java | 5 +-
.../TestAuthorizationPreEventListener.java | 23 +-
.../TestClientSideAuthorizationProvider.java | 4 +-
.../TestMetastoreAuthorizationProvider.java | 4 +-
.../TestMultiAuthorizationPreEventListener.java | 4 +-
.../TestHiveAuthorizerCheckInvocation.java | 38 +
.../hive/ql/txn/compactor/TestCompactor.java | 4 +-
.../org/apache/hive/beeline/TestSchemaTool.java | 22 +-
.../apache/hive/jdbc/TestJdbcWithMiniHS2.java | 36 +-
.../hive/service/TestDFSErrorHandling.java | 152 +
.../operation/TestOperationLoggingLayout.java | 52 +
.../service/cli/session/TestQueryDisplay.java | 3 +-
itests/pom.xml | 2 +-
itests/qtest-accumulo/pom.xml | 3 +-
.../test/resources/testconfiguration.properties | 38 +-
.../hadoop/hive/accumulo/AccumuloQTestUtil.java | 2 +-
.../hive/cli/control/AbstractCliConfig.java | 4 +-
.../control/AbstractCoreBlobstoreCliDriver.java | 3 +-
.../hadoop/hive/cli/control/CliConfigs.java | 4 +-
.../hadoop/hive/cli/control/CoreCliDriver.java | 4 +-
.../hive/cli/control/CoreCompareCliDriver.java | 2 +-
.../hive/cli/control/CoreNegativeCliDriver.java | 2 +-
.../hive/cli/control/CorePerfCliDriver.java | 2 +-
.../hadoop/hive/hbase/HBaseQTestUtil.java | 2 +-
.../metastore/hbase/HBaseStoreTestUtil.java | 45 -
.../org/apache/hadoop/hive/ql/QTestUtil.java | 87 +-
.../hadoop/hive/ql/parse/CoreParseNegative.java | 3 +-
.../org/apache/hive/beeline/Parallelized.java | 9 +-
.../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 4 +-
.../hive/storage/jdbc/JdbcRecordReader.java | 7 +-
.../org/apache/hive/storage/jdbc/JdbcSerDe.java | 18 +-
.../storage/jdbc/dao/JdbcRecordIterator.java | 8 +-
.../src/test/java/org/apache/TestSuite.java | 29 -
.../org/apache/hive/storage/jdbc/TestSuite.java | 29 +
.../dao/GenericJdbcDatabaseAccessorTest.java | 8 +-
jdbc/pom.xml | 2 +-
.../org/apache/hive/jdbc/HiveBaseResultSet.java | 6 +-
.../org/apache/hive/jdbc/HiveConnection.java | 5 +-
.../java/org/apache/hive/jdbc/JdbcColumn.java | 8 +-
.../hadoop/hive/llap/LlapBaseRecordReader.java | 103 +-
.../apache/hadoop/hive/llap/SubmitWorkInfo.java | 16 +-
.../ext/LlapTaskUmbilicalExternalClient.java | 504 +-
.../apache/hadoop/hive/llap/io/api/LlapIo.java | 4 +-
.../hive/llap/registry/ServiceInstance.java | 2 +-
.../hive/llap/tez/LlapProtocolClientProxy.java | 423 +-
.../helpers/LlapTaskUmbilicalServer.java | 44 +-
.../hadoop/hive/llap/TestAsyncPbRpcProxy.java | 145 +
.../tez/TestLlapDaemonProtocolClientProxy.java | 144 -
.../daemon/rpc/LlapDaemonProtocolProtos.java | 1854 +-
.../hadoop/hive/llap/AsyncPbRpcProxy.java | 435 +
.../hive/llap/impl/LlapProtocolClientImpl.java | 20 +-
.../hadoop/hive/llap/io/ChunkedInputStream.java | 16 +-
.../hive/llap/io/ChunkedOutputStream.java | 9 +-
.../src/protobuf/LlapDaemonProtocol.proto | 16 +-
.../hive/llap/io/TestChunkedInputStream.java | 6 +-
.../hadoop/hive/llap/LlapBaseInputFormat.java | 2 -
llap-server/bin/llapDaemon.sh | 3 +
llap-server/pom.xml | 7 +-
.../hadoop/hive/llap/cache/BuddyAllocator.java | 1496 +-
.../hive/llap/cache/EvictionDispatcher.java | 23 +-
.../hive/llap/cache/LlapAllocatorBuffer.java | 414 +
.../hive/llap/cache/LlapCacheableBuffer.java | 6 +-
.../hadoop/hive/llap/cache/LlapDataBuffer.java | 119 +-
.../hive/llap/cache/LowLevelCacheImpl.java | 70 +-
.../llap/cache/LowLevelCacheMemoryManager.java | 7 -
.../hive/llap/cache/LowLevelCachePolicy.java | 2 -
.../llap/cache/LowLevelFifoCachePolicy.java | 16 +-
.../llap/cache/LowLevelLrfuCachePolicy.java | 95 +-
.../hadoop/hive/llap/cache/MemoryManager.java | 2 -
.../hive/llap/cache/SerDeLowLevelCacheImpl.java | 115 +-
.../hadoop/hive/llap/cache/SimpleAllocator.java | 22 +-
.../hive/llap/cache/SimpleBufferManager.java | 12 +-
.../hadoop/hive/llap/cli/LlapServiceDriver.java | 6 +-
.../configuration/LlapDaemonConfiguration.java | 9 +-
.../hive/llap/daemon/ContainerRunner.java | 7 +-
.../llap/daemon/impl/ContainerRunnerImpl.java | 26 +-
.../impl/EvictingPriorityBlockingQueue.java | 29 +-
.../hive/llap/daemon/impl/LlapDaemon.java | 8 +
.../daemon/impl/LlapProtocolServerImpl.java | 12 +
.../llap/daemon/impl/QueryFragmentInfo.java | 24 +-
.../hadoop/hive/llap/daemon/impl/QueryInfo.java | 7 +-
.../hadoop/hive/llap/daemon/impl/Scheduler.java | 6 +-
.../impl/StaticPermanentFunctionChecker.java | 3 +-
.../llap/daemon/impl/TaskExecutorService.java | 451 +-
.../llap/daemon/impl/TaskRunnerCallable.java | 22 +-
.../comparator/FirstInFirstOutComparator.java | 14 +-
.../comparator/LlapQueueComparatorBase.java | 48 +
.../comparator/ShortestJobFirstComparator.java | 17 +-
.../daemon/services/impl/LlapWebServices.java | 1 +
.../impl/SystemConfigurationServlet.java | 196 +
.../hive/llap/io/api/impl/LlapInputFormat.java | 63 +-
.../hive/llap/io/api/impl/LlapIoImpl.java | 108 +-
.../hive/llap/io/api/impl/LlapRecordReader.java | 88 +-
.../llap/io/decode/EncodedDataConsumer.java | 1 +
.../io/decode/GenericColumnVectorProducer.java | 9 +-
.../llap/io/decode/OrcColumnVectorProducer.java | 15 +-
.../llap/io/decode/OrcEncodedDataConsumer.java | 12 +
.../llap/io/encoded/OrcEncodedDataReader.java | 122 +-
.../llap/io/encoded/SerDeEncodedDataReader.java | 75 +-
.../io/encoded/VectorDeserializeOrcWriter.java | 3 +
.../llap/io/metadata/OrcFileEstimateErrors.java | 4 +-
.../hive/llap/io/metadata/OrcFileMetadata.java | 4 +-
.../llap/io/metadata/OrcStripeMetadata.java | 4 +-
.../io/metadata/ParquetMetadataCacheImpl.java | 353 +
.../hive/llap/metrics/LlapDaemonJvmInfo.java | 3 +-
.../hive/llap/metrics/LlapDaemonJvmMetrics.java | 7 +-
.../llap/shufflehandler/FadvisedFileRegion.java | 14 +-
.../llap/shufflehandler/ShuffleHandler.java | 23 +-
llap-server/src/main/resources/templates.py | 2 +-
.../hive/llap/cache/TestBuddyAllocator.java | 73 +-
.../cache/TestBuddyAllocatorForceEvict.java | 470 +
.../hive/llap/cache/TestLowLevelCacheImpl.java | 29 +-
.../llap/cache/TestLowLevelLrfuCachePolicy.java | 56 -
.../hive/llap/cache/TestOrcMetadataCache.java | 12 -
.../daemon/impl/TaskExecutorTestHelpers.java | 97 +-
.../impl/TestEvictingPriorityBlockingQueue.java | 12 +-
.../daemon/impl/TestTaskExecutorService.java | 165 +-
.../TestFirstInFirstOutComparator.java | 36 +-
.../TestShortestJobFirstComparator.java | 35 +-
metastore/if/hive_metastore.thrift | 1593 -
metastore/pom.xml | 90 +-
.../upgrade/derby/022-HIVE-11107.derby.sql | 2 +-
.../upgrade/derby/043-HIVE-16922.derby.sql | 3 +
.../upgrade/derby/044-HIVE-16997.derby.sql | 2 +
.../upgrade/derby/hive-schema-3.0.0.derby.sql | 2 +-
.../derby/upgrade-2.3.0-to-3.0.0.derby.sql | 2 +
.../upgrade/hive/hive-schema-3.0.0.hive.sql | 24 +
.../upgrade/mssql/028-HIVE-16922.mssql.sql | 3 +
.../upgrade/mssql/029-HIVE-16997.mssql.sql | 2 +
.../upgrade/mssql/hive-schema-3.0.0.mssql.sql | 2 +
.../mssql/upgrade-2.3.0-to-3.0.0.mssql.sql | 2 +
.../upgrade/mysql/043-HIVE-16922.mysql.sql | 3 +
.../upgrade/mysql/044-HIVE-16997.mysql.sql | 2 +
.../upgrade/mysql/hive-schema-3.0.0.mysql.sql | 4 +-
.../mysql/upgrade-2.3.0-to-3.0.0.mysql.sql | 2 +
.../upgrade/oracle/043-HIVE-16922.oracle.sql | 3 +
.../upgrade/oracle/044-HIVE-16997.oracle.sql | 2 +
.../upgrade/oracle/hive-schema-3.0.0.oracle.sql | 2 +
.../oracle/upgrade-2.3.0-to-3.0.0.oracle.sql | 2 +
.../postgres/042-HIVE-16922.postgres.sql | 3 +
.../postgres/043-HIVE-16997.postgres.sql | 2 +
.../postgres/hive-schema-3.0.0.postgres.sql | 2 +
.../upgrade-2.3.0-to-3.0.0.postgres.sql | 2 +
.../metastore/hbase/HbaseMetastoreProto.java | 46709 ----
.../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 73240 ------
.../gen/thrift/gen-cpp/ThriftHiveMetastore.h | 23202 --
.../ThriftHiveMetastore_server.skeleton.cpp | 834 -
.../thrift/gen-cpp/hive_metastore_constants.cpp | 65 -
.../thrift/gen-cpp/hive_metastore_constants.h | 48 -
.../gen/thrift/gen-cpp/hive_metastore_types.cpp | 21453 --
.../gen/thrift/gen-cpp/hive_metastore_types.h | 8861 -
.../hive/metastore/api/AbortTxnRequest.java | 387 -
.../hive/metastore/api/AbortTxnsRequest.java | 438 -
.../metastore/api/AddDynamicPartitions.java | 864 -
.../metastore/api/AddForeignKeyRequest.java | 443 -
.../api/AddNotNullConstraintRequest.java | 443 -
.../metastore/api/AddPartitionsRequest.java | 849 -
.../hive/metastore/api/AddPartitionsResult.java | 447 -
.../metastore/api/AddPrimaryKeyRequest.java | 443 -
.../api/AddUniqueConstraintRequest.java | 443 -
.../hadoop/hive/metastore/api/AggrStats.java | 542 -
.../metastore/api/AlreadyExistsException.java | 395 -
.../metastore/api/BinaryColumnStatsData.java | 687 -
.../metastore/api/BooleanColumnStatsData.java | 687 -
.../metastore/api/CacheFileMetadataRequest.java | 703 -
.../metastore/api/CacheFileMetadataResult.java | 387 -
.../hive/metastore/api/CheckLockRequest.java | 589 -
.../metastore/api/ClearFileMetadataRequest.java | 438 -
.../metastore/api/ClearFileMetadataResult.java | 283 -
.../hive/metastore/api/ClientCapabilities.java | 441 -
.../hive/metastore/api/ClientCapability.java | 45 -
.../hive/metastore/api/ColumnStatistics.java | 549 -
.../metastore/api/ColumnStatisticsData.java | 675 -
.../metastore/api/ColumnStatisticsDesc.java | 798 -
.../hive/metastore/api/ColumnStatisticsObj.java | 593 -
.../hive/metastore/api/CommitTxnRequest.java | 387 -
.../hive/metastore/api/CompactionRequest.java | 977 -
.../hive/metastore/api/CompactionResponse.java | 583 -
.../hive/metastore/api/CompactionType.java | 45 -
.../api/ConfigValSecurityException.java | 395 -
.../api/CurrentNotificationEventId.java | 387 -
.../hive/metastore/api/DataOperationType.java | 57 -
.../hadoop/hive/metastore/api/Database.java | 1095 -
.../apache/hadoop/hive/metastore/api/Date.java | 387 -
.../hive/metastore/api/DateColumnStatsData.java | 814 -
.../hadoop/hive/metastore/api/Decimal.java | 497 -
.../metastore/api/DecimalColumnStatsData.java | 814 -
.../metastore/api/DoubleColumnStatsData.java | 790 -
.../metastore/api/DropConstraintRequest.java | 591 -
.../hive/metastore/api/DropPartitionsExpr.java | 505 -
.../metastore/api/DropPartitionsRequest.java | 1112 -
.../metastore/api/DropPartitionsResult.java | 447 -
.../hive/metastore/api/EnvironmentContext.java | 447 -
.../hive/metastore/api/EventRequestType.java | 48 -
.../hadoop/hive/metastore/api/FieldSchema.java | 603 -
.../metastore/api/FileMetadataExprType.java | 42 -
.../hive/metastore/api/FireEventRequest.java | 861 -
.../metastore/api/FireEventRequestData.java | 309 -
.../hive/metastore/api/FireEventResponse.java | 283 -
.../hive/metastore/api/ForeignKeysRequest.java | 707 -
.../hive/metastore/api/ForeignKeysResponse.java | 443 -
.../hadoop/hive/metastore/api/Function.java | 1199 -
.../hadoop/hive/metastore/api/FunctionType.java | 42 -
.../metastore/api/GetAllFunctionsResponse.java | 447 -
.../api/GetFileMetadataByExprRequest.java | 773 -
.../api/GetFileMetadataByExprResult.java | 553 -
.../metastore/api/GetFileMetadataRequest.java | 438 -
.../metastore/api/GetFileMetadataResult.java | 540 -
.../metastore/api/GetOpenTxnsInfoResponse.java | 542 -
.../hive/metastore/api/GetOpenTxnsResponse.java | 750 -
.../api/GetPrincipalsInRoleRequest.java | 389 -
.../api/GetPrincipalsInRoleResponse.java | 443 -
.../api/GetRoleGrantsForPrincipalRequest.java | 502 -
.../api/GetRoleGrantsForPrincipalResponse.java | 443 -
.../hive/metastore/api/GetTableRequest.java | 605 -
.../hive/metastore/api/GetTableResult.java | 394 -
.../hive/metastore/api/GetTablesRequest.java | 659 -
.../hive/metastore/api/GetTablesResult.java | 443 -
.../api/GrantRevokePrivilegeRequest.java | 620 -
.../api/GrantRevokePrivilegeResponse.java | 390 -
.../metastore/api/GrantRevokeRoleRequest.java | 1059 -
.../metastore/api/GrantRevokeRoleResponse.java | 390 -
.../hive/metastore/api/GrantRevokeType.java | 45 -
.../hive/metastore/api/HeartbeatRequest.java | 489 -
.../metastore/api/HeartbeatTxnRangeRequest.java | 482 -
.../api/HeartbeatTxnRangeResponse.java | 588 -
.../hive/metastore/api/HiveObjectPrivilege.java | 729 -
.../hive/metastore/api/HiveObjectRef.java | 872 -
.../hive/metastore/api/HiveObjectType.java | 54 -
.../apache/hadoop/hive/metastore/api/Index.java | 1374 -
.../api/IndexAlreadyExistsException.java | 395 -
.../metastore/api/InsertEventRequestData.java | 700 -
.../metastore/api/InvalidInputException.java | 395 -
.../metastore/api/InvalidObjectException.java | 395 -
.../api/InvalidOperationException.java | 395 -
.../api/InvalidPartitionException.java | 395 -
.../hive/metastore/api/LockComponent.java | 1158 -
.../hadoop/hive/metastore/api/LockLevel.java | 48 -
.../hadoop/hive/metastore/api/LockRequest.java | 861 -
.../hadoop/hive/metastore/api/LockResponse.java | 500 -
.../hadoop/hive/metastore/api/LockState.java | 51 -
.../hadoop/hive/metastore/api/LockType.java | 48 -
.../hive/metastore/api/LongColumnStatsData.java | 790 -
.../hive/metastore/api/MetaException.java | 395 -
.../hive/metastore/api/MetadataPpdResult.java | 517 -
.../hive/metastore/api/NoSuchLockException.java | 395 -
.../metastore/api/NoSuchObjectException.java | 395 -
.../hive/metastore/api/NoSuchTxnException.java | 395 -
.../api/NotNullConstraintsRequest.java | 490 -
.../api/NotNullConstraintsResponse.java | 443 -
.../hive/metastore/api/NotificationEvent.java | 1006 -
.../metastore/api/NotificationEventRequest.java | 490 -
.../api/NotificationEventResponse.java | 443 -
.../hive/metastore/api/OpenTxnRequest.java | 702 -
.../hive/metastore/api/OpenTxnsResponse.java | 438 -
.../apache/hadoop/hive/metastore/api/Order.java | 497 -
.../hadoop/hive/metastore/api/Partition.java | 1229 -
.../hive/metastore/api/PartitionEventType.java | 42 -
.../api/PartitionListComposingSpec.java | 449 -
.../hive/metastore/api/PartitionSpec.java | 826 -
.../api/PartitionSpecWithSharedSD.java | 558 -
.../hive/metastore/api/PartitionWithoutSD.java | 1016 -
.../metastore/api/PartitionsByExprRequest.java | 815 -
.../metastore/api/PartitionsByExprResult.java | 542 -
.../metastore/api/PartitionsStatsRequest.java | 790 -
.../metastore/api/PartitionsStatsResult.java | 490 -
.../hive/metastore/api/PrimaryKeysRequest.java | 490 -
.../hive/metastore/api/PrimaryKeysResponse.java | 443 -
.../metastore/api/PrincipalPrivilegeSet.java | 906 -
.../hive/metastore/api/PrincipalType.java | 48 -
.../hadoop/hive/metastore/api/PrivilegeBag.java | 449 -
.../hive/metastore/api/PrivilegeGrantInfo.java | 815 -
.../metastore/api/PutFileMetadataRequest.java | 710 -
.../metastore/api/PutFileMetadataResult.java | 283 -
.../hive/metastore/api/RequestPartsSpec.java | 438 -
.../hadoop/hive/metastore/api/ResourceType.java | 48 -
.../hadoop/hive/metastore/api/ResourceUri.java | 511 -
.../apache/hadoop/hive/metastore/api/Role.java | 601 -
.../hive/metastore/api/RolePrincipalGrant.java | 1035 -
.../hive/metastore/api/SQLForeignKey.java | 1715 -
.../metastore/api/SQLNotNullConstraint.java | 1005 -
.../hive/metastore/api/SQLPrimaryKey.java | 1103 -
.../hive/metastore/api/SQLUniqueConstraint.java | 1103 -
.../hadoop/hive/metastore/api/Schema.java | 605 -
.../hadoop/hive/metastore/api/SerDeInfo.java | 655 -
.../api/SetPartitionsStatsRequest.java | 550 -
.../hive/metastore/api/ShowCompactRequest.java | 283 -
.../hive/metastore/api/ShowCompactResponse.java | 443 -
.../api/ShowCompactResponseElement.java | 1641 -
.../hive/metastore/api/ShowLocksRequest.java | 710 -
.../hive/metastore/api/ShowLocksResponse.java | 449 -
.../metastore/api/ShowLocksResponseElement.java | 1929 -
.../hadoop/hive/metastore/api/SkewedInfo.java | 834 -
.../hive/metastore/api/StorageDescriptor.java | 1748 -
.../metastore/api/StringColumnStatsData.java | 782 -
.../apache/hadoop/hive/metastore/api/Table.java | 1948 -
.../hadoop/hive/metastore/api/TableMeta.java | 701 -
.../hive/metastore/api/TableStatsRequest.java | 640 -
.../hive/metastore/api/TableStatsResult.java | 443 -
.../hive/metastore/api/ThriftHiveMetastore.java | 186370 ---------------
.../hive/metastore/api/TxnAbortedException.java | 395 -
.../hadoop/hive/metastore/api/TxnInfo.java | 1220 -
.../hive/metastore/api/TxnOpenException.java | 395 -
.../hadoop/hive/metastore/api/TxnState.java | 48 -
.../apache/hadoop/hive/metastore/api/Type.java | 768 -
.../metastore/api/UniqueConstraintsRequest.java | 490 -
.../api/UniqueConstraintsResponse.java | 443 -
.../hive/metastore/api/UnknownDBException.java | 395 -
.../api/UnknownPartitionException.java | 395 -
.../metastore/api/UnknownTableException.java | 395 -
.../hive/metastore/api/UnlockRequest.java | 387 -
.../hadoop/hive/metastore/api/Version.java | 499 -
.../metastore/api/hive_metastoreConstants.java | 87 -
.../gen-php/metastore/ThriftHiveMetastore.php | 46667 ----
.../src/gen/thrift/gen-php/metastore/Types.php | 20941 --
.../hive_metastore/ThriftHiveMetastore-remote | 1291 -
.../hive_metastore/ThriftHiveMetastore.py | 37904 ----
.../thrift/gen-py/hive_metastore/__init__.py | 1 -
.../thrift/gen-py/hive_metastore/constants.py | 35 -
.../gen/thrift/gen-py/hive_metastore/ttypes.py | 14886 --
.../thrift/gen-rb/hive_metastore_constants.rb | 57 -
.../gen/thrift/gen-rb/hive_metastore_types.rb | 3444 -
.../gen/thrift/gen-rb/thrift_hive_metastore.rb | 10557 -
.../hadoop/hive/metastore/DatabaseProduct.java | 2 +-
.../apache/hadoop/hive/metastore/Deadline.java | 4 +-
.../hive/metastore/FileMetadataManager.java | 7 +-
.../hive/metastore/HMSMetricsListener.java | 47 +-
.../hadoop/hive/metastore/HiveAlterHandler.java | 22 +-
.../hadoop/hive/metastore/HiveMetaHook.java | 4 +
.../hadoop/hive/metastore/HiveMetaStore.java | 606 +-
.../hive/metastore/HiveMetaStoreClient.java | 78 +-
.../hadoop/hive/metastore/IMetaStoreClient.java | 89 +-
.../hive/metastore/IMetaStoreSchemaInfo.java | 2 +-
.../hive/metastore/MetaStoreDirectSql.java | 464 +-
.../hive/metastore/MetaStoreEventListener.java | 45 +-
.../MetaStoreEventListenerConstants.java | 41 +
.../hive/metastore/MetaStoreFilterHook.java | 5 +-
.../metastore/MetaStoreListenerNotifier.java | 52 +-
.../metastore/MetaStorePreEventListener.java | 5 +-
.../hive/metastore/MetaStoreSchemaInfo.java | 4 +-
.../hadoop/hive/metastore/MetaStoreThread.java | 2 +-
.../hadoop/hive/metastore/MetaStoreUtils.java | 314 +-
.../metastore/NumDistinctValueEstimator.java | 367 -
.../hadoop/hive/metastore/ObjectStore.java | 178 +-
.../hive/metastore/PartFilterExprUtil.java | 2 +-
.../apache/hadoop/hive/metastore/RawStore.java | 23 +-
.../hive/metastore/ReplChangeManager.java | 153 +-
.../hive/metastore/RetryingHMSHandler.java | 4 +-
.../hive/metastore/StatObjectConverter.java | 87 +-
.../hive/metastore/TUGIBasedProcessor.java | 2 +-
.../TransactionalMetaStoreEventListener.java | 39 +
.../TransactionalValidationListener.java | 44 +-
.../apache/hadoop/hive/metastore/Warehouse.java | 21 +-
.../hive/metastore/cache/CachedStore.java | 169 +-
.../hive/metastore/cache/SharedCache.java | 10 +-
.../aggr/BinaryColumnStatsAggregator.java | 64 +
.../aggr/BooleanColumnStatsAggregator.java | 65 +
.../columnstats/aggr/ColumnStatsAggregator.java | 33 +
.../aggr/ColumnStatsAggregatorFactory.java | 113 +
.../aggr/DateColumnStatsAggregator.java | 362 +
.../aggr/DecimalColumnStatsAggregator.java | 375 +
.../aggr/DoubleColumnStatsAggregator.java | 349 +
.../aggr/IExtrapolatePartStatus.java | 47 +
.../aggr/LongColumnStatsAggregator.java | 348 +
.../aggr/StringColumnStatsAggregator.java | 305 +
.../cache/DateColumnStatsDataInspector.java | 124 +
.../cache/DecimalColumnStatsDataInspector.java | 124 +
.../cache/DoubleColumnStatsDataInspector.java | 124 +
.../cache/LongColumnStatsDataInspector.java | 124 +
.../cache/StringColumnStatsDataInspector.java | 125 +
.../merge/BinaryColumnStatsMerger.java | 35 +
.../merge/BooleanColumnStatsMerger.java | 35 +
.../columnstats/merge/ColumnStatsMerger.java | 31 +
.../merge/ColumnStatsMergerFactory.java | 120 +
.../merge/DateColumnStatsMerger.java | 59 +
.../merge/DecimalColumnStatsMerger.java | 61 +
.../merge/DoubleColumnStatsMerger.java | 54 +
.../merge/LongColumnStatsMerger.java | 54 +
.../merge/StringColumnStatsMerger.java | 54 +
.../datasource/BoneCPDataSourceProvider.java | 95 +
.../datasource/DataSourceProvider.java | 79 +
.../datasource/DataSourceProviderFactory.java | 47 +
.../datasource/HikariCPDataSourceProvider.java | 97 +
.../hive/metastore/datasource/package-info.java | 23 +
.../metastore/events/AddForeignKeyEvent.java | 37 +
.../hive/metastore/events/AddIndexEvent.java | 4 +
.../events/AddNotNullConstraintEvent.java | 37 +
.../metastore/events/AddPartitionEvent.java | 4 +
.../metastore/events/AddPrimaryKeyEvent.java | 38 +
.../events/AddUniqueConstraintEvent.java | 37 +
.../hive/metastore/events/AlterIndexEvent.java | 4 +
.../metastore/events/AlterPartitionEvent.java | 4 +
.../hive/metastore/events/AlterTableEvent.java | 4 +
.../metastore/events/ConfigChangeEvent.java | 4 +
.../metastore/events/CreateDatabaseEvent.java | 4 +
.../metastore/events/CreateFunctionEvent.java | 4 +
.../hive/metastore/events/CreateTableEvent.java | 4 +
.../metastore/events/DropConstraintEvent.java | 47 +
.../metastore/events/DropDatabaseEvent.java | 4 +
.../metastore/events/DropFunctionEvent.java | 4 +
.../hive/metastore/events/DropIndexEvent.java | 4 +
.../metastore/events/DropPartitionEvent.java | 4 +
.../hive/metastore/events/DropTableEvent.java | 4 +
.../hive/metastore/events/InsertEvent.java | 6 +-
.../hive/metastore/events/ListenerEvent.java | 5 +-
.../events/LoadPartitionDoneEvent.java | 4 +
.../hive/metastore/events/PreAddIndexEvent.java | 4 +
.../metastore/events/PreAddPartitionEvent.java | 4 +
.../metastore/events/PreAlterDatabaseEvent.java | 47 +
.../metastore/events/PreAlterIndexEvent.java | 4 +
.../events/PreAlterPartitionEvent.java | 4 +
.../metastore/events/PreAlterTableEvent.java | 4 +
.../events/PreAuthorizationCallEvent.java | 4 +
.../events/PreCreateDatabaseEvent.java | 4 +
.../metastore/events/PreCreateTableEvent.java | 4 +
.../metastore/events/PreDropDatabaseEvent.java | 4 +
.../metastore/events/PreDropIndexEvent.java | 4 +
.../metastore/events/PreDropPartitionEvent.java | 4 +
.../metastore/events/PreDropTableEvent.java | 4 +
.../hive/metastore/events/PreEventContext.java | 8 +-
.../events/PreLoadPartitionDoneEvent.java | 4 +
.../metastore/events/PreReadDatabaseEvent.java | 4 +
.../metastore/events/PreReadTableEvent.java | 4 +
.../hbase/AggrStatsInvalidatorFilter.java | 121 -
.../hadoop/hive/metastore/hbase/Counter.java | 53 -
.../hive/metastore/hbase/HBaseConnection.java | 96 -
.../metastore/hbase/HBaseFilterPlanUtil.java | 612 -
.../hive/metastore/hbase/HBaseImport.java | 619 -
.../hive/metastore/hbase/HBaseReadWrite.java | 2943 -
.../hive/metastore/hbase/HBaseSchemaTool.java | 192 -
.../hadoop/hive/metastore/hbase/HBaseStore.java | 2964 -
.../hadoop/hive/metastore/hbase/HBaseUtils.java | 1781 -
.../hive/metastore/hbase/MetadataStore.java | 2 +-
.../hive/metastore/hbase/ObjectCache.java | 81 -
.../hive/metastore/hbase/PartitionCache.java | 168 -
.../metastore/hbase/PartitionKeyComparator.java | 292 -
.../hbase/SharedStorageDescriptor.java | 251 -
.../hadoop/hive/metastore/hbase/StatsCache.java | 323 -
.../metastore/hbase/TephraHBaseConnection.java | 127 -
.../metastore/hbase/VanillaHBaseConnection.java | 137 -
.../stats/BinaryColumnStatsAggregator.java | 64 -
.../stats/BooleanColumnStatsAggregator.java | 65 -
.../hbase/stats/ColumnStatsAggregator.java | 34 -
.../stats/ColumnStatsAggregatorFactory.java | 104 -
.../stats/DecimalColumnStatsAggregator.java | 351 -
.../stats/DoubleColumnStatsAggregator.java | 325 -
.../hbase/stats/IExtrapolatePartStatus.java | 47 -
.../hbase/stats/LongColumnStatsAggregator.java | 323 -
.../stats/StringColumnStatsAggregator.java | 105 -
.../stats/merge/BinaryColumnStatsMerger.java | 35 -
.../stats/merge/BooleanColumnStatsMerger.java | 35 -
.../hbase/stats/merge/ColumnStatsMerger.java | 34 -
.../stats/merge/ColumnStatsMergerFactory.java | 150 -
.../stats/merge/DateColumnStatsMerger.java | 55 -
.../stats/merge/DecimalColumnStatsMerger.java | 55 -
.../stats/merge/DoubleColumnStatsMerger.java | 48 -
.../stats/merge/LongColumnStatsMerger.java | 48 -
.../stats/merge/StringColumnStatsMerger.java | 49 -
.../metastore/hooks/JDOConnectionURLHook.java | 4 +
.../messaging/AddForeignKeyMessage.java | 36 +
.../messaging/AddNotNullConstraintMessage.java | 36 +
.../messaging/AddPartitionMessage.java | 2 +
.../messaging/AddPrimaryKeyMessage.java | 35 +
.../messaging/AddUniqueConstraintMessage.java | 36 +
.../messaging/AlterPartitionMessage.java | 2 +
.../metastore/messaging/AlterTableMessage.java | 2 +
.../metastore/messaging/CreateTableMessage.java | 2 +
.../messaging/DropConstraintMessage.java | 29 +
.../messaging/DropPartitionMessage.java | 2 +
.../metastore/messaging/DropTableMessage.java | 2 +
.../hive/metastore/messaging/EventMessage.java | 7 +-
.../hive/metastore/messaging/EventUtils.java | 13 +
.../hive/metastore/messaging/InsertMessage.java | 2 +
.../messaging/MessageDeserializer.java | 36 +-
.../metastore/messaging/MessageFactory.java | 53 +-
.../event/filters/DatabaseAndTableFilter.java | 2 +-
.../json/JSONAddForeignKeyMessage.java | 101 +
.../json/JSONAddNotNullConstraintMessage.java | 96 +
.../messaging/json/JSONAddPartitionMessage.java | 8 +-
.../json/JSONAddPrimaryKeyMessage.java | 101 +
.../json/JSONAddUniqueConstraintMessage.java | 98 +
.../json/JSONAlterPartitionMessage.java | 8 +-
.../messaging/json/JSONAlterTableMessage.java | 8 +-
.../messaging/json/JSONCreateTableMessage.java | 18 +-
.../json/JSONDropConstraintMessage.java | 90 +
.../json/JSONDropPartitionMessage.java | 15 +-
.../messaging/json/JSONDropTableMessage.java | 13 +-
.../messaging/json/JSONInsertMessage.java | 8 +-
.../messaging/json/JSONMessageDeserializer.java | 50 +
.../messaging/json/JSONMessageFactory.java | 61 +-
.../hive/metastore/parser/ExpressionTree.java | 4 +-
.../hive/metastore/tools/HiveSchemaHelper.java | 2 +-
.../hadoop/hive/metastore/txn/TxnDbUtil.java | 5 +-
.../hadoop/hive/metastore/txn/TxnHandler.java | 254 +-
.../hadoop/hive/metastore/txn/TxnStore.java | 2 +-
.../hadoop/hive/metastore/txn/TxnUtils.java | 2 +-
.../src/main/resources/thrift-replacements.txt | 60 -
.../hive/metastore/model/MColumnDescriptor.java | 51 -
.../hive/metastore/model/MConstraint.java | 188 -
.../hive/metastore/model/MDBPrivilege.java | 132 -
.../hadoop/hive/metastore/model/MDatabase.java | 128 -
.../hive/metastore/model/MDelegationToken.java | 45 -
.../hive/metastore/model/MFieldSchema.java | 80 -
.../hadoop/hive/metastore/model/MFunction.java | 119 -
.../hive/metastore/model/MGlobalPrivilege.java | 120 -
.../hadoop/hive/metastore/model/MIndex.java | 200 -
.../hadoop/hive/metastore/model/MMasterKey.java | 55 -
.../metastore/model/MMetastoreDBProperties.java | 56 -
.../hive/metastore/model/MNotificationLog.java | 98 -
.../metastore/model/MNotificationNextId.java | 42 -
.../hadoop/hive/metastore/model/MOrder.java | 62 -
.../hadoop/hive/metastore/model/MPartition.java | 155 -
.../model/MPartitionColumnPrivilege.java | 161 -
.../model/MPartitionColumnStatistics.java | 258 -
.../hive/metastore/model/MPartitionEvent.java | 90 -
.../metastore/model/MPartitionPrivilege.java | 139 -
.../hive/metastore/model/MPrincipalDesc.java | 59 -
.../hive/metastore/model/MResourceUri.java | 49 -
.../hadoop/hive/metastore/model/MRole.java | 80 -
.../hadoop/hive/metastore/model/MRoleMap.java | 120 -
.../hadoop/hive/metastore/model/MSerDeInfo.java | 81 -
.../metastore/model/MStorageDescriptor.java | 277 -
.../hive/metastore/model/MStringList.java | 62 -
.../hadoop/hive/metastore/model/MTable.java | 256 -
.../metastore/model/MTableColumnPrivilege.java | 160 -
.../metastore/model/MTableColumnStatistics.java | 249 -
.../hive/metastore/model/MTablePrivilege.java | 139 -
.../hadoop/hive/metastore/model/MType.java | 105 -
.../hive/metastore/model/MVersionTable.java | 57 -
metastore/src/model/package.jdo | 1081 -
.../metastore/hbase/hbase_metastore_proto.proto | 368 -
.../DummyRawStoreControlledCommit.java | 29 +-
.../DummyRawStoreForJdoConnection.java | 26 +-
.../InjectableBehaviourObjectStore.java | 26 +-
.../hadoop/hive/metastore/TestObjectStore.java | 25 +-
.../hadoop/hive/metastore/TestOldSchema.java | 220 +
.../hive/metastore/cache/TestCachedStore.java | 230 +-
.../TestDataSourceProviderFactory.java | 167 +
.../hadoop/hive/metastore/hbase/MockUtils.java | 219 -
.../hbase/TestHBaseAggregateStatsCache.java | 316 -
...stHBaseAggregateStatsCacheWithBitVector.java | 184 -
.../TestHBaseAggregateStatsExtrapolation.java | 716 -
.../TestHBaseAggregateStatsNDVUniformDist.java | 580 -
.../hbase/TestHBaseFilterPlanUtil.java | 483 -
.../hive/metastore/hbase/TestHBaseStore.java | 1978 -
.../hbase/TestHBaseStoreBitVector.java | 622 -
.../metastore/hbase/TestHBaseStoreCached.java | 365 -
.../hbase/TestSharedStorageDescriptor.java | 152 -
pom.xml | 55 +-
ql/if/queryplan.thrift | 3 +
ql/pom.xml | 10 +-
ql/src/gen/thrift/gen-cpp/queryplan_types.cpp | 12 +-
ql/src/gen/thrift/gen-cpp/queryplan_types.h | 5 +-
.../hadoop/hive/ql/plan/api/StageType.java | 11 +-
ql/src/gen/thrift/gen-php/Types.php | 6 +
ql/src/gen/thrift/gen-py/queryplan/ttypes.py | 9 +
ql/src/gen/thrift/gen-rb/queryplan_types.rb | 7 +-
.../UDAFTemplates/VectorUDAFAvg.txt | 255 +-
.../UDAFTemplates/VectorUDAFAvgDecimal.txt | 566 +
.../UDAFTemplates/VectorUDAFAvgDecimalMerge.txt | 597 +
.../UDAFTemplates/VectorUDAFAvgMerge.txt | 547 +
.../UDAFTemplates/VectorUDAFAvgTimestamp.txt | 517 +
.../UDAFTemplates/VectorUDAFMinMax.txt | 184 +-
.../UDAFTemplates/VectorUDAFMinMaxDecimal.txt | 30 +-
.../VectorUDAFMinMaxIntervalDayTime.txt | 26 +-
.../UDAFTemplates/VectorUDAFMinMaxString.txt | 29 +-
.../UDAFTemplates/VectorUDAFMinMaxTimestamp.txt | 26 +-
.../UDAFTemplates/VectorUDAFSum.txt | 139 +-
.../UDAFTemplates/VectorUDAFVar.txt | 194 +-
.../UDAFTemplates/VectorUDAFVarDecimal.txt | 219 +-
.../UDAFTemplates/VectorUDAFVarMerge.txt | 573 +
.../UDAFTemplates/VectorUDAFVarTimestamp.txt | 477 +
.../hadoop/hive/llap/ChannelOutputStream.java | 37 +-
.../hadoop/hive/llap/LlapCacheAwareFs.java | 422 +
.../hive/llap/LlapOutputFormatService.java | 4 +-
.../hadoop/hive/llap/LlapRecordWriter.java | 6 +-
.../hadoop/hive/ql/CompilationOpContext.java | 13 +-
.../java/org/apache/hadoop/hive/ql/Context.java | 2 +-
.../java/org/apache/hadoop/hive/ql/Driver.java | 12 +
.../org/apache/hadoop/hive/ql/ErrorMsg.java | 40 +-
.../hadoop/hive/ql/HiveDriverRunHook.java | 4 +
.../hive/ql/HiveDriverRunHookContext.java | 4 +
.../apache/hadoop/hive/ql/QueryProperties.java | 1 +
.../hadoop/hive/ql/exec/CollectOperator.java | 5 +
.../hadoop/hive/ql/exec/ColumnStatsTask.java | 46 +-
.../hive/ql/exec/ColumnStatsUpdateTask.java | 24 +-
.../hadoop/hive/ql/exec/CommonJoinOperator.java | 17 +-
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 152 +-
.../hadoop/hive/ql/exec/DemuxOperator.java | 5 +
.../apache/hadoop/hive/ql/exec/Description.java | 5 +
.../apache/hadoop/hive/ql/exec/ExplainTask.java | 27 +-
.../hadoop/hive/ql/exec/FetchOperator.java | 6 -
.../hadoop/hive/ql/exec/FileSinkOperator.java | 77 +-
.../hadoop/hive/ql/exec/ForwardOperator.java | 5 +
.../hadoop/hive/ql/exec/FunctionRegistry.java | 4 +-
.../hadoop/hive/ql/exec/FunctionTask.java | 28 +-
.../hadoop/hive/ql/exec/GroupByOperator.java | 2 +-
.../hive/ql/exec/HashTableDummyOperator.java | 5 +-
.../hive/ql/exec/HashTableSinkOperator.java | 10 +-
.../hadoop/hive/ql/exec/KeyWrapperFactory.java | 6 +
.../ql/exec/LateralViewForwardOperator.java | 5 +
.../hadoop/hive/ql/exec/ListSinkOperator.java | 5 +
.../hadoop/hive/ql/exec/MapJoinOperator.java | 18 +
.../apache/hadoop/hive/ql/exec/MapOperator.java | 3 +
.../hadoop/hive/ql/exec/MapredContext.java | 2 +
.../apache/hadoop/hive/ql/exec/MoveTask.java | 48 +-
.../apache/hadoop/hive/ql/exec/MuxOperator.java | 5 +
.../apache/hadoop/hive/ql/exec/NodeUtils.java | 68 +-
.../hadoop/hive/ql/exec/ObjectCacheFactory.java | 9 +-
.../apache/hadoop/hive/ql/exec/Operator.java | 75 +
.../hadoop/hive/ql/exec/OperatorFactory.java | 3 +
.../hadoop/hive/ql/exec/OperatorUtils.java | 39 +-
.../hive/ql/exec/OrcFileMergeOperator.java | 38 +-
.../hadoop/hive/ql/exec/ReplCopyTask.java | 157 +-
.../hive/ql/exec/SerializationUtilities.java | 7 +-
.../ql/exec/SparkHashTableSinkOperator.java | 17 +-
.../hadoop/hive/ql/exec/StatsNoJobTask.java | 5 -
.../hadoop/hive/ql/exec/TableScanOperator.java | 11 +-
.../apache/hadoop/hive/ql/exec/TaskFactory.java | 10 +-
.../org/apache/hadoop/hive/ql/exec/UDF.java | 3 +
.../hive/ql/exec/UDFArgumentException.java | 4 +
.../ql/exec/UDFArgumentLengthException.java | 5 +
.../hive/ql/exec/UDFArgumentTypeException.java | 6 +-
.../hadoop/hive/ql/exec/UDFMethodResolver.java | 1 +
.../hadoop/hive/ql/exec/UnionOperator.java | 5 +
.../apache/hadoop/hive/ql/exec/Utilities.java | 509 +-
.../hive/ql/exec/errors/ErrorAndSolution.java | 5 +-
.../hadoop/hive/ql/exec/mr/MapRedTask.java | 2 +-
.../ql/exec/persistence/FlatRowContainer.java | 2 +-
.../hive/ql/exec/persistence/RowContainer.java | 10 +-
.../hadoop/hive/ql/exec/repl/ReplDumpTask.java | 325 +
.../hadoop/hive/ql/exec/repl/ReplDumpWork.java | 75 +
.../hive/ql/exec/repl/ReplStateLogTask.java | 51 +
.../hive/ql/exec/repl/ReplStateLogWork.java | 100 +
.../ql/exec/repl/bootstrap/ReplLoadTask.java | 289 +
.../ql/exec/repl/bootstrap/ReplLoadWork.java | 70 +
.../repl/bootstrap/events/BootstrapEvent.java | 28 +
.../repl/bootstrap/events/DatabaseEvent.java | 34 +
.../repl/bootstrap/events/FunctionEvent.java | 33 +
.../repl/bootstrap/events/PartitionEvent.java | 26 +
.../exec/repl/bootstrap/events/TableEvent.java | 42 +
.../filesystem/BootstrapEventsIterator.java | 181 +
.../filesystem/DatabaseEventsIterator.java | 145 +
.../events/filesystem/FSDatabaseEvent.java | 88 +
.../events/filesystem/FSFunctionEvent.java | 39 +
.../events/filesystem/FSPartitionEvent.java | 84 +
.../events/filesystem/FSTableEvent.java | 123 +
.../exec/repl/bootstrap/load/LoadDatabase.java | 129 +
.../exec/repl/bootstrap/load/LoadFunction.java | 87 +
.../repl/bootstrap/load/ReplicationState.java | 58 +
.../exec/repl/bootstrap/load/TaskTracker.java | 120 +
.../bootstrap/load/table/LoadPartitions.java | 316 +
.../repl/bootstrap/load/table/LoadTable.java | 245 +
.../repl/bootstrap/load/table/TableContext.java | 49 +
.../exec/repl/bootstrap/load/util/Context.java | 37 +
.../repl/bootstrap/load/util/PathUtils.java | 105 +
.../hive/ql/exec/spark/HashTableLoader.java | 2 +-
.../ql/exec/spark/LocalHiveSparkClient.java | 2 +-
.../ql/exec/spark/RemoteHiveSparkClient.java | 19 +-
.../hive/ql/exec/spark/RepartitionShuffler.java | 42 -
.../exec/spark/SparkDynamicPartitionPruner.java | 13 +-
.../hive/ql/exec/spark/SparkPlanGenerator.java | 6 +-
.../ql/exec/spark/SparkReduceRecordHandler.java | 331 +-
.../hive/ql/exec/spark/SparkUtilities.java | 20 +
.../ql/exec/spark/session/SparkSessionImpl.java | 11 +-
.../hadoop/hive/ql/exec/tez/DagUtils.java | 36 +-
.../hive/ql/exec/tez/MapRecordSource.java | 12 +
.../hadoop/hive/ql/exec/tez/ObjectCache.java | 5 +
.../hive/ql/exec/tez/ReduceRecordSource.java | 80 +-
.../ql/exec/tez/RestrictedConfigChecker.java | 79 +
.../ql/exec/tez/SessionExpirationTracker.java | 235 +
.../hadoop/hive/ql/exec/tez/TezSessionPool.java | 165 +
.../hive/ql/exec/tez/TezSessionPoolManager.java | 551 +-
.../hive/ql/exec/tez/TezSessionPoolSession.java | 158 +
.../apache/hadoop/hive/ql/exec/tez/TezTask.java | 84 +-
.../ql/exec/tez/tools/KeyValueInputMerger.java | 50 +-
.../vector/VectorAppMasterEventOperator.java | 3 +-
.../hive/ql/exec/vector/VectorAssignRow.java | 410 +-
.../hive/ql/exec/vector/VectorBatchDebug.java | 105 +
.../ql/exec/vector/VectorDeserializeRow.java | 492 +-
.../hive/ql/exec/vector/VectorExtractRow.java | 40 +-
.../ql/exec/vector/VectorFilterOperator.java | 2 +-
.../ql/exec/vector/VectorGroupByOperator.java | 69 +-
.../ql/exec/vector/VectorLimitOperator.java | 2 +-
.../exec/vector/VectorMapJoinBaseOperator.java | 3 +-
.../hive/ql/exec/vector/VectorMapOperator.java | 48 +-
.../exec/vector/VectorSMBMapJoinOperator.java | 2 +-
.../ql/exec/vector/VectorSelectOperator.java | 12 +-
.../ql/exec/vector/VectorizationContext.java | 247 +-
.../ql/exec/vector/VectorizedBatchUtil.java | 61 +-
.../ql/exec/vector/VectorizedRowBatchCtx.java | 55 +-
.../vector/expressions/CastDateToTimestamp.java | 131 +
.../vector/expressions/IfExprColumnNull.java | 109 +
.../vector/expressions/IfExprNullColumn.java | 109 +
.../VectorExpressionWriterFactory.java | 188 +-
.../VectorInBloomFilterColDynamicValue.java | 18 +-
.../aggregates/VectorAggregateExpression.java | 17 +-
.../aggregates/VectorUDAFAvgDecimal.java | 521 -
.../aggregates/VectorUDAFAvgTimestamp.java | 488 -
.../aggregates/VectorUDAFBloomFilter.java | 60 +-
.../aggregates/VectorUDAFBloomFilterMerge.java | 43 +-
.../expressions/aggregates/VectorUDAFCount.java | 28 +-
.../aggregates/VectorUDAFCountMerge.java | 28 +-
.../aggregates/VectorUDAFCountStar.java | 20 +-
.../aggregates/VectorUDAFStdPopTimestamp.java | 533 -
.../aggregates/VectorUDAFStdSampTimestamp.java | 533 -
.../aggregates/VectorUDAFSumDecimal.java | 39 +-
.../aggregates/VectorUDAFSumTimestamp.java | 434 +
.../aggregates/VectorUDAFVarPopTimestamp.java | 533 -
.../aggregates/VectorUDAFVarSampTimestamp.java | 533 -
.../mapjoin/VectorMapJoinCommonOperator.java | 35 +-
.../VectorMapJoinGenerateResultOperator.java | 6 +-
.../fast/VectorMapJoinFastTableContainer.java | 8 +-
.../VectorMapJoinOptimizedCreateHashTable.java | 6 +-
.../exec/vector/ptf/VectorPTFEvaluatorBase.java | 122 +
.../vector/ptf/VectorPTFEvaluatorCount.java | 108 +
.../vector/ptf/VectorPTFEvaluatorCountStar.java | 79 +
.../ptf/VectorPTFEvaluatorDecimalAvg.java | 162 +
.../VectorPTFEvaluatorDecimalFirstValue.java | 117 +
.../ptf/VectorPTFEvaluatorDecimalLastValue.java | 113 +
.../ptf/VectorPTFEvaluatorDecimalMax.java | 146 +
.../ptf/VectorPTFEvaluatorDecimalMin.java | 146 +
.../ptf/VectorPTFEvaluatorDecimalSum.java | 140 +
.../vector/ptf/VectorPTFEvaluatorDenseRank.java | 77 +
.../vector/ptf/VectorPTFEvaluatorDoubleAvg.java | 153 +
.../ptf/VectorPTFEvaluatorDoubleFirstValue.java | 113 +
.../ptf/VectorPTFEvaluatorDoubleLastValue.java | 109 +
.../vector/ptf/VectorPTFEvaluatorDoubleMax.java | 136 +
.../vector/ptf/VectorPTFEvaluatorDoubleMin.java | 136 +
.../vector/ptf/VectorPTFEvaluatorDoubleSum.java | 134 +
.../vector/ptf/VectorPTFEvaluatorLongAvg.java | 153 +
.../ptf/VectorPTFEvaluatorLongFirstValue.java | 113 +
.../ptf/VectorPTFEvaluatorLongLastValue.java | 109 +
.../vector/ptf/VectorPTFEvaluatorLongMax.java | 136 +
.../vector/ptf/VectorPTFEvaluatorLongMin.java | 136 +
.../vector/ptf/VectorPTFEvaluatorLongSum.java | 134 +
.../exec/vector/ptf/VectorPTFEvaluatorRank.java | 81 +
.../vector/ptf/VectorPTFEvaluatorRowNumber.java | 77 +
.../exec/vector/ptf/VectorPTFGroupBatches.java | 216 +
.../ql/exec/vector/ptf/VectorPTFOperator.java | 570 +
.../ql/exec/vector/udf/VectorUDFAdaptor.java | 171 +-
.../hive/ql/hooks/ExecuteWithHookContext.java | 5 +
.../org/apache/hadoop/hive/ql/hooks/Hook.java | 6 +
.../hadoop/hive/ql/hooks/HookContext.java | 4 +
.../hadoop/hive/ql/hooks/PostExecute.java | 4 +
.../apache/hadoop/hive/ql/hooks/PreExecute.java | 4 +
.../hadoop/hive/ql/hooks/QueryLifeTimeHook.java | 5 +
.../hive/ql/hooks/QueryLifeTimeHookContext.java | 4 +
.../hooks/QueryLifeTimeHookWithParseHooks.java | 5 +
.../hadoop/hive/ql/io/AcidInputFormat.java | 11 +-
.../hadoop/hive/ql/io/AcidOutputFormat.java | 33 +-
.../org/apache/hadoop/hive/ql/io/AcidUtils.java | 152 +-
.../apache/hadoop/hive/ql/io/BucketCodec.java | 128 +
.../hive/ql/io/CombineHiveInputFormat.java | 2 +-
.../org/apache/hadoop/hive/ql/io/HdfsUtils.java | 40 +-
.../hadoop/hive/ql/io/HiveFileFormatUtils.java | 2 +-
.../hadoop/hive/ql/io/HiveInputFormat.java | 106 +-
.../io/LlapCacheOnlyInputFormatInterface.java | 28 +
.../hadoop/hive/ql/io/RecordIdentifier.java | 21 +-
.../hadoop/hive/ql/io/orc/ExternalCache.java | 2 +-
.../hive/ql/io/orc/OrcFileKeyWrapper.java | 36 +-
.../io/orc/OrcFileStripeMergeRecordReader.java | 5 +-
.../hadoop/hive/ql/io/orc/OrcInputFormat.java | 190 +-
.../hadoop/hive/ql/io/orc/OrcOutputFormat.java | 2 +-
.../hive/ql/io/orc/OrcRawRecordMerger.java | 773 +-
.../hadoop/hive/ql/io/orc/OrcRecordUpdater.java | 160 +-
.../apache/hadoop/hive/ql/io/orc/OrcSplit.java | 10 +-
.../io/orc/VectorizedOrcAcidRowBatchReader.java | 207 +-
.../hive/ql/io/orc/encoded/EncodedReader.java | 3 +-
.../ql/io/orc/encoded/EncodedReaderImpl.java | 538 +-
.../hive/ql/io/orc/encoded/IncompleteCb.java | 7 +
.../hadoop/hive/ql/io/orc/encoded/IoTrace.java | 478 +
.../hadoop/hive/ql/io/orc/encoded/Reader.java | 2 +-
.../hive/ql/io/orc/encoded/ReaderImpl.java | 6 +-
.../ql/io/parquet/MapredParquetInputFormat.java | 47 +-
.../io/parquet/MapredParquetOutputFormat.java | 30 +-
.../ql/io/parquet/ParquetRecordReaderBase.java | 54 +-
.../hive/ql/io/parquet/ProjectionPusher.java | 4 +-
.../parquet/VectorizedParquetInputFormat.java | 22 +-
.../ql/io/parquet/convert/ETypeConverter.java | 24 +-
.../hive/ql/io/parquet/convert/Repeated.java | 1 -
.../parquet/read/DataWritableReadSupport.java | 21 +-
.../read/ParquetFilterPredicateConverter.java | 13 +-
.../read/ParquetRecordReaderWrapper.java | 26 +-
.../ql/io/parquet/serde/ParquetTableUtils.java | 31 +-
.../ql/io/parquet/timestamp/NanoTimeUtils.java | 247 +-
.../vector/ParquetFooterInputFromCache.java | 196 +
.../vector/VectorizedParquetRecordReader.java | 202 +-
.../vector/VectorizedPrimitiveColumnReader.java | 21 +-
.../parquet/write/DataWritableWriteSupport.java | 14 +-
.../ql/io/parquet/write/DataWritableWriter.java | 10 +-
.../write/ParquetRecordWriterWrapper.java | 24 +
.../apache/hadoop/hive/ql/lib/RuleRegExp.java | 3 +
.../hadoop/hive/ql/lockmgr/DbTxnManager.java | 18 +-
.../hive/ql/lockmgr/EmbeddedLockManager.java | 22 +-
.../zookeeper/ZooKeeperHiveLockManager.java | 13 +-
.../hadoop/hive/ql/log/LogDivertAppender.java | 15 +-
.../hive/ql/log/LogDivertAppenderForTest.java | 8 +-
.../apache/hadoop/hive/ql/metadata/Hive.java | 174 +-
.../hadoop/hive/ql/metadata/HiveException.java | 38 +-
.../metadata/HiveMaterializedViewsRegistry.java | 9 +-
.../hive/ql/metadata/HiveStorageHandler.java | 4 +
.../metadata/HiveStoragePredicateHandler.java | 6 +
.../hadoop/hive/ql/metadata/InputEstimator.java | 4 +
.../ql/metadata/SessionHiveMetaStoreClient.java | 13 +
.../hadoop/hive/ql/metadata/VirtualColumn.java | 14 +
.../formatting/MetaDataFormatUtils.java | 35 +-
.../BucketingSortingReduceSinkOptimizer.java | 7 +-
.../optimizer/ConstantPropagateProcFactory.java | 6 +-
.../hive/ql/optimizer/ConvertJoinMapJoin.java | 89 +-
.../ql/optimizer/CountDistinctRewriteProc.java | 64 +-
.../DynamicPartitionPruningOptimization.java | 151 +-
.../hive/ql/optimizer/GenMapRedUtils.java | 5 +
.../hive/ql/optimizer/MapJoinProcessor.java | 5 +-
.../ql/optimizer/OperatorComparatorFactory.java | 552 -
.../hadoop/hive/ql/optimizer/Optimizer.java | 4 +-
.../ql/optimizer/ReduceSinkMapJoinProc.java | 6 +-
.../hive/ql/optimizer/SharedScanOptimizer.java | 626 -
.../hive/ql/optimizer/SharedWorkOptimizer.java | 1177 +
.../optimizer/SortedDynPartitionOptimizer.java | 1 +
.../ql/optimizer/SparkRemoveDynamicPruning.java | 80 +
.../SparkRemoveDynamicPruningBySize.java | 65 -
.../hive/ql/optimizer/StatsOptimizer.java | 17 +-
.../ql/optimizer/calcite/HiveCalciteUtil.java | 9 +-
.../calcite/HiveConfPlannerContext.java | 34 +
.../optimizer/calcite/HivePlannerContext.java | 7 +-
.../hive/ql/optimizer/calcite/HiveType.java | 48 +
.../optimizer/calcite/HiveTypeSystemImpl.java | 17 +-
.../ql/optimizer/calcite/RelOptHiveTable.java | 63 +-
.../calcite/cost/HiveAlgorithmsUtil.java | 10 +-
.../calcite/cost/HiveDefaultCostModel.java | 2 +-
.../calcite/cost/HiveOnTezCostModel.java | 35 +-
.../calcite/reloperators/HiveAggregate.java | 13 +-
.../calcite/reloperators/HiveIntersect.java | 10 +
.../calcite/reloperators/HiveJoin.java | 5 +-
.../calcite/reloperators/HiveTableScan.java | 50 +-
.../calcite/reloperators/HiveUnion.java | 9 +
.../rules/HiveAggregateJoinTransposeRule.java | 2 +-
.../calcite/rules/HiveAggregateReduceRule.java | 126 +
.../HiveDruidProjectFilterTransposeRule.java | 48 -
.../rules/HiveExpandDistinctAggregatesRule.java | 3 +-
.../rules/HiveFilterSetOpTransposeRule.java | 98 +-
.../calcite/rules/HiveJoinAddNotNullRule.java | 42 +-
.../HiveJoinPushTransitivePredicatesRule.java | 2 +-
.../rules/HiveReduceExpressionsRule.java | 2 +-
.../HiveReduceExpressionsWithStatsRule.java | 2 +-
.../calcite/rules/HiveRelDecorrelator.java | 306 +-
.../calcite/rules/HiveRelFieldTrimmer.java | 2 +-
.../calcite/rules/HiveRemoveSqCountCheck.java | 142 +
.../calcite/rules/HiveSemiJoinRule.java | 96 +-
.../calcite/rules/HiveSortJoinReduceRule.java | 3 +-
.../rules/HiveSortLimitPullUpConstantsRule.java | 2 +-
.../calcite/rules/HiveSortRemoveRule.java | 2 +-
.../calcite/rules/HiveSortUnionReduceRule.java | 2 +-
.../calcite/rules/HiveSubQueryRemoveRule.java | 620 +-
.../calcite/rules/HiveUnionMergeRule.java | 84 +
.../rules/HiveUnionPullUpConstantsRule.java | 2 +-
.../HiveMaterializedViewFilterScanRule.java | 92 -
.../rules/views/HiveMaterializedViewRule.java | 38 +
.../MaterializedViewSubstitutionVisitor.java | 292 -
.../rules/views/SubstitutionVisitor.java | 2458 -
.../calcite/stats/HiveRelMdSelectivity.java | 33 +-
.../calcite/translator/ASTBuilder.java | 31 +-
.../calcite/translator/ExprNodeConverter.java | 32 +-
.../calcite/translator/RexNodeConverter.java | 65 +-
.../translator/SqlFunctionConverter.java | 7 +-
.../calcite/translator/TypeConverter.java | 17 +-
.../correlation/CorrelationUtilities.java | 51 +-
.../correlation/ReduceSinkDeDuplication.java | 476 +-
.../ReduceSinkDeDuplicationUtils.java | 584 +
.../ReduceSinkJoinDeDuplication.java | 255 +
.../ql/optimizer/lineage/ExprProcFactory.java | 48 +-
.../hive/ql/optimizer/lineage/Generator.java | 27 +
.../ql/optimizer/lineage/OpProcFactory.java | 36 +-
.../hive/ql/optimizer/physical/LlapDecider.java | 2 +-
.../hive/ql/optimizer/physical/Vectorizer.java | 944 +-
.../hive/ql/optimizer/ppr/PartitionPruner.java | 23 +-
.../spark/CombineEquivalentWorkResolver.java | 79 +-
.../spark/SetSparkReducerParallelism.java | 141 +-
.../optimizer/spark/SparkMapJoinOptimizer.java | 50 +
.../spark/SparkPartitionPruningSinkDesc.java | 17 +-
.../stats/annotation/StatsRulesProcFactory.java | 263 +-
.../hadoop/hive/ql/parse/CalcitePlanner.java | 70 +-
.../hadoop/hive/ql/parse/ColumnStatsList.java | 64 +
.../ql/parse/ColumnStatsSemanticAnalyzer.java | 36 +-
.../hive/ql/parse/DDLSemanticAnalyzer.java | 49 +-
.../apache/hadoop/hive/ql/parse/EximUtil.java | 39 +-
.../hive/ql/parse/ExportSemanticAnalyzer.java | 243 +-
.../hive/ql/parse/FunctionSemanticAnalyzer.java | 4 +-
.../hadoop/hive/ql/parse/GenTezUtils.java | 232 +-
.../org/apache/hadoop/hive/ql/parse/HiveLexer.g | 2 +-
.../apache/hadoop/hive/ql/parse/HiveParser.g | 19 +-
.../hive/ql/parse/HiveSemanticAnalyzerHook.java | 4 +
.../parse/HiveSemanticAnalyzerHookContext.java | 9 +
.../HiveSemanticAnalyzerHookContextImpl.java | 12 +
.../hive/ql/parse/ImportSemanticAnalyzer.java | 177 +-
.../hive/ql/parse/LoadSemanticAnalyzer.java | 5 +
.../hadoop/hive/ql/parse/ParseContext.java | 30 +-
.../hive/ql/parse/PrunedPartitionList.java | 21 +-
.../apache/hadoop/hive/ql/parse/QBSubQuery.java | 23 +-
.../ql/parse/ReplicationSemanticAnalyzer.java | 782 +-
.../hadoop/hive/ql/parse/ReplicationSpec.java | 146 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 158 +-
.../hive/ql/parse/SemanticAnalyzerFactory.java | 8 +-
.../hadoop/hive/ql/parse/SemanticException.java | 5 +-
.../hadoop/hive/ql/parse/SemiJoinHint.java | 13 +-
.../hadoop/hive/ql/parse/TezCompiler.java | 85 +-
.../hive/ql/parse/TypeCheckProcFactory.java | 21 +-
.../ql/parse/UpdateDeleteSemanticAnalyzer.java | 10 +-
.../hadoop/hive/ql/parse/repl/CopyUtils.java | 136 +
.../hadoop/hive/ql/parse/repl/DumpType.java | 35 +
.../hadoop/hive/ql/parse/repl/ReplLogger.java | 36 +
.../hadoop/hive/ql/parse/repl/ReplState.java | 57 +
.../dump/BootStrapReplicationSpecFunction.java | 5 +-
.../hive/ql/parse/repl/dump/HiveWrapper.java | 2 +-
.../ql/parse/repl/dump/PartitionExport.java | 119 +
.../hive/ql/parse/repl/dump/TableExport.java | 274 +
.../hadoop/hive/ql/parse/repl/dump/Utils.java | 32 +
.../repl/dump/events/AddForeignKeyHandler.java | 41 +
.../events/AddNotNullConstraintHandler.java | 41 +
.../repl/dump/events/AddPartitionHandler.java | 3 +-
.../repl/dump/events/AddPrimaryKeyHandler.java | 41 +
.../dump/events/AddUniqueConstraintHandler.java | 41 +
.../repl/dump/events/CreateTableHandler.java | 8 +-
.../repl/dump/events/DropConstraintHandler.java | 41 +
.../repl/dump/events/EventHandlerFactory.java | 5 +
.../ql/parse/repl/dump/io/FileOperations.java | 112 +
.../parse/repl/dump/io/FunctionSerializer.java | 2 +-
.../parse/repl/dump/io/PartitionSerializer.java | 11 +-
.../ql/parse/repl/dump/io/TableSerializer.java | 22 +-
.../repl/dump/log/BootstrapDumpLogger.java | 71 +
.../repl/dump/log/IncrementalDumpLogger.java | 55 +
.../repl/dump/log/state/BootstrapDumpBegin.java | 47 +
.../repl/dump/log/state/BootstrapDumpEnd.java | 59 +
.../dump/log/state/BootstrapDumpFunction.java | 44 +
.../repl/dump/log/state/BootstrapDumpTable.java | 52 +
.../dump/log/state/IncrementalDumpBegin.java | 43 +
.../repl/dump/log/state/IncrementalDumpEnd.java | 54 +
.../dump/log/state/IncrementalDumpEvent.java | 51 +
.../hive/ql/parse/repl/load/DumpMetaData.java | 11 +-
.../parse/repl/load/EventDumpDirComparator.java | 38 +
.../parse/repl/load/UpdatedMetaDataTracker.java | 76 +
.../repl/load/log/BootstrapLoadLogger.java | 66 +
.../repl/load/log/IncrementalLoadLogger.java | 55 +
.../repl/load/log/state/BootstrapLoadBegin.java | 51 +
.../repl/load/log/state/BootstrapLoadEnd.java | 59 +
.../load/log/state/BootstrapLoadFunction.java | 44 +
.../repl/load/log/state/BootstrapLoadTable.java | 52 +
.../load/log/state/IncrementalLoadBegin.java | 47 +
.../repl/load/log/state/IncrementalLoadEnd.java | 54 +
.../load/log/state/IncrementalLoadEvent.java | 51 +
.../load/message/AbstractMessageHandler.java | 25 +-
.../repl/load/message/AddForeignKeyHandler.java | 70 +
.../message/AddNotNullConstraintHandler.java | 71 +
.../repl/load/message/AddPrimaryKeyHandler.java | 68 +
.../message/AddUniqueConstraintHandler.java | 69 +
.../load/message/CreateFunctionHandler.java | 20 +-
.../load/message/DropConstraintHandler.java | 50 +
.../repl/load/message/DropFunctionHandler.java | 5 +-
.../repl/load/message/DropPartitionHandler.java | 5 +-
.../repl/load/message/DropTableHandler.java | 5 +-
.../parse/repl/load/message/InsertHandler.java | 3 +-
.../parse/repl/load/message/MessageHandler.java | 21 +-
.../load/message/RenamePartitionHandler.java | 15 +-
.../repl/load/message/RenameTableHandler.java | 24 +-
.../parse/repl/load/message/TableHandler.java | 23 +-
.../load/message/TruncatePartitionHandler.java | 11 +-
.../repl/load/message/TruncateTableHandler.java | 8 +-
.../ql/parse/spark/GenSparkProcContext.java | 5 +
.../hive/ql/parse/spark/GenSparkUtils.java | 124 +-
.../hive/ql/parse/spark/GenSparkWork.java | 2 +-
.../hive/ql/parse/spark/SparkCompiler.java | 32 +-
.../SparkPartitionPruningSinkOperator.java | 51 +-
.../parse/spark/SparkProcessAnalyzeTable.java | 17 +-
.../hive/ql/parse/spark/SplitOpTreeForDPP.java | 131 +-
.../hive/ql/plan/AbstractOperatorDesc.java | 9 +
.../hadoop/hive/ql/plan/AlterDatabaseDesc.java | 37 +-
.../hadoop/hive/ql/plan/AlterTableDesc.java | 48 +-
.../hadoop/hive/ql/plan/AppMasterEventDesc.java | 12 +
.../hadoop/hive/ql/plan/ColStatistics.java | 3 -
.../hive/ql/plan/CommonMergeJoinDesc.java | 9 +
.../ql/plan/ConditionalResolverMergeFiles.java | 11 +-
.../hadoop/hive/ql/plan/CreateFunctionDesc.java | 15 +-
.../hadoop/hive/ql/plan/DescTableDesc.java | 4 +-
.../hadoop/hive/ql/plan/DropFunctionDesc.java | 16 +-
.../hive/ql/plan/DynamicPruningEventDesc.java | 24 +
.../hadoop/hive/ql/plan/DynamicValue.java | 4 +
.../hadoop/hive/ql/plan/ExprNodeDescUtils.java | 21 +
.../hadoop/hive/ql/plan/FileSinkDesc.java | 29 +
.../apache/hadoop/hive/ql/plan/FilterDesc.java | 12 +
.../apache/hadoop/hive/ql/plan/GroupByDesc.java | 82 +-
.../hadoop/hive/ql/plan/HashTableSinkDesc.java | 10 +
.../hadoop/hive/ql/plan/HiveOperation.java | 4 +
.../hadoop/hive/ql/plan/ImportTableDesc.java | 71 +-
.../apache/hadoop/hive/ql/plan/JoinDesc.java | 15 +
.../hive/ql/plan/LateralViewJoinDesc.java | 11 +
.../apache/hadoop/hive/ql/plan/LimitDesc.java | 9 +
.../apache/hadoop/hive/ql/plan/MapJoinDesc.java | 19 +-
.../org/apache/hadoop/hive/ql/plan/MapWork.java | 49 +-
.../hadoop/hive/ql/plan/OperatorDesc.java | 1 +
.../org/apache/hadoop/hive/ql/plan/PTFDesc.java | 112 +
.../hadoop/hive/ql/plan/ReduceSinkDesc.java | 25 +
.../hive/ql/plan/RenamePartitionDesc.java | 22 +-
.../hadoop/hive/ql/plan/ReplCopyWork.java | 82 +-
.../apache/hadoop/hive/ql/plan/ScriptDesc.java | 11 +
.../apache/hadoop/hive/ql/plan/SelectDesc.java | 12 +
.../apache/hadoop/hive/ql/plan/Statistics.java | 23 +-
.../hadoop/hive/ql/plan/TableScanDesc.java | 26 +
.../hadoop/hive/ql/plan/TezEdgeProperty.java | 7 +-
.../hadoop/hive/ql/plan/TruncateTableDesc.java | 11 +-
.../apache/hadoop/hive/ql/plan/UDTFDesc.java | 12 +
.../hadoop/hive/ql/plan/VectorGroupByDesc.java | 18 +
.../hadoop/hive/ql/plan/VectorMapJoinDesc.java | 24 +-
.../hadoop/hive/ql/plan/VectorPTFDesc.java | 360 +
.../hadoop/hive/ql/plan/VectorPTFInfo.java | 161 +
.../hadoop/hive/ql/plan/ptf/WindowFrameDef.java | 6 +-
.../hive/ql/ppd/SyntheticJoinPredicate.java | 2 +-
.../hadoop/hive/ql/processors/DfsProcessor.java | 60 +-
.../hadoop/hive/ql/processors/SetProcessor.java | 11 +
.../AuthorizationPreEventListener.java | 19 +
.../authorization/plugin/HiveOperationType.java | 4 +
.../plugin/sqlstd/Operation2Privilege.java | 31 +
.../sqlstd/SQLStdHiveAccessController.java | 3 +-
.../SQLStdHiveAuthorizationValidator.java | 9 +
.../hadoop/hive/ql/session/LineageState.java | 14 +
.../hadoop/hive/ql/session/OperationLog.java | 6 +-
.../hive/ql/stats/ClientStatsPublisher.java | 6 +
.../hadoop/hive/ql/stats/StatsAggregator.java | 6 +-
.../hive/ql/stats/StatsCollectionContext.java | 5 +
.../hadoop/hive/ql/stats/StatsPublisher.java | 6 +-
.../apache/hadoop/hive/ql/stats/StatsUtils.java | 289 +-
.../hive/ql/txn/compactor/CompactorMR.java | 14 +-
.../apache/hadoop/hive/ql/udf/UDFToBoolean.java | 2 +-
.../apache/hadoop/hive/ql/udf/UDFToInteger.java | 5 +-
.../apache/hadoop/hive/ql/udf/UDFToString.java | 4 +-
.../generic/AbstractGenericUDAFResolver.java | 1 +
.../hadoop/hive/ql/udf/generic/Collector.java | 4 +
.../DecimalNumDistinctValueEstimator.java | 42 -
.../DoubleNumDistinctValueEstimator.java | 39 -
.../hive/ql/udf/generic/GenericUDAFAverage.java | 10 +-
.../ql/udf/generic/GenericUDAFBloomFilter.java | 32 +-
.../ql/udf/generic/GenericUDAFComputeStats.java | 381 +-
.../ql/udf/generic/GenericUDAFEvaluator.java | 8 +
.../GenericUDAFMkCollectionEvaluator.java | 2 +-
.../udf/generic/GenericUDAFParameterInfo.java | 4 +
.../ql/udf/generic/GenericUDAFResolver2.java | 1 +
.../ql/udf/generic/GenericUDAFStdSample.java | 13 +-
.../hive/ql/udf/generic/GenericUDAFSum.java | 25 +-
.../udf/generic/GenericUDAFVarianceSample.java | 13 +-
.../hadoop/hive/ql/udf/generic/GenericUDF.java | 12 +-
.../hive/ql/udf/generic/GenericUDFDate.java | 4 +-
.../ql/udf/generic/GenericUDFInBloomFilter.java | 16 +-
.../hive/ql/udf/generic/GenericUDFLeadLag.java | 4 +-
.../ql/udf/generic/GenericUDFTimestamp.java | 3 +-
.../generic/GenericUDFToTimestampLocalTZ.java | 113 +
.../ql/udf/generic/GenericUDFToTimestampTZ.java | 89 -
.../hadoop/hive/ql/udf/generic/GenericUDTF.java | 5 +-
.../ql/udf/generic/GenericUDTFGetSplits.java | 20 +-
.../generic/LongNumDistinctValueEstimator.java | 43 -
.../udf/generic/NumDistinctValueEstimator.java | 383 -
.../StringNumDistinctValueEstimator.java | 39 -
.../apache/hadoop/hive/ql/TestTxnCommands.java | 250 +-
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 553 +-
.../ql/TestTxnCommands2WithSplitUpdate.java | 586 -
...ommands2WithSplitUpdateAndVectorization.java | 4 +-
.../hadoop/hive/ql/TestTxnCommandsBase.java | 162 +
.../apache/hadoop/hive/ql/TestTxnNoBuckets.java | 297 +
.../hive/ql/exec/TestKeyWrapperFactory.java | 113 +
.../hadoop/hive/ql/exec/TestUtilities.java | 224 +
.../ql/exec/persistence/TestRowContainer.java | 75 +
.../repl/bootstrap/load/TaskTrackerTest.java | 46 +
.../hive/ql/exec/spark/TestSparkTask.java | 49 +
.../hive/ql/exec/tez/SampleTezSessionState.java | 3 +-
.../hadoop/hive/ql/exec/tez/TestDagUtils.java | 69 +
.../hive/ql/exec/tez/TestTezSessionPool.java | 46 +-
.../hadoop/hive/ql/exec/tez/TestTezTask.java | 6 +-
.../hive/ql/exec/util/DescriptionTest.java | 31 +
.../CollectorTestOperator.java | 47 +
.../CountCollectorTestOperator.java | 48 +
.../CountVectorCollectorTestOperator.java | 47 +
.../RowCollectorTestOperator.java | 52 +
.../RowCollectorTestOperatorBase.java | 32 +
.../RowVectorCollectorTestOperator.java | 63 +
.../ql/exec/util/rowobjects/RowTestObjects.java | 100 +
.../exec/util/rowobjects/RowTestObjectsMap.java | 77 +
.../util/rowobjects/RowTestObjectsMultiSet.java | 91 +
.../exec/vector/TestVectorGroupByOperator.java | 2 +-
.../exec/vector/TestVectorSelectOperator.java | 3 +-
.../expressions/TestVectorMathFunctions.java | 22 +
.../vector/expressions/TestVectorTypeCasts.java | 16 +
.../exec/vector/mapjoin/MapJoinTestConfig.java | 658 +
.../ql/exec/vector/mapjoin/MapJoinTestData.java | 272 +
.../vector/mapjoin/MapJoinTestDescription.java | 239 +
.../vector/mapjoin/TestMapJoinOperator.java | 549 +
.../util/batchgen/TestVectorBatchGenerate.java | 57 +
.../batchgen/VectorBatchGenerateStream.java | 63 +
.../util/batchgen/VectorBatchGenerateUtil.java | 92 +
.../util/batchgen/VectorBatchGenerator.java | 306 +
.../batchgen/VectorColumnGroupGenerator.java | 494 +
.../apache/hadoop/hive/ql/io/TestAcidUtils.java | 64 +-
.../hive/ql/io/orc/TestInputOutputFormat.java | 772 +-
.../hive/ql/io/orc/TestOrcRawRecordMerger.java | 745 +-
.../hive/ql/io/orc/TestOrcRecordUpdater.java | 51 +-
.../TestVectorizedOrcAcidRowBatchReader.java | 8 +-
.../io/orc/encoded/TestEncodedReaderImpl.java | 14 +-
.../io/parquet/AbstractTestParquetDirect.java | 7 +-
.../ql/io/parquet/TestDataWritableWriter.java | 65 +-
.../parquet/TestMapredParquetOutputFormat.java | 10 -
.../io/parquet/TestParquetRowGroupFilter.java | 3 -
.../io/parquet/TestVectorizedColumnReader.java | 2 +-
.../parquet/VectorizedColumnReaderTestBase.java | 16 +-
.../io/parquet/convert/TestETypeConverter.java | 124 -
.../read/TestParquetFilterPredicate.java | 56 +-
.../serde/TestParquetTimestampUtils.java | 263 +
.../io/parquet/timestamp/TestNanoTimeUtils.java | 256 -
.../TestParquetTimestampConverter.java | 264 -
.../hive/ql/lockmgr/TestDbTxnManager2.java | 6 -
...TestGenMapRedUtilsCreateConditionalTask.java | 2 +
.../calcite/TestCBORuleFiredOnlyOnce.java | 2 +-
.../ql/optimizer/physical/TestVectorizer.java | 10 +-
.../parse/TestReplicationSemanticAnalyzer.java | 71 +-
.../hive/ql/parse/repl/CopyUtilsTest.java | 47 +
.../repl/load/TestEventDumpDirComparator.java | 44 +
.../PrimaryToReplicaResourceFunctionTest.java | 18 +
.../processors/TestCommandProcessorFactory.java | 2 +
.../hive/ql/txn/compactor/CompactorTest.java | 8 +
.../hive/ql/txn/compactor/TestWorker.java | 125 +-
.../queries/clientnegative/acid_overwrite.q | 8 -
.../alter_table_non_partitioned_table_cascade.q | 4 +
.../queries/clientnegative/create_not_acid.q | 2 +-
.../clientnegative/load_data_into_acid.q | 22 +
.../parquet_alter_part_table_drop_columns.q | 22 +
.../parquet_int96_alter_invalid_timezone.q | 5 -
.../parquet_int96_create_invalid_timezone.q | 3 -
.../clientnegative/repl_dump_requires_admin.q | 38 +
.../clientnegative/repl_load_requires_admin.q | 41 +
.../subquery_scalar_implicit_gby.q | 1 -
.../clientpositive/acid_bucket_pruning.q | 8 +-
.../clientpositive/acid_insert_overwrite.q | 73 +
.../queries/clientpositive/acid_no_buckets.q | 210 +
.../alter_partition_onto_nocurrent_db.q | 15 +
...lter_table_update_status_disable_bitvector.q | 139 +
.../clientpositive/authorization_view_8.q | 23 +
ql/src/test/queries/clientpositive/bitvector.q | 3 +
.../clientpositive/bucket_num_reducers_acid.q | 30 +
.../clientpositive/bucket_num_reducers_acid2.q | 33 +
ql/src/test/queries/clientpositive/char_udf1.q | 4 +-
.../queries/clientpositive/cmdwithcomments.q | 2 +
.../queries/clientpositive/compute_stats_date.q | 2 +-
.../clientpositive/compute_stats_decimal.q | 2 +-
.../clientpositive/compute_stats_double.q | 2 +-
.../queries/clientpositive/compute_stats_long.q | 2 +-
.../clientpositive/compute_stats_string.q | 2 +-
ql/src/test/queries/clientpositive/concat_op.q | 8 +-
.../queries/clientpositive/constprog_semijoin.q | 2 +
.../clientpositive/correlationoptimizer14.q | 1 +
.../queries/clientpositive/count_dist_rewrite.q | 2 +
ql/src/test/queries/clientpositive/dfscmd.q | 7 +
.../clientpositive/dynamic_semijoin_reduction.q | 22 +-
.../dynamic_semijoin_reduction_3.q | 2 +-
.../dynamic_semijoin_reduction_sw.q | 60 +
.../clientpositive/explaindenpendencydiffengs.q | 10 +
.../test/queries/clientpositive/filter_union.q | 56 +
ql/src/test/queries/clientpositive/fm-sketch.q | 58 +
.../clientpositive/groupby_distinct_samekey.q | 1 +
ql/src/test/queries/clientpositive/hll.q | 57 +
ql/src/test/queries/clientpositive/innerjoin1.q | 19 +
ql/src/test/queries/clientpositive/input20.q | 8 +-
ql/src/test/queries/clientpositive/input33.q | 8 +-
.../test/queries/clientpositive/input3_limit.q | 6 +-
.../queries/clientpositive/insert_orig_table.q | 20 +-
.../clientpositive/insert_overwrite_directory.q | 4 +-
.../clientpositive/insert_values_orig_table.q | 20 +-
.../insert_values_orig_table_use_metadata.q | 23 +-
.../test/queries/clientpositive/jdbc_handler.q | 7 +-
ql/src/test/queries/clientpositive/join47.q | 204 +
.../queries/clientpositive/join_max_hashtable.q | 13 +
ql/src/test/queries/clientpositive/llap_smb.q | 53 +
.../test/queries/clientpositive/localtimezone.q | 95 +
ql/src/test/queries/clientpositive/mapjoin47.q | 206 +
.../clientpositive/materialized_view_create.q | 6 +
.../materialized_view_create_rewrite.q | 2 +
ql/src/test/queries/clientpositive/mm_all.q | 45 -
.../test/queries/clientpositive/mm_loaddata.q | 53 +
.../queries/clientpositive/orc_empty_table.q | 11 +
.../clientpositive/orc_merge_incompat_schema.q | 47 +
.../orc_merge_incompat_writer_version.q | 30 +
.../clientpositive/parquet_int96_timestamp.q | 85 -
.../clientpositive/parquet_no_row_serde.q | 25 +
.../clientpositive/parquet_ppd_decimal.q | 1 +
.../clientpositive/parquet_predicate_pushdown.q | 1 +
.../parquet_predicate_pushdown_2.q | 7 +
.../parquet_timestamp_conversion.q | 13 -
.../test/queries/clientpositive/parquet_types.q | 1 +
.../parquet_types_vectorization.q | 1 +
ql/src/test/queries/clientpositive/partialdhj.q | 54 +
.../test/queries/clientpositive/perf/query11.q | 77 +
.../test/queries/clientpositive/perf/query28.q | 2 +
ql/src/test/queries/clientpositive/ppd2.q | 2 +
.../queries/clientpositive/ppd_join_filter.q | 2 +
.../reduceSinkDeDuplication_pRS_key_empty.q | 10 +-
.../reduce_deduplicate_extended.q | 1 +
.../test/queries/clientpositive/semijoin_hint.q | 32 +-
.../clientpositive/set_processor_namespaces.q | 8 +
ql/src/test/queries/clientpositive/skewjoin.q | 1 +
.../queries/clientpositive/smb_mapjoin_20.q | 13 +
.../queries/clientpositive/smb_mapjoin_47.q | 211 +
.../clientpositive/spark_constprog_dpp.q | 17 +
.../spark_dynamic_partition_pruning.q | 56 +-
.../spark_dynamic_partition_pruning_2.q | 16 +
.../spark_dynamic_partition_pruning_3.q | 228 +
...ark_dynamic_partition_pruning_mapjoin_only.q | 50 +
.../spark_explain_groupbyshuffle.q | 8 +
.../spark_multi_insert_parallel_orderby.q | 67 +
.../queries/clientpositive/spark_union_merge.q | 41 +
.../spark_use_ts_stats_for_mapjoin.q | 1 +
...spark_vectorized_dynamic_partition_pruning.q | 2 +-
.../test/queries/clientpositive/subquery_in.q | 50 +-
.../queries/clientpositive/subquery_scalar.q | 49 +
ql/src/test/queries/clientpositive/sysdb.q | 12 +-
.../test/queries/clientpositive/timestamptz.q | 16 +-
.../test/queries/clientpositive/timestamptz_1.q | 4 +-
.../test/queries/clientpositive/timestamptz_2.q | 3 +-
.../clientpositive/truncate_column_buckets.q | 4 +-
.../queries/clientpositive/udf_stddev_samp.q | 9 +
.../test/queries/clientpositive/udf_var_samp.q | 9 +
.../test/queries/clientpositive/varchar_udf1.q | 4 +-
.../queries/clientpositive/vector_aggregate_9.q | 16 +-
.../clientpositive/vector_between_columns.q | 1 +
.../queries/clientpositive/vector_complex_all.q | 41 +
.../clientpositive/vector_groupby_cube1.q | 17 +-
.../vector_groupby_grouping_id1.q | 21 +
.../vector_groupby_grouping_id2.q | 52 +
.../vector_groupby_grouping_window.q | 1 +
.../clientpositive/vector_groupby_reduce.q | 73 +-
.../vector_outer_reference_windowed.q | 91 +
.../clientpositive/vector_ptf_part_simple.q | 240 +-
.../clientpositive/vector_tablesample_rows.q | 8 +-
.../test/queries/clientpositive/vector_udf1.q | 74 +-
.../queries/clientpositive/vector_windowing.q | 791 +
.../vector_windowing_expressions.q | 94 +
.../clientpositive/vector_windowing_gby.q | 21 +
.../clientpositive/vector_windowing_gby2.q | 48 +
.../vector_windowing_multipartitioning.q | 73 +
.../clientpositive/vector_windowing_navfn.q | 2 +
.../vector_windowing_order_null.q | 58 +
.../vector_windowing_range_multiorder.q | 68 +
.../clientpositive/vector_windowing_rank.q | 117 +
.../clientpositive/vector_windowing_streaming.q | 85 +
.../vector_windowing_windowspec.q | 70 +
.../vector_windowing_windowspec4.q | 37 +
.../queries/clientpositive/vectorization_0.q | 15 +-
.../queries/clientpositive/vectorization_1.q | 22 +
.../queries/clientpositive/vectorization_10.q | 24 +
.../queries/clientpositive/vectorization_11.q | 15 +
.../queries/clientpositive/vectorization_12.q | 32 +
.../queries/clientpositive/vectorization_13.q | 2 +-
.../queries/clientpositive/vectorization_14.q | 2 +-
.../queries/clientpositive/vectorization_15.q | 5 +-
.../queries/clientpositive/vectorization_16.q | 2 +-
.../queries/clientpositive/vectorization_17.q | 2 +-
.../queries/clientpositive/vectorization_2.q | 23 +
.../queries/clientpositive/vectorization_3.q | 25 +
.../queries/clientpositive/vectorization_4.q | 23 +
.../queries/clientpositive/vectorization_5.q | 20 +
.../queries/clientpositive/vectorization_6.q | 21 +
.../queries/clientpositive/vectorization_7.q | 2 +-
.../queries/clientpositive/vectorization_8.q | 2 +-
.../queries/clientpositive/vectorization_9.q | 2 +-
.../queries/clientpositive/vectorized_casts.q | 4 +-
.../clientpositive/vectorized_distinct_gby.q | 7 +-
.../vectorized_multi_output_select.q | 28 +
.../queries/clientpositive/vectorized_parquet.q | 4 +
.../clientpositive/vectorized_parquet_types.q | 1 +
.../queries/clientpositive/vectorized_ptf.q | 1 +
.../clientpositive/vectorized_timestamp.q | 27 +-
.../results/clientnegative/acid_overwrite.q.out | 19 -
...er_table_non_partitioned_table_cascade.q.out | 21 +
.../authorization_uri_export.q.out | 1 +
.../clientnegative/create_not_acid.q.out | 4 +-
.../clientnegative/delete_non_acid_table.q.out | 2 +-
.../results/clientnegative/dyn_part_max.q.out | 1 -
.../clientnegative/load_data_into_acid.q.out | 33 +
.../parquet_alter_part_table_drop_columns.q.out | 53 +
.../parquet_int96_alter_invalid_timezone.q.out | 13 -
.../parquet_int96_create_invalid_timezone.q.out | 5 -
.../repl_dump_requires_admin.q.out | 69 +
.../repl_load_requires_admin.q.out | 80 +
.../set_hiveconf_internal_variable0.q.out | 2 +-
.../set_hiveconf_internal_variable1.q.out | 2 +-
.../subquery_scalar_implicit_gby.q.out | 1 -
.../clientnegative/update_non_acid_table.q.out | 2 +-
.../clientpositive/acid_insert_overwrite.q.out | 395 +
.../clientpositive/acid_table_stats.q.out | 14 +-
.../clientpositive/alterColumnStats.q.out | 16 +-
.../clientpositive/alterColumnStatsPart.q.out | 6 +-
.../alter_partition_onto_nocurrent_db.q.out | 78 +
.../alter_partition_update_status.q.out | 30 +-
.../alter_table_column_stats.q.out | 672 +-
.../alter_table_update_status.q.out | 384 +-
..._table_update_status_disable_bitvector.q.out | 708 +
.../clientpositive/analyze_tbl_part.q.out | 18 +-
.../annotate_stats_deep_filters.q.out | 4 +-
.../clientpositive/annotate_stats_groupby.q.out | 10 +-
.../clientpositive/annotate_stats_join.q.out | 44 +-
.../annotate_stats_join_pkfk.q.out | 28 +-
.../clientpositive/annotate_stats_select.q.out | 4 +-
.../clientpositive/autoColumnStats_4.q.out | 16 +-
.../clientpositive/autoColumnStats_5.q.out | 50 +-
.../clientpositive/autoColumnStats_6.q.out | 6 +-
.../clientpositive/autoColumnStats_7.q.out | 12 +-
.../clientpositive/autoColumnStats_8.q.out | 22 +-
.../clientpositive/autoColumnStats_9.q.out | 32 +-
.../results/clientpositive/auto_join0.q.out | 109 +-
.../results/clientpositive/auto_join12.q.out | 22 +-
.../results/clientpositive/auto_join15.q.out | 47 +-
.../results/clientpositive/auto_join16.q.out | 16 +-
.../results/clientpositive/auto_join20.q.out | 94 +-
.../results/clientpositive/auto_join26.q.out | 8 +-
.../results/clientpositive/auto_join27.q.out | 8 +-
.../results/clientpositive/auto_join31.q.out | 286 +-
.../results/clientpositive/auto_join4.q.out | 6 +-
.../results/clientpositive/auto_join5.q.out | 6 +-
.../results/clientpositive/auto_join8.q.out | 6 +-
.../auto_join_without_localtask.q.out | 74 +-
.../results/clientpositive/avro_decimal.q.out | 8 +-
.../clientpositive/avro_decimal_native.q.out | 8 +-
.../materialized_view_create_rewrite.q.out | 8 +
.../clientpositive/beeline/smb_mapjoin_12.q.out | 75 +-
.../test/results/clientpositive/bitvector.q.out | 10 +
.../bucket_num_reducers_acid.q.out | 33 +
.../bucket_num_reducers_acid2.q.out | 44 +
.../bucketsortoptimize_insert_8.q.out | 342 +-
.../cbo_rp_annotate_stats_groupby.q.out | 10 +-
.../clientpositive/cbo_rp_auto_join0.q.out | 98 +-
.../clientpositive/cbo_rp_auto_join1.q.out | 18 +-
.../results/clientpositive/cbo_rp_join0.q.out | 12 +-
.../clientpositive/cbo_rp_outer_join_ppr.q.out | 44 +-
.../test/results/clientpositive/char_udf1.q.out | 10 +-
.../clientpositive/cmdwithcomments.q.out | 17 +-
.../clientpositive/colstats_all_nulls.q.out | 16 +-
...names_with_leading_and_trailing_spaces.q.out | 16 +-
.../column_pruner_multiple_children.q.out | 22 +-
.../clientpositive/columnstats_partlvl.q.out | 92 +-
.../clientpositive/columnstats_partlvl_dp.q.out | 84 +-
.../clientpositive/columnstats_quoting.q.out | 24 +-
.../clientpositive/columnstats_tbllvl.q.out | 144 +-
.../test/results/clientpositive/combine2.q.out | 6 +-
.../results/clientpositive/compustat_avro.q.out | 16 +-
.../clientpositive/compute_stats_date.q.out | 34 +-
.../clientpositive/compute_stats_decimal.q.out | Bin 1633 -> 1512 bytes
.../clientpositive/compute_stats_double.q.out | Bin 1539 -> 1443 bytes
.../compute_stats_empty_table.q.out | 6 +-
.../clientpositive/compute_stats_long.q.out | Bin 1498 -> 1376 bytes
.../clientpositive/compute_stats_string.q.out | Bin 1517 -> 1448 bytes
.../test/results/clientpositive/concat_op.q.out | 24 +-
.../confirm_initial_tbl_stats.q.out | 86 +-
.../test/results/clientpositive/constGby.q.out | 2 +-
.../clientpositive/constant_prop_2.q.out | 6 +-
.../clientpositive/constantfolding.q.out | 4 +-
.../clientpositive/constprog_partitioner.q.out | 10 +-
.../clientpositive/correlated_join_keys.q.out | 10 +-
.../clientpositive/correlationoptimizer10.q.out | 4 +-
.../clientpositive/correlationoptimizer11.q.out | 8 +-
.../clientpositive/correlationoptimizer12.q.out | 4 +-
.../clientpositive/correlationoptimizer13.q.out | 32 +-
.../clientpositive/correlationoptimizer15.q.out | 4 +-
.../clientpositive/correlationoptimizer7.q.out | 24 +-
.../clientpositive/correlationoptimizer8.q.out | 252 +-
.../clientpositive/correlationoptimizer9.q.out | 156 +-
.../clientpositive/count_dist_rewrite.q.out | 12 +-
.../results/clientpositive/create_view.q.out | 4 +-
.../clientpositive/cross_join_merge.q.out | 80 +-
.../results/clientpositive/ctas_colname.q.out | 4 +-
.../results/clientpositive/decimal_stats.q.out | 8 +-
.../results/clientpositive/decimal_udf.q.out | 24 +-
.../results/clientpositive/deleteAnalyze.q.out | 8 +-
.../clientpositive/describe_syntax.q.out | 24 +-
.../results/clientpositive/describe_table.q.out | 32 +-
ql/src/test/results/clientpositive/dfscmd.q.out | 1 +
.../display_colstats_tbllvl.q.out | 128 +-
.../clientpositive/distinct_windowing.q.out | 8 +-
.../distinct_windowing_no_cbo.q.out | 12 +-
.../results/clientpositive/druid_basic2.q.out | 103 +-
.../clientpositive/druid_intervals.q.out | 130 +-
.../clientpositive/druid_timeseries.q.out | 6 +-
.../results/clientpositive/druid_topn.q.out | 268 +-
.../clientpositive/dynamic_rdd_cache.q.out | 1431 -
.../dynpart_sort_optimization_acid.q.out | 1475 -
.../encrypted/encryption_move_tbl.q.out | 36 +-
.../results/clientpositive/except_all.q.out | 48 +-
.../exec_parallel_column_stats.q.out | 12 +-
.../clientpositive/explain_logical.q.out | 32 +-
.../explaindenpendencydiffengs.q.out | 52 +
.../extrapolate_part_stats_full.q.out | 6 +-
.../extrapolate_part_stats_partial.q.out | 20 +-
.../results/clientpositive/filter_aggr.q.out | 228 +-
.../results/clientpositive/filter_union.q.out | 593 +
.../test/results/clientpositive/fm-sketch.q.out | 323 +
.../test/results/clientpositive/fold_case.q.out | 14 +-
.../clientpositive/fouter_join_ppr.q.out | 84 +-
.../results/clientpositive/groupby4_map.q.out | 4 +-
.../clientpositive/groupby4_map_skew.q.out | 4 +-
.../results/clientpositive/groupby_cube1.q.out | 20 +-
.../groupby_distinct_samekey.q.out | 45 +-
.../groupby_grouping_window.q.out | 2 +-
.../clientpositive/groupby_position.q.out | 56 +-
.../clientpositive/groupby_rollup1.q.out | 12 +-
.../clientpositive/groupby_sort_11.q.out | 18 +-
.../clientpositive/groupby_sort_1_23.q.out | 254 +-
.../results/clientpositive/groupby_sort_2.q.out | 6 +-
.../results/clientpositive/groupby_sort_3.q.out | 12 +-
.../results/clientpositive/groupby_sort_4.q.out | 12 +-
.../results/clientpositive/groupby_sort_5.q.out | 18 +-
.../results/clientpositive/groupby_sort_6.q.out | 20 +-
.../results/clientpositive/groupby_sort_7.q.out | 6 +-
.../results/clientpositive/groupby_sort_9.q.out | 6 +-
.../clientpositive/groupby_sort_skew_1_23.q.out | 254 +-
.../clientpositive/groupby_sort_test_1.q.out | 6 +-
ql/src/test/results/clientpositive/hll.q.out | 323 +
.../identity_project_remove_skip.q.out | 189 +-
.../clientpositive/index_auto_mult_tables.q.out | 52 +-
.../index_auto_mult_tables_compact.q.out | 52 +-
.../clientpositive/infer_bucket_sort.q.out | 4 +-
.../infer_bucket_sort_grouping_operators.q.out | 18 +-
.../results/clientpositive/innerjoin1.q.out | 214 +
.../test/results/clientpositive/input20.q.out | 21 +-
.../test/results/clientpositive/input30.q.out | 6 +-
.../test/results/clientpositive/input31.q.out | 4 +-
.../test/results/clientpositive/input32.q.out | 4 +-
.../test/results/clientpositive/input33.q.out | 21 +-
.../test/results/clientpositive/input39.q.out | 2 +-
.../results/clientpositive/input3_limit.q.out | 44 +-
.../test/results/clientpositive/input41.q.out | 2 +-
.../clientpositive/insert_orig_table.q.out | 68 +-
.../insert_overwrite_directory.q.out | 8 +-
.../insert_values_orig_table.q.out | 68 +-
.../insert_values_orig_table_use_metadata.q.out | 116 +-
ql/src/test/results/clientpositive/join12.q.out | 30 +-
ql/src/test/results/clientpositive/join16.q.out | 22 +-
ql/src/test/results/clientpositive/join29.q.out | 48 +-
ql/src/test/results/clientpositive/join30.q.out | 8 +-
ql/src/test/results/clientpositive/join31.q.out | 6 +-
ql/src/test/results/clientpositive/join32.q.out | 80 +-
ql/src/test/results/clientpositive/join33.q.out | 80 +-
ql/src/test/results/clientpositive/join35.q.out | 64 +-
ql/src/test/results/clientpositive/join38.q.out | 28 +-
ql/src/test/results/clientpositive/join4.q.out | 8 +-
ql/src/test/results/clientpositive/join40.q.out | 6 +-
ql/src/test/results/clientpositive/join47.q.out | 1715 +
ql/src/test/results/clientpositive/join5.q.out | 8 +-
ql/src/test/results/clientpositive/join8.q.out | 8 +-
.../clientpositive/join_alt_syntax.q.out | 272 +-
.../clientpositive/join_cond_pushdown_2.q.out | 112 +-
.../clientpositive/join_cond_pushdown_4.q.out | 112 +-
.../clientpositive/join_filters_overlap.q.out | 4 +-
.../results/clientpositive/join_parse.q.out | 102 +-
.../limit_pushdown_negative.q.out | 96 +-
.../list_bucket_query_oneskew_2.q.out | 10 +-
.../llap/acid_bucket_pruning.q.out | 30 +-
.../clientpositive/llap/acid_no_buckets.q.out | 1976 +
.../llap/authorization_view_8.q.out | 69 +
.../clientpositive/llap/autoColumnStats_2.q.out | 64 +-
.../clientpositive/llap/auto_join0.q.out | 51 +-
.../clientpositive/llap/auto_join1.q.out | 6 +-
.../clientpositive/llap/auto_join21.q.out | 10 +-
.../clientpositive/llap/auto_join29.q.out | 90 +-
.../clientpositive/llap/auto_join30.q.out | 630 +-
.../llap/auto_smb_mapjoin_14.q.out | 44 +-
.../llap/auto_sortmerge_join_9.q.out | 234 +-
.../clientpositive/llap/bucket_groupby.q.out | 72 +-
.../llap/bucket_map_join_tez1.q.out | 7 -
.../llap/bucket_map_join_tez2.q.out | 34 +-
.../results/clientpositive/llap/cluster.q.out | 40 +-
...names_with_leading_and_trailing_spaces.q.out | 16 +-
.../llap/column_table_stats.q.out | 68 +-
.../llap/column_table_stats_orc.q.out | 50 +-
.../llap/columnstats_part_coltype.q.out | 126 +-
.../llap/correlationoptimizer1.q.out | 376 +-
.../llap/correlationoptimizer2.q.out | 186 +-
.../llap/correlationoptimizer3.q.out | 296 +-
.../llap/correlationoptimizer4.q.out | 220 +-
.../llap/correlationoptimizer6.q.out | 532 +-
.../results/clientpositive/llap/count.q.out | 124 +-
.../llap/count_dist_rewrite.q.out | 66 +-
.../clientpositive/llap/cross_join.q.out | 8 +-
.../clientpositive/llap/deleteAnalyze.q.out | 8 +-
.../llap/dynamic_partition_pruning.q.out | 317 +-
.../llap/dynamic_semijoin_reduction.q.out | 863 +-
.../llap/dynamic_semijoin_reduction_3.q.out | 4 +-
.../llap/dynamic_semijoin_reduction_sw.q.out | 518 +
.../llap/dynamic_semijoin_user_level.q.out | 172 +-
.../llap/dynpart_sort_optimization2.q.out | 20 +-
.../clientpositive/llap/except_distinct.q.out | 52 +-
.../clientpositive/llap/explainuser_1.q.out | 1505 +-
.../clientpositive/llap/explainuser_2.q.out | 931 +-
.../clientpositive/llap/explainuser_4.q.out | 14 +-
.../extrapolate_part_stats_partial_ndv.q.out | 132 +-
.../clientpositive/llap/filter_union.q.out | 539 +
.../results/clientpositive/llap/groupby1.q.out | 6 +-
.../results/clientpositive/llap/groupby2.q.out | 6 +-
.../llap/groupby_resolution.q.out | 78 +-
.../results/clientpositive/llap/having.q.out | 66 +-
.../llap/hybridgrace_hashjoin_1.q.out | 12 +-
.../llap/hybridgrace_hashjoin_2.q.out | 24 +-
.../llap/identity_project_remove_skip.q.out | 71 +-
.../clientpositive/llap/insert_orig_table.q.out | 68 +-
.../clientpositive/llap/intersect_all.q.out | 40 +-
.../llap/intersect_distinct.q.out | 40 +-
.../clientpositive/llap/intersect_merge.q.out | 204 +-
.../clientpositive/llap/jdbc_handler.q.out | 32 +-
.../results/clientpositive/llap/join1.q.out | 6 +-
.../clientpositive/llap/join32_lessSize.q.out | 175 +-
.../llap/join_max_hashtable.q.out | 222 +-
.../llap/limit_join_transpose.q.out | 214 +-
.../clientpositive/llap/limit_pushdown.q.out | 68 +-
.../clientpositive/llap/limit_pushdown3.q.out | 64 +-
.../results/clientpositive/llap/lineage2.q.out | 4 +-
.../results/clientpositive/llap/lineage3.q.out | 4 +-
.../results/clientpositive/llap/llap_smb.q.out | 359 +
.../clientpositive/llap/llap_stats.q.out | 16 +-
.../llap/llap_vector_nohybridgrace.q.out | 12 +-
.../clientpositive/llap/llapdecider.q.out | 88 +-
.../results/clientpositive/llap/merge1.q.out | 16 +-
.../results/clientpositive/llap/merge2.q.out | 12 +-
.../results/clientpositive/llap/mergejoin.q.out | 16 +-
.../llap/metadata_only_queries.q.out | 112 +-
.../test/results/clientpositive/llap/mrr.q.out | 153 +-
.../clientpositive/llap/multiMapJoin2.q.out | 255 +-
.../clientpositive/llap/offset_limit.q.out | 10 +-
.../llap/offset_limit_ppd_optimizer.q.out | 58 +-
.../clientpositive/llap/orc_create.q.out | 2 +-
.../llap/orc_merge_incompat_schema.q.out | 161 +
.../orc_merge_incompat_writer_version.q.out | 108 +
.../clientpositive/llap/orc_ppd_basic.q.out | 234 +-
.../results/clientpositive/llap/parallel.q.out | 2 +-
.../clientpositive/llap/parallel_colstats.q.out | 26 +-
.../llap/parquet_predicate_pushdown.q.out | 18 +-
.../llap/parquet_types_vectorization.q.out | 849 +
.../clientpositive/llap/partialdhj.q.out | 460 +
.../llap/partition_shared_scan.q.out | 7 -
.../test/results/clientpositive/llap/ptf.q.out | 154 +-
.../clientpositive/llap/ptf_streaming.q.out | 96 +-
.../llap/reduce_deduplicate_extended.q.out | 74 +-
.../schema_evol_orc_vec_part_all_complex.q.out | 72 +-
.../schema_evol_text_vec_part_all_complex.q.out | 72 +-
...hema_evol_text_vecrow_part_all_complex.q.out | 72 +-
.../clientpositive/llap/semijoin_hint.q.out | 175 +-
.../results/clientpositive/llap/skewjoin.q.out | 12 +-
.../clientpositive/llap/skewjoinopt15.q.out | 4 +-
.../clientpositive/llap/smb_mapjoin_14.q.out | 54 +-
.../clientpositive/llap/smb_mapjoin_15.q.out | 4 +-
.../clientpositive/llap/stats_only_null.q.out | 6 +-
.../clientpositive/llap/subquery_exists.q.out | 22 +-
.../clientpositive/llap/subquery_in.q.out | 1015 +-
.../clientpositive/llap/subquery_multi.q.out | 531 +-
.../clientpositive/llap/subquery_notin.q.out | 740 +-
.../clientpositive/llap/subquery_null_agg.q.out | 55 +-
.../clientpositive/llap/subquery_scalar.q.out | 2227 +-
.../clientpositive/llap/subquery_select.q.out | 244 +-
.../clientpositive/llap/subquery_views.q.out | 111 +-
.../results/clientpositive/llap/sysdb.q.out | 252 +-
.../llap/table_access_keys_stats.q.out | 36 +-
.../results/clientpositive/llap/tez_dml.q.out | 12 +-
.../llap/tez_dynpart_hashjoin_1.q.out | 24 +-
.../llap/tez_dynpart_hashjoin_2.q.out | 24 +-
.../llap/tez_dynpart_hashjoin_3.q.out | 4 +-
.../clientpositive/llap/tez_join_hash.q.out | 20 +-
.../clientpositive/llap/tez_join_tests.q.out | 37 +-
.../clientpositive/llap/tez_joins_explain.q.out | 37 +-
.../clientpositive/llap/tez_smb_empty.q.out | 16 +-
.../clientpositive/llap/tez_smb_main.q.out | 4 +-
.../results/clientpositive/llap/tez_union.q.out | 64 +-
.../clientpositive/llap/tez_union2.q.out | 16 +-
.../llap/tez_union_group_by.q.out | 2 +-
.../llap/tez_union_multiinsert.q.out | 68 +-
.../llap/tez_vector_dynpart_hashjoin_1.q.out | 24 +-
.../llap/tez_vector_dynpart_hashjoin_2.q.out | 24 +-
.../results/clientpositive/llap/union2.q.out | 4 +-
.../results/clientpositive/llap/union4.q.out | 6 +-
.../results/clientpositive/llap/union5.q.out | 4 +-
.../results/clientpositive/llap/union6.q.out | 4 +-
.../results/clientpositive/llap/union7.q.out | 20 +-
.../results/clientpositive/llap/union9.q.out | 6 +-
.../clientpositive/llap/unionDistinct_1.q.out | 283 +-
.../clientpositive/llap/union_remove_26.q.out | 54 +-
.../clientpositive/llap/union_top_level.q.out | 30 +-
.../clientpositive/llap/varchar_udf1.q.out | 10 +-
.../llap/vector_adaptor_usage_mode.q.out | 43 +-
.../llap/vector_aggregate_9.q.out | 311 +-
.../llap/vector_auto_smb_mapjoin_14.q.out | 104 +-
.../clientpositive/llap/vector_between_in.q.out | 44 +-
.../llap/vector_binary_join_groupby.q.out | 8 +
.../llap/vector_cast_constant.q.out | 34 +-
.../clientpositive/llap/vector_char_2.q.out | 8 +
.../clientpositive/llap/vector_coalesce_2.q.out | 8 +
.../llap/vector_complex_all.q.out | 1209 +-
.../llap/vector_complex_join.q.out | 77 +-
.../clientpositive/llap/vector_count.q.out | 70 +-
.../llap/vector_count_distinct.q.out | 8 +
.../clientpositive/llap/vector_date_1.q.out | 2 +-
.../llap/vector_decimal_aggregate.q.out | 45 +-
.../llap/vector_decimal_precision.q.out | 32 +-
.../llap/vector_decimal_round.q.out | 44 +-
.../llap/vector_decimal_udf.q.out | 6 +-
.../clientpositive/llap/vector_distinct_2.q.out | 4 +
.../llap/vector_empty_where.q.out | 32 +
.../clientpositive/llap/vector_groupby4.q.out | 4 +
.../clientpositive/llap/vector_groupby6.q.out | 4 +
.../clientpositive/llap/vector_groupby_3.q.out | 4 +
.../llap/vector_groupby_cube1.q.out | 196 +-
.../llap/vector_groupby_grouping_id1.q.out | 800 +
.../llap/vector_groupby_grouping_id2.q.out | 2117 +
.../llap/vector_groupby_grouping_id3.q.out | 8 +
.../llap/vector_groupby_grouping_sets3.q.out | 8 +-
.../llap/vector_groupby_grouping_sets4.q.out | 27 -
.../vector_groupby_grouping_sets_grouping.q.out | 2 +-
.../llap/vector_groupby_grouping_window.q.out | 4 +-
.../llap/vector_groupby_mapjoin.q.out | 22 +-
.../llap/vector_groupby_reduce.q.out | 2568 +-
.../llap/vector_groupby_rollup1.q.out | 12 +-
.../llap/vector_grouping_sets.q.out | 8 +
.../llap/vector_include_no_sel.q.out | 4 +
.../clientpositive/llap/vector_inner_join.q.out | 2 +
.../llap/vector_interval_mapjoin.q.out | 8 +-
.../clientpositive/llap/vector_join30.q.out | 1170 +-
.../llap/vector_left_outer_join.q.out | 4 +-
.../llap/vector_leftsemi_mapjoin.q.out | 168 +
.../llap/vector_mapjoin_reduce.q.out | 52 +-
.../llap/vector_mr_diff_schema_alias.q.out | 34 +-
.../llap/vector_null_projection.q.out | 8 +-
.../llap/vector_number_compare_projection.q.out | 186 +-
.../clientpositive/llap/vector_nvl.q.out | 2 +-
.../clientpositive/llap/vector_orderby_5.q.out | 4 +
.../llap/vector_outer_join1.q.out | 16 +-
.../llap/vector_outer_join2.q.out | 8 +-
.../llap/vector_outer_reference_windowed.q.out | 1435 +
.../llap/vector_partition_diff_num_cols.q.out | 20 +
.../llap/vector_partitioned_date_time.q.out | 40 +-
.../llap/vector_ptf_part_simple.q.out | 3898 +-
.../llap/vector_reduce_groupby_decimal.q.out | 4 +
.../llap/vector_string_concat.q.out | 4 +
.../llap/vector_tablesample_rows.q.out | 93 +-
.../clientpositive/llap/vector_udf1.q.out | 1137 +-
.../llap/vector_when_case_null.q.out | 12 +-
.../clientpositive/llap/vector_windowing.q.out | 9768 +
.../llap/vector_windowing_expressions.q.out | 2022 +
.../llap/vector_windowing_gby.q.out | 314 +
.../llap/vector_windowing_gby2.q.out | 1158 +
.../vector_windowing_multipartitioning.q.out | 11587 +
.../llap/vector_windowing_navfn.q.out | 218 +-
.../llap/vector_windowing_order_null.q.out | 1232 +
.../vector_windowing_range_multiorder.q.out | 12584 ++
.../llap/vector_windowing_rank.q.out | 1860 +
.../llap/vector_windowing_streaming.q.out | 1036 +
.../llap/vector_windowing_windowspec.q.out | 2374 +
.../llap/vector_windowing_windowspec4.q.out | 223 +
.../clientpositive/llap/vectorization_0.q.out | 281 +-
.../clientpositive/llap/vectorization_1.q.out | 175 +
.../clientpositive/llap/vectorization_10.q.out | 118 +
.../clientpositive/llap/vectorization_11.q.out | 100 +
.../clientpositive/llap/vectorization_12.q.out | 240 +
.../clientpositive/llap/vectorization_13.q.out | 100 +-
.../clientpositive/llap/vectorization_14.q.out | 86 +-
.../clientpositive/llap/vectorization_15.q.out | 155 +-
.../clientpositive/llap/vectorization_16.q.out | 68 +-
.../clientpositive/llap/vectorization_17.q.out | 41 +-
.../clientpositive/llap/vectorization_2.q.out | 179 +
.../clientpositive/llap/vectorization_3.q.out | 184 +
.../clientpositive/llap/vectorization_4.q.out | 178 +
.../clientpositive/llap/vectorization_5.q.out | 173 +
.../clientpositive/llap/vectorization_6.q.out | 112 +
.../clientpositive/llap/vectorization_7.q.out | 18 +-
.../clientpositive/llap/vectorization_8.q.out | 34 +-
.../clientpositive/llap/vectorization_9.q.out | 68 +-
.../llap/vectorization_limit.q.out | 83 +-
.../llap/vectorization_pushdown.q.out | 10 +-
.../llap/vectorization_short_regress.q.out | 376 +-
.../llap/vectorized_bucketmapjoin1.q.out | 21 +-
.../clientpositive/llap/vectorized_case.q.out | 20 +-
.../clientpositive/llap/vectorized_casts.q.out | 89 +-
.../llap/vectorized_date_funcs.q.out | 4 +
.../llap/vectorized_distinct_gby.q.out | 150 +-
.../vectorized_dynamic_partition_pruning.q.out | 340 +-
.../vectorized_dynamic_semijoin_reduction.q.out | 60 +-
...vectorized_dynamic_semijoin_reduction2.q.out | 30 +-
.../llap/vectorized_mapjoin.q.out | 36 +-
.../llap/vectorized_mapjoin2.q.out | 8 +-
.../llap/vectorized_multi_output_select.q.out | 201 +
.../llap/vectorized_nested_mapjoin.q.out | 6 +-
.../llap/vectorized_parquet.q.out | 28 +-
.../llap/vectorized_parquet_types.q.out | 36 +-
.../clientpositive/llap/vectorized_ptf.q.out | 298 +-
.../llap/vectorized_shufflejoin.q.out | 26 +-
.../llap/vectorized_timestamp.q.out | 630 +-
.../llap/vectorized_timestamp_funcs.q.out | 70 +-
.../results/clientpositive/llap/windowing.q.out | 2 +-
.../clientpositive/llap/windowing_gby.q.out | 4 +-
.../llap/windowing_windowspec.q.out | 955 +
.../results/clientpositive/localtimezone.q.out | 558 +
.../clientpositive/louter_join_ppr.q.out | 84 +-
.../test/results/clientpositive/mapjoin47.q.out | 1790 +
.../test/results/clientpositive/masking_4.q.out | 16 +-
.../materialized_view_create.q.out | 40 +
.../materialized_view_create_rewrite.q.out | 8 +
ql/src/test/results/clientpositive/merge1.q.out | 6 +-
ql/src/test/results/clientpositive/merge2.q.out | 6 +-
.../clientpositive/metadata_only_queries.q.out | 112 +-
ql/src/test/results/clientpositive/mm_all.q.out | 296 -
.../results/clientpositive/mm_loaddata.q.out | 296 +
.../clientpositive/multi_insert_gby2.q.out | 40 +-
.../clientpositive/multi_insert_gby3.q.out | 189 +-
.../clientpositive/nested_column_pruning.q.out | 42 +-
.../results/clientpositive/notable_alias1.q.out | 24 +-
.../results/clientpositive/notable_alias2.q.out | 24 +-
.../test/results/clientpositive/nullgroup.q.out | 8 +-
.../results/clientpositive/nullgroup2.q.out | 76 +-
.../results/clientpositive/nullgroup3.q.out | 8 +-
.../results/clientpositive/nullgroup4.q.out | 26 +-
.../nullgroup4_multi_distinct.q.out | 12 +-
.../results/clientpositive/orc_create.q.out | 2 +-
.../clientpositive/orc_empty_table.q.out | 68 +
.../orc_merge_incompat_schema.q.out | 161 +
.../orc_merge_incompat_writer_version.q.out | 108 +
.../results/clientpositive/outer_join_ppr.q.out | 40 +-
.../outer_reference_windowed.q.out | 10 +-
.../clientpositive/parallel_colstats.q.out | 24 +-
.../clientpositive/parquet_analyze.q.out | 4 +-
.../parquet_int96_timestamp.q.out | 191 +-
.../parquet_mixed_partition_formats.q.out | 6 +-
.../clientpositive/parquet_no_row_serde.q.out | 190 +
.../parquet_predicate_pushdown_2.q.out | 38 +
.../parquet_timestamp_conversion.q.out | 68 -
.../clientpositive/partial_column_stats.q.out | 20 +-
.../clientpositive/partition_boolexpr.q.out | 4 +-
.../partition_coltype_literals.q.out | 44 +-
ql/src/test/results/clientpositive/pcs.q.out | 2 +-
.../results/clientpositive/perf/query1.q.out | 40 +-
.../results/clientpositive/perf/query10.q.out | 52 +-
.../results/clientpositive/perf/query11.q.out | 351 +
.../results/clientpositive/perf/query14.q.out | 1876 +-
.../results/clientpositive/perf/query16.q.out | 265 +-
.../results/clientpositive/perf/query17.q.out | 236 +-
.../results/clientpositive/perf/query2.q.out | 22 +-
.../results/clientpositive/perf/query23.q.out | 180 +-
.../results/clientpositive/perf/query24.q.out | 162 +-
.../results/clientpositive/perf/query25.q.out | 232 +-
.../results/clientpositive/perf/query28.q.out | 184 +-
.../results/clientpositive/perf/query29.q.out | 232 +-
.../results/clientpositive/perf/query30.q.out | 102 +-
.../results/clientpositive/perf/query31.q.out | 90 +-
.../results/clientpositive/perf/query32.q.out | 18 +-
.../results/clientpositive/perf/query33.q.out | 178 +-
.../results/clientpositive/perf/query35.q.out | 32 +-
.../results/clientpositive/perf/query38.q.out | 270 +-
.../results/clientpositive/perf/query39.q.out | 30 +-
.../results/clientpositive/perf/query4.q.out | 1118 +-
.../results/clientpositive/perf/query44.q.out | 29 +-
.../results/clientpositive/perf/query45.q.out | 6 +-
.../results/clientpositive/perf/query46.q.out | 6 +-
.../results/clientpositive/perf/query47.q.out | 222 +-
.../results/clientpositive/perf/query49.q.out | 22 +-
.../results/clientpositive/perf/query5.q.out | 122 +-
.../results/clientpositive/perf/query50.q.out | 72 +-
.../results/clientpositive/perf/query51.q.out | 6 +-
.../results/clientpositive/perf/query54.q.out | 458 +-
.../results/clientpositive/perf/query56.q.out | 178 +-
.../results/clientpositive/perf/query57.q.out | 222 +-
.../results/clientpositive/perf/query58.q.out | 96 +-
.../results/clientpositive/perf/query59.q.out | 20 +-
.../results/clientpositive/perf/query6.q.out | 306 +-
.../results/clientpositive/perf/query60.q.out | 178 +-
.../results/clientpositive/perf/query61.q.out | 30 +-
.../results/clientpositive/perf/query64.q.out | 734 +-
.../results/clientpositive/perf/query65.q.out | 6 +-
.../results/clientpositive/perf/query66.q.out | 40 +-
.../results/clientpositive/perf/query68.q.out | 6 +-
.../results/clientpositive/perf/query69.q.out | 32 +-
.../results/clientpositive/perf/query70.q.out | 22 +-
.../results/clientpositive/perf/query71.q.out | 80 +-
.../results/clientpositive/perf/query72.q.out | 270 +-
.../results/clientpositive/perf/query74.q.out | 544 +-
.../results/clientpositive/perf/query75.q.out | 110 +-
.../results/clientpositive/perf/query76.q.out | 84 +-
.../results/clientpositive/perf/query77.q.out | 188 +-
.../results/clientpositive/perf/query78.q.out | 104 +-
.../results/clientpositive/perf/query8.q.out | 10 +-
.../results/clientpositive/perf/query80.q.out | 172 +-
.../results/clientpositive/perf/query81.q.out | 104 +-
.../results/clientpositive/perf/query83.q.out | 311 +-
.../results/clientpositive/perf/query85.q.out | 148 +-
.../results/clientpositive/perf/query87.q.out | 248 +-
.../results/clientpositive/perf/query88.q.out | 138 +-
.../results/clientpositive/perf/query9.q.out | 90 +-
.../results/clientpositive/perf/query90.q.out | 30 +-
.../results/clientpositive/perf/query92.q.out | 18 +-
.../results/clientpositive/perf/query94.q.out | 265 +-
.../results/clientpositive/perf/query95.q.out | 80 +-
.../results/clientpositive/perf/query97.q.out | 16 +-
.../test/results/clientpositive/plan_json.q.out | 2 +-
ql/src/test/results/clientpositive/ppd2.q.out | 52 +-
.../test/results/clientpositive/ppd_gby.q.out | 32 +-
.../test/results/clientpositive/ppd_gby2.q.out | 44 +-
.../results/clientpositive/ppd_gby_join.q.out | 64 +-
.../test/results/clientpositive/ppd_join.q.out | 48 +-
.../test/results/clientpositive/ppd_join4.q.out | 23 +-
.../clientpositive/ppd_outer_join1.q.out | 40 +-
.../clientpositive/ppd_outer_join2.q.out | 40 +-
.../clientpositive/ppd_outer_join3.q.out | 40 +-
.../clientpositive/ppd_outer_join4.q.out | 60 +-
.../test/results/clientpositive/ppd_union.q.out | 82 +-
ql/src/test/results/clientpositive/ppd_vc.q.out | 18 +-
.../results/clientpositive/ppd_windowing1.q.out | 66 +-
.../results/clientpositive/ptfgroupbyjoin.q.out | 2 +-
.../results/clientpositive/quotedid_basic.q.out | 4 +-
.../clientpositive/rcfile_null_value.q.out | 8 +-
.../reduceSinkDeDuplication_pRS_key_empty.q.out | Bin 10132 -> 10596 bytes
.../reduce_deduplicate_extended2.q.out | 89 +-
.../clientpositive/remove_exprs_stats.q.out | 12 +-
.../rename_external_partition_location.q.out | 24 +-
.../rename_table_update_column_stats.q.out | 96 +-
.../clientpositive/router_join_ppr.q.out | 84 +-
.../test/results/clientpositive/row__id.q.out | 18 +-
.../test/results/clientpositive/semijoin2.q.out | 4 +-
.../test/results/clientpositive/semijoin4.q.out | 4 +-
.../test/results/clientpositive/semijoin5.q.out | 8 +-
.../set_processor_namespaces.q.out | 2 +
.../results/clientpositive/setop_subq.q.out | 36 +-
.../show_create_table_delimited.q.out | 2 +-
.../clientpositive/skewjoin_mapjoin1.q.out | 16 +-
.../clientpositive/skewjoin_mapjoin10.q.out | 16 +-
.../clientpositive/skewjoin_mapjoin5.q.out | 24 +-
.../results/clientpositive/skewjoinopt1.q.out | 8 +-
.../results/clientpositive/skewjoinopt2.q.out | 8 +-
.../results/clientpositive/skewjoinopt9.q.out | 24 +-
.../results/clientpositive/smb_mapjoin_12.q.out | 75 +-
.../results/clientpositive/smb_mapjoin_20.q.out | 1060 +-
.../results/clientpositive/smb_mapjoin_47.q.out | 1542 +
.../spark/annotate_stats_join.q.out | 44 +-
.../clientpositive/spark/auto_join0.q.out | 43 +-
.../clientpositive/spark/auto_join12.q.out | 22 +-
.../clientpositive/spark/auto_join15.q.out | 35 +-
.../clientpositive/spark/auto_join16.q.out | 16 +-
.../clientpositive/spark/auto_join20.q.out | 62 +-
.../clientpositive/spark/auto_join26.q.out | 4 +-
.../clientpositive/spark/auto_join27.q.out | 2 +-
.../clientpositive/spark/auto_join30.q.out | 793 +-
.../clientpositive/spark/auto_join31.q.out | 116 +-
.../clientpositive/spark/auto_join4.q.out | 6 +-
.../clientpositive/spark/auto_join5.q.out | 6 +-
.../clientpositive/spark/auto_join8.q.out | 6 +-
.../spark/auto_join_reordering_values.q.out | 10 +-
.../spark/auto_join_without_localtask.q.out | 28 +-
.../spark/auto_smb_mapjoin_14.q.out | 24 +-
.../spark/auto_sortmerge_join_1.q.out | 6 +-
.../spark/auto_sortmerge_join_12.q.out | 10 +-
.../spark/auto_sortmerge_join_3.q.out | 6 +-
.../spark/auto_sortmerge_join_4.q.out | 6 +-
.../spark/auto_sortmerge_join_5.q.out | 4 +-
.../spark/auto_sortmerge_join_7.q.out | 8 +-
.../spark/auto_sortmerge_join_8.q.out | 8 +-
.../spark/auto_sortmerge_join_9.q.out | 66 +-
.../spark/avro_decimal_native.q.out | 8 +-
.../spark/bucket_map_join_spark1.q.out | 8 +-
.../spark/bucket_map_join_spark2.q.out | 8 +-
.../spark/bucket_map_join_spark3.q.out | 8 +-
.../spark/bucket_map_join_spark4.q.out | 12 +-
.../spark/bucket_map_join_tez2.q.out | 28 +-
.../clientpositive/spark/bucketmapjoin1.q.out | 8 +-
.../clientpositive/spark/bucketmapjoin2.q.out | 14 +-
.../clientpositive/spark/bucketmapjoin3.q.out | 8 +-
.../clientpositive/spark/bucketmapjoin4.q.out | 8 +-
.../spark/constprog_partitioner.q.out | 10 +-
.../spark/constprog_semijoin.q.out | 20 +-
.../results/clientpositive/spark/count.q.out | 124 +-
.../spark/dynamic_rdd_cache.q.out | 75 +-
.../spark/explaindenpendencydiffengs.q.out | 52 +
.../spark/filter_join_breaktask.q.out | 6 +-
.../results/clientpositive/spark/groupby3.q.out | 2 +-
.../clientpositive/spark/groupby4_map.q.out | 4 +-
.../spark/groupby4_map_skew.q.out | 4 +-
.../clientpositive/spark/groupby_cube1.q.out | 20 +-
.../clientpositive/spark/groupby_map_ppr.q.out | 4 +-
.../spark/groupby_map_ppr_multi_distinct.q.out | 4 +-
.../clientpositive/spark/groupby_position.q.out | 62 +-
.../clientpositive/spark/groupby_ppr.q.out | 4 +-
.../spark/groupby_ppr_multi_distinct.q.out | 4 +-
.../spark/groupby_resolution.q.out | 2 +-
.../clientpositive/spark/groupby_rollup1.q.out | 12 +-
.../spark/groupby_sort_1_23.q.out | 240 +-
.../spark/groupby_sort_skew_1_23.q.out | 240 +-
.../spark/identity_project_remove_skip.q.out | 70 +-
.../clientpositive/spark/index_bitmap3.q.out | 4 +-
.../spark/index_bitmap_auto.q.out | 4 +-
.../spark/infer_bucket_sort_map_operators.q.out | 8 +-
.../infer_bucket_sort_reducers_power_two.q.out | 2 +-
.../results/clientpositive/spark/join12.q.out | 30 +-
.../results/clientpositive/spark/join16.q.out | 22 +-
.../results/clientpositive/spark/join17.q.out | 4 +-
.../results/clientpositive/spark/join26.q.out | 6 +-
.../results/clientpositive/spark/join29.q.out | 48 +-
.../results/clientpositive/spark/join30.q.out | 4 +-
.../results/clientpositive/spark/join31.q.out | 2 +-
.../results/clientpositive/spark/join32.q.out | 84 +-
.../clientpositive/spark/join32_lessSize.q.out | 104 +-
.../results/clientpositive/spark/join33.q.out | 84 +-
.../results/clientpositive/spark/join34.q.out | 6 +-
.../results/clientpositive/spark/join35.q.out | 66 +-
.../results/clientpositive/spark/join38.q.out | 24 +-
.../results/clientpositive/spark/join4.q.out | 8 +-
.../results/clientpositive/spark/join5.q.out | 8 +-
.../results/clientpositive/spark/join8.q.out | 8 +-
.../results/clientpositive/spark/join9.q.out | 4 +-
.../clientpositive/spark/join_alt_syntax.q.out | 220 +-
.../spark/join_cond_pushdown_2.q.out | 92 +-
.../spark/join_cond_pushdown_4.q.out | 92 +-
.../spark/join_filters_overlap.q.out | 38 +-
.../clientpositive/spark/limit_pushdown.q.out | 72 +-
.../clientpositive/spark/louter_join_ppr.q.out | 108 +-
.../clientpositive/spark/mapjoin_mapjoin.q.out | 12 +-
.../results/clientpositive/spark/merge1.q.out | 6 +-
.../results/clientpositive/spark/merge2.q.out | 6 +-
.../spark/metadata_only_queries.q.out | 112 +-
.../spark/multi_insert_gby2.q.out | 20 +-
.../spark/multi_insert_gby3.q.out | 126 +-
.../clientpositive/spark/nullgroup.q.out | 8 +-
.../clientpositive/spark/nullgroup2.q.out | 76 +-
.../clientpositive/spark/nullgroup4.q.out | 26 +-
.../spark/nullgroup4_multi_distinct.q.out | 12 +-
.../spark/optimize_nullscan.q.out | 58 +-
.../clientpositive/spark/outer_join_ppr.q.out | 52 +-
.../spark/parquet_int96_timestamp.q.out | 718 -
.../test/results/clientpositive/spark/pcr.q.out | 20 +-
.../clientpositive/spark/ppd_gby_join.q.out | 64 +-
.../results/clientpositive/spark/ppd_join.q.out | 48 +-
.../clientpositive/spark/ppd_join_filter.q.out | 16 +-
.../clientpositive/spark/ppd_outer_join1.q.out | 40 +-
.../clientpositive/spark/ppd_outer_join2.q.out | 40 +-
.../clientpositive/spark/ppd_outer_join3.q.out | 40 +-
.../clientpositive/spark/ppd_outer_join4.q.out | 60 +-
.../test/results/clientpositive/spark/ptf.q.out | 114 +-
.../clientpositive/spark/ptf_streaming.q.out | 62 +-
.../clientpositive/spark/router_join_ppr.q.out | 108 +-
.../clientpositive/spark/skewjoinopt1.q.out | 8 +-
.../clientpositive/spark/skewjoinopt15.q.out | 8 +-
.../clientpositive/spark/skewjoinopt2.q.out | 8 +-
.../clientpositive/spark/skewjoinopt9.q.out | 24 +-
.../clientpositive/spark/smb_mapjoin_14.q.out | 54 +-
.../clientpositive/spark/smb_mapjoin_15.q.out | 20 +-
.../clientpositive/spark/smb_mapjoin_20.q.out | 1063 +-
.../spark/spark_constprog_dpp.q.out | 138 +
.../spark/spark_dynamic_partition_pruning.q.out | 3343 +-
.../spark_dynamic_partition_pruning_2.q.out | 213 +-
.../spark_dynamic_partition_pruning_3.q.out | 1791 +
...dynamic_partition_pruning_mapjoin_only.q.out | 492 +
.../spark/spark_explain_groupbyshuffle.q.out | 116 +
.../spark/spark_explainuser_1.q.out | 1106 +-
.../spark_multi_insert_parallel_orderby.q.out | 2013 +
.../spark/spark_union_merge.q.out | 564 +
.../spark/spark_use_op_stats.q.out | 46 +-
.../spark/spark_use_ts_stats_for_mapjoin.q.out | 10 +-
.../results/clientpositive/spark/stats1.q.out | 4 +-
.../clientpositive/spark/stats_only_null.q.out | 6 +-
.../clientpositive/spark/subquery_exists.q.out | 10 +-
.../clientpositive/spark/subquery_in.q.out | 660 +-
.../spark/table_access_keys_stats.q.out | 36 +-
.../clientpositive/spark/tez_join_tests.q.out | 26 +-
.../spark/tez_joins_explain.q.out | 26 +-
.../clientpositive/spark/transform_ppr1.q.out | 8 +-
.../clientpositive/spark/transform_ppr2.q.out | 4 +-
.../spark/truncate_column_buckets.q.out | 8 +-
.../results/clientpositive/spark/union10.q.out | 8 +-
.../results/clientpositive/spark/union11.q.out | 6 +-
.../results/clientpositive/spark/union12.q.out | 8 +-
.../results/clientpositive/spark/union14.q.out | 4 +-
.../results/clientpositive/spark/union15.q.out | 6 +-
.../results/clientpositive/spark/union16.q.out | 2 +-
.../results/clientpositive/spark/union17.q.out | 8 +-
.../results/clientpositive/spark/union18.q.out | 8 +-
.../results/clientpositive/spark/union19.q.out | 8 +-
.../results/clientpositive/spark/union2.q.out | 4 +-
.../results/clientpositive/spark/union20.q.out | 4 +-
.../results/clientpositive/spark/union22.q.out | 6 +-
.../results/clientpositive/spark/union24.q.out | 56 +-
.../results/clientpositive/spark/union25.q.out | 4 +-
.../results/clientpositive/spark/union30.q.out | 46 +-
.../results/clientpositive/spark/union31.q.out | 18 +-
.../results/clientpositive/spark/union4.q.out | 6 +-
.../results/clientpositive/spark/union5.q.out | 4 +-
.../results/clientpositive/spark/union6.q.out | 4 +-
.../results/clientpositive/spark/union7.q.out | 4 +-
.../results/clientpositive/spark/union9.q.out | 6 +-
.../clientpositive/spark/union_ppr.q.out | 8 +-
.../clientpositive/spark/union_remove_1.q.out | 6 +-
.../clientpositive/spark/union_remove_10.q.out | 8 +-
.../clientpositive/spark/union_remove_13.q.out | 8 +-
.../clientpositive/spark/union_remove_15.q.out | 8 +-
.../clientpositive/spark/union_remove_16.q.out | 8 +-
.../clientpositive/spark/union_remove_18.q.out | 6 +-
.../clientpositive/spark/union_remove_19.q.out | 54 +-
.../clientpositive/spark/union_remove_2.q.out | 6 +-
.../clientpositive/spark/union_remove_20.q.out | 6 +-
.../clientpositive/spark/union_remove_22.q.out | 6 +-
.../clientpositive/spark/union_remove_23.q.out | 10 +-
.../clientpositive/spark/union_remove_24.q.out | 6 +-
.../clientpositive/spark/union_remove_25.q.out | 6 +-
.../clientpositive/spark/union_remove_4.q.out | 6 +-
.../clientpositive/spark/union_remove_5.q.out | 6 +-
.../clientpositive/spark/union_remove_6.q.out | 6 +-
.../spark/union_remove_6_subq.q.out | 16 +-
.../clientpositive/spark/union_remove_7.q.out | 8 +-
.../clientpositive/spark/union_remove_8.q.out | 8 +-
.../clientpositive/spark/union_remove_9.q.out | 8 +-
.../clientpositive/spark/union_view.q.out | 24 +-
.../spark/vector_between_in.q.out | 52 +-
.../spark/vector_cast_constant.q.out | 33 +-
.../spark/vector_count_distinct.q.out | 10 +-
.../spark/vector_decimal_aggregate.q.out | 46 +-
.../spark/vector_distinct_2.q.out | 6 +-
.../clientpositive/spark/vector_elt.q.out | 12 +-
.../clientpositive/spark/vector_groupby_3.q.out | 6 +-
.../spark/vector_inner_join.q.out | 2 +
.../spark/vector_left_outer_join.q.out | 16 +-
.../spark/vector_mapjoin_reduce.q.out | 26 +-
.../clientpositive/spark/vector_orderby_5.q.out | 6 +-
.../spark/vector_outer_join1.q.out | 44 +-
.../spark/vector_outer_join2.q.out | 20 +-
.../spark/vector_string_concat.q.out | 6 +-
.../spark/vector_tablesample_rows.q.out | 424 +
.../clientpositive/spark/vectorization_0.q.out | 365 +-
.../clientpositive/spark/vectorization_1.q.out | 173 +
.../clientpositive/spark/vectorization_10.q.out | 117 +
.../clientpositive/spark/vectorization_11.q.out | 99 +
.../clientpositive/spark/vectorization_12.q.out | 238 +
.../clientpositive/spark/vectorization_13.q.out | 142 +-
.../clientpositive/spark/vectorization_14.q.out | 105 +-
.../clientpositive/spark/vectorization_15.q.out | 174 +-
.../clientpositive/spark/vectorization_16.q.out | 83 +-
.../clientpositive/spark/vectorization_17.q.out | 53 +-
.../clientpositive/spark/vectorization_2.q.out | 177 +
.../clientpositive/spark/vectorization_3.q.out | 182 +
.../clientpositive/spark/vectorization_4.q.out | 176 +
.../clientpositive/spark/vectorization_5.q.out | 171 +
.../clientpositive/spark/vectorization_6.q.out | 111 +
.../clientpositive/spark/vectorization_7.q.out | 18 +-
.../clientpositive/spark/vectorization_8.q.out | 18 +-
.../clientpositive/spark/vectorization_9.q.out | 83 +-
.../spark/vectorization_div0.q.out | 36 +-
.../spark/vectorization_pushdown.q.out | 15 +-
.../spark/vectorization_short_regress.q.out | 490 +-
.../clientpositive/spark/vectorized_case.q.out | 44 +-
.../spark/vectorized_mapjoin.q.out | 47 +-
.../spark/vectorized_math_funcs.q.out | 8 +-
.../spark/vectorized_nested_mapjoin.q.out | 24 +-
.../clientpositive/spark/vectorized_ptf.q.out | 331 +-
.../spark/vectorized_shufflejoin.q.out | 43 +-
.../spark/vectorized_string_funcs.q.out | 8 +-
.../spark/vectorized_timestamp_funcs.q.out | 68 +-
.../clientpositive/spark/windowing.q.out | 2 +-
ql/src/test/results/clientpositive/stats1.q.out | 4 +-
.../clientpositive/stats_only_null.q.out | 6 +-
ql/src/test/results/clientpositive/subq2.q.out | 24 +-
.../clientpositive/subquery_exists.q.out | 10 +-
.../clientpositive/subquery_exists_having.q.out | 28 +-
.../clientpositive/subquery_in_having.q.out | 1439 +-
.../clientpositive/subquery_notexists.q.out | 32 +-
.../subquery_notexists_having.q.out | 32 +-
.../clientpositive/subquery_notin_having.q.out | 26 +-
.../subquery_unqualcolumnrefs.q.out | 86 +-
.../symlink_text_input_format.q.out | 4 +-
.../temp_table_display_colstats_tbllvl.q.out | 116 +-
.../clientpositive/tez/explainanalyze_1.q.out | 10 +-
.../clientpositive/tez/explainanalyze_2.q.out | 370 +-
.../clientpositive/tez/explainanalyze_3.q.out | 4 +-
.../clientpositive/tez/explainanalyze_4.q.out | 12 +-
.../clientpositive/tez/explainanalyze_5.q.out | 20 +-
.../clientpositive/tez/explainuser_3.q.out | 6 +-
.../tez/hybridgrace_hashjoin_1.q.out | 12 +-
.../tez/hybridgrace_hashjoin_2.q.out | 24 +-
.../tez/vectorization_limit.q.out | 82 +-
.../results/clientpositive/timestamptz.q.out | 48 +-
.../results/clientpositive/timestamptz_1.q.out | 26 +-
.../results/clientpositive/timestamptz_2.q.out | 34 +-
.../truncate_column_buckets.q.out | 8 +-
.../results/clientpositive/tunable_ndv.q.out | 60 +-
.../clientpositive/udf_stddev_samp.q.out | 66 +-
.../results/clientpositive/udf_var_samp.q.out | 66 +-
.../results/clientpositive/udtf_explode.q.out | 4 +-
.../test/results/clientpositive/union10.q.out | 8 +-
.../test/results/clientpositive/union11.q.out | 6 +-
.../test/results/clientpositive/union12.q.out | 8 +-
.../test/results/clientpositive/union14.q.out | 4 +-
.../test/results/clientpositive/union15.q.out | 6 +-
.../test/results/clientpositive/union16.q.out | 50 +-
.../test/results/clientpositive/union17.q.out | 8 +-
.../test/results/clientpositive/union18.q.out | 8 +-
.../test/results/clientpositive/union19.q.out | 8 +-
ql/src/test/results/clientpositive/union2.q.out | 4 +-
.../test/results/clientpositive/union20.q.out | 4 +-
.../test/results/clientpositive/union24.q.out | 56 +-
.../test/results/clientpositive/union25.q.out | 4 +-
.../test/results/clientpositive/union30.q.out | 52 +-
.../test/results/clientpositive/union31.q.out | 18 +-
.../test/results/clientpositive/union34.q.out | 4 +-
ql/src/test/results/clientpositive/union4.q.out | 6 +-
ql/src/test/results/clientpositive/union5.q.out | 4 +-
ql/src/test/results/clientpositive/union6.q.out | 4 +-
ql/src/test/results/clientpositive/union7.q.out | 4 +-
ql/src/test/results/clientpositive/union9.q.out | 6 +-
.../clientpositive/union_pos_alias.q.out | 10 +-
.../results/clientpositive/union_remove_1.q.out | 12 +-
.../clientpositive/union_remove_10.q.out | 6 +-
.../clientpositive/union_remove_13.q.out | 6 +-
.../clientpositive/union_remove_15.q.out | 12 +-
.../clientpositive/union_remove_16.q.out | 12 +-
.../clientpositive/union_remove_18.q.out | 12 +-
.../clientpositive/union_remove_19.q.out | 108 +-
.../results/clientpositive/union_remove_2.q.out | 6 +-
.../clientpositive/union_remove_20.q.out | 12 +-
.../clientpositive/union_remove_22.q.out | 12 +-
.../clientpositive/union_remove_23.q.out | 10 +-
.../clientpositive/union_remove_24.q.out | 12 +-
.../clientpositive/union_remove_25.q.out | 12 +-
.../results/clientpositive/union_remove_4.q.out | 12 +-
.../results/clientpositive/union_remove_5.q.out | 6 +-
.../results/clientpositive/union_remove_6.q.out | 12 +-
.../clientpositive/union_remove_6_subq.q.out | 30 +-
.../results/clientpositive/union_remove_7.q.out | 12 +-
.../results/clientpositive/union_remove_8.q.out | 6 +-
.../results/clientpositive/union_remove_9.q.out | 6 +-
.../results/clientpositive/union_view.q.out | 24 +-
.../clientpositive/unionall_unbalancedppd.q.out | 35 +-
.../clientpositive/vector_aggregate_9.q.out | 239 +-
.../vector_binary_join_groupby.q.out | 8 +
.../clientpositive/vector_cast_constant.q.out | 16 +-
.../results/clientpositive/vector_char_2.q.out | 8 +
.../clientpositive/vector_coalesce_2.q.out | 8 +
.../clientpositive/vector_complex_all.q.out | 6 +-
.../clientpositive/vector_complex_join.q.out | 28 +-
.../results/clientpositive/vector_count.q.out | 78 +-
.../results/clientpositive/vector_date_1.q.out | 2 +-
.../vector_decimal_aggregate.q.out | 20 +-
.../vector_decimal_precision.q.out | 16 +-
.../clientpositive/vector_decimal_round.q.out | 44 +-
.../clientpositive/vector_distinct_2.q.out | 4 +
.../clientpositive/vector_empty_where.q.out | 32 +
.../clientpositive/vector_groupby4.q.out | 4 +
.../clientpositive/vector_groupby6.q.out | 4 +
.../clientpositive/vector_groupby_3.q.out | 4 +
.../clientpositive/vector_groupby_mapjoin.q.out | 8 +
.../clientpositive/vector_groupby_reduce.q.out | 2522 +-
.../clientpositive/vector_grouping_sets.q.out | 8 +
.../clientpositive/vector_include_no_sel.q.out | 4 +
.../vector_interval_mapjoin.q.out | 6 +-
.../clientpositive/vector_mapjoin_reduce.q.out | 20 +-
.../vector_mr_diff_schema_alias.q.out | 26 +-
.../clientpositive/vector_null_projection.q.out | 8 +-
.../results/clientpositive/vector_nvl.q.out | 2 +-
.../clientpositive/vector_orderby_5.q.out | 4 +
.../clientpositive/vector_outer_join1.q.out | 4 +
.../clientpositive/vector_outer_join2.q.out | 4 +
.../clientpositive/vector_outer_join3.q.out | 6 +-
.../clientpositive/vector_outer_join4.q.out | 2 +-
.../vector_outer_reference_windowed.q.out | 1221 +
.../clientpositive/vector_ptf_part_simple.q.out | 2895 +
.../vector_reduce_groupby_decimal.q.out | 4 +
.../clientpositive/vector_string_concat.q.out | 4 +
.../vector_tablesample_rows.q.out | 100 +-
.../clientpositive/vector_when_case_null.q.out | 12 +-
.../clientpositive/vector_windowing.q.out | 9149 +
.../vector_windowing_expressions.q.out | 1822 +
.../clientpositive/vector_windowing_gby.q.out | 252 +
.../clientpositive/vector_windowing_gby2.q.out | 1015 +
.../vector_windowing_multipartitioning.q.out | 11288 +
.../vector_windowing_order_null.q.out | 989 +
.../vector_windowing_range_multiorder.q.out | 12239 +
.../clientpositive/vector_windowing_rank.q.out | 1543 +
.../vector_windowing_streaming.q.out | 836 +
.../vector_windowing_windowspec.q.out | 2060 +
.../vector_windowing_windowspec4.q.out | 211 +
.../clientpositive/vectorization_1.q.out | 146 +
.../clientpositive/vectorization_10.q.out | 114 +
.../clientpositive/vectorization_11.q.out | 96 +
.../clientpositive/vectorization_12.q.out | 1284 +-
.../clientpositive/vectorization_13.q.out | 43 +-
.../clientpositive/vectorization_14.q.out | 1301 +-
.../clientpositive/vectorization_15.q.out | 155 +-
.../clientpositive/vectorization_16.q.out | 364 +-
.../clientpositive/vectorization_17.q.out | 27 +-
.../clientpositive/vectorization_2.q.out | 150 +
.../clientpositive/vectorization_3.q.out | 155 +
.../clientpositive/vectorization_4.q.out | 149 +
.../clientpositive/vectorization_5.q.out | 144 +
.../clientpositive/vectorization_6.q.out | 108 +
.../clientpositive/vectorization_7.q.out | 10 +-
.../clientpositive/vectorization_8.q.out | 10 +-
.../clientpositive/vectorization_9.q.out | 364 +-
.../clientpositive/vectorization_limit.q.out | 28 +-
.../clientpositive/vectorization_pushdown.q.out | 2 +-
.../clientpositive/vectorized_case.q.out | 20 +-
.../clientpositive/vectorized_casts.q.out | 85 +-
.../clientpositive/vectorized_date_funcs.q.out | 4 +
.../vectorized_distinct_gby.q.out | 78 +-
.../clientpositive/vectorized_mapjoin.q.out | 16 +-
.../clientpositive/vectorized_mapjoin2.q.out | 12 +-
.../vectorized_parquet_types.q.out | 16 +-
.../clientpositive/vectorized_shufflejoin.q.out | 19 +-
.../clientpositive/vectorized_timestamp.q.out | 314 +-
.../vectorized_timestamp_funcs.q.out | 49 +-
.../test/results/clientpositive/view_cbo.q.out | 12 +-
.../results/clientpositive/windowing_gby2.q.out | 12 +-
.../clientpositive/windowing_navfn.q.out | 2 +-
.../clientpositive/windowing_streaming.q.out | 6 +-
serde/if/serde.thrift | 6 +-
.../src/gen/thrift/gen-cpp/serde_constants.cpp | 6 +-
serde/src/gen/thrift/gen-cpp/serde_constants.h | 2 +-
.../hadoop/hive/serde/serdeConstants.java | 6 +-
.../org/apache/hadoop/hive/serde/Types.php | 10 +-
.../org_apache_hadoop_hive_serde/constants.py | 6 +-
serde/src/gen/thrift/gen-rb/serde_constants.rb | 6 +-
.../apache/hadoop/hive/serde2/SerDeUtils.java | 6 +-
.../binarysortable/BinarySortableSerDe.java | 25 +-
.../serde2/io/TimestampLocalTZWritable.java | 449 +
.../hive/serde2/io/TimestampTZWritable.java | 427 -
.../hadoop/hive/serde2/lazy/LazyFactory.java | 6 +-
.../hive/serde2/lazy/LazySerDeParameters.java | 19 +-
.../hive/serde2/lazy/LazyTimestampLocalTZ.java | 102 +
.../hive/serde2/lazy/LazyTimestampTZ.java | 91 -
.../hadoop/hive/serde2/lazy/LazyUtils.java | 6 +-
.../lazy/fast/LazySimpleDeserializeRead.java | 14 +-
.../LazyPrimitiveObjectInspectorFactory.java | 8 +-
.../LazyTimestampLocalTZObjectInspector.java | 52 +
.../LazyTimestampTZObjectInspector.java | 43 -
.../serde2/lazybinary/LazyBinaryFactory.java | 6 +-
.../hive/serde2/lazybinary/LazyBinarySerDe.java | 8 +-
.../lazybinary/LazyBinaryTimestampLocalTZ.java | 43 +
.../lazybinary/LazyBinaryTimestampTZ.java | 36 -
.../hive/serde2/lazybinary/LazyBinaryUtils.java | 6 +-
.../ObjectInspectorConverters.java | 14 +-
.../objectinspector/ObjectInspectorUtils.java | 22 +-
.../PrimitiveObjectInspector.java | 6 +-
.../JavaTimestampLocalTZObjectInspector.java | 103 +
.../JavaTimestampTZObjectInspector.java | 76 -
.../PrimitiveObjectInspectorConverter.java | 23 +-
.../PrimitiveObjectInspectorFactory.java | 32 +-
.../PrimitiveObjectInspectorUtils.java | 123 +-
...SettableTimestampLocalTZObjectInspector.java | 34 +
.../SettableTimestampTZObjectInspector.java | 34 -
.../TimestampLocalTZObjectInspector.java | 29 +
.../primitive/TimestampTZObjectInspector.java | 29 -
...ConstantTimestampLocalTZObjectInspector.java | 40 +
...tableConstantTimestampTZObjectInspector.java | 36 -
...WritableTimestampLocalTZObjectInspector.java | 107 +
.../WritableTimestampTZObjectInspector.java | 79 -
.../serde2/thrift/ThriftJDBCBinarySerDe.java | 2 +-
.../apache/hadoop/hive/serde2/thrift/Type.java | 10 +-
.../typeinfo/TimestampLocalTZTypeInfo.java | 104 +
.../hive/serde2/typeinfo/TypeInfoFactory.java | 21 +-
.../hive/serde2/typeinfo/TypeInfoUtils.java | 28 +-
.../hive/serde2/io/TestTimestampTZWritable.java | 20 +-
.../hive/serde2/typeinfo/TestTypeInfoUtils.java | 34 +-
service-rpc/if/TCLIService.thrift | 8 +-
.../thrift/gen-cpp/TCLIService_constants.cpp | 2 +-
.../gen/thrift/gen-cpp/TCLIService_types.cpp | 4 +-
.../src/gen/thrift/gen-cpp/TCLIService_types.h | 2 +-
.../rpc/thrift/TCLIServiceConstants.java | 4 +-
.../apache/hive/service/rpc/thrift/TTypeId.java | 4 +-
service-rpc/src/gen/thrift/gen-php/Types.php | 6 +-
.../gen/thrift/gen-py/TCLIService/constants.py | 2 +-
.../src/gen/thrift/gen-py/TCLIService/ttypes.py | 6 +-
.../thrift/gen-rb/t_c_l_i_service_constants.rb | 2 +-
.../gen/thrift/gen-rb/t_c_l_i_service_types.rb | 6 +-
service/pom.xml | 11 +
.../org/apache/hive/tmpl/QueryProfileTmpl.jamon | 14 +-
.../apache/hive/service/cli/ColumnValue.java | 2 +-
.../apache/hive/service/cli/TypeDescriptor.java | 2 +-
.../operation/ExecuteStatementOperation.java | 8 +-
.../hive/service/cli/operation/Operation.java | 16 +-
.../service/cli/operation/SQLOperation.java | 5 +-
.../service/cli/session/HiveSessionHook.java | 4 +
.../service/cli/thrift/ThriftCLIService.java | 6 +-
.../cli/thrift/ThriftHttpCLIService.java | 19 +-
.../service/servlet/QueryProfileServlet.java | 4 +-
.../apache/hadoop/hive/shims/Hadoop23Shims.java | 60 +-
.../hadoop/hive/shims/TestHadoop23Shims.java | 14 +-
.../apache/hadoop/hive/shims/HadoopShims.java | 10 +-
.../hive/thrift/HadoopThriftAuthBridge.java | 16 +
.../apache/hive/spark/client/SparkClient.java | 9 +-
.../hive/spark/client/SparkClientFactory.java | 22 +-
.../hive/spark/client/SparkClientImpl.java | 68 +-
.../hive/spark/client/rpc/RpcConfiguration.java | 6 +-
.../hive/spark/client/rpc/RpcDispatcher.java | 7 +-
.../apache/hive/spark/client/rpc/RpcServer.java | 1 -
.../apache/hive/spark/client/rpc/TestRpc.java | 55 +-
standalone-metastore/pom.xml | 232 +
.../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 74036 ++++++
.../gen/thrift/gen-cpp/ThriftHiveMetastore.h | 23462 ++
.../ThriftHiveMetastore_server.skeleton.cpp | 844 +
.../thrift/gen-cpp/hive_metastore_constants.cpp | 65 +
.../thrift/gen-cpp/hive_metastore_constants.h | 48 +
.../gen/thrift/gen-cpp/hive_metastore_types.cpp | 21819 ++
.../gen/thrift/gen-cpp/hive_metastore_types.h | 9034 +
.../hive/metastore/api/AbortTxnRequest.java | 387 +
.../hive/metastore/api/AbortTxnsRequest.java | 438 +
.../metastore/api/AddDynamicPartitions.java | 864 +
.../metastore/api/AddForeignKeyRequest.java | 443 +
.../api/AddNotNullConstraintRequest.java | 443 +
.../metastore/api/AddPartitionsRequest.java | 849 +
.../hive/metastore/api/AddPartitionsResult.java | 447 +
.../metastore/api/AddPrimaryKeyRequest.java | 443 +
.../api/AddUniqueConstraintRequest.java | 443 +
.../hadoop/hive/metastore/api/AggrStats.java | 542 +
.../metastore/api/AlreadyExistsException.java | 395 +
.../metastore/api/BinaryColumnStatsData.java | 696 +
.../metastore/api/BooleanColumnStatsData.java | 696 +
.../metastore/api/CacheFileMetadataRequest.java | 703 +
.../metastore/api/CacheFileMetadataResult.java | 387 +
.../hive/metastore/api/CheckLockRequest.java | 589 +
.../metastore/api/ClearFileMetadataRequest.java | 438 +
.../metastore/api/ClearFileMetadataResult.java | 283 +
.../hive/metastore/api/ClientCapabilities.java | 441 +
.../hive/metastore/api/ClientCapability.java | 45 +
.../hive/metastore/api/CmRecycleRequest.java | 488 +
.../hive/metastore/api/CmRecycleResponse.java | 283 +
.../hive/metastore/api/ColumnStatistics.java | 549 +
.../metastore/api/ColumnStatisticsData.java | 675 +
.../metastore/api/ColumnStatisticsDesc.java | 798 +
.../hive/metastore/api/ColumnStatisticsObj.java | 593 +
.../hive/metastore/api/CommitTxnRequest.java | 387 +
.../hive/metastore/api/CompactionRequest.java | 977 +
.../hive/metastore/api/CompactionResponse.java | 583 +
.../hive/metastore/api/CompactionType.java | 45 +
.../api/ConfigValSecurityException.java | 395 +
.../api/CurrentNotificationEventId.java | 387 +
.../hive/metastore/api/DataOperationType.java | 57 +
.../hadoop/hive/metastore/api/Database.java | 1095 +
.../apache/hadoop/hive/metastore/api/Date.java | 387 +
.../hive/metastore/api/DateColumnStatsData.java | 823 +
.../hadoop/hive/metastore/api/Decimal.java | 497 +
.../metastore/api/DecimalColumnStatsData.java | 823 +
.../metastore/api/DoubleColumnStatsData.java | 799 +
.../metastore/api/DropConstraintRequest.java | 591 +
.../hive/metastore/api/DropPartitionsExpr.java | 505 +
.../metastore/api/DropPartitionsRequest.java | 1112 +
.../metastore/api/DropPartitionsResult.java | 447 +
.../hive/metastore/api/EnvironmentContext.java | 447 +
.../hive/metastore/api/EventRequestType.java | 48 +
.../hadoop/hive/metastore/api/FieldSchema.java | 603 +
.../metastore/api/FileMetadataExprType.java | 42 +
.../hive/metastore/api/FireEventRequest.java | 861 +
.../metastore/api/FireEventRequestData.java | 309 +
.../hive/metastore/api/FireEventResponse.java | 283 +
.../hive/metastore/api/ForeignKeysRequest.java | 707 +
.../hive/metastore/api/ForeignKeysResponse.java | 443 +
.../hadoop/hive/metastore/api/Function.java | 1199 +
.../hadoop/hive/metastore/api/FunctionType.java | 42 +
.../metastore/api/GetAllFunctionsResponse.java | 447 +
.../api/GetFileMetadataByExprRequest.java | 773 +
.../api/GetFileMetadataByExprResult.java | 553 +
.../metastore/api/GetFileMetadataRequest.java | 438 +
.../metastore/api/GetFileMetadataResult.java | 540 +
.../metastore/api/GetOpenTxnsInfoResponse.java | 542 +
.../hive/metastore/api/GetOpenTxnsResponse.java | 750 +
.../api/GetPrincipalsInRoleRequest.java | 389 +
.../api/GetPrincipalsInRoleResponse.java | 443 +
.../api/GetRoleGrantsForPrincipalRequest.java | 502 +
.../api/GetRoleGrantsForPrincipalResponse.java | 443 +
.../hive/metastore/api/GetTableRequest.java | 605 +
.../hive/metastore/api/GetTableResult.java | 394 +
.../hive/metastore/api/GetTablesRequest.java | 659 +
.../hive/metastore/api/GetTablesResult.java | 443 +
.../api/GrantRevokePrivilegeRequest.java | 620 +
.../api/GrantRevokePrivilegeResponse.java | 390 +
.../metastore/api/GrantRevokeRoleRequest.java | 1059 +
.../metastore/api/GrantRevokeRoleResponse.java | 390 +
.../hive/metastore/api/GrantRevokeType.java | 45 +
.../hive/metastore/api/HeartbeatRequest.java | 489 +
.../metastore/api/HeartbeatTxnRangeRequest.java | 482 +
.../api/HeartbeatTxnRangeResponse.java | 588 +
.../hive/metastore/api/HiveObjectPrivilege.java | 729 +
.../hive/metastore/api/HiveObjectRef.java | 872 +
.../hive/metastore/api/HiveObjectType.java | 54 +
.../apache/hadoop/hive/metastore/api/Index.java | 1374 +
.../api/IndexAlreadyExistsException.java | 395 +
.../metastore/api/InsertEventRequestData.java | 700 +
.../metastore/api/InvalidInputException.java | 395 +
.../metastore/api/InvalidObjectException.java | 395 +
.../api/InvalidOperationException.java | 395 +
.../api/InvalidPartitionException.java | 395 +
.../hive/metastore/api/LockComponent.java | 1158 +
.../hadoop/hive/metastore/api/LockLevel.java | 48 +
.../hadoop/hive/metastore/api/LockRequest.java | 861 +
.../hadoop/hive/metastore/api/LockResponse.java | 500 +
.../hadoop/hive/metastore/api/LockState.java | 51 +
.../hadoop/hive/metastore/api/LockType.java | 48 +
.../hive/metastore/api/LongColumnStatsData.java | 799 +
.../hive/metastore/api/MetaException.java | 395 +
.../hive/metastore/api/MetadataPpdResult.java | 517 +
.../hive/metastore/api/NoSuchLockException.java | 395 +
.../metastore/api/NoSuchObjectException.java | 395 +
.../hive/metastore/api/NoSuchTxnException.java | 395 +
.../api/NotNullConstraintsRequest.java | 490 +
.../api/NotNullConstraintsResponse.java | 443 +
.../hive/metastore/api/NotificationEvent.java | 1006 +
.../metastore/api/NotificationEventRequest.java | 490 +
.../api/NotificationEventResponse.java | 443 +
.../api/NotificationEventsCountRequest.java | 488 +
.../api/NotificationEventsCountResponse.java | 387 +
.../hive/metastore/api/OpenTxnRequest.java | 702 +
.../hive/metastore/api/OpenTxnsResponse.java | 438 +
.../apache/hadoop/hive/metastore/api/Order.java | 497 +
.../hadoop/hive/metastore/api/Partition.java | 1229 +
.../hive/metastore/api/PartitionEventType.java | 42 +
.../api/PartitionListComposingSpec.java | 449 +
.../hive/metastore/api/PartitionSpec.java | 826 +
.../api/PartitionSpecWithSharedSD.java | 558 +
.../hive/metastore/api/PartitionWithoutSD.java | 1016 +
.../metastore/api/PartitionsByExprRequest.java | 815 +
.../metastore/api/PartitionsByExprResult.java | 542 +
.../metastore/api/PartitionsStatsRequest.java | 790 +
.../metastore/api/PartitionsStatsResult.java | 490 +
.../hive/metastore/api/PrimaryKeysRequest.java | 490 +
.../hive/metastore/api/PrimaryKeysResponse.java | 443 +
.../metastore/api/PrincipalPrivilegeSet.java | 906 +
.../hive/metastore/api/PrincipalType.java | 48 +
.../hadoop/hive/metastore/api/PrivilegeBag.java | 449 +
.../hive/metastore/api/PrivilegeGrantInfo.java | 815 +
.../metastore/api/PutFileMetadataRequest.java | 710 +
.../metastore/api/PutFileMetadataResult.java | 283 +
.../hive/metastore/api/RequestPartsSpec.java | 438 +
.../hadoop/hive/metastore/api/ResourceType.java | 48 +
.../hadoop/hive/metastore/api/ResourceUri.java | 511 +
.../apache/hadoop/hive/metastore/api/Role.java | 601 +
.../hive/metastore/api/RolePrincipalGrant.java | 1035 +
.../hive/metastore/api/SQLForeignKey.java | 1715 +
.../metastore/api/SQLNotNullConstraint.java | 1005 +
.../hive/metastore/api/SQLPrimaryKey.java | 1103 +
.../hive/metastore/api/SQLUniqueConstraint.java | 1103 +
.../hadoop/hive/metastore/api/Schema.java | 605 +
.../hadoop/hive/metastore/api/SerDeInfo.java | 655 +
.../api/SetPartitionsStatsRequest.java | 550 +
.../hive/metastore/api/ShowCompactRequest.java | 283 +
.../hive/metastore/api/ShowCompactResponse.java | 443 +
.../api/ShowCompactResponseElement.java | 1641 +
.../hive/metastore/api/ShowLocksRequest.java | 710 +
.../hive/metastore/api/ShowLocksResponse.java | 449 +
.../metastore/api/ShowLocksResponseElement.java | 1929 +
.../hadoop/hive/metastore/api/SkewedInfo.java | 834 +
.../hive/metastore/api/StorageDescriptor.java | 1748 +
.../metastore/api/StringColumnStatsData.java | 791 +
.../apache/hadoop/hive/metastore/api/Table.java | 1948 +
.../hadoop/hive/metastore/api/TableMeta.java | 701 +
.../hive/metastore/api/TableStatsRequest.java | 640 +
.../hive/metastore/api/TableStatsResult.java | 443 +
.../hive/metastore/api/ThriftHiveMetastore.java | 188205 ++++++++++++++++
.../hive/metastore/api/TxnAbortedException.java | 395 +
.../hadoop/hive/metastore/api/TxnInfo.java | 1220 +
.../hive/metastore/api/TxnOpenException.java | 395 +
.../hadoop/hive/metastore/api/TxnState.java | 48 +
.../apache/hadoop/hive/metastore/api/Type.java | 768 +
.../metastore/api/UniqueConstraintsRequest.java | 490 +
.../api/UniqueConstraintsResponse.java | 443 +
.../hive/metastore/api/UnknownDBException.java | 395 +
.../api/UnknownPartitionException.java | 395 +
.../metastore/api/UnknownTableException.java | 395 +
.../hive/metastore/api/UnlockRequest.java | 387 +
.../hadoop/hive/metastore/api/Version.java | 499 +
.../metastore/api/hive_metastoreConstants.java | 87 +
.../gen-php/metastore/ThriftHiveMetastore.php | 47128 ++++
.../src/gen/thrift/gen-php/metastore/Types.php | 21262 ++
.../hive_metastore/ThriftHiveMetastore-remote | 1305 +
.../hive_metastore/ThriftHiveMetastore.py | 38297 ++++
.../thrift/gen-py/hive_metastore/__init__.py | 1 +
.../thrift/gen-py/hive_metastore/constants.py | 35 +
.../gen/thrift/gen-py/hive_metastore/ttypes.py | 15163 ++
.../thrift/gen-rb/hive_metastore_constants.rb | 57 +
.../gen/thrift/gen-rb/hive_metastore_types.rb | 3516 +
.../gen/thrift/gen-rb/thrift_hive_metastore.rb | 10672 +
.../hadoop/hive/metastore/ThreadPool.java | 63 +
.../hive/metastore/conf/MetastoreConf.java | 1514 +
.../hive/metastore/metrics/JsonReporter.java | 166 +
.../hive/metastore/metrics/JvmPauseMonitor.java | 222 +
.../hadoop/hive/metastore/metrics/Metrics.java | 243 +
.../metastore/metrics/MetricsConstants.java | 46 +
.../hive/metastore/metrics/PerfLogger.java | 194 +
.../hive/metastore/model/MColumnDescriptor.java | 51 +
.../hive/metastore/model/MConstraint.java | 188 +
.../hive/metastore/model/MDBPrivilege.java | 132 +
.../hadoop/hive/metastore/model/MDatabase.java | 128 +
.../hive/metastore/model/MDelegationToken.java | 45 +
.../hive/metastore/model/MFieldSchema.java | 80 +
.../hadoop/hive/metastore/model/MFunction.java | 119 +
.../hive/metastore/model/MGlobalPrivilege.java | 120 +
.../hadoop/hive/metastore/model/MIndex.java | 200 +
.../hadoop/hive/metastore/model/MMasterKey.java | 55 +
.../metastore/model/MMetastoreDBProperties.java | 56 +
.../hive/metastore/model/MNotificationLog.java | 98 +
.../metastore/model/MNotificationNextId.java | 42 +
.../hadoop/hive/metastore/model/MOrder.java | 62 +
.../hadoop/hive/metastore/model/MPartition.java | 155 +
.../model/MPartitionColumnPrivilege.java | 161 +
.../model/MPartitionColumnStatistics.java | 272 +
.../hive/metastore/model/MPartitionEvent.java | 90 +
.../metastore/model/MPartitionPrivilege.java | 139 +
.../hive/metastore/model/MPrincipalDesc.java | 59 +
.../hive/metastore/model/MResourceUri.java | 49 +
.../hadoop/hive/metastore/model/MRole.java | 80 +
.../hadoop/hive/metastore/model/MRoleMap.java | 120 +
.../hadoop/hive/metastore/model/MSerDeInfo.java | 81 +
.../metastore/model/MStorageDescriptor.java | 277 +
.../hive/metastore/model/MStringList.java | 62 +
.../hadoop/hive/metastore/model/MTable.java | 256 +
.../metastore/model/MTableColumnPrivilege.java | 160 +
.../metastore/model/MTableColumnStatistics.java | 263 +
.../hive/metastore/model/MTablePrivilege.java | 139 +
.../hadoop/hive/metastore/model/MType.java | 105 +
.../hive/metastore/model/MVersionTable.java | 57 +
.../security/DelegationTokenIdentifier.java | 52 +
.../security/DelegationTokenSecretManager.java | 125 +
.../security/DelegationTokenStore.java | 118 +
.../security/HadoopThriftAuthBridge.java | 684 +
.../security/HadoopThriftAuthBridge23.java | 114 +
.../metastore/security/MemoryTokenStore.java | 136 +
.../MetastoreDelegationTokenManager.java | 172 +
.../metastore/security/TFilterTransport.java | 99 +
.../security/TUGIAssumingTransport.java | 73 +
.../security/TUGIContainingTransport.java | 96 +
.../TokenStoreDelegationTokenSecretManager.java | 333 +
.../hadoop/hive/metastore/utils/HdfsUtils.java | 125 +
.../hive/metastore/utils/SecurityUtils.java | 40 +
.../hive/metastore/utils/StringUtils.java | 112 +
.../MetastoreDelegationTokenSupport.java | 68 +
.../src/main/resources/package.jdo | 1087 +
.../src/main/resources/thrift-replacements.txt | 69 +
.../src/main/thrift/hive_metastore.thrift | 1613 +
.../hive/metastore/conf/TestMetastoreConf.java | 414 +
.../hive/metastore/metrics/TestMetrics.java | 200 +
.../hive/metastore/utils/TestHdfsUtils.java | 193 +
.../apache/hadoop/hive/common/io/Allocator.java | 17 +
.../apache/hadoop/hive/common/io/DataCache.java | 6 +
.../hadoop/hive/common/io/DiskRangeList.java | 80 +-
.../hive/common/io/FileMetadataCache.java | 51 +
.../io/encoded/MemoryBufferOrBuffers.java | 24 +
.../hive/common/type/FastHiveDecimal.java | 18 +
.../hive/common/type/FastHiveDecimalImpl.java | 71 +-
.../hive/ql/io/sarg/SearchArgumentImpl.java | 5 +-
.../hive/serde2/io/HiveDecimalWritable.java | 58 +
.../apache/hive/common/util/BloomFilter.java | 37 +-
.../apache/hive/common/util/BloomKFilter.java | 472 +
.../hive/common/io/TestDiskRangeList.java | 106 +
.../hive/common/type/HiveDecimalTestBase.java | 10 +
.../hive/common/type/TestHiveDecimal.java | 73 +
.../hive/common/util/TestBloomFilter.java | 136 +-
.../hive/common/util/TestBloomKFilter.java | 699 +
.../ptest2/conf/deployed/master-mr2.properties | 57 +-
.../ptest2/conf/server-env.properties.example | 23 +
.../ptest/api/server/ExecutionController.java | 5 +-
.../hive/ptest/api/server/TestExecutor.java | 16 +-
.../org/apache/hive/ptest/execution/PTest.java | 18 +-
.../hive/ptest/execution/conf/Context.java | 84 +
.../conf/ExecutionContextConfiguration.java | 20 +-
.../ptest/execution/conf/TestConfiguration.java | 20 +-
.../hive/ptest/execution/conf/TestParser.java | 3 +-
.../hive/ptest/execution/TestJIRAService.java | 6 +-
.../hive/ptest/execution/conf/TestContext.java | 68 +
.../execution/conf/TestTestConfiguration.java | 10 +-
.../apache/hadoop/hive/tools/GenVectorCode.java | 345 +-
2788 files changed, 803465 insertions(+), 684284 deletions(-)
----------------------------------------------------------------------
[21/50] [abbrv] hive git commit: HIVE-17100 : Improve HS2 operation
logs for REPL commands (Sankar Hariappan, reviewed by Anishek Agarwal,
Thejas Nair)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEventsCountRequest.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEventsCountRequest.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEventsCountRequest.java
new file mode 100644
index 0000000..3644d6c
--- /dev/null
+++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEventsCountRequest.java
@@ -0,0 +1,488 @@
+/**
+ * Autogenerated by Thrift Compiler (0.9.3)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+package org.apache.hadoop.hive.metastore.api;
+
+import org.apache.thrift.scheme.IScheme;
+import org.apache.thrift.scheme.SchemeFactory;
+import org.apache.thrift.scheme.StandardScheme;
+
+import org.apache.thrift.scheme.TupleScheme;
+import org.apache.thrift.protocol.TTupleProtocol;
+import org.apache.thrift.protocol.TProtocolException;
+import org.apache.thrift.EncodingUtils;
+import org.apache.thrift.TException;
+import org.apache.thrift.async.AsyncMethodCallback;
+import org.apache.thrift.server.AbstractNonblockingServer.*;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.EnumMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.EnumSet;
+import java.util.Collections;
+import java.util.BitSet;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import javax.annotation.Generated;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
+@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
+public class NotificationEventsCountRequest implements org.apache.thrift.TBase<NotificationEventsCountRequest, NotificationEventsCountRequest._Fields>, java.io.Serializable, Cloneable, Comparable<NotificationEventsCountRequest> {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("NotificationEventsCountRequest");
+
+ private static final org.apache.thrift.protocol.TField FROM_EVENT_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("fromEventId", org.apache.thrift.protocol.TType.I64, (short)1);
+ private static final org.apache.thrift.protocol.TField DB_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("dbName", org.apache.thrift.protocol.TType.STRING, (short)2);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new NotificationEventsCountRequestStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new NotificationEventsCountRequestTupleSchemeFactory());
+ }
+
+ private long fromEventId; // required
+ private String dbName; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ FROM_EVENT_ID((short)1, "fromEventId"),
+ DB_NAME((short)2, "dbName");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 1: // FROM_EVENT_ID
+ return FROM_EVENT_ID;
+ case 2: // DB_NAME
+ return DB_NAME;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ private static final int __FROMEVENTID_ISSET_ID = 0;
+ private byte __isset_bitfield = 0;
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.FROM_EVENT_ID, new org.apache.thrift.meta_data.FieldMetaData("fromEventId", org.apache.thrift.TFieldRequirementType.REQUIRED,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
+ tmpMap.put(_Fields.DB_NAME, new org.apache.thrift.meta_data.FieldMetaData("dbName", org.apache.thrift.TFieldRequirementType.REQUIRED,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(NotificationEventsCountRequest.class, metaDataMap);
+ }
+
+ public NotificationEventsCountRequest() {
+ }
+
+ public NotificationEventsCountRequest(
+ long fromEventId,
+ String dbName)
+ {
+ this();
+ this.fromEventId = fromEventId;
+ setFromEventIdIsSet(true);
+ this.dbName = dbName;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public NotificationEventsCountRequest(NotificationEventsCountRequest other) {
+ __isset_bitfield = other.__isset_bitfield;
+ this.fromEventId = other.fromEventId;
+ if (other.isSetDbName()) {
+ this.dbName = other.dbName;
+ }
+ }
+
+ public NotificationEventsCountRequest deepCopy() {
+ return new NotificationEventsCountRequest(this);
+ }
+
+ @Override
+ public void clear() {
+ setFromEventIdIsSet(false);
+ this.fromEventId = 0;
+ this.dbName = null;
+ }
+
+ public long getFromEventId() {
+ return this.fromEventId;
+ }
+
+ public void setFromEventId(long fromEventId) {
+ this.fromEventId = fromEventId;
+ setFromEventIdIsSet(true);
+ }
+
+ public void unsetFromEventId() {
+ __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __FROMEVENTID_ISSET_ID);
+ }
+
+ /** Returns true if field fromEventId is set (has been assigned a value) and false otherwise */
+ public boolean isSetFromEventId() {
+ return EncodingUtils.testBit(__isset_bitfield, __FROMEVENTID_ISSET_ID);
+ }
+
+ public void setFromEventIdIsSet(boolean value) {
+ __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __FROMEVENTID_ISSET_ID, value);
+ }
+
+ public String getDbName() {
+ return this.dbName;
+ }
+
+ public void setDbName(String dbName) {
+ this.dbName = dbName;
+ }
+
+ public void unsetDbName() {
+ this.dbName = null;
+ }
+
+ /** Returns true if field dbName is set (has been assigned a value) and false otherwise */
+ public boolean isSetDbName() {
+ return this.dbName != null;
+ }
+
+ public void setDbNameIsSet(boolean value) {
+ if (!value) {
+ this.dbName = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case FROM_EVENT_ID:
+ if (value == null) {
+ unsetFromEventId();
+ } else {
+ setFromEventId((Long)value);
+ }
+ break;
+
+ case DB_NAME:
+ if (value == null) {
+ unsetDbName();
+ } else {
+ setDbName((String)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case FROM_EVENT_ID:
+ return getFromEventId();
+
+ case DB_NAME:
+ return getDbName();
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case FROM_EVENT_ID:
+ return isSetFromEventId();
+ case DB_NAME:
+ return isSetDbName();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof NotificationEventsCountRequest)
+ return this.equals((NotificationEventsCountRequest)that);
+ return false;
+ }
+
+ public boolean equals(NotificationEventsCountRequest that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_fromEventId = true;
+ boolean that_present_fromEventId = true;
+ if (this_present_fromEventId || that_present_fromEventId) {
+ if (!(this_present_fromEventId && that_present_fromEventId))
+ return false;
+ if (this.fromEventId != that.fromEventId)
+ return false;
+ }
+
+ boolean this_present_dbName = true && this.isSetDbName();
+ boolean that_present_dbName = true && that.isSetDbName();
+ if (this_present_dbName || that_present_dbName) {
+ if (!(this_present_dbName && that_present_dbName))
+ return false;
+ if (!this.dbName.equals(that.dbName))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ List<Object> list = new ArrayList<Object>();
+
+ boolean present_fromEventId = true;
+ list.add(present_fromEventId);
+ if (present_fromEventId)
+ list.add(fromEventId);
+
+ boolean present_dbName = true && (isSetDbName());
+ list.add(present_dbName);
+ if (present_dbName)
+ list.add(dbName);
+
+ return list.hashCode();
+ }
+
+ @Override
+ public int compareTo(NotificationEventsCountRequest other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+
+ lastComparison = Boolean.valueOf(isSetFromEventId()).compareTo(other.isSetFromEventId());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetFromEventId()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.fromEventId, other.fromEventId);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetDbName()).compareTo(other.isSetDbName());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetDbName()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.dbName, other.dbName);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("NotificationEventsCountRequest(");
+ boolean first = true;
+
+ sb.append("fromEventId:");
+ sb.append(this.fromEventId);
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("dbName:");
+ if (this.dbName == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.dbName);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ if (!isSetFromEventId()) {
+ throw new org.apache.thrift.protocol.TProtocolException("Required field 'fromEventId' is unset! Struct:" + toString());
+ }
+
+ if (!isSetDbName()) {
+ throw new org.apache.thrift.protocol.TProtocolException("Required field 'dbName' is unset! Struct:" + toString());
+ }
+
+ // check for sub-struct validity
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+ __isset_bitfield = 0;
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class NotificationEventsCountRequestStandardSchemeFactory implements SchemeFactory {
+ public NotificationEventsCountRequestStandardScheme getScheme() {
+ return new NotificationEventsCountRequestStandardScheme();
+ }
+ }
+
+ private static class NotificationEventsCountRequestStandardScheme extends StandardScheme<NotificationEventsCountRequest> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, NotificationEventsCountRequest struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 1: // FROM_EVENT_ID
+ if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
+ struct.fromEventId = iprot.readI64();
+ struct.setFromEventIdIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 2: // DB_NAME
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.dbName = iprot.readString();
+ struct.setDbNameIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, NotificationEventsCountRequest struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ oprot.writeFieldBegin(FROM_EVENT_ID_FIELD_DESC);
+ oprot.writeI64(struct.fromEventId);
+ oprot.writeFieldEnd();
+ if (struct.dbName != null) {
+ oprot.writeFieldBegin(DB_NAME_FIELD_DESC);
+ oprot.writeString(struct.dbName);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class NotificationEventsCountRequestTupleSchemeFactory implements SchemeFactory {
+ public NotificationEventsCountRequestTupleScheme getScheme() {
+ return new NotificationEventsCountRequestTupleScheme();
+ }
+ }
+
+ private static class NotificationEventsCountRequestTupleScheme extends TupleScheme<NotificationEventsCountRequest> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, NotificationEventsCountRequest struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ oprot.writeI64(struct.fromEventId);
+ oprot.writeString(struct.dbName);
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, NotificationEventsCountRequest struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ struct.fromEventId = iprot.readI64();
+ struct.setFromEventIdIsSet(true);
+ struct.dbName = iprot.readString();
+ struct.setDbNameIsSet(true);
+ }
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEventsCountResponse.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEventsCountResponse.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEventsCountResponse.java
new file mode 100644
index 0000000..1ddee44
--- /dev/null
+++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEventsCountResponse.java
@@ -0,0 +1,387 @@
+/**
+ * Autogenerated by Thrift Compiler (0.9.3)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+package org.apache.hadoop.hive.metastore.api;
+
+import org.apache.thrift.scheme.IScheme;
+import org.apache.thrift.scheme.SchemeFactory;
+import org.apache.thrift.scheme.StandardScheme;
+
+import org.apache.thrift.scheme.TupleScheme;
+import org.apache.thrift.protocol.TTupleProtocol;
+import org.apache.thrift.protocol.TProtocolException;
+import org.apache.thrift.EncodingUtils;
+import org.apache.thrift.TException;
+import org.apache.thrift.async.AsyncMethodCallback;
+import org.apache.thrift.server.AbstractNonblockingServer.*;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.EnumMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.EnumSet;
+import java.util.Collections;
+import java.util.BitSet;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import javax.annotation.Generated;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
+@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
+public class NotificationEventsCountResponse implements org.apache.thrift.TBase<NotificationEventsCountResponse, NotificationEventsCountResponse._Fields>, java.io.Serializable, Cloneable, Comparable<NotificationEventsCountResponse> {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("NotificationEventsCountResponse");
+
+ private static final org.apache.thrift.protocol.TField EVENTS_COUNT_FIELD_DESC = new org.apache.thrift.protocol.TField("eventsCount", org.apache.thrift.protocol.TType.I64, (short)1);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new NotificationEventsCountResponseStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new NotificationEventsCountResponseTupleSchemeFactory());
+ }
+
+ private long eventsCount; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ EVENTS_COUNT((short)1, "eventsCount");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 1: // EVENTS_COUNT
+ return EVENTS_COUNT;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ private static final int __EVENTSCOUNT_ISSET_ID = 0;
+ private byte __isset_bitfield = 0;
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.EVENTS_COUNT, new org.apache.thrift.meta_data.FieldMetaData("eventsCount", org.apache.thrift.TFieldRequirementType.REQUIRED,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(NotificationEventsCountResponse.class, metaDataMap);
+ }
+
+ public NotificationEventsCountResponse() {
+ }
+
+ public NotificationEventsCountResponse(
+ long eventsCount)
+ {
+ this();
+ this.eventsCount = eventsCount;
+ setEventsCountIsSet(true);
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public NotificationEventsCountResponse(NotificationEventsCountResponse other) {
+ __isset_bitfield = other.__isset_bitfield;
+ this.eventsCount = other.eventsCount;
+ }
+
+ public NotificationEventsCountResponse deepCopy() {
+ return new NotificationEventsCountResponse(this);
+ }
+
+ @Override
+ public void clear() {
+ setEventsCountIsSet(false);
+ this.eventsCount = 0;
+ }
+
+ public long getEventsCount() {
+ return this.eventsCount;
+ }
+
+ public void setEventsCount(long eventsCount) {
+ this.eventsCount = eventsCount;
+ setEventsCountIsSet(true);
+ }
+
+ public void unsetEventsCount() {
+ __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __EVENTSCOUNT_ISSET_ID);
+ }
+
+ /** Returns true if field eventsCount is set (has been assigned a value) and false otherwise */
+ public boolean isSetEventsCount() {
+ return EncodingUtils.testBit(__isset_bitfield, __EVENTSCOUNT_ISSET_ID);
+ }
+
+ public void setEventsCountIsSet(boolean value) {
+ __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __EVENTSCOUNT_ISSET_ID, value);
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case EVENTS_COUNT:
+ if (value == null) {
+ unsetEventsCount();
+ } else {
+ setEventsCount((Long)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case EVENTS_COUNT:
+ return getEventsCount();
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case EVENTS_COUNT:
+ return isSetEventsCount();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof NotificationEventsCountResponse)
+ return this.equals((NotificationEventsCountResponse)that);
+ return false;
+ }
+
+ public boolean equals(NotificationEventsCountResponse that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_eventsCount = true;
+ boolean that_present_eventsCount = true;
+ if (this_present_eventsCount || that_present_eventsCount) {
+ if (!(this_present_eventsCount && that_present_eventsCount))
+ return false;
+ if (this.eventsCount != that.eventsCount)
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ List<Object> list = new ArrayList<Object>();
+
+ boolean present_eventsCount = true;
+ list.add(present_eventsCount);
+ if (present_eventsCount)
+ list.add(eventsCount);
+
+ return list.hashCode();
+ }
+
+ @Override
+ public int compareTo(NotificationEventsCountResponse other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+
+ lastComparison = Boolean.valueOf(isSetEventsCount()).compareTo(other.isSetEventsCount());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetEventsCount()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.eventsCount, other.eventsCount);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("NotificationEventsCountResponse(");
+ boolean first = true;
+
+ sb.append("eventsCount:");
+ sb.append(this.eventsCount);
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ if (!isSetEventsCount()) {
+ throw new org.apache.thrift.protocol.TProtocolException("Required field 'eventsCount' is unset! Struct:" + toString());
+ }
+
+ // check for sub-struct validity
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+ __isset_bitfield = 0;
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class NotificationEventsCountResponseStandardSchemeFactory implements SchemeFactory {
+ public NotificationEventsCountResponseStandardScheme getScheme() {
+ return new NotificationEventsCountResponseStandardScheme();
+ }
+ }
+
+ private static class NotificationEventsCountResponseStandardScheme extends StandardScheme<NotificationEventsCountResponse> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, NotificationEventsCountResponse struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 1: // EVENTS_COUNT
+ if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
+ struct.eventsCount = iprot.readI64();
+ struct.setEventsCountIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, NotificationEventsCountResponse struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ oprot.writeFieldBegin(EVENTS_COUNT_FIELD_DESC);
+ oprot.writeI64(struct.eventsCount);
+ oprot.writeFieldEnd();
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class NotificationEventsCountResponseTupleSchemeFactory implements SchemeFactory {
+ public NotificationEventsCountResponseTupleScheme getScheme() {
+ return new NotificationEventsCountResponseTupleScheme();
+ }
+ }
+
+ private static class NotificationEventsCountResponseTupleScheme extends TupleScheme<NotificationEventsCountResponse> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, NotificationEventsCountResponse struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ oprot.writeI64(struct.eventsCount);
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, NotificationEventsCountResponse struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ struct.eventsCount = iprot.readI64();
+ struct.setEventsCountIsSet(true);
+ }
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
index 541cbb9..47335b4 100644
--- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
+++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
@@ -344,6 +344,8 @@ public class ThriftHiveMetastore {
public CurrentNotificationEventId get_current_notificationEventId() throws org.apache.thrift.TException;
+ public NotificationEventsCountResponse get_notification_events_count(NotificationEventsCountRequest rqst) throws org.apache.thrift.TException;
+
public FireEventResponse fire_listener_event(FireEventRequest rqst) throws org.apache.thrift.TException;
public void flushCache() throws org.apache.thrift.TException;
@@ -668,6 +670,8 @@ public class ThriftHiveMetastore {
public void get_current_notificationEventId(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
+ public void get_notification_events_count(NotificationEventsCountRequest rqst, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
+
public void fire_listener_event(FireEventRequest rqst, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
public void flushCache(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
@@ -5090,6 +5094,29 @@ public class ThriftHiveMetastore {
throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "get_current_notificationEventId failed: unknown result");
}
+ public NotificationEventsCountResponse get_notification_events_count(NotificationEventsCountRequest rqst) throws org.apache.thrift.TException
+ {
+ send_get_notification_events_count(rqst);
+ return recv_get_notification_events_count();
+ }
+
+ public void send_get_notification_events_count(NotificationEventsCountRequest rqst) throws org.apache.thrift.TException
+ {
+ get_notification_events_count_args args = new get_notification_events_count_args();
+ args.setRqst(rqst);
+ sendBase("get_notification_events_count", args);
+ }
+
+ public NotificationEventsCountResponse recv_get_notification_events_count() throws org.apache.thrift.TException
+ {
+ get_notification_events_count_result result = new get_notification_events_count_result();
+ receiveBase(result, "get_notification_events_count");
+ if (result.isSetSuccess()) {
+ return result.success;
+ }
+ throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "get_notification_events_count failed: unknown result");
+ }
+
public FireEventResponse fire_listener_event(FireEventRequest rqst) throws org.apache.thrift.TException
{
send_fire_listener_event(rqst);
@@ -10652,6 +10679,38 @@ public class ThriftHiveMetastore {
}
}
+ public void get_notification_events_count(NotificationEventsCountRequest rqst, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
+ checkReady();
+ get_notification_events_count_call method_call = new get_notification_events_count_call(rqst, resultHandler, this, ___protocolFactory, ___transport);
+ this.___currentMethod = method_call;
+ ___manager.call(method_call);
+ }
+
+ public static class get_notification_events_count_call extends org.apache.thrift.async.TAsyncMethodCall {
+ private NotificationEventsCountRequest rqst;
+ public get_notification_events_count_call(NotificationEventsCountRequest rqst, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+ super(client, protocolFactory, transport, resultHandler, false);
+ this.rqst = rqst;
+ }
+
+ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+ prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("get_notification_events_count", org.apache.thrift.protocol.TMessageType.CALL, 0));
+ get_notification_events_count_args args = new get_notification_events_count_args();
+ args.setRqst(rqst);
+ args.write(prot);
+ prot.writeMessageEnd();
+ }
+
+ public NotificationEventsCountResponse getResult() throws org.apache.thrift.TException {
+ if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+ throw new IllegalStateException("Method call not finished!");
+ }
+ org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+ org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+ return (new Client(prot)).recv_get_notification_events_count();
+ }
+ }
+
public void fire_listener_event(FireEventRequest rqst, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
checkReady();
fire_listener_event_call method_call = new fire_listener_event_call(rqst, resultHandler, this, ___protocolFactory, ___transport);
@@ -11098,6 +11157,7 @@ public class ThriftHiveMetastore {
processMap.put("add_dynamic_partitions", new add_dynamic_partitions());
processMap.put("get_next_notification", new get_next_notification());
processMap.put("get_current_notificationEventId", new get_current_notificationEventId());
+ processMap.put("get_notification_events_count", new get_notification_events_count());
processMap.put("fire_listener_event", new fire_listener_event());
processMap.put("flushCache", new flushCache());
processMap.put("cm_recycle", new cm_recycle());
@@ -14994,6 +15054,26 @@ public class ThriftHiveMetastore {
}
}
+ public static class get_notification_events_count<I extends Iface> extends org.apache.thrift.ProcessFunction<I, get_notification_events_count_args> {
+ public get_notification_events_count() {
+ super("get_notification_events_count");
+ }
+
+ public get_notification_events_count_args getEmptyArgsInstance() {
+ return new get_notification_events_count_args();
+ }
+
+ protected boolean isOneway() {
+ return false;
+ }
+
+ public get_notification_events_count_result getResult(I iface, get_notification_events_count_args args) throws org.apache.thrift.TException {
+ get_notification_events_count_result result = new get_notification_events_count_result();
+ result.success = iface.get_notification_events_count(args.rqst);
+ return result;
+ }
+ }
+
public static class fire_listener_event<I extends Iface> extends org.apache.thrift.ProcessFunction<I, fire_listener_event_args> {
public fire_listener_event() {
super("fire_listener_event");
@@ -15346,6 +15426,7 @@ public class ThriftHiveMetastore {
processMap.put("add_dynamic_partitions", new add_dynamic_partitions());
processMap.put("get_next_notification", new get_next_notification());
processMap.put("get_current_notificationEventId", new get_current_notificationEventId());
+ processMap.put("get_notification_events_count", new get_notification_events_count());
processMap.put("fire_listener_event", new fire_listener_event());
processMap.put("flushCache", new flushCache());
processMap.put("cm_recycle", new cm_recycle());
@@ -24604,6 +24685,57 @@ public class ThriftHiveMetastore {
}
}
+ public static class get_notification_events_count<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, get_notification_events_count_args, NotificationEventsCountResponse> {
+ public get_notification_events_count() {
+ super("get_notification_events_count");
+ }
+
+ public get_notification_events_count_args getEmptyArgsInstance() {
+ return new get_notification_events_count_args();
+ }
+
+ public AsyncMethodCallback<NotificationEventsCountResponse> getResultHandler(final AsyncFrameBuffer fb, final int seqid) {
+ final org.apache.thrift.AsyncProcessFunction fcall = this;
+ return new AsyncMethodCallback<NotificationEventsCountResponse>() {
+ public void onComplete(NotificationEventsCountResponse o) {
+ get_notification_events_count_result result = new get_notification_events_count_result();
+ result.success = o;
+ try {
+ fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid);
+ return;
+ } catch (Exception e) {
+ LOGGER.error("Exception writing to internal frame buffer", e);
+ }
+ fb.close();
+ }
+ public void onError(Exception e) {
+ byte msgType = org.apache.thrift.protocol.TMessageType.REPLY;
+ org.apache.thrift.TBase msg;
+ get_notification_events_count_result result = new get_notification_events_count_result();
+ {
+ msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION;
+ msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage());
+ }
+ try {
+ fcall.sendResponse(fb,msg,msgType,seqid);
+ return;
+ } catch (Exception ex) {
+ LOGGER.error("Exception writing to internal frame buffer", ex);
+ }
+ fb.close();
+ }
+ };
+ }
+
+ protected boolean isOneway() {
+ return false;
+ }
+
+ public void start(I iface, get_notification_events_count_args args, org.apache.thrift.async.AsyncMethodCallback<NotificationEventsCountResponse> resultHandler) throws TException {
+ iface.get_notification_events_count(args.rqst,resultHandler);
+ }
+ }
+
public static class fire_listener_event<I extends AsyncIface> extends org.apache.thrift.AsyncProcessFunction<I, fire_listener_event_args, FireEventResponse> {
public fire_listener_event() {
super("fire_listener_event");
@@ -180954,6 +181086,732 @@ public class ThriftHiveMetastore {
}
+ public static class get_notification_events_count_args implements org.apache.thrift.TBase<get_notification_events_count_args, get_notification_events_count_args._Fields>, java.io.Serializable, Cloneable, Comparable<get_notification_events_count_args> {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("get_notification_events_count_args");
+
+ private static final org.apache.thrift.protocol.TField RQST_FIELD_DESC = new org.apache.thrift.protocol.TField("rqst", org.apache.thrift.protocol.TType.STRUCT, (short)-1);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new get_notification_events_count_argsStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new get_notification_events_count_argsTupleSchemeFactory());
+ }
+
+ private NotificationEventsCountRequest rqst; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ RQST((short)-1, "rqst");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case -1: // RQST
+ return RQST;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.RQST, new org.apache.thrift.meta_data.FieldMetaData("rqst", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, NotificationEventsCountRequest.class)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_notification_events_count_args.class, metaDataMap);
+ }
+
+ public get_notification_events_count_args() {
+ }
+
+ public get_notification_events_count_args(
+ NotificationEventsCountRequest rqst)
+ {
+ this();
+ this.rqst = rqst;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public get_notification_events_count_args(get_notification_events_count_args other) {
+ if (other.isSetRqst()) {
+ this.rqst = new NotificationEventsCountRequest(other.rqst);
+ }
+ }
+
+ public get_notification_events_count_args deepCopy() {
+ return new get_notification_events_count_args(this);
+ }
+
+ @Override
+ public void clear() {
+ this.rqst = null;
+ }
+
+ public NotificationEventsCountRequest getRqst() {
+ return this.rqst;
+ }
+
+ public void setRqst(NotificationEventsCountRequest rqst) {
+ this.rqst = rqst;
+ }
+
+ public void unsetRqst() {
+ this.rqst = null;
+ }
+
+ /** Returns true if field rqst is set (has been assigned a value) and false otherwise */
+ public boolean isSetRqst() {
+ return this.rqst != null;
+ }
+
+ public void setRqstIsSet(boolean value) {
+ if (!value) {
+ this.rqst = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case RQST:
+ if (value == null) {
+ unsetRqst();
+ } else {
+ setRqst((NotificationEventsCountRequest)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case RQST:
+ return getRqst();
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case RQST:
+ return isSetRqst();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof get_notification_events_count_args)
+ return this.equals((get_notification_events_count_args)that);
+ return false;
+ }
+
+ public boolean equals(get_notification_events_count_args that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_rqst = true && this.isSetRqst();
+ boolean that_present_rqst = true && that.isSetRqst();
+ if (this_present_rqst || that_present_rqst) {
+ if (!(this_present_rqst && that_present_rqst))
+ return false;
+ if (!this.rqst.equals(that.rqst))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ List<Object> list = new ArrayList<Object>();
+
+ boolean present_rqst = true && (isSetRqst());
+ list.add(present_rqst);
+ if (present_rqst)
+ list.add(rqst);
+
+ return list.hashCode();
+ }
+
+ @Override
+ public int compareTo(get_notification_events_count_args other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+
+ lastComparison = Boolean.valueOf(isSetRqst()).compareTo(other.isSetRqst());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetRqst()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.rqst, other.rqst);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("get_notification_events_count_args(");
+ boolean first = true;
+
+ sb.append("rqst:");
+ if (this.rqst == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.rqst);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ if (rqst != null) {
+ rqst.validate();
+ }
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class get_notification_events_count_argsStandardSchemeFactory implements SchemeFactory {
+ public get_notification_events_count_argsStandardScheme getScheme() {
+ return new get_notification_events_count_argsStandardScheme();
+ }
+ }
+
+ private static class get_notification_events_count_argsStandardScheme extends StandardScheme<get_notification_events_count_args> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, get_notification_events_count_args struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case -1: // RQST
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
+ struct.rqst = new NotificationEventsCountRequest();
+ struct.rqst.read(iprot);
+ struct.setRqstIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, get_notification_events_count_args struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ if (struct.rqst != null) {
+ oprot.writeFieldBegin(RQST_FIELD_DESC);
+ struct.rqst.write(oprot);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class get_notification_events_count_argsTupleSchemeFactory implements SchemeFactory {
+ public get_notification_events_count_argsTupleScheme getScheme() {
+ return new get_notification_events_count_argsTupleScheme();
+ }
+ }
+
+ private static class get_notification_events_count_argsTupleScheme extends TupleScheme<get_notification_events_count_args> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, get_notification_events_count_args struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ BitSet optionals = new BitSet();
+ if (struct.isSetRqst()) {
+ optionals.set(0);
+ }
+ oprot.writeBitSet(optionals, 1);
+ if (struct.isSetRqst()) {
+ struct.rqst.write(oprot);
+ }
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, get_notification_events_count_args struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ BitSet incoming = iprot.readBitSet(1);
+ if (incoming.get(0)) {
+ struct.rqst = new NotificationEventsCountRequest();
+ struct.rqst.read(iprot);
+ struct.setRqstIsSet(true);
+ }
+ }
+ }
+
+ }
+
+ public static class get_notification_events_count_result implements org.apache.thrift.TBase<get_notification_events_count_result, get_notification_events_count_result._Fields>, java.io.Serializable, Cloneable, Comparable<get_notification_events_count_result> {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("get_notification_events_count_result");
+
+ private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new get_notification_events_count_resultStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new get_notification_events_count_resultTupleSchemeFactory());
+ }
+
+ private NotificationEventsCountResponse success; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ SUCCESS((short)0, "success");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 0: // SUCCESS
+ return SUCCESS;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, NotificationEventsCountResponse.class)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_notification_events_count_result.class, metaDataMap);
+ }
+
+ public get_notification_events_count_result() {
+ }
+
+ public get_notification_events_count_result(
+ NotificationEventsCountResponse success)
+ {
+ this();
+ this.success = success;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public get_notification_events_count_result(get_notification_events_count_result other) {
+ if (other.isSetSuccess()) {
+ this.success = new NotificationEventsCountResponse(other.success);
+ }
+ }
+
+ public get_notification_events_count_result deepCopy() {
+ return new get_notification_events_count_result(this);
+ }
+
+ @Override
+ public void clear() {
+ this.success = null;
+ }
+
+ public NotificationEventsCountResponse getSuccess() {
+ return this.success;
+ }
+
+ public void setSuccess(NotificationEventsCountResponse success) {
+ this.success = success;
+ }
+
+ public void unsetSuccess() {
+ this.success = null;
+ }
+
+ /** Returns true if field success is set (has been assigned a value) and false otherwise */
+ public boolean isSetSuccess() {
+ return this.success != null;
+ }
+
+ public void setSuccessIsSet(boolean value) {
+ if (!value) {
+ this.success = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case SUCCESS:
+ if (value == null) {
+ unsetSuccess();
+ } else {
+ setSuccess((NotificationEventsCountResponse)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case SUCCESS:
+ return getSuccess();
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case SUCCESS:
+ return isSetSuccess();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof get_notification_events_count_result)
+ return this.equals((get_notification_events_count_result)that);
+ return false;
+ }
+
+ public boolean equals(get_notification_events_count_result that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_success = true && this.isSetSuccess();
+ boolean that_present_success = true && that.isSetSuccess();
+ if (this_present_success || that_present_success) {
+ if (!(this_present_success && that_present_success))
+ return false;
+ if (!this.success.equals(that.success))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ List<Object> list = new ArrayList<Object>();
+
+ boolean present_success = true && (isSetSuccess());
+ list.add(present_success);
+ if (present_success)
+ list.add(success);
+
+ return list.hashCode();
+ }
+
+ @Override
+ public int compareTo(get_notification_events_count_result other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+
+ lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetSuccess()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("get_notification_events_count_result(");
+ boolean first = true;
+
+ sb.append("success:");
+ if (this.success == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.success);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ if (success != null) {
+ success.validate();
+ }
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class get_notification_events_count_resultStandardSchemeFactory implements SchemeFactory {
+ public get_notification_events_count_resultStandardScheme getScheme() {
+ return new get_notification_events_count_resultStandardScheme();
+ }
+ }
+
+ private static class get_notification_events_count_resultStandardScheme extends StandardScheme<get_notification_events_count_result> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, get_notification_events_count_result struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 0: // SUCCESS
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
+ struct.success = new NotificationEventsCountResponse();
+ struct.success.read(iprot);
+ struct.setSuccessIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, get_notification_events_count_result struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ if (struct.success != null) {
+ oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
+ struct.success.write(oprot);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class get_notification_events_count_resultTupleSchemeFactory implements SchemeFactory {
+ public get_notification_events_count_resultTupleScheme getScheme() {
+ return new get_notification_events_count_resultTupleScheme();
+ }
+ }
+
+ private static class get_notification_events_count_resultTupleScheme extends TupleScheme<get_notification_events_count_result> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, get_notification_events_count_result struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ BitSet optionals = new BitSet();
+ if (struct.isSetSuccess()) {
+ optionals.set(0);
+ }
+ oprot.writeBitSet(optionals, 1);
+ if (struct.isSetSuccess()) {
+ struct.success.write(oprot);
+ }
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, get_notification_events_count_result struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ BitSet incoming = iprot.readBitSet(1);
+ if (incoming.get(0)) {
+ struct.success = new NotificationEventsCountResponse();
+ struct.success.read(iprot);
+ struct.setSuccessIsSet(true);
+ }
+ }
+ }
+
+ }
+
public static class fire_listener_event_args implements org.apache.thrift.TBase<fire_listener_event_args, fire_listener_event_args._Fields>, java.io.Serializable, Cloneable, Comparable<fire_listener_event_args> {
private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("fire_listener_event_args");
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/standalone-metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php b/standalone-metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
index cc5e120..644def0 100644
--- a/standalone-metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
+++ b/standalone-metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
@@ -1191,6 +1191,11 @@ interface ThriftHiveMetastoreIf extends \FacebookServiceIf {
*/
public function get_current_notificationEventId();
/**
+ * @param \metastore\NotificationEventsCountRequest $rqst
+ * @return \metastore\NotificationEventsCountResponse
+ */
+ public function get_notification_events_count(\metastore\NotificationEventsCountRequest $rqst);
+ /**
* @param \metastore\FireEventRequest $rqst
* @return \metastore\FireEventResponse
*/
@@ -9851,6 +9856,57 @@ class ThriftHiveMetastoreClient extends \FacebookServiceClient implements \metas
throw new \Exception("get_current_notificationEventId failed: unknown result");
}
+ public function get_notification_events_count(\metastore\NotificationEventsCountRequest $rqst)
+ {
+ $this->send_get_notification_events_count($rqst);
+ return $this->recv_get_notification_events_count();
+ }
+
+ public function send_get_notification_events_count(\metastore\NotificationEventsCountRequest $rqst)
+ {
+ $args = new \metastore\ThriftHiveMetastore_get_notification_events_count_args();
+ $args->rqst = $rqst;
+ $bin_accel = ($this->output_ instanceof TBinaryProtocolAccelerated) && function_exists('thrift_protocol_write_binary');
+ if ($bin_accel)
+ {
+ thrift_protocol_write_binary($this->output_, 'get_notification_events_count', TMessageType::CALL, $args, $this->seqid_, $this->output_->isStrictWrite());
+ }
+ else
+ {
+ $this->output_->writeMessageBegin('get_notification_events_count', TMessageType::CALL, $this->seqid_);
+ $args->write($this->output_);
+ $this->output_->writeMessageEnd();
+ $this->output_->getTransport()->flush();
+ }
+ }
+
+ public function recv_get_notification_events_count()
+ {
+ $bin_accel = ($this->input_ instanceof TBinaryProtocolAccelerated) && function_exists('thrift_protocol_read_binary');
+ if ($bin_accel) $result = thrift_protocol_read_binary($this->input_, '\metastore\ThriftHiveMetastore_get_notification_events_count_result', $this->input_->isStrictRead());
+ else
+ {
+ $rseqid = 0;
+ $fname = null;
+ $mtype = 0;
+
+ $this->input_->readMessageBegin($fname, $mtype, $rseqid);
+ if ($mtype == TMessageType::EXCEPTION) {
+ $x = new TApplicationException();
+ $x->read($this->input_);
+ $this->input_->readMessageEnd();
+ throw $x;
+ }
+ $result = new \metastore\ThriftHiveMetastore_get_notification_events_count_result();
+ $result->read($this->input_);
+ $this->input_->readMessageEnd();
+ }
+ if ($result->success !== null) {
+ return $result->success;
+ }
+ throw new \Exception("get_notification_events_count failed: unknown result");
+ }
+
public function fire_listener_event(\metastore\FireEventRequest $rqst)
{
$this->send_fire_listener_event($rqst);
@@ -45514,6 +45570,166 @@ class ThriftHiveMetastore_get_current_notificationEventId_result {
}
+class ThriftHiveMetastore_get_notification_events_count_args {
+ static $_TSPEC;
+
+ /**
+ * @var \metastore\NotificationEventsCountRequest
+ */
+ public $rqst = null;
+
+ public function __construct($vals=null) {
+ if (!isset(self::$_TSPEC)) {
+ self::$_TSPEC = array(
+ -1 => array(
+ 'var' => 'rqst',
+ 'type' => TType::STRUCT,
+ 'class' => '\metastore\NotificationEventsCountRequest',
+ ),
+ );
+ }
+ if (is_array($vals)) {
+ if (isset($vals['rqst'])) {
+ $this->rqst = $vals['rqst'];
+ }
+ }
+ }
+
+ public function getName() {
+ return 'ThriftHiveMetastore_get_notification_events_count_args';
+ }
+
+ public function read($input)
+ {
+ $xfer = 0;
+ $fname = null;
+ $ftype = 0;
+ $fid = 0;
+ $xfer += $input->readStructBegin($fname);
+ while (true)
+ {
+ $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+ if ($ftype == TType::STOP) {
+ break;
+ }
+ switch ($fid)
+ {
+ case -1:
+ if ($ftype == TType::STRUCT) {
+ $this->rqst = new \metastore\NotificationEventsCountRequest();
+ $xfer += $this->rqst->read($input);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ default:
+ $xfer += $input->skip($ftype);
+ break;
+ }
+ $xfer += $input->readFieldEnd();
+ }
+ $xfer += $input->readStructEnd();
+ return $xfer;
+ }
+
+ public function write($output) {
+ $xfer = 0;
+ $xfer += $output->writeStructBegin('ThriftHiveMetastore_get_notification_events_count_args');
+ if ($this->rqst !== null) {
+ if (!is_object($this->rqst)) {
+ throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA);
+ }
+ $xfer += $output->writeFieldBegin('rqst', TType::STRUCT, -1);
+ $xfer += $this->rqst->write($output);
+ $xfer += $output->writeFieldEnd();
+ }
+ $xfer += $output->writeFieldStop();
+ $xfer += $output->writeStructEnd();
+ return $xfer;
+ }
+
+}
+
+class ThriftHiveMetastore_get_notification_events_count_result {
+ static $_TSPEC;
+
+ /**
+ * @var \metastore\NotificationEventsCountResponse
+ */
+ public $success = null;
+
+ public function __construct($vals=null) {
+ if (!isset(self::$_TSPEC)) {
+ self::$_TSPEC = array(
+ 0 => array(
+ 'var' => 'success',
+ 'type' => TType::STRUCT,
+ 'class' => '\metastore\NotificationEventsCountResponse',
+ ),
+ );
+ }
+ if (is_array($vals)) {
+ if (isset($vals['success'])) {
+ $this->success = $vals['success'];
+ }
+ }
+ }
+
+ public function getName() {
+ return 'ThriftHiveMetastore_get_notification_events_count_result';
+ }
+
+ public function read($input)
+ {
+ $xfer = 0;
+ $fname = null;
+ $ftype = 0;
+ $fid = 0;
+ $xfer += $input->readStructBegin($fname);
+ while (true)
+ {
+ $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+ if ($ftype == TType::STOP) {
+ break;
+ }
+ switch ($fid)
+ {
+ case 0:
+ if ($ftype == TType::STRUCT) {
+ $this->success = new \metastore\NotificationEventsCountResponse();
+ $xfer += $this->success->read($input);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ default:
+ $xfer += $input->skip($ftype);
+ break;
+ }
+ $xfer += $input->readFieldEnd();
+ }
+ $xfer += $input->readStructEnd();
+ return $xfer;
+ }
+
+ public function write($output) {
+ $xfer = 0;
+ $xfer += $output->writeStructBegin('ThriftHiveMetastore_get_notification_events_count_result');
+ if ($this->success !== null) {
+ if (!is_object($this->success)) {
+ throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA);
+ }
+ $xfer += $output->writeFieldBegin('success', TType::STRUCT, 0);
+ $xfer += $this->success->write($output);
+ $xfer += $output->writeFieldEnd();
+ }
+ $xfer += $output->writeFieldStop();
+ $xfer += $output->writeStructEnd();
+ return $xfer;
+ }
+
+}
+
class ThriftHiveMetastore_fire_listener_event_args {
static $_TSPEC;
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php b/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
index aff783e..f0f0a57 100644
--- a/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
+++ b/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
@@ -17075,6 +17075,179 @@ class CurrentNotificationEventId {
}
+class NotificationEventsCountRequest {
+ static $_TSPEC;
+
+ /**
+ * @var int
+ */
+ public $fromEventId = null;
+ /**
+ * @var string
+ */
+ public $dbName = null;
+
+ public function __construct($vals=null) {
+ if (!isset(self::$_TSPEC)) {
+ self::$_TSPEC = array(
+ 1 => array(
+ 'var' => 'fromEventId',
+ 'type' => TType::I64,
+ ),
+ 2 => array(
+ 'var' => 'dbName',
+ 'type' => TType::STRING,
+ ),
+ );
+ }
+ if (is_array($vals)) {
+ if (isset($vals['fromEventId'])) {
+ $this->fromEventId = $vals['fromEventId'];
+ }
+ if (isset($vals['dbName'])) {
+ $this->dbName = $vals['dbName'];
+ }
+ }
+ }
+
+ public function getName() {
+ return 'NotificationEventsCountRequest';
+ }
+
+ public function read($input)
+ {
+ $xfer = 0;
+ $fname = null;
+ $ftype = 0;
+ $fid = 0;
+ $xfer += $input->readStructBegin($fname);
+ while (true)
+ {
+ $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+ if ($ftype == TType::STOP) {
+ break;
+ }
+ switch ($fid)
+ {
+ case 1:
+ if ($ftype == TType::I64) {
+ $xfer += $input->readI64($this->fromEventId);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ case 2:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->dbName);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ default:
+ $xfer += $input->skip($ftype);
+ break;
+ }
+ $xfer += $input->readFieldEnd();
+ }
+ $xfer += $input->readStructEnd();
+ return $xfer;
+ }
+
+ public function write($output) {
+ $xfer = 0;
+ $xfer += $output->writeStructBegin('NotificationEventsCountRequest');
+ if ($this->fromEventId !== null) {
+ $xfer += $output->writeFieldBegin('fromEventId', TType::I64, 1);
+ $xfer += $output->writeI64($this->fromEventId);
+ $xfer += $output->writeFieldEnd();
+ }
+ if ($this->dbName !== null) {
+ $xfer += $output->writeFieldBegin('dbName', TType::STRING, 2);
+ $xfer += $output->writeString($this->dbName);
+ $xfer += $output->writeFieldEnd();
+ }
+ $xfer += $output->writeFieldStop();
+ $xfer += $output->writeStructEnd();
+ return $xfer;
+ }
+
+}
+
+class NotificationEventsCountResponse {
+ static $_TSPEC;
+
+ /**
+ * @var int
+ */
+ public $eventsCount = null;
+
+ public function __construct($vals=null) {
+ if (!isset(self::$_TSPEC)) {
+ self::$_TSPEC = array(
+ 1 => array(
+ 'var' => 'eventsCount',
+ 'type' => TType::I64,
+ ),
+ );
+ }
+ if (is_array($vals)) {
+ if (isset($vals['eventsCount'])) {
+ $this->eventsCount = $vals['eventsCount'];
+ }
+ }
+ }
+
+ public function getName() {
+ return 'NotificationEventsCountResponse';
+ }
+
+ public function read($input)
+ {
+ $xfer = 0;
+ $fname = null;
+ $ftype = 0;
+ $fid = 0;
+ $xfer += $input->readStructBegin($fname);
+ while (true)
+ {
+ $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+ if ($ftype == TType::STOP) {
+ break;
+ }
+ switch ($fid)
+ {
+ case 1:
+ if ($ftype == TType::I64) {
+ $xfer += $input->readI64($this->eventsCount);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
+ default:
+ $xfer += $input->skip($ftype);
+ break;
+ }
+ $xfer += $input->readFieldEnd();
+ }
+ $xfer += $input->readStructEnd();
+ return $xfer;
+ }
+
+ public function write($output) {
+ $xfer = 0;
+ $xfer += $output->writeStructBegin('NotificationEventsCountResponse');
+ if ($this->eventsCount !== null) {
+ $xfer += $output->writeFieldBegin('eventsCount', TType::I64, 1);
+ $xfer += $output->writeI64($this->eventsCount);
+ $xfer += $output->writeFieldEnd();
+ }
+ $xfer += $output->writeFieldStop();
+ $xfer += $output->writeStructEnd();
+ return $xfer;
+ }
+
+}
+
class InsertEventRequestData {
static $_TSPEC;
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote
index 46dcac9..14527e6 100755
--- a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote
+++ b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote
@@ -175,6 +175,7 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help':
print(' void add_dynamic_partitions(AddDynamicPartitions rqst)')
print(' NotificationEventResponse get_next_notification(NotificationEventRequest rqst)')
print(' CurrentNotificationEventId get_current_notificationEventId()')
+ print(' NotificationEventsCountResponse get_notification_events_count(NotificationEventsCountRequest rqst)')
print(' FireEventResponse fire_listener_event(FireEventRequest rqst)')
print(' void flushCache()')
print(' CmRecycleResponse cm_recycle(CmRecycleRequest request)')
@@ -1159,6 +1160,12 @@ elif cmd == 'get_current_notificationEventId':
sys.exit(1)
pp.pprint(client.get_current_notificationEventId())
+elif cmd == 'get_notification_events_count':
+ if len(args) != 1:
+ print('get_notification_events_count requires 1 args')
+ sys.exit(1)
+ pp.pprint(client.get_notification_events_count(eval(args[0]),))
+
elif cmd == 'fire_listener_event':
if len(args) != 1:
print('fire_listener_event requires 1 args')
[39/50] [abbrv] hive git commit: HIVE-14671 : merge master into
hive-14535 (Wei Zheng)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
----------------------------------------------------------------------
diff --cc standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
index 0000000,ea73b34..1dae310
mode 000000,100644..100644
--- a/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
+++ b/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
@@@ -1,0 -1,3515 +1,3516 @@@
+ #
+ # Autogenerated by Thrift Compiler (0.9.3)
+ #
+ # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ #
+
+ require 'thrift'
+ require 'fb303_types'
+
+
+ module HiveObjectType
+ GLOBAL = 1
+ DATABASE = 2
+ TABLE = 3
+ PARTITION = 4
+ COLUMN = 5
+ VALUE_MAP = {1 => "GLOBAL", 2 => "DATABASE", 3 => "TABLE", 4 => "PARTITION", 5 => "COLUMN"}
+ VALID_VALUES = Set.new([GLOBAL, DATABASE, TABLE, PARTITION, COLUMN]).freeze
+ end
+
+ module PrincipalType
+ USER = 1
+ ROLE = 2
+ GROUP = 3
+ VALUE_MAP = {1 => "USER", 2 => "ROLE", 3 => "GROUP"}
+ VALID_VALUES = Set.new([USER, ROLE, GROUP]).freeze
+ end
+
+ module PartitionEventType
+ LOAD_DONE = 1
+ VALUE_MAP = {1 => "LOAD_DONE"}
+ VALID_VALUES = Set.new([LOAD_DONE]).freeze
+ end
+
+ module TxnState
+ COMMITTED = 1
+ ABORTED = 2
+ OPEN = 3
+ VALUE_MAP = {1 => "COMMITTED", 2 => "ABORTED", 3 => "OPEN"}
+ VALID_VALUES = Set.new([COMMITTED, ABORTED, OPEN]).freeze
+ end
+
+ module LockLevel
+ DB = 1
+ TABLE = 2
+ PARTITION = 3
+ VALUE_MAP = {1 => "DB", 2 => "TABLE", 3 => "PARTITION"}
+ VALID_VALUES = Set.new([DB, TABLE, PARTITION]).freeze
+ end
+
+ module LockState
+ ACQUIRED = 1
+ WAITING = 2
+ ABORT = 3
+ NOT_ACQUIRED = 4
+ VALUE_MAP = {1 => "ACQUIRED", 2 => "WAITING", 3 => "ABORT", 4 => "NOT_ACQUIRED"}
+ VALID_VALUES = Set.new([ACQUIRED, WAITING, ABORT, NOT_ACQUIRED]).freeze
+ end
+
+ module LockType
+ SHARED_READ = 1
+ SHARED_WRITE = 2
+ EXCLUSIVE = 3
+ VALUE_MAP = {1 => "SHARED_READ", 2 => "SHARED_WRITE", 3 => "EXCLUSIVE"}
+ VALID_VALUES = Set.new([SHARED_READ, SHARED_WRITE, EXCLUSIVE]).freeze
+ end
+
+ module CompactionType
+ MINOR = 1
+ MAJOR = 2
+ VALUE_MAP = {1 => "MINOR", 2 => "MAJOR"}
+ VALID_VALUES = Set.new([MINOR, MAJOR]).freeze
+ end
+
+ module GrantRevokeType
+ GRANT = 1
+ REVOKE = 2
+ VALUE_MAP = {1 => "GRANT", 2 => "REVOKE"}
+ VALID_VALUES = Set.new([GRANT, REVOKE]).freeze
+ end
+
+ module DataOperationType
+ SELECT = 1
+ INSERT = 2
+ UPDATE = 3
+ DELETE = 4
+ UNSET = 5
+ NO_TXN = 6
+ VALUE_MAP = {1 => "SELECT", 2 => "INSERT", 3 => "UPDATE", 4 => "DELETE", 5 => "UNSET", 6 => "NO_TXN"}
+ VALID_VALUES = Set.new([SELECT, INSERT, UPDATE, DELETE, UNSET, NO_TXN]).freeze
+ end
+
+ module EventRequestType
+ INSERT = 1
+ UPDATE = 2
+ DELETE = 3
+ VALUE_MAP = {1 => "INSERT", 2 => "UPDATE", 3 => "DELETE"}
+ VALID_VALUES = Set.new([INSERT, UPDATE, DELETE]).freeze
+ end
+
+ module FunctionType
+ JAVA = 1
+ VALUE_MAP = {1 => "JAVA"}
+ VALID_VALUES = Set.new([JAVA]).freeze
+ end
+
+ module ResourceType
+ JAR = 1
+ FILE = 2
+ ARCHIVE = 3
+ VALUE_MAP = {1 => "JAR", 2 => "FILE", 3 => "ARCHIVE"}
+ VALID_VALUES = Set.new([JAR, FILE, ARCHIVE]).freeze
+ end
+
+ module FileMetadataExprType
+ ORC_SARG = 1
+ VALUE_MAP = {1 => "ORC_SARG"}
+ VALID_VALUES = Set.new([ORC_SARG]).freeze
+ end
+
+ module ClientCapability
+ TEST_CAPABILITY = 1
- VALUE_MAP = {1 => "TEST_CAPABILITY"}
- VALID_VALUES = Set.new([TEST_CAPABILITY]).freeze
++ INSERT_ONLY_TABLES = 2
++ VALUE_MAP = {1 => "TEST_CAPABILITY", 2 => "INSERT_ONLY_TABLES"}
++ VALID_VALUES = Set.new([TEST_CAPABILITY, INSERT_ONLY_TABLES]).freeze
+ end
+
+ class Version
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ VERSION = 1
+ COMMENTS = 2
+
+ FIELDS = {
+ VERSION => {:type => ::Thrift::Types::STRING, :name => 'version'},
+ COMMENTS => {:type => ::Thrift::Types::STRING, :name => 'comments'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class FieldSchema
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ NAME = 1
+ TYPE = 2
+ COMMENT = 3
+
+ FIELDS = {
+ NAME => {:type => ::Thrift::Types::STRING, :name => 'name'},
+ TYPE => {:type => ::Thrift::Types::STRING, :name => 'type'},
+ COMMENT => {:type => ::Thrift::Types::STRING, :name => 'comment'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class SQLPrimaryKey
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ TABLE_DB = 1
+ TABLE_NAME = 2
+ COLUMN_NAME = 3
+ KEY_SEQ = 4
+ PK_NAME = 5
+ ENABLE_CSTR = 6
+ VALIDATE_CSTR = 7
+ RELY_CSTR = 8
+
+ FIELDS = {
+ TABLE_DB => {:type => ::Thrift::Types::STRING, :name => 'table_db'},
+ TABLE_NAME => {:type => ::Thrift::Types::STRING, :name => 'table_name'},
+ COLUMN_NAME => {:type => ::Thrift::Types::STRING, :name => 'column_name'},
+ KEY_SEQ => {:type => ::Thrift::Types::I32, :name => 'key_seq'},
+ PK_NAME => {:type => ::Thrift::Types::STRING, :name => 'pk_name'},
+ ENABLE_CSTR => {:type => ::Thrift::Types::BOOL, :name => 'enable_cstr'},
+ VALIDATE_CSTR => {:type => ::Thrift::Types::BOOL, :name => 'validate_cstr'},
+ RELY_CSTR => {:type => ::Thrift::Types::BOOL, :name => 'rely_cstr'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class SQLForeignKey
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ PKTABLE_DB = 1
+ PKTABLE_NAME = 2
+ PKCOLUMN_NAME = 3
+ FKTABLE_DB = 4
+ FKTABLE_NAME = 5
+ FKCOLUMN_NAME = 6
+ KEY_SEQ = 7
+ UPDATE_RULE = 8
+ DELETE_RULE = 9
+ FK_NAME = 10
+ PK_NAME = 11
+ ENABLE_CSTR = 12
+ VALIDATE_CSTR = 13
+ RELY_CSTR = 14
+
+ FIELDS = {
+ PKTABLE_DB => {:type => ::Thrift::Types::STRING, :name => 'pktable_db'},
+ PKTABLE_NAME => {:type => ::Thrift::Types::STRING, :name => 'pktable_name'},
+ PKCOLUMN_NAME => {:type => ::Thrift::Types::STRING, :name => 'pkcolumn_name'},
+ FKTABLE_DB => {:type => ::Thrift::Types::STRING, :name => 'fktable_db'},
+ FKTABLE_NAME => {:type => ::Thrift::Types::STRING, :name => 'fktable_name'},
+ FKCOLUMN_NAME => {:type => ::Thrift::Types::STRING, :name => 'fkcolumn_name'},
+ KEY_SEQ => {:type => ::Thrift::Types::I32, :name => 'key_seq'},
+ UPDATE_RULE => {:type => ::Thrift::Types::I32, :name => 'update_rule'},
+ DELETE_RULE => {:type => ::Thrift::Types::I32, :name => 'delete_rule'},
+ FK_NAME => {:type => ::Thrift::Types::STRING, :name => 'fk_name'},
+ PK_NAME => {:type => ::Thrift::Types::STRING, :name => 'pk_name'},
+ ENABLE_CSTR => {:type => ::Thrift::Types::BOOL, :name => 'enable_cstr'},
+ VALIDATE_CSTR => {:type => ::Thrift::Types::BOOL, :name => 'validate_cstr'},
+ RELY_CSTR => {:type => ::Thrift::Types::BOOL, :name => 'rely_cstr'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class SQLUniqueConstraint
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ TABLE_DB = 1
+ TABLE_NAME = 2
+ COLUMN_NAME = 3
+ KEY_SEQ = 4
+ UK_NAME = 5
+ ENABLE_CSTR = 6
+ VALIDATE_CSTR = 7
+ RELY_CSTR = 8
+
+ FIELDS = {
+ TABLE_DB => {:type => ::Thrift::Types::STRING, :name => 'table_db'},
+ TABLE_NAME => {:type => ::Thrift::Types::STRING, :name => 'table_name'},
+ COLUMN_NAME => {:type => ::Thrift::Types::STRING, :name => 'column_name'},
+ KEY_SEQ => {:type => ::Thrift::Types::I32, :name => 'key_seq'},
+ UK_NAME => {:type => ::Thrift::Types::STRING, :name => 'uk_name'},
+ ENABLE_CSTR => {:type => ::Thrift::Types::BOOL, :name => 'enable_cstr'},
+ VALIDATE_CSTR => {:type => ::Thrift::Types::BOOL, :name => 'validate_cstr'},
+ RELY_CSTR => {:type => ::Thrift::Types::BOOL, :name => 'rely_cstr'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class SQLNotNullConstraint
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ TABLE_DB = 1
+ TABLE_NAME = 2
+ COLUMN_NAME = 3
+ NN_NAME = 4
+ ENABLE_CSTR = 5
+ VALIDATE_CSTR = 6
+ RELY_CSTR = 7
+
+ FIELDS = {
+ TABLE_DB => {:type => ::Thrift::Types::STRING, :name => 'table_db'},
+ TABLE_NAME => {:type => ::Thrift::Types::STRING, :name => 'table_name'},
+ COLUMN_NAME => {:type => ::Thrift::Types::STRING, :name => 'column_name'},
+ NN_NAME => {:type => ::Thrift::Types::STRING, :name => 'nn_name'},
+ ENABLE_CSTR => {:type => ::Thrift::Types::BOOL, :name => 'enable_cstr'},
+ VALIDATE_CSTR => {:type => ::Thrift::Types::BOOL, :name => 'validate_cstr'},
+ RELY_CSTR => {:type => ::Thrift::Types::BOOL, :name => 'rely_cstr'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class Type
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ NAME = 1
+ TYPE1 = 2
+ TYPE2 = 3
+ FIELDS = 4
+
+ FIELDS = {
+ NAME => {:type => ::Thrift::Types::STRING, :name => 'name'},
+ TYPE1 => {:type => ::Thrift::Types::STRING, :name => 'type1', :optional => true},
+ TYPE2 => {:type => ::Thrift::Types::STRING, :name => 'type2', :optional => true},
+ FIELDS => {:type => ::Thrift::Types::LIST, :name => 'fields', :element => {:type => ::Thrift::Types::STRUCT, :class => ::FieldSchema}, :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class HiveObjectRef
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ OBJECTTYPE = 1
+ DBNAME = 2
+ OBJECTNAME = 3
+ PARTVALUES = 4
+ COLUMNNAME = 5
+
+ FIELDS = {
+ OBJECTTYPE => {:type => ::Thrift::Types::I32, :name => 'objectType', :enum_class => ::HiveObjectType},
+ DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+ OBJECTNAME => {:type => ::Thrift::Types::STRING, :name => 'objectName'},
+ PARTVALUES => {:type => ::Thrift::Types::LIST, :name => 'partValues', :element => {:type => ::Thrift::Types::STRING}},
+ COLUMNNAME => {:type => ::Thrift::Types::STRING, :name => 'columnName'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ unless @objectType.nil? || ::HiveObjectType::VALID_VALUES.include?(@objectType)
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field objectType!')
+ end
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class PrivilegeGrantInfo
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ PRIVILEGE = 1
+ CREATETIME = 2
+ GRANTOR = 3
+ GRANTORTYPE = 4
+ GRANTOPTION = 5
+
+ FIELDS = {
+ PRIVILEGE => {:type => ::Thrift::Types::STRING, :name => 'privilege'},
+ CREATETIME => {:type => ::Thrift::Types::I32, :name => 'createTime'},
+ GRANTOR => {:type => ::Thrift::Types::STRING, :name => 'grantor'},
+ GRANTORTYPE => {:type => ::Thrift::Types::I32, :name => 'grantorType', :enum_class => ::PrincipalType},
+ GRANTOPTION => {:type => ::Thrift::Types::BOOL, :name => 'grantOption'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ unless @grantorType.nil? || ::PrincipalType::VALID_VALUES.include?(@grantorType)
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field grantorType!')
+ end
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class HiveObjectPrivilege
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ HIVEOBJECT = 1
+ PRINCIPALNAME = 2
+ PRINCIPALTYPE = 3
+ GRANTINFO = 4
+
+ FIELDS = {
+ HIVEOBJECT => {:type => ::Thrift::Types::STRUCT, :name => 'hiveObject', :class => ::HiveObjectRef},
+ PRINCIPALNAME => {:type => ::Thrift::Types::STRING, :name => 'principalName'},
+ PRINCIPALTYPE => {:type => ::Thrift::Types::I32, :name => 'principalType', :enum_class => ::PrincipalType},
+ GRANTINFO => {:type => ::Thrift::Types::STRUCT, :name => 'grantInfo', :class => ::PrivilegeGrantInfo}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ unless @principalType.nil? || ::PrincipalType::VALID_VALUES.include?(@principalType)
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field principalType!')
+ end
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class PrivilegeBag
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ PRIVILEGES = 1
+
+ FIELDS = {
+ PRIVILEGES => {:type => ::Thrift::Types::LIST, :name => 'privileges', :element => {:type => ::Thrift::Types::STRUCT, :class => ::HiveObjectPrivilege}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class PrincipalPrivilegeSet
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ USERPRIVILEGES = 1
+ GROUPPRIVILEGES = 2
+ ROLEPRIVILEGES = 3
+
+ FIELDS = {
+ USERPRIVILEGES => {:type => ::Thrift::Types::MAP, :name => 'userPrivileges', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::LIST, :element => {:type => ::Thrift::Types::STRUCT, :class => ::PrivilegeGrantInfo}}},
+ GROUPPRIVILEGES => {:type => ::Thrift::Types::MAP, :name => 'groupPrivileges', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::LIST, :element => {:type => ::Thrift::Types::STRUCT, :class => ::PrivilegeGrantInfo}}},
+ ROLEPRIVILEGES => {:type => ::Thrift::Types::MAP, :name => 'rolePrivileges', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::LIST, :element => {:type => ::Thrift::Types::STRUCT, :class => ::PrivilegeGrantInfo}}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class GrantRevokePrivilegeRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ REQUESTTYPE = 1
+ PRIVILEGES = 2
+ REVOKEGRANTOPTION = 3
+
+ FIELDS = {
+ REQUESTTYPE => {:type => ::Thrift::Types::I32, :name => 'requestType', :enum_class => ::GrantRevokeType},
+ PRIVILEGES => {:type => ::Thrift::Types::STRUCT, :name => 'privileges', :class => ::PrivilegeBag},
+ REVOKEGRANTOPTION => {:type => ::Thrift::Types::BOOL, :name => 'revokeGrantOption', :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ unless @requestType.nil? || ::GrantRevokeType::VALID_VALUES.include?(@requestType)
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field requestType!')
+ end
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class GrantRevokePrivilegeResponse
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ SUCCESS = 1
+
+ FIELDS = {
+ SUCCESS => {:type => ::Thrift::Types::BOOL, :name => 'success', :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class Role
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ ROLENAME = 1
+ CREATETIME = 2
+ OWNERNAME = 3
+
+ FIELDS = {
+ ROLENAME => {:type => ::Thrift::Types::STRING, :name => 'roleName'},
+ CREATETIME => {:type => ::Thrift::Types::I32, :name => 'createTime'},
+ OWNERNAME => {:type => ::Thrift::Types::STRING, :name => 'ownerName'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class RolePrincipalGrant
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ ROLENAME = 1
+ PRINCIPALNAME = 2
+ PRINCIPALTYPE = 3
+ GRANTOPTION = 4
+ GRANTTIME = 5
+ GRANTORNAME = 6
+ GRANTORPRINCIPALTYPE = 7
+
+ FIELDS = {
+ ROLENAME => {:type => ::Thrift::Types::STRING, :name => 'roleName'},
+ PRINCIPALNAME => {:type => ::Thrift::Types::STRING, :name => 'principalName'},
+ PRINCIPALTYPE => {:type => ::Thrift::Types::I32, :name => 'principalType', :enum_class => ::PrincipalType},
+ GRANTOPTION => {:type => ::Thrift::Types::BOOL, :name => 'grantOption'},
+ GRANTTIME => {:type => ::Thrift::Types::I32, :name => 'grantTime'},
+ GRANTORNAME => {:type => ::Thrift::Types::STRING, :name => 'grantorName'},
+ GRANTORPRINCIPALTYPE => {:type => ::Thrift::Types::I32, :name => 'grantorPrincipalType', :enum_class => ::PrincipalType}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ unless @principalType.nil? || ::PrincipalType::VALID_VALUES.include?(@principalType)
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field principalType!')
+ end
+ unless @grantorPrincipalType.nil? || ::PrincipalType::VALID_VALUES.include?(@grantorPrincipalType)
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field grantorPrincipalType!')
+ end
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class GetRoleGrantsForPrincipalRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ PRINCIPAL_NAME = 1
+ PRINCIPAL_TYPE = 2
+
+ FIELDS = {
+ PRINCIPAL_NAME => {:type => ::Thrift::Types::STRING, :name => 'principal_name'},
+ PRINCIPAL_TYPE => {:type => ::Thrift::Types::I32, :name => 'principal_type', :enum_class => ::PrincipalType}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field principal_name is unset!') unless @principal_name
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field principal_type is unset!') unless @principal_type
+ unless @principal_type.nil? || ::PrincipalType::VALID_VALUES.include?(@principal_type)
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field principal_type!')
+ end
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class GetRoleGrantsForPrincipalResponse
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ PRINCIPALGRANTS = 1
+
+ FIELDS = {
+ PRINCIPALGRANTS => {:type => ::Thrift::Types::LIST, :name => 'principalGrants', :element => {:type => ::Thrift::Types::STRUCT, :class => ::RolePrincipalGrant}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field principalGrants is unset!') unless @principalGrants
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class GetPrincipalsInRoleRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ ROLENAME = 1
+
+ FIELDS = {
+ ROLENAME => {:type => ::Thrift::Types::STRING, :name => 'roleName'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field roleName is unset!') unless @roleName
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class GetPrincipalsInRoleResponse
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ PRINCIPALGRANTS = 1
+
+ FIELDS = {
+ PRINCIPALGRANTS => {:type => ::Thrift::Types::LIST, :name => 'principalGrants', :element => {:type => ::Thrift::Types::STRUCT, :class => ::RolePrincipalGrant}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field principalGrants is unset!') unless @principalGrants
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class GrantRevokeRoleRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ REQUESTTYPE = 1
+ ROLENAME = 2
+ PRINCIPALNAME = 3
+ PRINCIPALTYPE = 4
+ GRANTOR = 5
+ GRANTORTYPE = 6
+ GRANTOPTION = 7
+
+ FIELDS = {
+ REQUESTTYPE => {:type => ::Thrift::Types::I32, :name => 'requestType', :enum_class => ::GrantRevokeType},
+ ROLENAME => {:type => ::Thrift::Types::STRING, :name => 'roleName'},
+ PRINCIPALNAME => {:type => ::Thrift::Types::STRING, :name => 'principalName'},
+ PRINCIPALTYPE => {:type => ::Thrift::Types::I32, :name => 'principalType', :enum_class => ::PrincipalType},
+ GRANTOR => {:type => ::Thrift::Types::STRING, :name => 'grantor', :optional => true},
+ GRANTORTYPE => {:type => ::Thrift::Types::I32, :name => 'grantorType', :optional => true, :enum_class => ::PrincipalType},
+ GRANTOPTION => {:type => ::Thrift::Types::BOOL, :name => 'grantOption', :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ unless @requestType.nil? || ::GrantRevokeType::VALID_VALUES.include?(@requestType)
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field requestType!')
+ end
+ unless @principalType.nil? || ::PrincipalType::VALID_VALUES.include?(@principalType)
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field principalType!')
+ end
+ unless @grantorType.nil? || ::PrincipalType::VALID_VALUES.include?(@grantorType)
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field grantorType!')
+ end
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class GrantRevokeRoleResponse
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ SUCCESS = 1
+
+ FIELDS = {
+ SUCCESS => {:type => ::Thrift::Types::BOOL, :name => 'success', :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class Database
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ NAME = 1
+ DESCRIPTION = 2
+ LOCATIONURI = 3
+ PARAMETERS = 4
+ PRIVILEGES = 5
+ OWNERNAME = 6
+ OWNERTYPE = 7
+
+ FIELDS = {
+ NAME => {:type => ::Thrift::Types::STRING, :name => 'name'},
+ DESCRIPTION => {:type => ::Thrift::Types::STRING, :name => 'description'},
+ LOCATIONURI => {:type => ::Thrift::Types::STRING, :name => 'locationUri'},
+ PARAMETERS => {:type => ::Thrift::Types::MAP, :name => 'parameters', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}},
+ PRIVILEGES => {:type => ::Thrift::Types::STRUCT, :name => 'privileges', :class => ::PrincipalPrivilegeSet, :optional => true},
+ OWNERNAME => {:type => ::Thrift::Types::STRING, :name => 'ownerName', :optional => true},
+ OWNERTYPE => {:type => ::Thrift::Types::I32, :name => 'ownerType', :optional => true, :enum_class => ::PrincipalType}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ unless @ownerType.nil? || ::PrincipalType::VALID_VALUES.include?(@ownerType)
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field ownerType!')
+ end
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class SerDeInfo
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ NAME = 1
+ SERIALIZATIONLIB = 2
+ PARAMETERS = 3
+
+ FIELDS = {
+ NAME => {:type => ::Thrift::Types::STRING, :name => 'name'},
+ SERIALIZATIONLIB => {:type => ::Thrift::Types::STRING, :name => 'serializationLib'},
+ PARAMETERS => {:type => ::Thrift::Types::MAP, :name => 'parameters', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class Order
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ COL = 1
+ ORDER = 2
+
+ FIELDS = {
+ COL => {:type => ::Thrift::Types::STRING, :name => 'col'},
+ ORDER => {:type => ::Thrift::Types::I32, :name => 'order'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class SkewedInfo
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ SKEWEDCOLNAMES = 1
+ SKEWEDCOLVALUES = 2
+ SKEWEDCOLVALUELOCATIONMAPS = 3
+
+ FIELDS = {
+ SKEWEDCOLNAMES => {:type => ::Thrift::Types::LIST, :name => 'skewedColNames', :element => {:type => ::Thrift::Types::STRING}},
+ SKEWEDCOLVALUES => {:type => ::Thrift::Types::LIST, :name => 'skewedColValues', :element => {:type => ::Thrift::Types::LIST, :element => {:type => ::Thrift::Types::STRING}}},
+ SKEWEDCOLVALUELOCATIONMAPS => {:type => ::Thrift::Types::MAP, :name => 'skewedColValueLocationMaps', :key => {:type => ::Thrift::Types::LIST, :element => {:type => ::Thrift::Types::STRING}}, :value => {:type => ::Thrift::Types::STRING}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class StorageDescriptor
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ COLS = 1
+ LOCATION = 2
+ INPUTFORMAT = 3
+ OUTPUTFORMAT = 4
+ COMPRESSED = 5
+ NUMBUCKETS = 6
+ SERDEINFO = 7
+ BUCKETCOLS = 8
+ SORTCOLS = 9
+ PARAMETERS = 10
+ SKEWEDINFO = 11
+ STOREDASSUBDIRECTORIES = 12
+
+ FIELDS = {
+ COLS => {:type => ::Thrift::Types::LIST, :name => 'cols', :element => {:type => ::Thrift::Types::STRUCT, :class => ::FieldSchema}},
+ LOCATION => {:type => ::Thrift::Types::STRING, :name => 'location'},
+ INPUTFORMAT => {:type => ::Thrift::Types::STRING, :name => 'inputFormat'},
+ OUTPUTFORMAT => {:type => ::Thrift::Types::STRING, :name => 'outputFormat'},
+ COMPRESSED => {:type => ::Thrift::Types::BOOL, :name => 'compressed'},
+ NUMBUCKETS => {:type => ::Thrift::Types::I32, :name => 'numBuckets'},
+ SERDEINFO => {:type => ::Thrift::Types::STRUCT, :name => 'serdeInfo', :class => ::SerDeInfo},
+ BUCKETCOLS => {:type => ::Thrift::Types::LIST, :name => 'bucketCols', :element => {:type => ::Thrift::Types::STRING}},
+ SORTCOLS => {:type => ::Thrift::Types::LIST, :name => 'sortCols', :element => {:type => ::Thrift::Types::STRUCT, :class => ::Order}},
+ PARAMETERS => {:type => ::Thrift::Types::MAP, :name => 'parameters', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}},
+ SKEWEDINFO => {:type => ::Thrift::Types::STRUCT, :name => 'skewedInfo', :class => ::SkewedInfo, :optional => true},
+ STOREDASSUBDIRECTORIES => {:type => ::Thrift::Types::BOOL, :name => 'storedAsSubDirectories', :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class Table
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ TABLENAME = 1
+ DBNAME = 2
+ OWNER = 3
+ CREATETIME = 4
+ LASTACCESSTIME = 5
+ RETENTION = 6
+ SD = 7
+ PARTITIONKEYS = 8
+ PARAMETERS = 9
+ VIEWORIGINALTEXT = 10
+ VIEWEXPANDEDTEXT = 11
+ TABLETYPE = 12
+ PRIVILEGES = 13
+ TEMPORARY = 14
+ REWRITEENABLED = 15
+
+ FIELDS = {
+ TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tableName'},
+ DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+ OWNER => {:type => ::Thrift::Types::STRING, :name => 'owner'},
+ CREATETIME => {:type => ::Thrift::Types::I32, :name => 'createTime'},
+ LASTACCESSTIME => {:type => ::Thrift::Types::I32, :name => 'lastAccessTime'},
+ RETENTION => {:type => ::Thrift::Types::I32, :name => 'retention'},
+ SD => {:type => ::Thrift::Types::STRUCT, :name => 'sd', :class => ::StorageDescriptor},
+ PARTITIONKEYS => {:type => ::Thrift::Types::LIST, :name => 'partitionKeys', :element => {:type => ::Thrift::Types::STRUCT, :class => ::FieldSchema}},
+ PARAMETERS => {:type => ::Thrift::Types::MAP, :name => 'parameters', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}},
+ VIEWORIGINALTEXT => {:type => ::Thrift::Types::STRING, :name => 'viewOriginalText'},
+ VIEWEXPANDEDTEXT => {:type => ::Thrift::Types::STRING, :name => 'viewExpandedText'},
+ TABLETYPE => {:type => ::Thrift::Types::STRING, :name => 'tableType'},
+ PRIVILEGES => {:type => ::Thrift::Types::STRUCT, :name => 'privileges', :class => ::PrincipalPrivilegeSet, :optional => true},
+ TEMPORARY => {:type => ::Thrift::Types::BOOL, :name => 'temporary', :default => false, :optional => true},
+ REWRITEENABLED => {:type => ::Thrift::Types::BOOL, :name => 'rewriteEnabled', :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class Partition
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ VALUES = 1
+ DBNAME = 2
+ TABLENAME = 3
+ CREATETIME = 4
+ LASTACCESSTIME = 5
+ SD = 6
+ PARAMETERS = 7
+ PRIVILEGES = 8
+
+ FIELDS = {
+ VALUES => {:type => ::Thrift::Types::LIST, :name => 'values', :element => {:type => ::Thrift::Types::STRING}},
+ DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+ TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tableName'},
+ CREATETIME => {:type => ::Thrift::Types::I32, :name => 'createTime'},
+ LASTACCESSTIME => {:type => ::Thrift::Types::I32, :name => 'lastAccessTime'},
+ SD => {:type => ::Thrift::Types::STRUCT, :name => 'sd', :class => ::StorageDescriptor},
+ PARAMETERS => {:type => ::Thrift::Types::MAP, :name => 'parameters', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}},
+ PRIVILEGES => {:type => ::Thrift::Types::STRUCT, :name => 'privileges', :class => ::PrincipalPrivilegeSet, :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class PartitionWithoutSD
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ VALUES = 1
+ CREATETIME = 2
+ LASTACCESSTIME = 3
+ RELATIVEPATH = 4
+ PARAMETERS = 5
+ PRIVILEGES = 6
+
+ FIELDS = {
+ VALUES => {:type => ::Thrift::Types::LIST, :name => 'values', :element => {:type => ::Thrift::Types::STRING}},
+ CREATETIME => {:type => ::Thrift::Types::I32, :name => 'createTime'},
+ LASTACCESSTIME => {:type => ::Thrift::Types::I32, :name => 'lastAccessTime'},
+ RELATIVEPATH => {:type => ::Thrift::Types::STRING, :name => 'relativePath'},
+ PARAMETERS => {:type => ::Thrift::Types::MAP, :name => 'parameters', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}},
+ PRIVILEGES => {:type => ::Thrift::Types::STRUCT, :name => 'privileges', :class => ::PrincipalPrivilegeSet, :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class PartitionSpecWithSharedSD
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ PARTITIONS = 1
+ SD = 2
+
+ FIELDS = {
+ PARTITIONS => {:type => ::Thrift::Types::LIST, :name => 'partitions', :element => {:type => ::Thrift::Types::STRUCT, :class => ::PartitionWithoutSD}},
+ SD => {:type => ::Thrift::Types::STRUCT, :name => 'sd', :class => ::StorageDescriptor}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class PartitionListComposingSpec
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ PARTITIONS = 1
+
+ FIELDS = {
+ PARTITIONS => {:type => ::Thrift::Types::LIST, :name => 'partitions', :element => {:type => ::Thrift::Types::STRUCT, :class => ::Partition}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class PartitionSpec
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ DBNAME = 1
+ TABLENAME = 2
+ ROOTPATH = 3
+ SHAREDSDPARTITIONSPEC = 4
+ PARTITIONLIST = 5
+
+ FIELDS = {
+ DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+ TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tableName'},
+ ROOTPATH => {:type => ::Thrift::Types::STRING, :name => 'rootPath'},
+ SHAREDSDPARTITIONSPEC => {:type => ::Thrift::Types::STRUCT, :name => 'sharedSDPartitionSpec', :class => ::PartitionSpecWithSharedSD, :optional => true},
+ PARTITIONLIST => {:type => ::Thrift::Types::STRUCT, :name => 'partitionList', :class => ::PartitionListComposingSpec, :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class Index
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ INDEXNAME = 1
+ INDEXHANDLERCLASS = 2
+ DBNAME = 3
+ ORIGTABLENAME = 4
+ CREATETIME = 5
+ LASTACCESSTIME = 6
+ INDEXTABLENAME = 7
+ SD = 8
+ PARAMETERS = 9
+ DEFERREDREBUILD = 10
+
+ FIELDS = {
+ INDEXNAME => {:type => ::Thrift::Types::STRING, :name => 'indexName'},
+ INDEXHANDLERCLASS => {:type => ::Thrift::Types::STRING, :name => 'indexHandlerClass'},
+ DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+ ORIGTABLENAME => {:type => ::Thrift::Types::STRING, :name => 'origTableName'},
+ CREATETIME => {:type => ::Thrift::Types::I32, :name => 'createTime'},
+ LASTACCESSTIME => {:type => ::Thrift::Types::I32, :name => 'lastAccessTime'},
+ INDEXTABLENAME => {:type => ::Thrift::Types::STRING, :name => 'indexTableName'},
+ SD => {:type => ::Thrift::Types::STRUCT, :name => 'sd', :class => ::StorageDescriptor},
+ PARAMETERS => {:type => ::Thrift::Types::MAP, :name => 'parameters', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}},
+ DEFERREDREBUILD => {:type => ::Thrift::Types::BOOL, :name => 'deferredRebuild'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class BooleanColumnStatsData
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ NUMTRUES = 1
+ NUMFALSES = 2
+ NUMNULLS = 3
+ BITVECTORS = 4
+
+ FIELDS = {
+ NUMTRUES => {:type => ::Thrift::Types::I64, :name => 'numTrues'},
+ NUMFALSES => {:type => ::Thrift::Types::I64, :name => 'numFalses'},
+ NUMNULLS => {:type => ::Thrift::Types::I64, :name => 'numNulls'},
+ BITVECTORS => {:type => ::Thrift::Types::STRING, :name => 'bitVectors', :binary => true, :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numTrues is unset!') unless @numTrues
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numFalses is unset!') unless @numFalses
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numNulls is unset!') unless @numNulls
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class DoubleColumnStatsData
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ LOWVALUE = 1
+ HIGHVALUE = 2
+ NUMNULLS = 3
+ NUMDVS = 4
+ BITVECTORS = 5
+
+ FIELDS = {
+ LOWVALUE => {:type => ::Thrift::Types::DOUBLE, :name => 'lowValue', :optional => true},
+ HIGHVALUE => {:type => ::Thrift::Types::DOUBLE, :name => 'highValue', :optional => true},
+ NUMNULLS => {:type => ::Thrift::Types::I64, :name => 'numNulls'},
+ NUMDVS => {:type => ::Thrift::Types::I64, :name => 'numDVs'},
+ BITVECTORS => {:type => ::Thrift::Types::STRING, :name => 'bitVectors', :binary => true, :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numNulls is unset!') unless @numNulls
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numDVs is unset!') unless @numDVs
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class LongColumnStatsData
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ LOWVALUE = 1
+ HIGHVALUE = 2
+ NUMNULLS = 3
+ NUMDVS = 4
+ BITVECTORS = 5
+
+ FIELDS = {
+ LOWVALUE => {:type => ::Thrift::Types::I64, :name => 'lowValue', :optional => true},
+ HIGHVALUE => {:type => ::Thrift::Types::I64, :name => 'highValue', :optional => true},
+ NUMNULLS => {:type => ::Thrift::Types::I64, :name => 'numNulls'},
+ NUMDVS => {:type => ::Thrift::Types::I64, :name => 'numDVs'},
+ BITVECTORS => {:type => ::Thrift::Types::STRING, :name => 'bitVectors', :binary => true, :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numNulls is unset!') unless @numNulls
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numDVs is unset!') unless @numDVs
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class StringColumnStatsData
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ MAXCOLLEN = 1
+ AVGCOLLEN = 2
+ NUMNULLS = 3
+ NUMDVS = 4
+ BITVECTORS = 5
+
+ FIELDS = {
+ MAXCOLLEN => {:type => ::Thrift::Types::I64, :name => 'maxColLen'},
+ AVGCOLLEN => {:type => ::Thrift::Types::DOUBLE, :name => 'avgColLen'},
+ NUMNULLS => {:type => ::Thrift::Types::I64, :name => 'numNulls'},
+ NUMDVS => {:type => ::Thrift::Types::I64, :name => 'numDVs'},
+ BITVECTORS => {:type => ::Thrift::Types::STRING, :name => 'bitVectors', :binary => true, :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field maxColLen is unset!') unless @maxColLen
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field avgColLen is unset!') unless @avgColLen
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numNulls is unset!') unless @numNulls
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numDVs is unset!') unless @numDVs
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class BinaryColumnStatsData
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ MAXCOLLEN = 1
+ AVGCOLLEN = 2
+ NUMNULLS = 3
+ BITVECTORS = 4
+
+ FIELDS = {
+ MAXCOLLEN => {:type => ::Thrift::Types::I64, :name => 'maxColLen'},
+ AVGCOLLEN => {:type => ::Thrift::Types::DOUBLE, :name => 'avgColLen'},
+ NUMNULLS => {:type => ::Thrift::Types::I64, :name => 'numNulls'},
+ BITVECTORS => {:type => ::Thrift::Types::STRING, :name => 'bitVectors', :binary => true, :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field maxColLen is unset!') unless @maxColLen
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field avgColLen is unset!') unless @avgColLen
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numNulls is unset!') unless @numNulls
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class Decimal
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ UNSCALED = 1
+ SCALE = 3
+
+ FIELDS = {
+ UNSCALED => {:type => ::Thrift::Types::STRING, :name => 'unscaled', :binary => true},
+ SCALE => {:type => ::Thrift::Types::I16, :name => 'scale'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field unscaled is unset!') unless @unscaled
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field scale is unset!') unless @scale
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class DecimalColumnStatsData
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ LOWVALUE = 1
+ HIGHVALUE = 2
+ NUMNULLS = 3
+ NUMDVS = 4
+ BITVECTORS = 5
+
+ FIELDS = {
+ LOWVALUE => {:type => ::Thrift::Types::STRUCT, :name => 'lowValue', :class => ::Decimal, :optional => true},
+ HIGHVALUE => {:type => ::Thrift::Types::STRUCT, :name => 'highValue', :class => ::Decimal, :optional => true},
+ NUMNULLS => {:type => ::Thrift::Types::I64, :name => 'numNulls'},
+ NUMDVS => {:type => ::Thrift::Types::I64, :name => 'numDVs'},
+ BITVECTORS => {:type => ::Thrift::Types::STRING, :name => 'bitVectors', :binary => true, :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numNulls is unset!') unless @numNulls
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numDVs is unset!') unless @numDVs
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class Date
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ DAYSSINCEEPOCH = 1
+
+ FIELDS = {
+ DAYSSINCEEPOCH => {:type => ::Thrift::Types::I64, :name => 'daysSinceEpoch'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field daysSinceEpoch is unset!') unless @daysSinceEpoch
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class DateColumnStatsData
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ LOWVALUE = 1
+ HIGHVALUE = 2
+ NUMNULLS = 3
+ NUMDVS = 4
+ BITVECTORS = 5
+
+ FIELDS = {
+ LOWVALUE => {:type => ::Thrift::Types::STRUCT, :name => 'lowValue', :class => ::Date, :optional => true},
+ HIGHVALUE => {:type => ::Thrift::Types::STRUCT, :name => 'highValue', :class => ::Date, :optional => true},
+ NUMNULLS => {:type => ::Thrift::Types::I64, :name => 'numNulls'},
+ NUMDVS => {:type => ::Thrift::Types::I64, :name => 'numDVs'},
+ BITVECTORS => {:type => ::Thrift::Types::STRING, :name => 'bitVectors', :binary => true, :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numNulls is unset!') unless @numNulls
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numDVs is unset!') unless @numDVs
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class ColumnStatisticsData < ::Thrift::Union
+ include ::Thrift::Struct_Union
+ class << self
+ def booleanStats(val)
+ ColumnStatisticsData.new(:booleanStats, val)
+ end
+
+ def longStats(val)
+ ColumnStatisticsData.new(:longStats, val)
+ end
+
+ def doubleStats(val)
+ ColumnStatisticsData.new(:doubleStats, val)
+ end
+
+ def stringStats(val)
+ ColumnStatisticsData.new(:stringStats, val)
+ end
+
+ def binaryStats(val)
+ ColumnStatisticsData.new(:binaryStats, val)
+ end
+
+ def decimalStats(val)
+ ColumnStatisticsData.new(:decimalStats, val)
+ end
+
+ def dateStats(val)
+ ColumnStatisticsData.new(:dateStats, val)
+ end
+ end
+
+ BOOLEANSTATS = 1
+ LONGSTATS = 2
+ DOUBLESTATS = 3
+ STRINGSTATS = 4
+ BINARYSTATS = 5
+ DECIMALSTATS = 6
+ DATESTATS = 7
+
+ FIELDS = {
+ BOOLEANSTATS => {:type => ::Thrift::Types::STRUCT, :name => 'booleanStats', :class => ::BooleanColumnStatsData},
+ LONGSTATS => {:type => ::Thrift::Types::STRUCT, :name => 'longStats', :class => ::LongColumnStatsData},
+ DOUBLESTATS => {:type => ::Thrift::Types::STRUCT, :name => 'doubleStats', :class => ::DoubleColumnStatsData},
+ STRINGSTATS => {:type => ::Thrift::Types::STRUCT, :name => 'stringStats', :class => ::StringColumnStatsData},
+ BINARYSTATS => {:type => ::Thrift::Types::STRUCT, :name => 'binaryStats', :class => ::BinaryColumnStatsData},
+ DECIMALSTATS => {:type => ::Thrift::Types::STRUCT, :name => 'decimalStats', :class => ::DecimalColumnStatsData},
+ DATESTATS => {:type => ::Thrift::Types::STRUCT, :name => 'dateStats', :class => ::DateColumnStatsData}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise(StandardError, 'Union fields are not set.') if get_set_field.nil? || get_value.nil?
+ end
+
+ ::Thrift::Union.generate_accessors self
+ end
+
+ class ColumnStatisticsObj
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ COLNAME = 1
+ COLTYPE = 2
+ STATSDATA = 3
+
+ FIELDS = {
+ COLNAME => {:type => ::Thrift::Types::STRING, :name => 'colName'},
+ COLTYPE => {:type => ::Thrift::Types::STRING, :name => 'colType'},
+ STATSDATA => {:type => ::Thrift::Types::STRUCT, :name => 'statsData', :class => ::ColumnStatisticsData}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field colName is unset!') unless @colName
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field colType is unset!') unless @colType
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field statsData is unset!') unless @statsData
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class ColumnStatisticsDesc
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ ISTBLLEVEL = 1
+ DBNAME = 2
+ TABLENAME = 3
+ PARTNAME = 4
+ LASTANALYZED = 5
+
+ FIELDS = {
+ ISTBLLEVEL => {:type => ::Thrift::Types::BOOL, :name => 'isTblLevel'},
+ DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+ TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tableName'},
+ PARTNAME => {:type => ::Thrift::Types::STRING, :name => 'partName', :optional => true},
+ LASTANALYZED => {:type => ::Thrift::Types::I64, :name => 'lastAnalyzed', :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field isTblLevel is unset!') if @isTblLevel.nil?
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbName is unset!') unless @dbName
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tableName is unset!') unless @tableName
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class ColumnStatistics
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ STATSDESC = 1
+ STATSOBJ = 2
+
+ FIELDS = {
+ STATSDESC => {:type => ::Thrift::Types::STRUCT, :name => 'statsDesc', :class => ::ColumnStatisticsDesc},
+ STATSOBJ => {:type => ::Thrift::Types::LIST, :name => 'statsObj', :element => {:type => ::Thrift::Types::STRUCT, :class => ::ColumnStatisticsObj}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field statsDesc is unset!') unless @statsDesc
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field statsObj is unset!') unless @statsObj
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class AggrStats
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ COLSTATS = 1
+ PARTSFOUND = 2
+
+ FIELDS = {
+ COLSTATS => {:type => ::Thrift::Types::LIST, :name => 'colStats', :element => {:type => ::Thrift::Types::STRUCT, :class => ::ColumnStatisticsObj}},
+ PARTSFOUND => {:type => ::Thrift::Types::I64, :name => 'partsFound'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field colStats is unset!') unless @colStats
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field partsFound is unset!') unless @partsFound
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class SetPartitionsStatsRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ COLSTATS = 1
+ NEEDMERGE = 2
+
+ FIELDS = {
+ COLSTATS => {:type => ::Thrift::Types::LIST, :name => 'colStats', :element => {:type => ::Thrift::Types::STRUCT, :class => ::ColumnStatistics}},
+ NEEDMERGE => {:type => ::Thrift::Types::BOOL, :name => 'needMerge', :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field colStats is unset!') unless @colStats
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class Schema
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ FIELDSCHEMAS = 1
+ PROPERTIES = 2
+
+ FIELDS = {
+ FIELDSCHEMAS => {:type => ::Thrift::Types::LIST, :name => 'fieldSchemas', :element => {:type => ::Thrift::Types::STRUCT, :class => ::FieldSchema}},
+ PROPERTIES => {:type => ::Thrift::Types::MAP, :name => 'properties', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class EnvironmentContext
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ PROPERTIES = 1
+
+ FIELDS = {
+ PROPERTIES => {:type => ::Thrift::Types::MAP, :name => 'properties', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class PrimaryKeysRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ DB_NAME = 1
+ TBL_NAME = 2
+
+ FIELDS = {
+ DB_NAME => {:type => ::Thrift::Types::STRING, :name => 'db_name'},
+ TBL_NAME => {:type => ::Thrift::Types::STRING, :name => 'tbl_name'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field db_name is unset!') unless @db_name
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tbl_name is unset!') unless @tbl_name
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class PrimaryKeysResponse
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ PRIMARYKEYS = 1
+
+ FIELDS = {
+ PRIMARYKEYS => {:type => ::Thrift::Types::LIST, :name => 'primaryKeys', :element => {:type => ::Thrift::Types::STRUCT, :class => ::SQLPrimaryKey}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field primaryKeys is unset!') unless @primaryKeys
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class ForeignKeysRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ PARENT_DB_NAME = 1
+ PARENT_TBL_NAME = 2
+ FOREIGN_DB_NAME = 3
+ FOREIGN_TBL_NAME = 4
+
+ FIELDS = {
+ PARENT_DB_NAME => {:type => ::Thrift::Types::STRING, :name => 'parent_db_name'},
+ PARENT_TBL_NAME => {:type => ::Thrift::Types::STRING, :name => 'parent_tbl_name'},
+ FOREIGN_DB_NAME => {:type => ::Thrift::Types::STRING, :name => 'foreign_db_name'},
+ FOREIGN_TBL_NAME => {:type => ::Thrift::Types::STRING, :name => 'foreign_tbl_name'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class ForeignKeysResponse
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ FOREIGNKEYS = 1
+
+ FIELDS = {
+ FOREIGNKEYS => {:type => ::Thrift::Types::LIST, :name => 'foreignKeys', :element => {:type => ::Thrift::Types::STRUCT, :class => ::SQLForeignKey}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field foreignKeys is unset!') unless @foreignKeys
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class UniqueConstraintsRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ DB_NAME = 1
+ TBL_NAME = 2
+
+ FIELDS = {
+ DB_NAME => {:type => ::Thrift::Types::STRING, :name => 'db_name'},
+ TBL_NAME => {:type => ::Thrift::Types::STRING, :name => 'tbl_name'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field db_name is unset!') unless @db_name
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tbl_name is unset!') unless @tbl_name
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class UniqueConstraintsResponse
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ UNIQUECONSTRAINTS = 1
+
+ FIELDS = {
+ UNIQUECONSTRAINTS => {:type => ::Thrift::Types::LIST, :name => 'uniqueConstraints', :element => {:type => ::Thrift::Types::STRUCT, :class => ::SQLUniqueConstraint}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field uniqueConstraints is unset!') unless @uniqueConstraints
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class NotNullConstraintsRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ DB_NAME = 1
+ TBL_NAME = 2
+
+ FIELDS = {
+ DB_NAME => {:type => ::Thrift::Types::STRING, :name => 'db_name'},
+ TBL_NAME => {:type => ::Thrift::Types::STRING, :name => 'tbl_name'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field db_name is unset!') unless @db_name
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tbl_name is unset!') unless @tbl_name
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class NotNullConstraintsResponse
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ NOTNULLCONSTRAINTS = 1
+
+ FIELDS = {
+ NOTNULLCONSTRAINTS => {:type => ::Thrift::Types::LIST, :name => 'notNullConstraints', :element => {:type => ::Thrift::Types::STRUCT, :class => ::SQLNotNullConstraint}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field notNullConstraints is unset!') unless @notNullConstraints
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class DropConstraintRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ DBNAME = 1
+ TABLENAME = 2
+ CONSTRAINTNAME = 3
+
+ FIELDS = {
+ DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbname'},
+ TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tablename'},
+ CONSTRAINTNAME => {:type => ::Thrift::Types::STRING, :name => 'constraintname'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbname is unset!') unless @dbname
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tablename is unset!') unless @tablename
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field constraintname is unset!') unless @constraintname
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class AddPrimaryKeyRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ PRIMARYKEYCOLS = 1
+
+ FIELDS = {
+ PRIMARYKEYCOLS => {:type => ::Thrift::Types::LIST, :name => 'primaryKeyCols', :element => {:type => ::Thrift::Types::STRUCT, :class => ::SQLPrimaryKey}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field primaryKeyCols is unset!') unless @primaryKeyCols
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class AddForeignKeyRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ FOREIGNKEYCOLS = 1
+
+ FIELDS = {
+ FOREIGNKEYCOLS => {:type => ::Thrift::Types::LIST, :name => 'foreignKeyCols', :element => {:type => ::Thrift::Types::STRUCT, :class => ::SQLForeignKey}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field foreignKeyCols is unset!') unless @foreignKeyCols
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class AddUniqueConstraintRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ UNIQUECONSTRAINTCOLS = 1
+
+ FIELDS = {
+ UNIQUECONSTRAINTCOLS => {:type => ::Thrift::Types::LIST, :name => 'uniqueConstraintCols', :element => {:type => ::Thrift::Types::STRUCT, :class => ::SQLUniqueConstraint}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field uniqueConstraintCols is unset!') unless @uniqueConstraintCols
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class AddNotNullConstraintRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ NOTNULLCONSTRAINTCOLS = 1
+
+ FIELDS = {
+ NOTNULLCONSTRAINTCOLS => {:type => ::Thrift::Types::LIST, :name => 'notNullConstraintCols', :element => {:type => ::Thrift::Types::STRUCT, :class => ::SQLNotNullConstraint}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field notNullConstraintCols is unset!') unless @notNullConstraintCols
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class PartitionsByExprResult
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ PARTITIONS = 1
+ HASUNKNOWNPARTITIONS = 2
+
+ FIELDS = {
+ PARTITIONS => {:type => ::Thrift::Types::LIST, :name => 'partitions', :element => {:type => ::Thrift::Types::STRUCT, :class => ::Partition}},
+ HASUNKNOWNPARTITIONS => {:type => ::Thrift::Types::BOOL, :name => 'hasUnknownPartitions'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field partitions is unset!') unless @partitions
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field hasUnknownPartitions is unset!') if @hasUnknownPartitions.nil?
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class PartitionsByExprRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ DBNAME = 1
+ TBLNAME = 2
+ EXPR = 3
+ DEFAULTPARTITIONNAME = 4
+ MAXPARTS = 5
+
+ FIELDS = {
+ DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+ TBLNAME => {:type => ::Thrift::Types::STRING, :name => 'tblName'},
+ EXPR => {:type => ::Thrift::Types::STRING, :name => 'expr', :binary => true},
+ DEFAULTPARTITIONNAME => {:type => ::Thrift::Types::STRING, :name => 'defaultPartitionName', :optional => true},
+ MAXPARTS => {:type => ::Thrift::Types::I16, :name => 'maxParts', :default => -1, :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbName is unset!') unless @dbName
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tblName is unset!') unless @tblName
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field expr is unset!') unless @expr
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class TableStatsResult
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ TABLESTATS = 1
+
+ FIELDS = {
+ TABLESTATS => {:type => ::Thrift::Types::LIST, :name => 'tableStats', :element => {:type => ::Thrift::Types::STRUCT, :class => ::ColumnStatisticsObj}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tableStats is unset!') unless @tableStats
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class PartitionsStatsResult
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ PARTSTATS = 1
+
+ FIELDS = {
+ PARTSTATS => {:type => ::Thrift::Types::MAP, :name => 'partStats', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::LIST, :element => {:type => ::Thrift::Types::STRUCT, :class => ::ColumnStatisticsObj}}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field partStats is unset!') unless @partStats
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class TableStatsRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ DBNAME = 1
+ TBLNAME = 2
+ COLNAMES = 3
+
+ FIELDS = {
+ DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+ TBLNAME => {:type => ::Thrift::Types::STRING, :name => 'tblName'},
+ COLNAMES => {:type => ::Thrift::Types::LIST, :name => 'colNames', :element => {:type => ::Thrift::Types::STRING}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbName is unset!') unless @dbName
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tblName is unset!') unless @tblName
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field colNames is unset!') unless @colNames
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class PartitionsStatsRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ DBNAME = 1
+ TBLNAME = 2
+ COLNAMES = 3
+ PARTNAMES = 4
+
+ FIELDS = {
+ DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+ TBLNAME => {:type => ::Thrift::Types::STRING, :name => 'tblName'},
+ COLNAMES => {:type => ::Thrift::Types::LIST, :name => 'colNames', :element => {:type => ::Thrift::Types::STRING}},
+ PARTNAMES => {:type => ::Thrift::Types::LIST, :name => 'partNames', :element => {:type => ::Thrift::Types::STRING}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbName is unset!') unless @dbName
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tblName is unset!') unless @tblName
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field colNames is unset!') unless @colNames
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field partNames is unset!') unless @partNames
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class AddPartitionsResult
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ PARTITIONS = 1
+
+ FIELDS = {
+ PARTITIONS => {:type => ::Thrift::Types::LIST, :name => 'partitions', :element => {:type => ::Thrift::Types::STRUCT, :class => ::Partition}, :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class AddPartitionsRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ DBNAME = 1
+ TBLNAME = 2
+ PARTS = 3
+ IFNOTEXISTS = 4
+ NEEDRESULT = 5
+
+ FIELDS = {
+ DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+ TBLNAME => {:type => ::Thrift::Types::STRING, :name => 'tblName'},
+ PARTS => {:type => ::Thrift::Types::LIST, :name => 'parts', :element => {:type => ::Thrift::Types::STRUCT, :class => ::Partition}},
+ IFNOTEXISTS => {:type => ::Thrift::Types::BOOL, :name => 'ifNotExists'},
+ NEEDRESULT => {:type => ::Thrift::Types::BOOL, :name => 'needResult', :default => true, :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbName is unset!') unless @dbName
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tblName is unset!') unless @tblName
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field parts is unset!') unless @parts
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field ifNotExists is unset!') if @ifNotExists.nil?
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class DropPartitionsResult
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ PARTITIONS = 1
+
+ FIELDS = {
+ PARTITIONS => {:type => ::Thrift::Types::LIST, :name => 'partitions', :element => {:type => ::Thrift::Types::STRUCT, :class => ::Partition}, :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class DropPartitionsExpr
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ EXPR = 1
+ PARTARCHIVELEVEL = 2
+
+ FIELDS = {
+ EXPR => {:type => ::Thrift::Types::STRING, :name => 'expr', :binary => true},
+ PARTARCHIVELEVEL => {:type => ::Thrift::Types::I32, :name => 'partArchiveLevel', :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field expr is unset!') unless @expr
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class RequestPartsSpec < ::Thrift::Union
+ include ::Thrift::Struct_Union
+ class << self
+ def names(val)
+ RequestPartsSpec.new(:names, val)
+ end
+
+ def exprs(val)
+ RequestPartsSpec.new(:exprs, val)
+ end
+ end
+
+ NAMES = 1
+ EXPRS = 2
+
+ FIELDS = {
+ NAMES => {:type => ::Thrift::Types::LIST, :name => 'names', :element => {:type => ::Thrift::Types::STRING}},
+ EXPRS => {:type => ::Thrift::Types::LIST, :name => 'exprs', :element => {:type => ::Thrift::Types::STRUCT, :class => ::DropPartitionsExpr}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise(StandardError, 'Union fields are not set.') if get_set_field.nil? || get_value.nil?
+ end
+
+ ::Thrift::Union.generate_accessors self
+ end
+
+ class DropPartitionsRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ DBNAME = 1
+ TBLNAME = 2
+ PARTS = 3
+ DELETEDATA = 4
+ IFEXISTS = 5
+ IGNOREPROTECTION = 6
+ ENVIRONMENTCONTEXT = 7
+ NEEDRESULT = 8
+
+ FIELDS = {
+ DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+ TBLNAME => {:type => ::Thrift::Types::STRING, :name => 'tblName'},
+ PARTS => {:type => ::Thrift::Types::STRUCT, :name => 'parts', :class => ::RequestPartsSpec},
+ DELETEDATA => {:type => ::Thrift::Types::BOOL, :name => 'deleteData', :optional => true},
+ IFEXISTS => {:type => ::Thrift::Types::BOOL, :name => 'ifExists', :default => true, :optional => true},
+ IGNOREPROTECTION => {:type => ::Thrift::Types::BOOL, :name => 'ignoreProtection', :optional => true},
+ ENVIRONMENTCONTEXT => {:type => ::Thrift::Types::STRUCT, :name => 'environmentContext', :class => ::EnvironmentContext, :optional => true},
+ NEEDRESULT => {:type => ::Thrift::Types::BOOL, :name => 'needResult', :default => true, :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbName is unset!') unless @dbName
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tblName is unset!') unless @tblName
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field parts is unset!') unless @parts
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class ResourceUri
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ RESOURCETYPE = 1
+ URI = 2
+
+ FIELDS = {
+ RESOURCETYPE => {:type => ::Thrift::Types::I32, :name => 'resourceType', :enum_class => ::ResourceType},
+ URI => {:type => ::Thrift::Types::STRING, :name => 'uri'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ unless @resourceType.nil? || ::ResourceType::VALID_VALUES.include?(@resourceType)
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field resourceType!')
+ end
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class Function
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ FUNCTIONNAME = 1
+ DBNAME = 2
+ CLASSNAME = 3
+ OWNERNAME = 4
+ OWNERTYPE = 5
+ CREATETIME = 6
+ FUNCTIONTYPE = 7
+ RESOURCEURIS = 8
+
+ FIELDS = {
+ FUNCTIONNAME => {:type => ::Thrift::Types::STRING, :name => 'functionName'},
+ DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+ CLASSNAME => {:type => ::Thrift::Types::STRING, :name => 'className'},
+ OWNERNAME => {:type => ::Thrift::Types::STRING, :name => 'ownerName'},
+ OWNERTYPE => {:type => ::Thrift::Types::I32, :name => 'ownerType', :enum_class => ::PrincipalType},
+ CREATETIME => {:type => ::Thrift::Types::I32, :name => 'createTime'},
+ FUNCTIONTYPE => {:type => ::Thrift::Types::I32, :name => 'functionType', :enum_class => ::FunctionType},
+ RESOURCEURIS => {:type => ::Thrift::Types::LIST, :name => 'resourceUris', :element => {:type => ::Thrift::Types::STRUCT, :class => ::ResourceUri}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ unless @ownerType.nil? || ::PrincipalType::VALID_VALUES.include?(@ownerType)
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field ownerType!')
+ end
+ unless @functionType.nil? || ::FunctionType::VALID_VALUES.include?(@functionType)
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field functionType!')
+ end
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class TxnInfo
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ ID = 1
+ STATE = 2
+ USER = 3
+ HOSTNAME = 4
+ AGENTINFO = 5
+ HEARTBEATCOUNT = 6
+ METAINFO = 7
+ STARTEDTIME = 8
+ LASTHEARTBEATTIME = 9
+
+ FIELDS = {
+ ID => {:type => ::Thrift::Types::I64, :name => 'id'},
+ STATE => {:type => ::Thrift::Types::I32, :name => 'state', :enum_class => ::TxnState},
+ USER => {:type => ::Thrift::Types::STRING, :name => 'user'},
+ HOSTNAME => {:type => ::Thrift::Types::STRING, :name => 'hostname'},
+ AGENTINFO => {:type => ::Thrift::Types::STRING, :name => 'agentInfo', :default => %q"Unknown", :optional => true},
+ HEARTBEATCOUNT => {:type => ::Thrift::Types::I32, :name => 'heartbeatCount', :default => 0, :optional => true},
+ METAINFO => {:type => ::Thrift::Types::STRING, :name => 'metaInfo', :optional => true},
+ STARTEDTIME => {:type => ::Thrift::Types::I64, :name => 'startedTime', :optional => true},
+ LASTHEARTBEATTIME => {:type => ::Thrift::Types::I64, :name => 'lastHeartbeatTime', :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field id is unset!') unless @id
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field state is unset!') unless @state
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field user is unset!') unless @user
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field hostname is unset!') unless @hostname
+ unless @state.nil? || ::TxnState::VALID_VALUES.include?(@state)
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field state!')
+ end
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class GetOpenTxnsInfoResponse
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ TXN_HIGH_WATER_MARK = 1
+ OPEN_TXNS = 2
+
+ FIELDS = {
+ TXN_HIGH_WATER_MARK => {:type => ::Thrift::Types::I64, :name => 'txn_high_water_mark'},
+ OPEN_TXNS => {:type => ::Thrift::Types::LIST, :name => 'open_txns', :element => {:type => ::Thrift::Types::STRUCT, :class => ::TxnInfo}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field txn_high_water_mark is unset!') unless @txn_high_water_mark
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field open_txns is unset!') unless @open_txns
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class GetOpenTxnsResponse
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ TXN_HIGH_WATER_MARK = 1
+ OPEN_TXNS = 2
+ MIN_OPEN_TXN = 3
+ ABORTEDBITS = 4
+
+ FIELDS = {
+ TXN_HIGH_WATER_MARK => {:type => ::Thrift::Types::I64, :name => 'txn_high_water_mark'},
+ OPEN_TXNS => {:type => ::Thrift::Types::LIST, :name => 'open_txns', :element => {:type => ::Thrift::Types::I64}},
+ MIN_OPEN_TXN => {:type => ::Thrift::Types::I64, :name => 'min_open_txn', :optional => true},
+ ABORTEDBITS => {:type => ::Thrift::Types::STRING, :name => 'abortedBits', :binary => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field txn_high_water_mark is unset!') unless @txn_high_water_mark
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field open_txns is unset!') unless @open_txns
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field abortedBits is unset!') unless @abortedBits
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class OpenTxnRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ NUM_TXNS = 1
+ USER = 2
+ HOSTNAME = 3
+ AGENTINFO = 4
+
+ FIELDS = {
+ NUM_TXNS => {:type => ::Thrift::Types::I32, :name => 'num_txns'},
+ USER => {:type => ::Thrift::Types::STRING, :name => 'user'},
+ HOSTNAME => {:type => ::Thrift::Types::STRING, :name => 'hostname'},
+ AGENTINFO => {:type => ::Thrift::Types::STRING, :name => 'agentInfo', :default => %q"Unknown", :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field num_txns is unset!') unless @num_txns
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field user is unset!') unless @user
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field hostname is unset!') unless @hostname
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class OpenTxnsResponse
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ TXN_IDS = 1
+
+ FIELDS = {
+ TXN_IDS => {:type => ::Thrift::Types::LIST, :name => 'txn_ids', :element => {:type => ::Thrift::Types::I64}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field txn_ids is unset!') unless @txn_ids
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class AbortTxnRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ TXNID = 1
+
+ FIELDS = {
+ TXNID => {:type => ::Thrift::Types::I64, :name => 'txnid'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field txnid is unset!') unless @txnid
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class AbortTxnsRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ TXN_IDS = 1
+
+ FIELDS = {
+ TXN_IDS => {:type => ::Thrift::Types::LIST, :name => 'txn_ids', :element => {:type => ::Thrift::Types::I64}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field txn_ids is unset!') unless @txn_ids
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class CommitTxnRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ TXNID = 1
+
+ FIELDS = {
+ TXNID => {:type => ::Thrift::Types::I64, :name => 'txnid'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field txnid is unset!') unless @txnid
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class LockComponent
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ TYPE = 1
+ LEVEL = 2
+ DBNAME = 3
+ TABLENAME = 4
+ PARTITIONNAME = 5
+ OPERATIONTYPE = 6
+ ISACID = 7
+ ISDYNAMICPARTITIONWRITE = 8
+
+ FIELDS = {
+ TYPE => {:type => ::Thrift::Types::I32, :name => 'type', :enum_class => ::LockType},
+ LEVEL => {:type => ::Thrift::Types::I32, :name => 'level', :enum_class => ::LockLevel},
+ DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbname'},
+ TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tablename', :optional => true},
+ PARTITIONNAME => {:type => ::Thrift::Types::STRING, :name => 'partitionname', :optional => true},
+ OPERATIONTYPE => {:type => ::Thrift::Types::I32, :name => 'operationType', :default => 5, :optional => true, :enum_class => ::DataOperationType},
+ ISACID => {:type => ::Thrift::Types::BOOL, :name => 'isAcid', :default => false, :optional => true},
+ ISDYNAMICPARTITIONWRITE => {:type => ::Thrift::Types::BOOL, :name => 'isDynamicPartitionWrite', :default => false, :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field type is unset!') unless @type
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field level is unset!') unless @level
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbname is unset!') unless @dbname
+ unless @type.nil? || ::LockType::VALID_VALUES.include?(@type)
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field type!')
+ end
+ unless @level.nil? || ::LockLevel::VALID_VALUES.include?(@level)
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field level!')
+ end
+ unless @operationType.nil? || ::DataOperationType::VALID_VALUES.include?(@operationType)
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field operationType!')
+ end
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class LockRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ COMPONENT = 1
+ TXNID = 2
+ USER = 3
+ HOSTNAME = 4
+ AGENTINFO = 5
+
+ FIELDS = {
+ COMPONENT => {:type => ::Thrift::Types::LIST, :name => 'component', :element => {:type => ::Thrift::Types::STRUCT, :class => ::LockComponent}},
+ TXNID => {:type => ::Thrift::Types::I64, :name => 'txnid', :optional => true},
+ USER => {:type => ::Thrift::Types::STRING, :name => 'user'},
+ HOSTNAME => {:type => ::Thrift::Types::STRING, :name => 'hostname'},
+ AGENTINFO => {:type => ::Thrift::Types::STRING, :name => 'agentInfo', :default => %q"Unknown", :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field component is unset!') unless @component
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field user is unset!') unless @user
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field hostname is unset!') unless @hostname
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class LockResponse
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ LOCKID = 1
+ STATE = 2
+
+ FIELDS = {
+ LOCKID => {:type => ::Thrift::Types::I64, :name => 'lockid'},
+ STATE => {:type => ::Thrift::Types::I32, :name => 'state', :enum_class => ::LockState}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field lockid is unset!') unless @lockid
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field state is unset!') unless @state
+ unless @state.nil? || ::LockState::VALID_VALUES.include?(@state)
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field state!')
+ end
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class CheckLockRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ LOCKID = 1
+ TXNID = 2
+ ELAPSED_MS = 3
+
+ FIELDS = {
+ LOCKID => {:type => ::Thrift::Types::I64, :name => 'lockid'},
+ TXNID => {:type => ::Thrift::Types::I64, :name => 'txnid', :optional => true},
+ ELAPSED_MS => {:type => ::Thrift::Types::I64, :name => 'elapsed_ms', :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field lockid is unset!') unless @lockid
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class UnlockRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ LOCKID = 1
+
+ FIELDS = {
+ LOCKID => {:type => ::Thrift::Types::I64, :name => 'lockid'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field lockid is unset!') unless @lockid
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class ShowLocksRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ DBNAME = 1
+ TABLENAME = 2
+ PARTNAME = 3
+ ISEXTENDED = 4
+
+ FIELDS = {
+ DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbname', :optional => true},
+ TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tablename', :optional => true},
+ PARTNAME => {:type => ::Thrift::Types::STRING, :name => 'partname', :optional => true},
+ ISEXTENDED => {:type => ::Thrift::Types::BOOL, :name => 'isExtended', :default => false, :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class ShowLocksResponseElement
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ LOCKID = 1
+ DBNAME = 2
+ TABLENAME = 3
+ PARTNAME = 4
+ STATE = 5
+ TYPE = 6
+ TXNID = 7
+ LASTHEARTBEAT = 8
+ ACQUIREDAT = 9
+ USER = 10
+ HOSTNAME = 11
+ HEARTBEATCOUNT = 12
+ AGENTINFO = 13
+ BLOCKEDBYEXTID = 14
+ BLOCKEDBYINTID = 15
+ LOCKIDINTERNAL = 16
+
+ FIELDS = {
+ LOCKID => {:type => ::Thrift::Types::I64, :name => 'lockid'},
+ DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbname'},
+ TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tablename', :optional => true},
+ PARTNAME => {:type => ::Thrift::Types::STRING, :name => 'partname', :optional => true},
+ STATE => {:type => ::Thrift::Types::I32, :name => 'state', :enum_class => ::LockState},
+ TYPE => {:type => ::Thrift::Types::I32, :name => 'type', :enum_class => ::LockType},
+ TXNID => {:type => ::Thrift::Types::I64, :name => 'txnid', :optional => true},
+ LASTHEARTBEAT => {:type => ::Thrift::Types::I64, :name => 'lastheartbeat'},
+ ACQUIREDAT => {:type => ::Thrift::Types::I64, :name => 'acquiredat', :optional => true},
+ USER => {:type => ::Thrift::Types::STRING, :name => 'user'},
+ HOSTNAME => {:type => ::Thrift::Types::STRING, :name => 'hostname'},
+ HEARTBEATCOUNT => {:type => ::Thrift::Types::I32, :name => 'heartbeatCount', :default => 0, :optional => true},
+ AGENTINFO => {:type => ::Thrift::Types::STRING, :name => 'agentInfo', :optional => true},
+ BLOCKEDBYEXTID => {:type => ::Thrift::Types::I64, :name => 'blockedByExtId', :optional => true},
+ BLOCKEDBYINTID => {:type => ::Thrift::Types::I64, :name => 'blockedByIntId', :optional => true},
+ LOCKIDINTERNAL => {:type => ::Thrift::Types::I64, :name => 'lockIdInternal', :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field lockid is unset!') unless @lockid
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbname is unset!') unless @dbname
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field state is unset!') unless @state
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field type is unset!') unless @type
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field lastheartbeat is unset!') unless @lastheartbeat
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field user is unset!') unless @user
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field hostname is unset!') unless @hostname
+ unless @state.nil? || ::LockState::VALID_VALUES.include?(@state)
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field state!')
+ end
+ unless @type.nil? || ::LockType::VALID_VALUES.include?(@type)
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field type!')
+ end
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class ShowLocksResponse
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ LOCKS = 1
+
+ FIELDS = {
+ LOCKS => {:type => ::Thrift::Types::LIST, :name => 'locks', :element => {:type => ::Thrift::Types::STRUCT, :class => ::ShowLocksResponseElement}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class HeartbeatRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ LOCKID = 1
+ TXNID = 2
+
+ FIELDS = {
+ LOCKID => {:type => ::Thrift::Types::I64, :name => 'lockid', :optional => true},
+ TXNID => {:type => ::Thrift::Types::I64, :name => 'txnid', :optional => true}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class HeartbeatTxnRangeRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ MIN = 1
+ MAX = 2
+
+ FIELDS = {
+ MIN => {:type => ::Thrift::Types::I64, :name => 'min'},
+ MAX => {:type => ::Thrift::Types::I64, :name => 'max'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field min is unset!') unless @min
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field max is unset!') unless @max
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class HeartbeatTxnRangeResponse
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ ABORTED = 1
+ NOSUCH = 2
+
+ FIELDS = {
+ ABORTED => {:type => ::Thrift::Types::SET, :name => 'aborted', :element => {:type => ::Thrift::Types::I64}},
+ NOSUCH => {:type => ::Thrift::Types::SET, :name => 'nosuch', :element => {:type => ::Thrift::Types::I64}}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field aborted is unset!') unless @aborted
+ raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field nosuch is unset!') unless @nosuch
+ end
+
+ ::Thrift::Struct.generate_accessors self
+ end
+
+ class CompactionRequest
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ DBNAME = 1
+ TABLENAME = 2
+ PARTITIONNAME = 3
+ TYPE = 4
+ RUNAS = 5
+ PROPERTIES = 6
+
+
<TRUNCATED>
[33/50] [abbrv] hive git commit: HIVE-17323 SemiJoin: Detect DPP edge
as running parallel to semi-join (Deepak Jaiswal, reviewed by Gopal V)
Posted by we...@apache.org.
HIVE-17323 SemiJoin: Detect DPP edge as running parallel to semi-join (Deepak Jaiswal, reviewed by Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0d0fa42c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0d0fa42c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0d0fa42c
Branch: refs/heads/hive-14535
Commit: 0d0fa42c0742d6324543bacb554be4414cf7d969
Parents: 6c8f14b
Author: Deepak Jaiswal <de...@gmail.com>
Authored: Thu Aug 31 17:06:29 2017 -0700
Committer: Vineet Garg <vg...@apache.com>
Committed: Thu Aug 31 17:08:41 2017 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/parse/TezCompiler.java | 25 +-
.../clientpositive/dynamic_semijoin_reduction.q | 14 +-
.../llap/dynamic_semijoin_reduction.q.out | 441 +++++++++++++++++--
3 files changed, 443 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0d0fa42c/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
index 1671773..5921594 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
@@ -931,8 +931,8 @@ public class TezCompiler extends TaskCompiler {
}
// Remove the semijoin optimization branch along with ALL the mappings
// The parent GB2 has all the branches. Collect them and remove them.
- for (Operator<?> op : gbOp.getChildOperators()) {
- ReduceSinkOperator rsFinal = (ReduceSinkOperator) op;
+ for (Node node : gbOp.getChildren()) {
+ ReduceSinkOperator rsFinal = (ReduceSinkOperator) node;
TableScanOperator ts = pCtx.getRsToSemiJoinBranchInfo().
get(rsFinal).getTsOp();
if (LOG.isDebugEnabled()) {
@@ -1004,6 +1004,17 @@ public class TezCompiler extends TaskCompiler {
while (!(op instanceof ReduceSinkOperator) &&
!(op instanceof TableScanOperator) &&
!(op.getChildren() != null && op.getChildren().size() > 1)) {
+ if (op instanceof MapJoinOperator) {
+ // Pick the correct parent, only one of the parents is not
+ // ReduceSink, that is what we are looking for.
+ for (Operator<?> parentOp : op.getParentOperators()) {
+ if (parentOp instanceof ReduceSinkOperator) {
+ continue;
+ }
+ op = parentOp; // parent in current pipeline
+ continue;
+ }
+ }
op = op.getParentOperators().get(0);
}
@@ -1023,6 +1034,12 @@ public class TezCompiler extends TaskCompiler {
// If not ReduceSink Op, skip
if (!(child instanceof ReduceSinkOperator)) {
+ // This still could be DPP.
+ if (child instanceof AppMasterEventOperator &&
+ ((AppMasterEventOperator) child).getConf() instanceof DynamicPruningEventDesc) {
+ // DPP indeed, Set parallel edges true
+ parallelEdges = true;
+ }
continue;
}
@@ -1053,7 +1070,7 @@ public class TezCompiler extends TaskCompiler {
/*
* The algorithm looks at all the mapjoins in the operator pipeline until
* it hits RS Op and for each mapjoin examines if it has paralllel semijoin
- * edge.
+ * edge or dynamic partition pruning.
*/
private void removeSemijoinsParallelToMapJoin(OptimizeTezProcContext procCtx)
throws SemanticException {
@@ -1076,7 +1093,7 @@ public class TezCompiler extends TaskCompiler {
Deque<Operator<?>> deque = new LinkedList<>();
deque.add(parent);
while (!deque.isEmpty()) {
- Operator<?> op = deque.poll();
+ Operator<?> op = deque.pollLast();
if (op instanceof ReduceSinkOperator) {
// Done with this branch
continue;
http://git-wip-us.apache.org/repos/asf/hive/blob/0d0fa42c/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction.q b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction.q
index b22890b..a36e981 100644
--- a/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction.q
+++ b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction.q
@@ -101,10 +101,17 @@ EXPLAIN select count(*) from srcpart_date join srcpart_small on (srcpart_date.ke
select count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1) join alltypesorc_int on (srcpart_date.value = alltypesorc_int.cstring);
set hive.tez.dynamic.semijoin.reduction.for.mapjoin=true;
-- Enable semijoin parallel to mapjoins.
-EXPLAIN select count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1);
-select count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1);
+EXPLAIN select count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1) join alltypesorc_int on (srcpart_date.value = alltypesorc_int.cstring);
+select count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1) join alltypesorc_int on (srcpart_date.value = alltypesorc_int.cstring);
set hive.tez.dynamic.semijoin.reduction.for.mapjoin=false;
---set hive.tez.dynamic.semijoin.reduction=false;
+-- HIVE-17323 - with DPP, the 1st mapjoin is on a map with DPP and 2nd mapjoin is on a map which had semijoin but still removed.
+create table alltypesorc_int40 as select * from alltypesorc_int limit 40;
+set hive.tez.dynamic.semijoin.reduction=false;
+EXPLAIN select count(*) from srcpart_date join srcpart_small on (srcpart_date.ds = srcpart_small.ds) join alltypesorc_int40 on (srcpart_date.value = alltypesorc_int40.cstring);
+select count(*) from srcpart_date join srcpart_small on (srcpart_date.ds = srcpart_small.ds) join alltypesorc_int40 on (srcpart_date.value = alltypesorc_int40.cstring);
+set hive.tez.dynamic.semijoin.reduction=true;
+EXPLAIN select count(*) from srcpart_date join srcpart_small on (srcpart_date.ds = srcpart_small.ds) join alltypesorc_int40 on (srcpart_date.value = alltypesorc_int40.cstring);
+select count(*) from srcpart_date join srcpart_small on (srcpart_date.ds = srcpart_small.ds) join alltypesorc_int40 on (srcpart_date.value = alltypesorc_int40.cstring);
-- With unions
explain select * from alltypesorc_int join
@@ -113,6 +120,7 @@ explain select * from alltypesorc_int join
select srcpart_small.key1 as key from srcpart_small) unionsrc on (alltypesorc_int.cstring = unionsrc.key);
+
drop table srcpart_date;
drop table srcpart_small;
drop table alltypesorc_int;
http://git-wip-us.apache.org/repos/asf/hive/blob/0d0fa42c/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out
index 3bd35bf..61dcf3b 100644
--- a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out
@@ -2528,9 +2528,9 @@ POSTHOOK: Input: default@srcpart_small@ds=2008-04-08
POSTHOOK: Input: default@srcpart_small@ds=2008-04-09
#### A masked pattern was here ####
0
-PREHOOK: query: EXPLAIN select count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1)
+PREHOOK: query: EXPLAIN select count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1) join alltypesorc_int on (srcpart_date.value = alltypesorc_int.cstring)
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN select count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1)
+POSTHOOK: query: EXPLAIN select count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1) join alltypesorc_int on (srcpart_date.value = alltypesorc_int.cstring)
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -2542,41 +2542,53 @@ STAGE PLANS:
#### A masked pattern was here ####
Edges:
Map 1 <- Map 3 (BROADCAST_EDGE), Reducer 4 (BROADCAST_EDGE)
+ Map 5 <- Map 1 (BROADCAST_EDGE), Reducer 2 (BROADCAST_EDGE)
Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE)
+ Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
alias: srcpart_date
- filterExpr: (key is not null and (key BETWEEN DynamicValue(RS_7_srcpart_small_key1_min) AND DynamicValue(RS_7_srcpart_small_key1_max) and in_bloom_filter(key, DynamicValue(RS_7_srcpart_small_key1_bloom_filter)))) (type: boolean)
- Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE
+ filterExpr: (key is not null and value is not null and (key BETWEEN DynamicValue(RS_10_srcpart_small_key1_min) AND DynamicValue(RS_10_srcpart_small_key1_max) and in_bloom_filter(key, DynamicValue(RS_10_srcpart_small_key1_bloom_filter)))) (type: boolean)
+ Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key is not null and (key BETWEEN DynamicValue(RS_7_srcpart_small_key1_min) AND DynamicValue(RS_7_srcpart_small_key1_max) and in_bloom_filter(key, DynamicValue(RS_7_srcpart_small_key1_bloom_filter)))) (type: boolean)
- Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: (key is not null and value is not null and (key BETWEEN DynamicValue(RS_10_srcpart_small_key1_min) AND DynamicValue(RS_10_srcpart_small_key1_max) and in_bloom_filter(key, DynamicValue(RS_10_srcpart_small_key1_bloom_filter)))) (type: boolean)
+ Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE
Map Join Operator
condition map:
Inner Join 0 to 1
keys:
0 _col0 (type: string)
1 _col0 (type: string)
+ outputColumnNames: _col1
input vertices:
1 Map 3
- Statistics: Num rows: 129 Data size: 1032 Basic stats: COMPLETE Column stats: PARTIAL
- Group By Operator
- aggregations: count()
- mode: hash
+ Statistics: Num rows: 129 Data size: 11739 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 129 Data size: 11739 Basic stats: COMPLETE Column stats: PARTIAL
+ Select Operator
+ expressions: _col1 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL
- value expressions: _col0 (type: bigint)
+ Statistics: Num rows: 129 Data size: 11739 Basic stats: COMPLETE Column stats: PARTIAL
+ Group By Operator
+ aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=40)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: PARTIAL
+ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Execution mode: llap
LLAP IO: all inputs
Map 3
@@ -2612,10 +2624,67 @@ STAGE PLANS:
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Execution mode: llap
LLAP IO: all inputs
+ Map 5
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc_int
+ filterExpr: (cstring is not null and (cstring BETWEEN DynamicValue(RS_12_srcpart_date_value_min) AND DynamicValue(RS_12_srcpart_date_value_max) and in_bloom_filter(cstring, DynamicValue(RS_12_srcpart_date_value_bloom_filter)))) (type: boolean)
+ Statistics: Num rows: 12288 Data size: 862450 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: (cstring is not null and (cstring BETWEEN DynamicValue(RS_12_srcpart_date_value_min) AND DynamicValue(RS_12_srcpart_date_value_max) and in_bloom_filter(cstring, DynamicValue(RS_12_srcpart_date_value_bloom_filter)))) (type: boolean)
+ Statistics: Num rows: 9174 Data size: 643900 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: cstring (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 9174 Data size: 643900 Basic stats: COMPLETE Column stats: COMPLETE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: string)
+ 1 _col0 (type: string)
+ input vertices:
+ 0 Map 1
+ Statistics: Num rows: 261 Data size: 2088 Basic stats: COMPLETE Column stats: PARTIAL
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL
+ value expressions: _col0 (type: bigint)
+ Execution mode: llap
+ LLAP IO: all inputs
Reducer 2
Execution mode: llap
Reduce Operator Tree:
Group By Operator
+ aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=40)
+ mode: final
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: PARTIAL
+ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
+ Reducer 4
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=40)
+ mode: final
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: PARTIAL
+ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
+ Reducer 6
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
aggregations: count(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
@@ -2627,18 +2696,173 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Reducer 4
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1) join alltypesorc_int on (srcpart_date.value = alltypesorc_int.cstring)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc_int
+PREHOOK: Input: default@srcpart_date
+PREHOOK: Input: default@srcpart_date@ds=2008-04-08
+PREHOOK: Input: default@srcpart_date@ds=2008-04-09
+PREHOOK: Input: default@srcpart_small
+PREHOOK: Input: default@srcpart_small@ds=2008-04-08
+PREHOOK: Input: default@srcpart_small@ds=2008-04-09
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1) join alltypesorc_int on (srcpart_date.value = alltypesorc_int.cstring)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc_int
+POSTHOOK: Input: default@srcpart_date
+POSTHOOK: Input: default@srcpart_date@ds=2008-04-08
+POSTHOOK: Input: default@srcpart_date@ds=2008-04-09
+POSTHOOK: Input: default@srcpart_small
+POSTHOOK: Input: default@srcpart_small@ds=2008-04-08
+POSTHOOK: Input: default@srcpart_small@ds=2008-04-09
+#### A masked pattern was here ####
+0
+PREHOOK: query: create table alltypesorc_int40 as select * from alltypesorc_int limit 40
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@alltypesorc_int
+PREHOOK: Output: database:default
+PREHOOK: Output: default@alltypesorc_int40
+POSTHOOK: query: create table alltypesorc_int40 as select * from alltypesorc_int limit 40
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@alltypesorc_int
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@alltypesorc_int40
+POSTHOOK: Lineage: alltypesorc_int40.cint SIMPLE [(alltypesorc_int)alltypesorc_int.FieldSchema(name:cint, type:int, comment:null), ]
+POSTHOOK: Lineage: alltypesorc_int40.cstring SIMPLE [(alltypesorc_int)alltypesorc_int.FieldSchema(name:cstring, type:string, comment:null), ]
+PREHOOK: query: EXPLAIN select count(*) from srcpart_date join srcpart_small on (srcpart_date.ds = srcpart_small.ds) join alltypesorc_int40 on (srcpart_date.value = alltypesorc_int40.cstring)
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN select count(*) from srcpart_date join srcpart_small on (srcpart_date.ds = srcpart_small.ds) join alltypesorc_int40 on (srcpart_date.value = alltypesorc_int40.cstring)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_date
+ filterExpr: value is not null (type: boolean)
+ Statistics: Num rows: 2000 Data size: 550000 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: value is not null (type: boolean)
+ Statistics: Num rows: 2000 Data size: 550000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: value (type: string), ds (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2000 Data size: 550000 Basic stats: COMPLETE Column stats: COMPLETE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0
+ input vertices:
+ 1 Map 3
+ Statistics: Num rows: 20000 Data size: 1820000 Basic stats: COMPLETE Column stats: COMPLETE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ input vertices:
+ 1 Map 4
+ Statistics: Num rows: 22000 Data size: 2002000 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: llap
+ LLAP IO: all inputs
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_small
+ filterExpr: ds is not null (type: boolean)
+ Statistics: Num rows: 20 Data size: 7200 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: ds (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 20 Data size: 3680 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 20 Data size: 3680 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 20 Data size: 3680 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
+ Dynamic Partitioning Event Operator
+ Target column: ds (string)
+ Target Input: srcpart_date
+ Partition key expr: ds
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
+ Target Vertex: Map 1
+ Execution mode: llap
+ LLAP IO: all inputs
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc_int40
+ filterExpr: cstring is not null (type: boolean)
+ Statistics: Num rows: 40 Data size: 1040 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: cstring is not null (type: boolean)
+ Statistics: Num rows: 40 Data size: 1040 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cstring (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 40 Data size: 1040 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 40 Data size: 1040 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: llap
+ LLAP IO: no inputs
+ Reducer 2
Execution mode: llap
Reduce Operator Tree:
Group By Operator
- aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=40)
- mode: final
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: PARTIAL
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: PARTIAL
- value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -2646,8 +2870,9 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1)
+PREHOOK: query: select count(*) from srcpart_date join srcpart_small on (srcpart_date.ds = srcpart_small.ds) join alltypesorc_int40 on (srcpart_date.value = alltypesorc_int40.cstring)
PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc_int40
PREHOOK: Input: default@srcpart_date
PREHOOK: Input: default@srcpart_date@ds=2008-04-08
PREHOOK: Input: default@srcpart_date@ds=2008-04-09
@@ -2655,8 +2880,9 @@ PREHOOK: Input: default@srcpart_small
PREHOOK: Input: default@srcpart_small@ds=2008-04-08
PREHOOK: Input: default@srcpart_small@ds=2008-04-09
#### A masked pattern was here ####
-POSTHOOK: query: select count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1)
+POSTHOOK: query: select count(*) from srcpart_date join srcpart_small on (srcpart_date.ds = srcpart_small.ds) join alltypesorc_int40 on (srcpart_date.value = alltypesorc_int40.cstring)
POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc_int40
POSTHOOK: Input: default@srcpart_date
POSTHOOK: Input: default@srcpart_date@ds=2008-04-08
POSTHOOK: Input: default@srcpart_date@ds=2008-04-09
@@ -2664,7 +2890,162 @@ POSTHOOK: Input: default@srcpart_small
POSTHOOK: Input: default@srcpart_small@ds=2008-04-08
POSTHOOK: Input: default@srcpart_small@ds=2008-04-09
#### A masked pattern was here ####
-176
+0
+PREHOOK: query: EXPLAIN select count(*) from srcpart_date join srcpart_small on (srcpart_date.ds = srcpart_small.ds) join alltypesorc_int40 on (srcpart_date.value = alltypesorc_int40.cstring)
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN select count(*) from srcpart_date join srcpart_small on (srcpart_date.ds = srcpart_small.ds) join alltypesorc_int40 on (srcpart_date.value = alltypesorc_int40.cstring)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE)
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_date
+ filterExpr: value is not null (type: boolean)
+ Statistics: Num rows: 2000 Data size: 550000 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: value is not null (type: boolean)
+ Statistics: Num rows: 2000 Data size: 550000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: value (type: string), ds (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2000 Data size: 550000 Basic stats: COMPLETE Column stats: COMPLETE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0
+ input vertices:
+ 1 Map 3
+ Statistics: Num rows: 20000 Data size: 1820000 Basic stats: COMPLETE Column stats: COMPLETE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ input vertices:
+ 1 Map 4
+ Statistics: Num rows: 22000 Data size: 2002000 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: llap
+ LLAP IO: all inputs
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: srcpart_small
+ filterExpr: ds is not null (type: boolean)
+ Statistics: Num rows: 20 Data size: 7200 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: ds (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 20 Data size: 3680 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 20 Data size: 3680 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 20 Data size: 3680 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
+ Dynamic Partitioning Event Operator
+ Target column: ds (string)
+ Target Input: srcpart_date
+ Partition key expr: ds
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
+ Target Vertex: Map 1
+ Execution mode: llap
+ LLAP IO: all inputs
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc_int40
+ filterExpr: cstring is not null (type: boolean)
+ Statistics: Num rows: 40 Data size: 1040 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: cstring is not null (type: boolean)
+ Statistics: Num rows: 40 Data size: 1040 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cstring (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 40 Data size: 1040 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 40 Data size: 1040 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: llap
+ LLAP IO: no inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from srcpart_date join srcpart_small on (srcpart_date.ds = srcpart_small.ds) join alltypesorc_int40 on (srcpart_date.value = alltypesorc_int40.cstring)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc_int40
+PREHOOK: Input: default@srcpart_date
+PREHOOK: Input: default@srcpart_date@ds=2008-04-08
+PREHOOK: Input: default@srcpart_date@ds=2008-04-09
+PREHOOK: Input: default@srcpart_small
+PREHOOK: Input: default@srcpart_small@ds=2008-04-08
+PREHOOK: Input: default@srcpart_small@ds=2008-04-09
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from srcpart_date join srcpart_small on (srcpart_date.ds = srcpart_small.ds) join alltypesorc_int40 on (srcpart_date.value = alltypesorc_int40.cstring)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc_int40
+POSTHOOK: Input: default@srcpart_date
+POSTHOOK: Input: default@srcpart_date@ds=2008-04-08
+POSTHOOK: Input: default@srcpart_date@ds=2008-04-09
+POSTHOOK: Input: default@srcpart_small
+POSTHOOK: Input: default@srcpart_small@ds=2008-04-08
+POSTHOOK: Input: default@srcpart_small@ds=2008-04-09
+#### A masked pattern was here ####
+0
PREHOOK: query: explain select * from alltypesorc_int join
(select srcpart_date.key as key from srcpart_date
union all
[37/50] [abbrv] hive git commit: HIVE-14671 : merge master into
hive-14535 (Wei Zheng)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/standalone-metastore/src/main/thrift/hive_metastore.thrift
----------------------------------------------------------------------
diff --cc standalone-metastore/src/main/thrift/hive_metastore.thrift
index 0000000,7268d53..322e0ec
mode 000000,100644..100644
--- a/standalone-metastore/src/main/thrift/hive_metastore.thrift
+++ b/standalone-metastore/src/main/thrift/hive_metastore.thrift
@@@ -1,0 -1,1612 +1,1613 @@@
+ #!/usr/local/bin/thrift -java
+
+ /**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ #
+ # Thrift Service that the MetaStore is built on
+ #
+
+ include "share/fb303/if/fb303.thrift"
+
+ namespace java org.apache.hadoop.hive.metastore.api
+ namespace php metastore
+ namespace cpp Apache.Hadoop.Hive
+
+ const string DDL_TIME = "transient_lastDdlTime"
+
+ struct Version {
+ 1: string version,
+ 2: string comments
+ }
+
+ struct FieldSchema {
+ 1: string name, // name of the field
+ 2: string type, // type of the field. primitive types defined above, specify list<TYPE_NAME>, map<TYPE_NAME, TYPE_NAME> for lists & maps
+ 3: string comment
+ }
+
+ struct SQLPrimaryKey {
+ 1: string table_db, // table schema
+ 2: string table_name, // table name
+ 3: string column_name, // column name
+ 4: i32 key_seq, // sequence number within primary key
+ 5: string pk_name, // primary key name
+ 6: bool enable_cstr, // Enable/Disable
+ 7: bool validate_cstr, // Validate/No validate
+ 8: bool rely_cstr // Rely/No Rely
+ }
+
+ struct SQLForeignKey {
+ 1: string pktable_db, // primary key table schema
+ 2: string pktable_name, // primary key table name
+ 3: string pkcolumn_name, // primary key column name
+ 4: string fktable_db, // foreign key table schema
+ 5: string fktable_name, // foreign key table name
+ 6: string fkcolumn_name, // foreign key column name
+ 7: i32 key_seq, // sequence within foreign key
+ 8: i32 update_rule, // what happens to foreign key when parent key is updated
+ 9: i32 delete_rule, // what happens to foreign key when parent key is deleted
+ 10: string fk_name, // foreign key name
+ 11: string pk_name, // primary key name
+ 12: bool enable_cstr, // Enable/Disable
+ 13: bool validate_cstr, // Validate/No validate
+ 14: bool rely_cstr // Rely/No Rely
+ }
+
+ struct SQLUniqueConstraint {
+ 1: string table_db, // table schema
+ 2: string table_name, // table name
+ 3: string column_name, // column name
+ 4: i32 key_seq, // sequence number within unique constraint
+ 5: string uk_name, // unique key name
+ 6: bool enable_cstr, // Enable/Disable
+ 7: bool validate_cstr, // Validate/No validate
+ 8: bool rely_cstr // Rely/No Rely
+ }
+
+ struct SQLNotNullConstraint {
+ 1: string table_db, // table schema
+ 2: string table_name, // table name
+ 3: string column_name, // column name
+ 4: string nn_name, // not null name
+ 5: bool enable_cstr, // Enable/Disable
+ 6: bool validate_cstr, // Validate/No validate
+ 7: bool rely_cstr // Rely/No Rely
+ }
+
+ struct Type {
+ 1: string name, // one of the types in PrimitiveTypes or CollectionTypes or User defined types
+ 2: optional string type1, // object type if the name is 'list' (LIST_TYPE), key type if the name is 'map' (MAP_TYPE)
+ 3: optional string type2, // val type if the name is 'map' (MAP_TYPE)
+ 4: optional list<FieldSchema> fields // if the name is one of the user defined types
+ }
+
+ enum HiveObjectType {
+ GLOBAL = 1,
+ DATABASE = 2,
+ TABLE = 3,
+ PARTITION = 4,
+ COLUMN = 5,
+ }
+
+ enum PrincipalType {
+ USER = 1,
+ ROLE = 2,
+ GROUP = 3,
+ }
+
+ const string HIVE_FILTER_FIELD_OWNER = "hive_filter_field_owner__"
+ const string HIVE_FILTER_FIELD_PARAMS = "hive_filter_field_params__"
+ const string HIVE_FILTER_FIELD_LAST_ACCESS = "hive_filter_field_last_access__"
+
+ enum PartitionEventType {
+ LOAD_DONE = 1,
+ }
+
+ // Enums for transaction and lock management
+ enum TxnState {
+ COMMITTED = 1,
+ ABORTED = 2,
+ OPEN = 3,
+ }
+
+ enum LockLevel {
+ DB = 1,
+ TABLE = 2,
+ PARTITION = 3,
+ }
+
+ enum LockState {
+ ACQUIRED = 1, // requester has the lock
+ WAITING = 2, // requester is waiting for the lock and should call checklock at a later point to see if the lock has been obtained.
+ ABORT = 3, // the lock has been aborted, most likely due to timeout
+ NOT_ACQUIRED = 4, // returned only with lockNoWait, indicates the lock was not available and was not acquired
+ }
+
+ enum LockType {
+ SHARED_READ = 1,
+ SHARED_WRITE = 2,
+ EXCLUSIVE = 3,
+ }
+
+ enum CompactionType {
+ MINOR = 1,
+ MAJOR = 2,
+ }
+
+ enum GrantRevokeType {
+ GRANT = 1,
+ REVOKE = 2,
+ }
+
+ enum DataOperationType {
+ SELECT = 1,
+ INSERT = 2
+ UPDATE = 3,
+ DELETE = 4,
+ UNSET = 5,//this is the default to distinguish from NULL from old clients
+ NO_TXN = 6,//drop table, insert overwrite, etc - something non-transactional
+ }
+
+ // Types of events the client can request that the metastore fire. For now just support DML operations, as the metastore knows
+ // about DDL operations and there's no reason for the client to request such an event.
+ enum EventRequestType {
+ INSERT = 1,
+ UPDATE = 2,
+ DELETE = 3,
+ }
+
+ struct HiveObjectRef{
+ 1: HiveObjectType objectType,
+ 2: string dbName,
+ 3: string objectName,
+ 4: list<string> partValues,
+ 5: string columnName,
+ }
+
+ struct PrivilegeGrantInfo {
+ 1: string privilege,
+ 2: i32 createTime,
+ 3: string grantor,
+ 4: PrincipalType grantorType,
+ 5: bool grantOption,
+ }
+
+ struct HiveObjectPrivilege {
+ 1: HiveObjectRef hiveObject,
+ 2: string principalName,
+ 3: PrincipalType principalType,
+ 4: PrivilegeGrantInfo grantInfo,
+ }
+
+ struct PrivilegeBag {
+ 1: list<HiveObjectPrivilege> privileges,
+ }
+
+ struct PrincipalPrivilegeSet {
+ 1: map<string, list<PrivilegeGrantInfo>> userPrivileges, // user name -> privilege grant info
+ 2: map<string, list<PrivilegeGrantInfo>> groupPrivileges, // group name -> privilege grant info
+ 3: map<string, list<PrivilegeGrantInfo>> rolePrivileges, //role name -> privilege grant info
+ }
+
+ struct GrantRevokePrivilegeRequest {
+ 1: GrantRevokeType requestType;
+ 2: PrivilegeBag privileges;
+ 3: optional bool revokeGrantOption; // Only for revoke request
+ }
+
+ struct GrantRevokePrivilegeResponse {
+ 1: optional bool success;
+ }
+
+ struct Role {
+ 1: string roleName,
+ 2: i32 createTime,
+ 3: string ownerName,
+ }
+
+ // Representation of a grant for a principal to a role
+ struct RolePrincipalGrant {
+ 1: string roleName,
+ 2: string principalName,
+ 3: PrincipalType principalType,
+ 4: bool grantOption,
+ 5: i32 grantTime,
+ 6: string grantorName,
+ 7: PrincipalType grantorPrincipalType
+ }
+
+ struct GetRoleGrantsForPrincipalRequest {
+ 1: required string principal_name,
+ 2: required PrincipalType principal_type
+ }
+
+ struct GetRoleGrantsForPrincipalResponse {
+ 1: required list<RolePrincipalGrant> principalGrants;
+ }
+
+ struct GetPrincipalsInRoleRequest {
+ 1: required string roleName;
+ }
+
+ struct GetPrincipalsInRoleResponse {
+ 1: required list<RolePrincipalGrant> principalGrants;
+ }
+
+ struct GrantRevokeRoleRequest {
+ 1: GrantRevokeType requestType;
+ 2: string roleName;
+ 3: string principalName;
+ 4: PrincipalType principalType;
+ 5: optional string grantor; // Needed for grant
+ 6: optional PrincipalType grantorType; // Needed for grant
+ 7: optional bool grantOption;
+ }
+
+ struct GrantRevokeRoleResponse {
+ 1: optional bool success;
+ }
+
+ // namespace for tables
+ struct Database {
+ 1: string name,
+ 2: string description,
+ 3: string locationUri,
+ 4: map<string, string> parameters, // properties associated with the database
+ 5: optional PrincipalPrivilegeSet privileges,
+ 6: optional string ownerName,
+ 7: optional PrincipalType ownerType
+ }
+
+ // This object holds the information needed by SerDes
+ struct SerDeInfo {
+ 1: string name, // name of the serde, table name by default
+ 2: string serializationLib, // usually the class that implements the extractor & loader
+ 3: map<string, string> parameters // initialization parameters
+ }
+
+ // sort order of a column (column name along with asc(1)/desc(0))
+ struct Order {
+ 1: string col, // sort column name
+ 2: i32 order // asc(1) or desc(0)
+ }
+
+ // this object holds all the information about skewed table
+ struct SkewedInfo {
+ 1: list<string> skewedColNames, // skewed column names
+ 2: list<list<string>> skewedColValues, //skewed values
+ 3: map<list<string>, string> skewedColValueLocationMaps, //skewed value to location mappings
+ }
+
+ // this object holds all the information about physical storage of the data belonging to a table
+ struct StorageDescriptor {
+ 1: list<FieldSchema> cols, // required (refer to types defined above)
+ 2: string location, // defaults to <warehouse loc>/<db loc>/tablename
+ 3: string inputFormat, // SequenceFileInputFormat (binary) or TextInputFormat` or custom format
+ 4: string outputFormat, // SequenceFileOutputFormat (binary) or IgnoreKeyTextOutputFormat or custom format
+ 5: bool compressed, // compressed or not
+ 6: i32 numBuckets, // this must be specified if there are any dimension columns
+ 7: SerDeInfo serdeInfo, // serialization and deserialization information
+ 8: list<string> bucketCols, // reducer grouping columns and clustering columns and bucketing columns`
+ 9: list<Order> sortCols, // sort order of the data in each bucket
+ 10: map<string, string> parameters, // any user supplied key value hash
+ 11: optional SkewedInfo skewedInfo, // skewed information
+ 12: optional bool storedAsSubDirectories // stored as subdirectories or not
+ }
+
+ // table information
+ struct Table {
+ 1: string tableName, // name of the table
+ 2: string dbName, // database name ('default')
+ 3: string owner, // owner of this table
+ 4: i32 createTime, // creation time of the table
+ 5: i32 lastAccessTime, // last access time (usually this will be filled from HDFS and shouldn't be relied on)
+ 6: i32 retention, // retention time
+ 7: StorageDescriptor sd, // storage descriptor of the table
+ 8: list<FieldSchema> partitionKeys, // partition keys of the table. only primitive types are supported
+ 9: map<string, string> parameters, // to store comments or any other user level parameters
+ 10: string viewOriginalText, // original view text, null for non-view
+ 11: string viewExpandedText, // expanded view text, null for non-view
+ 12: string tableType, // table type enum, e.g. EXTERNAL_TABLE
+ 13: optional PrincipalPrivilegeSet privileges,
+ 14: optional bool temporary=false,
- 15: optional bool rewriteEnabled // rewrite enabled or not
++ 15: optional bool rewriteEnabled, // rewrite enabled or not
+ }
+
+ struct Partition {
+ 1: list<string> values // string value is converted to appropriate partition key type
+ 2: string dbName,
+ 3: string tableName,
+ 4: i32 createTime,
+ 5: i32 lastAccessTime,
+ 6: StorageDescriptor sd,
+ 7: map<string, string> parameters,
+ 8: optional PrincipalPrivilegeSet privileges
+ }
+
+ struct PartitionWithoutSD {
+ 1: list<string> values // string value is converted to appropriate partition key type
+ 2: i32 createTime,
+ 3: i32 lastAccessTime,
+ 4: string relativePath,
+ 5: map<string, string> parameters,
+ 6: optional PrincipalPrivilegeSet privileges
+ }
+
+ struct PartitionSpecWithSharedSD {
+ 1: list<PartitionWithoutSD> partitions,
+ 2: StorageDescriptor sd,
+ }
+
+ struct PartitionListComposingSpec {
+ 1: list<Partition> partitions
+ }
+
+ struct PartitionSpec {
+ 1: string dbName,
+ 2: string tableName,
+ 3: string rootPath,
+ 4: optional PartitionSpecWithSharedSD sharedSDPartitionSpec,
+ 5: optional PartitionListComposingSpec partitionList
+ }
+
+ struct Index {
+ 1: string indexName, // unique with in the whole database namespace
+ 2: string indexHandlerClass, // reserved
+ 3: string dbName,
+ 4: string origTableName,
+ 5: i32 createTime,
+ 6: i32 lastAccessTime,
+ 7: string indexTableName,
+ 8: StorageDescriptor sd,
+ 9: map<string, string> parameters,
+ 10: bool deferredRebuild
+ }
+
+ // column statistics
+ struct BooleanColumnStatsData {
+ 1: required i64 numTrues,
+ 2: required i64 numFalses,
+ 3: required i64 numNulls,
+ 4: optional binary bitVectors
+ }
+
+ struct DoubleColumnStatsData {
+ 1: optional double lowValue,
+ 2: optional double highValue,
+ 3: required i64 numNulls,
+ 4: required i64 numDVs,
+ 5: optional binary bitVectors
+ }
+
+ struct LongColumnStatsData {
+ 1: optional i64 lowValue,
+ 2: optional i64 highValue,
+ 3: required i64 numNulls,
+ 4: required i64 numDVs,
+ 5: optional binary bitVectors
+ }
+
+ struct StringColumnStatsData {
+ 1: required i64 maxColLen,
+ 2: required double avgColLen,
+ 3: required i64 numNulls,
+ 4: required i64 numDVs,
+ 5: optional binary bitVectors
+ }
+
+ struct BinaryColumnStatsData {
+ 1: required i64 maxColLen,
+ 2: required double avgColLen,
+ 3: required i64 numNulls,
+ 4: optional binary bitVectors
+ }
+
+
+ struct Decimal {
+ 1: required binary unscaled,
+ 3: required i16 scale
+ }
+
+ struct DecimalColumnStatsData {
+ 1: optional Decimal lowValue,
+ 2: optional Decimal highValue,
+ 3: required i64 numNulls,
+ 4: required i64 numDVs,
+ 5: optional binary bitVectors
+ }
+
+ struct Date {
+ 1: required i64 daysSinceEpoch
+ }
+
+ struct DateColumnStatsData {
+ 1: optional Date lowValue,
+ 2: optional Date highValue,
+ 3: required i64 numNulls,
+ 4: required i64 numDVs,
+ 5: optional binary bitVectors
+ }
+
+ union ColumnStatisticsData {
+ 1: BooleanColumnStatsData booleanStats,
+ 2: LongColumnStatsData longStats,
+ 3: DoubleColumnStatsData doubleStats,
+ 4: StringColumnStatsData stringStats,
+ 5: BinaryColumnStatsData binaryStats,
+ 6: DecimalColumnStatsData decimalStats,
+ 7: DateColumnStatsData dateStats
+ }
+
+ struct ColumnStatisticsObj {
+ 1: required string colName,
+ 2: required string colType,
+ 3: required ColumnStatisticsData statsData
+ }
+
+ struct ColumnStatisticsDesc {
+ 1: required bool isTblLevel,
+ 2: required string dbName,
+ 3: required string tableName,
+ 4: optional string partName,
+ 5: optional i64 lastAnalyzed
+ }
+
+ struct ColumnStatistics {
+ 1: required ColumnStatisticsDesc statsDesc,
+ 2: required list<ColumnStatisticsObj> statsObj;
+ }
+
+ struct AggrStats {
+ 1: required list<ColumnStatisticsObj> colStats,
+ 2: required i64 partsFound // number of partitions for which stats were found
+ }
+
+ struct SetPartitionsStatsRequest {
+ 1: required list<ColumnStatistics> colStats,
+ 2: optional bool needMerge //stats need to be merged with the existing stats
+ }
+
+ // schema of the table/query results etc.
+ struct Schema {
+ // column names, types, comments
+ 1: list<FieldSchema> fieldSchemas, // delimiters etc
+ 2: map<string, string> properties
+ }
+
+ // Key-value store to be used with selected
+ // Metastore APIs (create, alter methods).
+ // The client can pass environment properties / configs that can be
+ // accessed in hooks.
+ struct EnvironmentContext {
+ 1: map<string, string> properties
+ }
+
+ struct PrimaryKeysRequest {
+ 1: required string db_name,
+ 2: required string tbl_name
+ }
+
+ struct PrimaryKeysResponse {
+ 1: required list<SQLPrimaryKey> primaryKeys
+ }
+
+ struct ForeignKeysRequest {
+ 1: string parent_db_name,
+ 2: string parent_tbl_name,
+ 3: string foreign_db_name,
+ 4: string foreign_tbl_name
+ }
+
+ struct ForeignKeysResponse {
+ 1: required list<SQLForeignKey> foreignKeys
+ }
+
+ struct UniqueConstraintsRequest {
+ 1: required string db_name,
+ 2: required string tbl_name
+ }
+
+ struct UniqueConstraintsResponse {
+ 1: required list<SQLUniqueConstraint> uniqueConstraints
+ }
+
+ struct NotNullConstraintsRequest {
+ 1: required string db_name,
+ 2: required string tbl_name
+ }
+
+ struct NotNullConstraintsResponse {
+ 1: required list<SQLNotNullConstraint> notNullConstraints
+ }
+
+ struct DropConstraintRequest {
+ 1: required string dbname,
+ 2: required string tablename,
+ 3: required string constraintname
+ }
+
+ struct AddPrimaryKeyRequest {
+ 1: required list<SQLPrimaryKey> primaryKeyCols
+ }
+
+ struct AddForeignKeyRequest {
+ 1: required list<SQLForeignKey> foreignKeyCols
+ }
+
+ struct AddUniqueConstraintRequest {
+ 1: required list<SQLUniqueConstraint> uniqueConstraintCols
+ }
+
+ struct AddNotNullConstraintRequest {
+ 1: required list<SQLNotNullConstraint> notNullConstraintCols
+ }
+
+ // Return type for get_partitions_by_expr
+ struct PartitionsByExprResult {
+ 1: required list<Partition> partitions,
+ // Whether the results has any (currently, all) partitions which may or may not match
+ 2: required bool hasUnknownPartitions
+ }
+
+ struct PartitionsByExprRequest {
+ 1: required string dbName,
+ 2: required string tblName,
+ 3: required binary expr,
+ 4: optional string defaultPartitionName,
+ 5: optional i16 maxParts=-1
+ }
+
+ struct TableStatsResult {
+ 1: required list<ColumnStatisticsObj> tableStats
+ }
+
+ struct PartitionsStatsResult {
+ 1: required map<string, list<ColumnStatisticsObj>> partStats
+ }
+
+ struct TableStatsRequest {
+ 1: required string dbName,
+ 2: required string tblName,
+ 3: required list<string> colNames
+ }
+
+ struct PartitionsStatsRequest {
+ 1: required string dbName,
+ 2: required string tblName,
+ 3: required list<string> colNames,
+ 4: required list<string> partNames
+ }
+
+ // Return type for add_partitions_req
+ struct AddPartitionsResult {
+ 1: optional list<Partition> partitions,
+ }
+
+ // Request type for add_partitions_req
+ struct AddPartitionsRequest {
+ 1: required string dbName,
+ 2: required string tblName,
+ 3: required list<Partition> parts,
+ 4: required bool ifNotExists,
+ 5: optional bool needResult=true
+ }
+
+ // Return type for drop_partitions_req
+ struct DropPartitionsResult {
+ 1: optional list<Partition> partitions,
+ }
+
+ struct DropPartitionsExpr {
+ 1: required binary expr;
+ 2: optional i32 partArchiveLevel;
+ }
+
+ union RequestPartsSpec {
+ 1: list<string> names;
+ 2: list<DropPartitionsExpr> exprs;
+ }
+
+ // Request type for drop_partitions_req
+ // TODO: we might want to add "bestEffort" flag; where a subset can fail
+ struct DropPartitionsRequest {
+ 1: required string dbName,
+ 2: required string tblName,
+ 3: required RequestPartsSpec parts,
+ 4: optional bool deleteData,
+ 5: optional bool ifExists=true, // currently verified on client
+ 6: optional bool ignoreProtection,
+ 7: optional EnvironmentContext environmentContext,
+ 8: optional bool needResult=true
+ }
+
+ enum FunctionType {
+ JAVA = 1,
+ }
+
+ enum ResourceType {
+ JAR = 1,
+ FILE = 2,
+ ARCHIVE = 3,
+ }
+
+ struct ResourceUri {
+ 1: ResourceType resourceType,
+ 2: string uri,
+ }
+
+ // User-defined function
+ struct Function {
+ 1: string functionName,
+ 2: string dbName,
+ 3: string className,
+ 4: string ownerName,
+ 5: PrincipalType ownerType,
+ 6: i32 createTime,
+ 7: FunctionType functionType,
+ 8: list<ResourceUri> resourceUris,
+ }
+
+ // Structs for transaction and locks
+ struct TxnInfo {
+ 1: required i64 id,
+ 2: required TxnState state,
+ 3: required string user, // used in 'show transactions' to help admins find who has open transactions
+ 4: required string hostname, // used in 'show transactions' to help admins find who has open transactions
+ 5: optional string agentInfo = "Unknown",
+ 6: optional i32 heartbeatCount=0,
+ 7: optional string metaInfo,
+ 8: optional i64 startedTime,
+ 9: optional i64 lastHeartbeatTime,
+ }
+
+ struct GetOpenTxnsInfoResponse {
+ 1: required i64 txn_high_water_mark,
+ 2: required list<TxnInfo> open_txns,
+ }
+
+ struct GetOpenTxnsResponse {
+ 1: required i64 txn_high_water_mark,
+ 2: required list<i64> open_txns, // set<i64> changed to list<i64> since 3.0
+ 3: optional i64 min_open_txn, //since 1.3,2.2
+ 4: required binary abortedBits, // since 3.0
+ }
+
+ struct OpenTxnRequest {
+ 1: required i32 num_txns,
+ 2: required string user,
+ 3: required string hostname,
+ 4: optional string agentInfo = "Unknown",
+ }
+
+ struct OpenTxnsResponse {
+ 1: required list<i64> txn_ids,
+ }
+
+ struct AbortTxnRequest {
+ 1: required i64 txnid,
+ }
+
+ struct AbortTxnsRequest {
+ 1: required list<i64> txn_ids,
+ }
+
+ struct CommitTxnRequest {
+ 1: required i64 txnid,
+ }
+
+ struct LockComponent {
+ 1: required LockType type,
+ 2: required LockLevel level,
+ 3: required string dbname,
+ 4: optional string tablename,
+ 5: optional string partitionname,
+ 6: optional DataOperationType operationType = DataOperationType.UNSET,
+ 7: optional bool isAcid = false,
+ 8: optional bool isDynamicPartitionWrite = false
+ }
+
+ struct LockRequest {
+ 1: required list<LockComponent> component,
+ 2: optional i64 txnid,
+ 3: required string user, // used in 'show locks' to help admins find who has open locks
+ 4: required string hostname, // used in 'show locks' to help admins find who has open locks
+ 5: optional string agentInfo = "Unknown",
+ }
+
+ struct LockResponse {
+ 1: required i64 lockid,
+ 2: required LockState state,
+ }
+
+ struct CheckLockRequest {
+ 1: required i64 lockid,
+ 2: optional i64 txnid,
+ 3: optional i64 elapsed_ms,
+ }
+
+ struct UnlockRequest {
+ 1: required i64 lockid,
+ }
+
+ struct ShowLocksRequest {
+ 1: optional string dbname,
+ 2: optional string tablename,
+ 3: optional string partname,
+ 4: optional bool isExtended=false,
+ }
+
+ struct ShowLocksResponseElement {
+ 1: required i64 lockid,
+ 2: required string dbname,
+ 3: optional string tablename,
+ 4: optional string partname,
+ 5: required LockState state,
+ 6: required LockType type,
+ 7: optional i64 txnid,
+ 8: required i64 lastheartbeat,
+ 9: optional i64 acquiredat,
+ 10: required string user,
+ 11: required string hostname,
+ 12: optional i32 heartbeatCount = 0,
+ 13: optional string agentInfo,
+ 14: optional i64 blockedByExtId,
+ 15: optional i64 blockedByIntId,
+ 16: optional i64 lockIdInternal,
+ }
+
+ struct ShowLocksResponse {
+ 1: list<ShowLocksResponseElement> locks,
+ }
+
+ struct HeartbeatRequest {
+ 1: optional i64 lockid,
+ 2: optional i64 txnid
+ }
+
+ struct HeartbeatTxnRangeRequest {
+ 1: required i64 min,
+ 2: required i64 max
+ }
+
+ struct HeartbeatTxnRangeResponse {
+ 1: required set<i64> aborted,
+ 2: required set<i64> nosuch
+ }
+
+ struct CompactionRequest {
+ 1: required string dbname,
+ 2: required string tablename,
+ 3: optional string partitionname,
+ 4: required CompactionType type,
+ 5: optional string runas,
+ 6: optional map<string, string> properties
+ }
+
+ struct CompactionResponse {
+ 1: required i64 id,
+ 2: required string state,
+ 3: required bool accepted
+ }
+
+ struct ShowCompactRequest {
+ }
+
+ struct ShowCompactResponseElement {
+ 1: required string dbname,
+ 2: required string tablename,
+ 3: optional string partitionname,
+ 4: required CompactionType type,
+ 5: required string state,
+ 6: optional string workerid,
+ 7: optional i64 start,
+ 8: optional string runAs,
+ 9: optional i64 hightestTxnId, // Highest Txn ID handled by this compaction
+ 10: optional string metaInfo,
+ 11: optional i64 endTime,
+ 12: optional string hadoopJobId = "None",
+ 13: optional i64 id,
+ }
+
+ struct ShowCompactResponse {
+ 1: required list<ShowCompactResponseElement> compacts,
+ }
+
+ struct AddDynamicPartitions {
+ 1: required i64 txnid,
+ 2: required string dbname,
+ 3: required string tablename,
+ 4: required list<string> partitionnames,
+ 5: optional DataOperationType operationType = DataOperationType.UNSET
+ }
+
+ struct NotificationEventRequest {
+ 1: required i64 lastEvent,
+ 2: optional i32 maxEvents,
+ }
+
+ struct NotificationEvent {
+ 1: required i64 eventId,
+ 2: required i32 eventTime,
+ 3: required string eventType,
+ 4: optional string dbName,
+ 5: optional string tableName,
+ 6: required string message,
+ 7: optional string messageFormat,
+ }
+
+ struct NotificationEventResponse {
+ 1: required list<NotificationEvent> events,
+ }
+
+ struct CurrentNotificationEventId {
+ 1: required i64 eventId,
+ }
+
+ struct NotificationEventsCountRequest {
+ 1: required i64 fromEventId,
+ 2: required string dbName,
+ }
+
+ struct NotificationEventsCountResponse {
+ 1: required i64 eventsCount,
+ }
+
+ struct InsertEventRequestData {
+ 1: optional bool replace,
+ 2: required list<string> filesAdded,
+ // Checksum of files (hex string of checksum byte payload)
+ 3: optional list<string> filesAddedChecksum,
+ }
+
+ union FireEventRequestData {
+ 1: InsertEventRequestData insertData
+ }
+
+ struct FireEventRequest {
+ 1: required bool successful,
+ 2: required FireEventRequestData data
+ // dbname, tablename, and partition vals are included as optional in the top level event rather than placed in each type of
+ // subevent as I assume they'll be used across most event types.
+ 3: optional string dbName,
+ 4: optional string tableName,
+ 5: optional list<string> partitionVals,
+ }
+
+ struct FireEventResponse {
+ // NOP for now, this is just a place holder for future responses
+ }
+
+ struct MetadataPpdResult {
+ 1: optional binary metadata,
+ 2: optional binary includeBitset
+ }
+
+ // Return type for get_file_metadata_by_expr
+ struct GetFileMetadataByExprResult {
+ 1: required map<i64, MetadataPpdResult> metadata,
+ 2: required bool isSupported
+ }
+
+ enum FileMetadataExprType {
+ ORC_SARG = 1
+ }
+
+
+ // Request type for get_file_metadata_by_expr
+ struct GetFileMetadataByExprRequest {
+ 1: required list<i64> fileIds,
+ 2: required binary expr,
+ 3: optional bool doGetFooters,
+ 4: optional FileMetadataExprType type
+ }
+
+ // Return type for get_file_metadata
+ struct GetFileMetadataResult {
+ 1: required map<i64, binary> metadata,
+ 2: required bool isSupported
+ }
+
+ // Request type for get_file_metadata
+ struct GetFileMetadataRequest {
+ 1: required list<i64> fileIds
+ }
+
+ // Return type for put_file_metadata
+ struct PutFileMetadataResult {
+ }
+
+ // Request type for put_file_metadata
+ struct PutFileMetadataRequest {
+ 1: required list<i64> fileIds,
+ 2: required list<binary> metadata,
+ 3: optional FileMetadataExprType type
+ }
+
+ // Return type for clear_file_metadata
+ struct ClearFileMetadataResult {
+ }
+
+ // Request type for clear_file_metadata
+ struct ClearFileMetadataRequest {
+ 1: required list<i64> fileIds
+ }
+
+ // Return type for cache_file_metadata
+ struct CacheFileMetadataResult {
+ 1: required bool isSupported
+ }
+
+ // Request type for cache_file_metadata
+ struct CacheFileMetadataRequest {
+ 1: required string dbName,
+ 2: required string tblName,
+ 3: optional string partName,
+ 4: optional bool isAllParts
+ }
+
+ struct GetAllFunctionsResponse {
+ 1: optional list<Function> functions
+ }
+
+ enum ClientCapability {
- TEST_CAPABILITY = 1
++ TEST_CAPABILITY = 1,
++ INSERT_ONLY_TABLES = 2
+ }
+
+
+ struct ClientCapabilities {
+ 1: required list<ClientCapability> values
+ }
+
+ struct GetTableRequest {
+ 1: required string dbName,
+ 2: required string tblName,
+ 3: optional ClientCapabilities capabilities
+ }
+
+ struct GetTableResult {
+ 1: required Table table
+ }
+
+ struct GetTablesRequest {
+ 1: required string dbName,
+ 2: optional list<string> tblNames,
+ 3: optional ClientCapabilities capabilities
+ }
+
+ struct GetTablesResult {
+ 1: required list<Table> tables
+ }
+
+ // Request type for cm_recycle
+ struct CmRecycleRequest {
+ 1: required string dataPath,
+ 2: required bool purge
+ }
+
+ // Response type for cm_recycle
+ struct CmRecycleResponse {
+ }
+
+ struct TableMeta {
+ 1: required string dbName;
+ 2: required string tableName;
+ 3: required string tableType;
+ 4: optional string comments;
+ }
+
+ exception MetaException {
+ 1: string message
+ }
+
+ exception UnknownTableException {
+ 1: string message
+ }
+
+ exception UnknownDBException {
+ 1: string message
+ }
+
+ exception AlreadyExistsException {
+ 1: string message
+ }
+
+ exception InvalidPartitionException {
+ 1: string message
+ }
+
+ exception UnknownPartitionException {
+ 1: string message
+ }
+
+ exception InvalidObjectException {
+ 1: string message
+ }
+
+ exception NoSuchObjectException {
+ 1: string message
+ }
+
+ exception IndexAlreadyExistsException {
+ 1: string message
+ }
+
+ exception InvalidOperationException {
+ 1: string message
+ }
+
+ exception ConfigValSecurityException {
+ 1: string message
+ }
+
+ exception InvalidInputException {
+ 1: string message
+ }
+
+ // Transaction and lock exceptions
+ exception NoSuchTxnException {
+ 1: string message
+ }
+
+ exception TxnAbortedException {
+ 1: string message
+ }
+
+ exception TxnOpenException {
+ 1: string message
+ }
+
+ exception NoSuchLockException {
+ 1: string message
+ }
+
+ /**
+ * This interface is live.
+ */
+ service ThriftHiveMetastore extends fb303.FacebookService
+ {
+ string getMetaConf(1:string key) throws(1:MetaException o1)
+ void setMetaConf(1:string key, 2:string value) throws(1:MetaException o1)
+
+ void create_database(1:Database database) throws(1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3:MetaException o3)
+ Database get_database(1:string name) throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ void drop_database(1:string name, 2:bool deleteData, 3:bool cascade) throws(1:NoSuchObjectException o1, 2:InvalidOperationException o2, 3:MetaException o3)
+ list<string> get_databases(1:string pattern) throws(1:MetaException o1)
+ list<string> get_all_databases() throws(1:MetaException o1)
+ void alter_database(1:string dbname, 2:Database db) throws(1:MetaException o1, 2:NoSuchObjectException o2)
+
+ // returns the type with given name (make seperate calls for the dependent types if needed)
+ Type get_type(1:string name) throws(1:MetaException o1, 2:NoSuchObjectException o2)
+ bool create_type(1:Type type) throws(1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3:MetaException o3)
+ bool drop_type(1:string type) throws(1:MetaException o1, 2:NoSuchObjectException o2)
+ map<string, Type> get_type_all(1:string name)
+ throws(1:MetaException o2)
+
+ // Gets a list of FieldSchemas describing the columns of a particular table
+ list<FieldSchema> get_fields(1: string db_name, 2: string table_name) throws (1: MetaException o1, 2: UnknownTableException o2, 3: UnknownDBException o3),
+ list<FieldSchema> get_fields_with_environment_context(1: string db_name, 2: string table_name, 3:EnvironmentContext environment_context) throws (1: MetaException o1, 2: UnknownTableException o2, 3: UnknownDBException o3)
+
+ // Gets a list of FieldSchemas describing both the columns and the partition keys of a particular table
+ list<FieldSchema> get_schema(1: string db_name, 2: string table_name) throws (1: MetaException o1, 2: UnknownTableException o2, 3: UnknownDBException o3)
+ list<FieldSchema> get_schema_with_environment_context(1: string db_name, 2: string table_name, 3:EnvironmentContext environment_context) throws (1: MetaException o1, 2: UnknownTableException o2, 3: UnknownDBException o3)
+
+ // create a Hive table. Following fields must be set
+ // tableName
+ // database (only 'default' for now until Hive QL supports databases)
+ // owner (not needed, but good to have for tracking purposes)
+ // sd.cols (list of field schemas)
+ // sd.inputFormat (SequenceFileInputFormat (binary like falcon tables or u_full) or TextInputFormat)
+ // sd.outputFormat (SequenceFileInputFormat (binary) or TextInputFormat)
+ // sd.serdeInfo.serializationLib (SerDe class name eg org.apache.hadoop.hive.serde.simple_meta.MetadataTypedColumnsetSerDe
+ // * See notes on DDL_TIME
+ void create_table(1:Table tbl) throws(1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3:MetaException o3, 4:NoSuchObjectException o4)
+ void create_table_with_environment_context(1:Table tbl,
+ 2:EnvironmentContext environment_context)
+ throws (1:AlreadyExistsException o1,
+ 2:InvalidObjectException o2, 3:MetaException o3,
+ 4:NoSuchObjectException o4)
+ void create_table_with_constraints(1:Table tbl, 2: list<SQLPrimaryKey> primaryKeys, 3: list<SQLForeignKey> foreignKeys,
+ 4: list<SQLUniqueConstraint> uniqueConstraints, 5: list<SQLNotNullConstraint> notNullConstraints)
+ throws (1:AlreadyExistsException o1,
+ 2:InvalidObjectException o2, 3:MetaException o3,
+ 4:NoSuchObjectException o4)
+ void drop_constraint(1:DropConstraintRequest req)
+ throws(1:NoSuchObjectException o1, 2:MetaException o3)
+ void add_primary_key(1:AddPrimaryKeyRequest req)
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ void add_foreign_key(1:AddForeignKeyRequest req)
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ void add_unique_constraint(1:AddUniqueConstraintRequest req)
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ void add_not_null_constraint(1:AddNotNullConstraintRequest req)
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
+
+ // drops the table and all the partitions associated with it if the table has partitions
+ // delete data (including partitions) if deleteData is set to true
+ void drop_table(1:string dbname, 2:string name, 3:bool deleteData)
+ throws(1:NoSuchObjectException o1, 2:MetaException o3)
+ void drop_table_with_environment_context(1:string dbname, 2:string name, 3:bool deleteData,
+ 4:EnvironmentContext environment_context)
+ throws(1:NoSuchObjectException o1, 2:MetaException o3)
+ void truncate_table(1:string dbName, 2:string tableName, 3:list<string> partNames)
+ throws(1:MetaException o1)
+ list<string> get_tables(1: string db_name, 2: string pattern) throws (1: MetaException o1)
+ list<string> get_tables_by_type(1: string db_name, 2: string pattern, 3: string tableType) throws (1: MetaException o1)
+ list<TableMeta> get_table_meta(1: string db_patterns, 2: string tbl_patterns, 3: list<string> tbl_types)
+ throws (1: MetaException o1)
+ list<string> get_all_tables(1: string db_name) throws (1: MetaException o1)
+
+ Table get_table(1:string dbname, 2:string tbl_name)
+ throws (1:MetaException o1, 2:NoSuchObjectException o2)
+ list<Table> get_table_objects_by_name(1:string dbname, 2:list<string> tbl_names)
+ GetTableResult get_table_req(1:GetTableRequest req) throws (1:MetaException o1, 2:NoSuchObjectException o2)
+ GetTablesResult get_table_objects_by_name_req(1:GetTablesRequest req)
+ throws (1:MetaException o1, 2:InvalidOperationException o2, 3:UnknownDBException o3)
+
+ // Get a list of table names that match a filter.
+ // The filter operators are LIKE, <, <=, >, >=, =, <>
+ //
+ // In the filter statement, values interpreted as strings must be enclosed in quotes,
+ // while values interpreted as integers should not be. Strings and integers are the only
+ // supported value types.
+ //
+ // The currently supported key names in the filter are:
+ // Constants.HIVE_FILTER_FIELD_OWNER, which filters on the tables' owner's name
+ // and supports all filter operators
+ // Constants.HIVE_FILTER_FIELD_LAST_ACCESS, which filters on the last access times
+ // and supports all filter operators except LIKE
+ // Constants.HIVE_FILTER_FIELD_PARAMS, which filters on the tables' parameter keys and values
+ // and only supports the filter operators = and <>.
+ // Append the parameter key name to HIVE_FILTER_FIELD_PARAMS in the filter statement.
+ // For example, to filter on parameter keys called "retention", the key name in the filter
+ // statement should be Constants.HIVE_FILTER_FIELD_PARAMS + "retention"
+ // Also, = and <> only work for keys that exist
+ // in the tables. E.g., if you are looking for tables where key1 <> value, it will only
+ // look at tables that have a value for the parameter key1.
+ // Some example filter statements include:
+ // filter = Constants.HIVE_FILTER_FIELD_OWNER + " like \".*test.*\" and " +
+ // Constants.HIVE_FILTER_FIELD_LAST_ACCESS + " = 0";
+ // filter = Constants.HIVE_FILTER_FIELD_PARAMS + "retention = \"30\" or " +
+ // Constants.HIVE_FILTER_FIELD_PARAMS + "retention = \"90\""
+ // @param dbName
+ // The name of the database from which you will retrieve the table names
+ // @param filterType
+ // The type of filter
+ // @param filter
+ // The filter string
+ // @param max_tables
+ // The maximum number of tables returned
+ // @return A list of table names that match the desired filter
+ list<string> get_table_names_by_filter(1:string dbname, 2:string filter, 3:i16 max_tables=-1)
+ throws (1:MetaException o1, 2:InvalidOperationException o2, 3:UnknownDBException o3)
+
+ // alter table applies to only future partitions not for existing partitions
+ // * See notes on DDL_TIME
+ void alter_table(1:string dbname, 2:string tbl_name, 3:Table new_tbl)
+ throws (1:InvalidOperationException o1, 2:MetaException o2)
+ void alter_table_with_environment_context(1:string dbname, 2:string tbl_name,
+ 3:Table new_tbl, 4:EnvironmentContext environment_context)
+ throws (1:InvalidOperationException o1, 2:MetaException o2)
+ // alter table not only applies to future partitions but also cascade to existing partitions
+ void alter_table_with_cascade(1:string dbname, 2:string tbl_name, 3:Table new_tbl, 4:bool cascade)
+ throws (1:InvalidOperationException o1, 2:MetaException o2)
+ // the following applies to only tables that have partitions
+ // * See notes on DDL_TIME
+ Partition add_partition(1:Partition new_part)
+ throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+ Partition add_partition_with_environment_context(1:Partition new_part,
+ 2:EnvironmentContext environment_context)
+ throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2,
+ 3:MetaException o3)
+ i32 add_partitions(1:list<Partition> new_parts)
+ throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+ i32 add_partitions_pspec(1:list<PartitionSpec> new_parts)
+ throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+ Partition append_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals)
+ throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+ AddPartitionsResult add_partitions_req(1:AddPartitionsRequest request)
+ throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+ Partition append_partition_with_environment_context(1:string db_name, 2:string tbl_name,
+ 3:list<string> part_vals, 4:EnvironmentContext environment_context)
+ throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+ Partition append_partition_by_name(1:string db_name, 2:string tbl_name, 3:string part_name)
+ throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+ Partition append_partition_by_name_with_environment_context(1:string db_name, 2:string tbl_name,
+ 3:string part_name, 4:EnvironmentContext environment_context)
+ throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+ bool drop_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals, 4:bool deleteData)
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ bool drop_partition_with_environment_context(1:string db_name, 2:string tbl_name,
+ 3:list<string> part_vals, 4:bool deleteData, 5:EnvironmentContext environment_context)
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ bool drop_partition_by_name(1:string db_name, 2:string tbl_name, 3:string part_name, 4:bool deleteData)
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ bool drop_partition_by_name_with_environment_context(1:string db_name, 2:string tbl_name,
+ 3:string part_name, 4:bool deleteData, 5:EnvironmentContext environment_context)
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ DropPartitionsResult drop_partitions_req(1: DropPartitionsRequest req)
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
+
+ Partition get_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
+ Partition exchange_partition(1:map<string, string> partitionSpecs, 2:string source_db,
+ 3:string source_table_name, 4:string dest_db, 5:string dest_table_name)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2, 3:InvalidObjectException o3,
+ 4:InvalidInputException o4)
+
+ list<Partition> exchange_partitions(1:map<string, string> partitionSpecs, 2:string source_db,
+ 3:string source_table_name, 4:string dest_db, 5:string dest_table_name)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2, 3:InvalidObjectException o3,
+ 4:InvalidInputException o4)
+
+ Partition get_partition_with_auth(1:string db_name, 2:string tbl_name, 3:list<string> part_vals,
+ 4: string user_name, 5: list<string> group_names) throws(1:MetaException o1, 2:NoSuchObjectException o2)
+
+ Partition get_partition_by_name(1:string db_name 2:string tbl_name, 3:string part_name)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
+
+ // returns all the partitions for this table in reverse chronological order.
+ // If max parts is given then it will return only that many.
+ list<Partition> get_partitions(1:string db_name, 2:string tbl_name, 3:i16 max_parts=-1)
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ list<Partition> get_partitions_with_auth(1:string db_name, 2:string tbl_name, 3:i16 max_parts=-1,
+ 4: string user_name, 5: list<string> group_names) throws(1:NoSuchObjectException o1, 2:MetaException o2)
+
+ list<PartitionSpec> get_partitions_pspec(1:string db_name, 2:string tbl_name, 3:i32 max_parts=-1)
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
+
+ list<string> get_partition_names(1:string db_name, 2:string tbl_name, 3:i16 max_parts=-1)
+ throws(1:MetaException o2)
+
+ // get_partition*_ps methods allow filtering by a partial partition specification,
+ // as needed for dynamic partitions. The values that are not restricted should
+ // be empty strings. Nulls were considered (instead of "") but caused errors in
+ // generated Python code. The size of part_vals may be smaller than the
+ // number of partition columns - the unspecified values are considered the same
+ // as "".
+ list<Partition> get_partitions_ps(1:string db_name 2:string tbl_name
+ 3:list<string> part_vals, 4:i16 max_parts=-1)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
+ list<Partition> get_partitions_ps_with_auth(1:string db_name, 2:string tbl_name, 3:list<string> part_vals, 4:i16 max_parts=-1,
+ 5: string user_name, 6: list<string> group_names) throws(1:NoSuchObjectException o1, 2:MetaException o2)
+
+ list<string> get_partition_names_ps(1:string db_name,
+ 2:string tbl_name, 3:list<string> part_vals, 4:i16 max_parts=-1)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
+
+ // get the partitions matching the given partition filter
+ list<Partition> get_partitions_by_filter(1:string db_name 2:string tbl_name
+ 3:string filter, 4:i16 max_parts=-1)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
+
+ // List partitions as PartitionSpec instances.
+ list<PartitionSpec> get_part_specs_by_filter(1:string db_name 2:string tbl_name
+ 3:string filter, 4:i32 max_parts=-1)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
+
+ // get the partitions matching the given partition filter
+ // unlike get_partitions_by_filter, takes serialized hive expression, and with that can work
+ // with any filter (get_partitions_by_filter only works if the filter can be pushed down to JDOQL.
+ PartitionsByExprResult get_partitions_by_expr(1:PartitionsByExprRequest req)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
+
+ // get the partitions matching the given partition filter
+ i32 get_num_partitions_by_filter(1:string db_name 2:string tbl_name 3:string filter)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
+
+ // get partitions give a list of partition names
+ list<Partition> get_partitions_by_names(1:string db_name 2:string tbl_name 3:list<string> names)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
+
+ // changes the partition to the new partition object. partition is identified from the part values
+ // in the new_part
+ // * See notes on DDL_TIME
+ void alter_partition(1:string db_name, 2:string tbl_name, 3:Partition new_part)
+ throws (1:InvalidOperationException o1, 2:MetaException o2)
+
+ // change a list of partitions. All partitions are altered atomically and all
+ // prehooks are fired together followed by all post hooks
+ void alter_partitions(1:string db_name, 2:string tbl_name, 3:list<Partition> new_parts)
+ throws (1:InvalidOperationException o1, 2:MetaException o2)
+ void alter_partitions_with_environment_context(1:string db_name, 2:string tbl_name, 3:list<Partition> new_parts, 4:EnvironmentContext environment_context) throws (1:InvalidOperationException o1, 2:MetaException o2)
+
+ void alter_partition_with_environment_context(1:string db_name,
+ 2:string tbl_name, 3:Partition new_part,
+ 4:EnvironmentContext environment_context)
+ throws (1:InvalidOperationException o1, 2:MetaException o2)
+
+ // rename the old partition to the new partition object by changing old part values to the part values
+ // in the new_part. old partition is identified from part_vals.
+ // partition keys in new_part should be the same as those in old partition.
+ void rename_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals, 4:Partition new_part)
+ throws (1:InvalidOperationException o1, 2:MetaException o2)
+
+ // returns whether or not the partition name is valid based on the value of the config
+ // hive.metastore.partition.name.whitelist.pattern
+ bool partition_name_has_valid_characters(1:list<string> part_vals, 2:bool throw_exception)
+ throws(1: MetaException o1)
+
+ // gets the value of the configuration key in the metastore server. returns
+ // defaultValue if the key does not exist. if the configuration key does not
+ // begin with "hive", "mapred", or "hdfs", a ConfigValSecurityException is
+ // thrown.
+ string get_config_value(1:string name, 2:string defaultValue)
+ throws(1:ConfigValSecurityException o1)
+
+ // converts a partition name into a partition values array
+ list<string> partition_name_to_vals(1: string part_name)
+ throws(1: MetaException o1)
+ // converts a partition name into a partition specification (a mapping from
+ // the partition cols to the values)
+ map<string, string> partition_name_to_spec(1: string part_name)
+ throws(1: MetaException o1)
+
+ void markPartitionForEvent(1:string db_name, 2:string tbl_name, 3:map<string,string> part_vals,
+ 4:PartitionEventType eventType) throws (1: MetaException o1, 2: NoSuchObjectException o2,
+ 3: UnknownDBException o3, 4: UnknownTableException o4, 5: UnknownPartitionException o5,
+ 6: InvalidPartitionException o6)
+ bool isPartitionMarkedForEvent(1:string db_name, 2:string tbl_name, 3:map<string,string> part_vals,
+ 4: PartitionEventType eventType) throws (1: MetaException o1, 2:NoSuchObjectException o2,
+ 3: UnknownDBException o3, 4: UnknownTableException o4, 5: UnknownPartitionException o5,
+ 6: InvalidPartitionException o6)
+
+ //index
+ Index add_index(1:Index new_index, 2: Table index_table)
+ throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+ void alter_index(1:string dbname, 2:string base_tbl_name, 3:string idx_name, 4:Index new_idx)
+ throws (1:InvalidOperationException o1, 2:MetaException o2)
+ bool drop_index_by_name(1:string db_name, 2:string tbl_name, 3:string index_name, 4:bool deleteData)
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ Index get_index_by_name(1:string db_name 2:string tbl_name, 3:string index_name)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
+
+ list<Index> get_indexes(1:string db_name, 2:string tbl_name, 3:i16 max_indexes=-1)
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ list<string> get_index_names(1:string db_name, 2:string tbl_name, 3:i16 max_indexes=-1)
+ throws(1:MetaException o2)
+
+ //primary keys and foreign keys
+ PrimaryKeysResponse get_primary_keys(1:PrimaryKeysRequest request)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
+ ForeignKeysResponse get_foreign_keys(1:ForeignKeysRequest request)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
+ // other constraints
+ UniqueConstraintsResponse get_unique_constraints(1:UniqueConstraintsRequest request)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
+ NotNullConstraintsResponse get_not_null_constraints(1:NotNullConstraintsRequest request)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
+
+ // column statistics interfaces
+
+ // update APIs persist the column statistics object(s) that are passed in. If statistics already
+ // exists for one or more columns, the existing statistics will be overwritten. The update APIs
+ // validate that the dbName, tableName, partName, colName[] passed in as part of the ColumnStatistics
+ // struct are valid, throws InvalidInputException/NoSuchObjectException if found to be invalid
+ bool update_table_column_statistics(1:ColumnStatistics stats_obj) throws (1:NoSuchObjectException o1,
+ 2:InvalidObjectException o2, 3:MetaException o3, 4:InvalidInputException o4)
+ bool update_partition_column_statistics(1:ColumnStatistics stats_obj) throws (1:NoSuchObjectException o1,
+ 2:InvalidObjectException o2, 3:MetaException o3, 4:InvalidInputException o4)
+
+ // get APIs return the column statistics corresponding to db_name, tbl_name, [part_name], col_name if
+ // such statistics exists. If the required statistics doesn't exist, get APIs throw NoSuchObjectException
+ // For instance, if get_table_column_statistics is called on a partitioned table for which only
+ // partition level column stats exist, get_table_column_statistics will throw NoSuchObjectException
+ ColumnStatistics get_table_column_statistics(1:string db_name, 2:string tbl_name, 3:string col_name) throws
+ (1:NoSuchObjectException o1, 2:MetaException o2, 3:InvalidInputException o3, 4:InvalidObjectException o4)
+ ColumnStatistics get_partition_column_statistics(1:string db_name, 2:string tbl_name, 3:string part_name,
+ 4:string col_name) throws (1:NoSuchObjectException o1, 2:MetaException o2,
+ 3:InvalidInputException o3, 4:InvalidObjectException o4)
+ TableStatsResult get_table_statistics_req(1:TableStatsRequest request) throws
+ (1:NoSuchObjectException o1, 2:MetaException o2)
+ PartitionsStatsResult get_partitions_statistics_req(1:PartitionsStatsRequest request) throws
+ (1:NoSuchObjectException o1, 2:MetaException o2)
+ AggrStats get_aggr_stats_for(1:PartitionsStatsRequest request) throws
+ (1:NoSuchObjectException o1, 2:MetaException o2)
+ bool set_aggr_stats_for(1:SetPartitionsStatsRequest request) throws
+ (1:NoSuchObjectException o1, 2:InvalidObjectException o2, 3:MetaException o3, 4:InvalidInputException o4)
+
+
+ // delete APIs attempt to delete column statistics, if found, associated with a given db_name, tbl_name, [part_name]
+ // and col_name. If the delete API doesn't find the statistics record in the metastore, throws NoSuchObjectException
+ // Delete API validates the input and if the input is invalid throws InvalidInputException/InvalidObjectException.
+ bool delete_partition_column_statistics(1:string db_name, 2:string tbl_name, 3:string part_name, 4:string col_name) throws
+ (1:NoSuchObjectException o1, 2:MetaException o2, 3:InvalidObjectException o3,
+ 4:InvalidInputException o4)
+ bool delete_table_column_statistics(1:string db_name, 2:string tbl_name, 3:string col_name) throws
+ (1:NoSuchObjectException o1, 2:MetaException o2, 3:InvalidObjectException o3,
+ 4:InvalidInputException o4)
+
+ //
+ // user-defined functions
+ //
+
+ void create_function(1:Function func)
+ throws (1:AlreadyExistsException o1,
+ 2:InvalidObjectException o2,
+ 3:MetaException o3,
+ 4:NoSuchObjectException o4)
+
+ void drop_function(1:string dbName, 2:string funcName)
+ throws (1:NoSuchObjectException o1, 2:MetaException o3)
+
+ void alter_function(1:string dbName, 2:string funcName, 3:Function newFunc)
+ throws (1:InvalidOperationException o1, 2:MetaException o2)
+
+ list<string> get_functions(1:string dbName, 2:string pattern)
+ throws (1:MetaException o1)
+ Function get_function(1:string dbName, 2:string funcName)
+ throws (1:MetaException o1, 2:NoSuchObjectException o2)
+
+ GetAllFunctionsResponse get_all_functions() throws (1:MetaException o1)
+
+ //authorization privileges
+
+ bool create_role(1:Role role) throws(1:MetaException o1)
+ bool drop_role(1:string role_name) throws(1:MetaException o1)
+ list<string> get_role_names() throws(1:MetaException o1)
+ // Deprecated, use grant_revoke_role()
+ bool grant_role(1:string role_name, 2:string principal_name, 3:PrincipalType principal_type,
+ 4:string grantor, 5:PrincipalType grantorType, 6:bool grant_option) throws(1:MetaException o1)
+ // Deprecated, use grant_revoke_role()
+ bool revoke_role(1:string role_name, 2:string principal_name, 3:PrincipalType principal_type)
+ throws(1:MetaException o1)
+ list<Role> list_roles(1:string principal_name, 2:PrincipalType principal_type) throws(1:MetaException o1)
+ GrantRevokeRoleResponse grant_revoke_role(1:GrantRevokeRoleRequest request) throws(1:MetaException o1)
+
+ // get all role-grants for users/roles that have been granted the given role
+ // Note that in the returned list of RolePrincipalGrants, the roleName is
+ // redundant as it would match the role_name argument of this function
+ GetPrincipalsInRoleResponse get_principals_in_role(1: GetPrincipalsInRoleRequest request) throws(1:MetaException o1)
+
+ // get grant information of all roles granted to the given principal
+ // Note that in the returned list of RolePrincipalGrants, the principal name,type is
+ // redundant as it would match the principal name,type arguments of this function
+ GetRoleGrantsForPrincipalResponse get_role_grants_for_principal(1: GetRoleGrantsForPrincipalRequest request) throws(1:MetaException o1)
+
+ PrincipalPrivilegeSet get_privilege_set(1:HiveObjectRef hiveObject, 2:string user_name,
+ 3: list<string> group_names) throws(1:MetaException o1)
+ list<HiveObjectPrivilege> list_privileges(1:string principal_name, 2:PrincipalType principal_type,
+ 3: HiveObjectRef hiveObject) throws(1:MetaException o1)
+
+ // Deprecated, use grant_revoke_privileges()
+ bool grant_privileges(1:PrivilegeBag privileges) throws(1:MetaException o1)
+ // Deprecated, use grant_revoke_privileges()
+ bool revoke_privileges(1:PrivilegeBag privileges) throws(1:MetaException o1)
+ GrantRevokePrivilegeResponse grant_revoke_privileges(1:GrantRevokePrivilegeRequest request) throws(1:MetaException o1);
+
+ // this is used by metastore client to send UGI information to metastore server immediately
+ // after setting up a connection.
+ list<string> set_ugi(1:string user_name, 2:list<string> group_names) throws (1:MetaException o1)
+
+ //Authentication (delegation token) interfaces
+
+ // get metastore server delegation token for use from the map/reduce tasks to authenticate
+ // to metastore server
+ string get_delegation_token(1:string token_owner, 2:string renewer_kerberos_principal_name)
+ throws (1:MetaException o1)
+
+ // method to renew delegation token obtained from metastore server
+ i64 renew_delegation_token(1:string token_str_form) throws (1:MetaException o1)
+
+ // method to cancel delegation token obtained from metastore server
+ void cancel_delegation_token(1:string token_str_form) throws (1:MetaException o1)
+
+ // add a delegation token
+ bool add_token(1:string token_identifier, 2:string delegation_token)
+
+ // remove a delegation token
+ bool remove_token(1:string token_identifier)
+
+ // get a delegation token by identifier
+ string get_token(1:string token_identifier)
+
+ // get all delegation token identifiers
+ list<string> get_all_token_identifiers()
+
+ // add master key
+ i32 add_master_key(1:string key) throws (1:MetaException o1)
+
+ // update master key
+ void update_master_key(1:i32 seq_number, 2:string key) throws (1:NoSuchObjectException o1, 2:MetaException o2)
+
+ // remove master key
+ bool remove_master_key(1:i32 key_seq)
+
+ // get master keys
+ list<string> get_master_keys()
+
+ // Transaction and lock management calls
+ // Get just list of open transactions
+ GetOpenTxnsResponse get_open_txns()
+ // Get list of open transactions with state (open, aborted)
+ GetOpenTxnsInfoResponse get_open_txns_info()
+ OpenTxnsResponse open_txns(1:OpenTxnRequest rqst)
+ void abort_txn(1:AbortTxnRequest rqst) throws (1:NoSuchTxnException o1)
+ void abort_txns(1:AbortTxnsRequest rqst) throws (1:NoSuchTxnException o1)
+ void commit_txn(1:CommitTxnRequest rqst) throws (1:NoSuchTxnException o1, 2:TxnAbortedException o2)
+ LockResponse lock(1:LockRequest rqst) throws (1:NoSuchTxnException o1, 2:TxnAbortedException o2)
+ LockResponse check_lock(1:CheckLockRequest rqst)
+ throws (1:NoSuchTxnException o1, 2:TxnAbortedException o2, 3:NoSuchLockException o3)
+ void unlock(1:UnlockRequest rqst) throws (1:NoSuchLockException o1, 2:TxnOpenException o2)
+ ShowLocksResponse show_locks(1:ShowLocksRequest rqst)
+ void heartbeat(1:HeartbeatRequest ids) throws (1:NoSuchLockException o1, 2:NoSuchTxnException o2, 3:TxnAbortedException o3)
+ HeartbeatTxnRangeResponse heartbeat_txn_range(1:HeartbeatTxnRangeRequest txns)
+ void compact(1:CompactionRequest rqst)
+ CompactionResponse compact2(1:CompactionRequest rqst)
+ ShowCompactResponse show_compact(1:ShowCompactRequest rqst)
+ void add_dynamic_partitions(1:AddDynamicPartitions rqst) throws (1:NoSuchTxnException o1, 2:TxnAbortedException o2)
+
+ // Notification logging calls
+ NotificationEventResponse get_next_notification(1:NotificationEventRequest rqst)
+ CurrentNotificationEventId get_current_notificationEventId()
+ NotificationEventsCountResponse get_notification_events_count(NotificationEventsCountRequest rqst)
+ FireEventResponse fire_listener_event(1:FireEventRequest rqst)
+ void flushCache()
+
+ // Repl Change Management api
+ CmRecycleResponse cm_recycle(1:CmRecycleRequest request) throws(1:MetaException o1)
+
+ GetFileMetadataByExprResult get_file_metadata_by_expr(1:GetFileMetadataByExprRequest req)
+ GetFileMetadataResult get_file_metadata(1:GetFileMetadataRequest req)
+ PutFileMetadataResult put_file_metadata(1:PutFileMetadataRequest req)
+ ClearFileMetadataResult clear_file_metadata(1:ClearFileMetadataRequest req)
+ CacheFileMetadataResult cache_file_metadata(1:CacheFileMetadataRequest req)
+
+ // Metastore DB properties
+ string get_metastore_db_uuid() throws (1:MetaException o1)
+ }
+
+ // * Note about the DDL_TIME: When creating or altering a table or a partition,
+ // if the DDL_TIME is not set, the current time will be used.
+
+ // For storing info about archived partitions in parameters
+
+ // Whether the partition is archived
+ const string IS_ARCHIVED = "is_archived",
+ // The original location of the partition, before archiving. After archiving,
+ // this directory will contain the archive. When the partition
+ // is dropped, this directory will be deleted
+ const string ORIGINAL_LOCATION = "original_location",
+
+ // Whether or not the table is considered immutable - immutable tables can only be
+ // overwritten or created if unpartitioned, or if partitioned, partitions inside them
+ // can only be overwritten or created. Immutability supports write-once and replace
+ // semantics, but not append.
+ const string IS_IMMUTABLE = "immutable",
+
+ // these should be needed only for backward compatibility with filestore
+ const string META_TABLE_COLUMNS = "columns",
+ const string META_TABLE_COLUMN_TYPES = "columns.types",
+ const string BUCKET_FIELD_NAME = "bucket_field_name",
+ const string BUCKET_COUNT = "bucket_count",
+ const string FIELD_TO_DIMENSION = "field_to_dimension",
+ const string META_TABLE_NAME = "name",
+ const string META_TABLE_DB = "db",
+ const string META_TABLE_LOCATION = "location",
+ const string META_TABLE_SERDE = "serde",
+ const string META_TABLE_PARTITION_COLUMNS = "partition_columns",
+ const string META_TABLE_PARTITION_COLUMN_TYPES = "partition_columns.types",
+ const string FILE_INPUT_FORMAT = "file.inputformat",
+ const string FILE_OUTPUT_FORMAT = "file.outputformat",
+ const string META_TABLE_STORAGE = "storage_handler",
+ const string TABLE_IS_TRANSACTIONAL = "transactional",
+ const string TABLE_NO_AUTO_COMPACT = "no_auto_compaction",
+ const string TABLE_TRANSACTIONAL_PROPERTIES = "transactional_properties",
+
+
[09/50] [abbrv] hive git commit: HIVE-16949: Leak of threads from
Get-Input-Paths and Get-Input-Summary thread pool (Sahil Takiar,
reviewed by Vihang Karajgaonkar)
Posted by we...@apache.org.
HIVE-16949: Leak of threads from Get-Input-Paths and Get-Input-Summary thread pool (Sahil Takiar, reviewed by Vihang Karajgaonkar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3fc131c7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3fc131c7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3fc131c7
Branch: refs/heads/hive-14535
Commit: 3fc131c79a5329ea509f0c125e789ddd042b1797
Parents: 6be50b7
Author: Sahil Takiar <ta...@gmail.com>
Authored: Mon Aug 28 11:19:41 2017 -0700
Committer: Sahil Takiar <st...@cloudera.com>
Committed: Mon Aug 28 11:19:41 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/exec/Utilities.java | 305 ++++++++++---------
.../hadoop/hive/ql/exec/TestUtilities.java | 99 ++++++
2 files changed, 265 insertions(+), 139 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3fc131c7/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 99c85f2..aca99f2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -2233,153 +2233,164 @@ public final class Utilities {
// Process the case when name node call is needed
final Map<String, ContentSummary> resultMap = new ConcurrentHashMap<String, ContentSummary>();
- ArrayList<Future<?>> results = new ArrayList<Future<?>>();
final ExecutorService executor;
int numExecutors = getMaxExecutorsForInputListing(ctx.getConf(), pathNeedProcess.size());
if (numExecutors > 1) {
LOG.info("Using " + numExecutors + " threads for getContentSummary");
executor = Executors.newFixedThreadPool(numExecutors,
- new ThreadFactoryBuilder().setDaemon(true)
- .setNameFormat("Get-Input-Summary-%d").build());
+ new ThreadFactoryBuilder().setDaemon(true)
+ .setNameFormat("Get-Input-Summary-%d").build());
} else {
executor = null;
}
+ ContentSummary cs = getInputSummaryWithPool(ctx, pathNeedProcess, work, summary, executor);
+ perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.INPUT_SUMMARY);
+ return cs;
+ }
+ }
- HiveInterruptCallback interrup = HiveInterruptUtils.add(new HiveInterruptCallback() {
- @Override
- public void interrupt() {
- for (Path path : pathNeedProcess) {
- try {
- path.getFileSystem(ctx.getConf()).close();
- } catch (IOException ignore) {
- LOG.debug("Failed to close filesystem", ignore);
- }
- }
- if (executor != null) {
- executor.shutdownNow();
+ @VisibleForTesting
+ static ContentSummary getInputSummaryWithPool(final Context ctx, Set<Path> pathNeedProcess, MapWork work,
+ long[] summary, ExecutorService executor) throws IOException {
+ List<Future<?>> results = new ArrayList<Future<?>>();
+ final Map<String, ContentSummary> resultMap = new ConcurrentHashMap<String, ContentSummary>();
+
+ HiveInterruptCallback interrup = HiveInterruptUtils.add(new HiveInterruptCallback() {
+ @Override
+ public void interrupt() {
+ for (Path path : pathNeedProcess) {
+ try {
+ path.getFileSystem(ctx.getConf()).close();
+ } catch (IOException ignore) {
+ LOG.debug("Failed to close filesystem", ignore);
}
}
- });
- try {
- Configuration conf = ctx.getConf();
- JobConf jobConf = new JobConf(conf);
- for (Path path : pathNeedProcess) {
- final Path p = path;
- final String pathStr = path.toString();
- // All threads share the same Configuration and JobConf based on the
- // assumption that they are thread safe if only read operations are
- // executed. It is not stated in Hadoop's javadoc, the sourcce codes
- // clearly showed that they made efforts for it and we believe it is
- // thread safe. Will revisit this piece of codes if we find the assumption
- // is not correct.
- final Configuration myConf = conf;
- final JobConf myJobConf = jobConf;
- final Map<String, Operator<?>> aliasToWork = work.getAliasToWork();
- final Map<Path, ArrayList<String>> pathToAlias = work.getPathToAliases();
- final PartitionDesc partDesc = work.getPathToPartitionInfo().get(p);
- Runnable r = new Runnable() {
- @Override
- public void run() {
- try {
- Class<? extends InputFormat> inputFormatCls = partDesc
- .getInputFileFormatClass();
- InputFormat inputFormatObj = HiveInputFormat.getInputFormatFromCache(
- inputFormatCls, myJobConf);
- if (inputFormatObj instanceof ContentSummaryInputFormat) {
- ContentSummaryInputFormat cs = (ContentSummaryInputFormat) inputFormatObj;
- resultMap.put(pathStr, cs.getContentSummary(p, myJobConf));
- return;
- }
+ if (executor != null) {
+ executor.shutdownNow();
+ }
+ }
+ });
+ try {
+ Configuration conf = ctx.getConf();
+ JobConf jobConf = new JobConf(conf);
+ for (Path path : pathNeedProcess) {
+ final Path p = path;
+ final String pathStr = path.toString();
+ // All threads share the same Configuration and JobConf based on the
+ // assumption that they are thread safe if only read operations are
+ // executed. It is not stated in Hadoop's javadoc, the sourcce codes
+ // clearly showed that they made efforts for it and we believe it is
+ // thread safe. Will revisit this piece of codes if we find the assumption
+ // is not correct.
+ final Configuration myConf = conf;
+ final JobConf myJobConf = jobConf;
+ final Map<String, Operator<?>> aliasToWork = work.getAliasToWork();
+ final Map<Path, ArrayList<String>> pathToAlias = work.getPathToAliases();
+ final PartitionDesc partDesc = work.getPathToPartitionInfo().get(p);
+ Runnable r = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ Class<? extends InputFormat> inputFormatCls = partDesc
+ .getInputFileFormatClass();
+ InputFormat inputFormatObj = HiveInputFormat.getInputFormatFromCache(
+ inputFormatCls, myJobConf);
+ if (inputFormatObj instanceof ContentSummaryInputFormat) {
+ ContentSummaryInputFormat cs = (ContentSummaryInputFormat) inputFormatObj;
+ resultMap.put(pathStr, cs.getContentSummary(p, myJobConf));
+ return;
+ }
- String metaTableStorage = null;
- if (partDesc.getTableDesc() != null &&
- partDesc.getTableDesc().getProperties() != null) {
- metaTableStorage = partDesc.getTableDesc().getProperties()
- .getProperty(hive_metastoreConstants.META_TABLE_STORAGE, null);
- }
- if (partDesc.getProperties() != null) {
- metaTableStorage = partDesc.getProperties()
- .getProperty(hive_metastoreConstants.META_TABLE_STORAGE, metaTableStorage);
- }
+ String metaTableStorage = null;
+ if (partDesc.getTableDesc() != null &&
+ partDesc.getTableDesc().getProperties() != null) {
+ metaTableStorage = partDesc.getTableDesc().getProperties()
+ .getProperty(hive_metastoreConstants.META_TABLE_STORAGE, null);
+ }
+ if (partDesc.getProperties() != null) {
+ metaTableStorage = partDesc.getProperties()
+ .getProperty(hive_metastoreConstants.META_TABLE_STORAGE, metaTableStorage);
+ }
- HiveStorageHandler handler = HiveUtils.getStorageHandler(myConf, metaTableStorage);
- if (handler instanceof InputEstimator) {
- long total = 0;
- TableDesc tableDesc = partDesc.getTableDesc();
- InputEstimator estimator = (InputEstimator) handler;
- for (String alias : HiveFileFormatUtils.doGetAliasesFromPath(pathToAlias, p)) {
- JobConf jobConf = new JobConf(myJobConf);
- TableScanOperator scanOp = (TableScanOperator) aliasToWork.get(alias);
- Utilities.setColumnNameList(jobConf, scanOp, true);
- Utilities.setColumnTypeList(jobConf, scanOp, true);
- PlanUtils.configureInputJobPropertiesForStorageHandler(tableDesc);
- Utilities.copyTableJobPropertiesToConf(tableDesc, jobConf);
- total += estimator.estimate(jobConf, scanOp, -1).getTotalLength();
- }
- resultMap.put(pathStr, new ContentSummary(total, -1, -1));
- } else {
- // todo: should nullify summary for non-native tables,
- // not to be selected as a mapjoin target
- FileSystem fs = p.getFileSystem(myConf);
- resultMap.put(pathStr, fs.getContentSummary(p));
+ HiveStorageHandler handler = HiveUtils.getStorageHandler(myConf, metaTableStorage);
+ if (handler instanceof InputEstimator) {
+ long total = 0;
+ TableDesc tableDesc = partDesc.getTableDesc();
+ InputEstimator estimator = (InputEstimator) handler;
+ for (String alias : HiveFileFormatUtils.doGetAliasesFromPath(pathToAlias, p)) {
+ JobConf jobConf = new JobConf(myJobConf);
+ TableScanOperator scanOp = (TableScanOperator) aliasToWork.get(alias);
+ Utilities.setColumnNameList(jobConf, scanOp, true);
+ Utilities.setColumnTypeList(jobConf, scanOp, true);
+ PlanUtils.configureInputJobPropertiesForStorageHandler(tableDesc);
+ Utilities.copyTableJobPropertiesToConf(tableDesc, jobConf);
+ total += estimator.estimate(jobConf, scanOp, -1).getTotalLength();
}
- } catch (Exception e) {
- // We safely ignore this exception for summary data.
- // We don't update the cache to protect it from polluting other
- // usages. The worst case is that IOException will always be
- // retried for another getInputSummary(), which is fine as
- // IOException is not considered as a common case.
- LOG.info("Cannot get size of " + pathStr + ". Safely ignored.");
+ resultMap.put(pathStr, new ContentSummary(total, -1, -1));
+ } else {
+ // todo: should nullify summary for non-native tables,
+ // not to be selected as a mapjoin target
+ FileSystem fs = p.getFileSystem(myConf);
+ resultMap.put(pathStr, fs.getContentSummary(p));
}
+ } catch (Exception e) {
+ // We safely ignore this exception for summary data.
+ // We don't update the cache to protect it from polluting other
+ // usages. The worst case is that IOException will always be
+ // retried for another getInputSummary(), which is fine as
+ // IOException is not considered as a common case.
+ LOG.info("Cannot get size of " + pathStr + ". Safely ignored.");
}
- };
-
- if (executor == null) {
- r.run();
- } else {
- Future<?> result = executor.submit(r);
- results.add(result);
}
+ };
+
+ if (executor == null) {
+ r.run();
+ } else {
+ Future<?> result = executor.submit(r);
+ results.add(result);
}
+ }
- if (executor != null) {
- for (Future<?> result : results) {
- boolean executorDone = false;
- do {
- try {
- result.get();
- executorDone = true;
- } catch (InterruptedException e) {
- LOG.info("Interrupted when waiting threads: ", e);
- Thread.currentThread().interrupt();
- break;
- } catch (ExecutionException e) {
- throw new IOException(e);
- }
- } while (!executorDone);
- }
- executor.shutdown();
+ if (executor != null) {
+ for (Future<?> result : results) {
+ boolean executorDone = false;
+ do {
+ try {
+ result.get();
+ executorDone = true;
+ } catch (InterruptedException e) {
+ LOG.info("Interrupted when waiting threads: ", e);
+ Thread.currentThread().interrupt();
+ break;
+ } catch (ExecutionException e) {
+ throw new IOException(e);
+ }
+ } while (!executorDone);
}
- HiveInterruptUtils.checkInterrupted();
- for (Map.Entry<String, ContentSummary> entry : resultMap.entrySet()) {
- ContentSummary cs = entry.getValue();
+ executor.shutdown();
+ }
+ HiveInterruptUtils.checkInterrupted();
+ for (Map.Entry<String, ContentSummary> entry : resultMap.entrySet()) {
+ ContentSummary cs = entry.getValue();
- summary[0] += cs.getLength();
- summary[1] += cs.getFileCount();
- summary[2] += cs.getDirectoryCount();
+ summary[0] += cs.getLength();
+ summary[1] += cs.getFileCount();
+ summary[2] += cs.getDirectoryCount();
- ctx.addCS(entry.getKey(), cs);
- LOG.info("Cache Content Summary for " + entry.getKey() + " length: " + cs.getLength()
- + " file count: "
- + cs.getFileCount() + " directory count: " + cs.getDirectoryCount());
- }
+ ctx.addCS(entry.getKey(), cs);
+ LOG.info("Cache Content Summary for " + entry.getKey() + " length: " + cs.getLength()
+ + " file count: "
+ + cs.getFileCount() + " directory count: " + cs.getDirectoryCount());
+ }
- perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.INPUT_SUMMARY);
- return new ContentSummary(summary[0], summary[1], summary[2]);
- } finally {
- HiveInterruptUtils.remove(interrup);
+ return new ContentSummary(summary[0], summary[1], summary[2]);
+ } finally {
+ if (executor != null) {
+ executor.shutdownNow();
}
+ HiveInterruptUtils.remove(interrup);
}
}
@@ -3144,7 +3155,7 @@ public final class Utilities {
// Note: this copies the list because createDummyFileForEmptyPartition may modify the map.
for (Path file : new LinkedList<Path>(work.getPathToAliases().keySet())) {
if (lDrvStat != null && lDrvStat.driverState == DriverState.INTERRUPT)
- throw new IOException("Operation is Canceled. ");
+ throw new IOException("Operation is Canceled.");
List<String> aliases = work.getPathToAliases().get(file);
if (aliases.contains(alias)) {
@@ -3188,41 +3199,57 @@ public final class Utilities {
}
}
- ExecutorService pool = null;
+ List<Path> finalPathsToAdd = new LinkedList<>();
+
int numExecutors = getMaxExecutorsForInputListing(job, pathsToAdd.size());
if (numExecutors > 1) {
- pool = Executors.newFixedThreadPool(numExecutors,
- new ThreadFactoryBuilder().setDaemon(true).setNameFormat("Get-Input-Paths-%d").build());
- }
+ ExecutorService pool = Executors.newFixedThreadPool(numExecutors,
+ new ThreadFactoryBuilder().setDaemon(true).setNameFormat("Get-Input-Paths-%d").build());
- List<Path> finalPathsToAdd = new LinkedList<>();
- Map<GetInputPathsCallable, Future<Path>> getPathsCallableToFuture = new LinkedHashMap<>();
- for (final Path path : pathsToAdd) {
- if (lDrvStat != null && lDrvStat.driverState == DriverState.INTERRUPT) {
- throw new IOException("Operation is Canceled. ");
- }
- if (pool == null) {
+ finalPathsToAdd.addAll(getInputPathsWithPool(job, work, hiveScratchDir, ctx, skipDummy, pathsToAdd, pool));
+ } else {
+ for (final Path path : pathsToAdd) {
+ if (lDrvStat != null && lDrvStat.driverState == DriverState.INTERRUPT) {
+ throw new IOException("Operation is Canceled.");
+ }
Path newPath = new GetInputPathsCallable(path, job, work, hiveScratchDir, ctx, skipDummy).call();
updatePathForMapWork(newPath, work, path);
finalPathsToAdd.add(newPath);
- } else {
+ }
+ }
+
+ return finalPathsToAdd;
+ }
+
+ @VisibleForTesting
+ static List<Path> getInputPathsWithPool(JobConf job, MapWork work, Path hiveScratchDir,
+ Context ctx, boolean skipDummy, List<Path> pathsToAdd,
+ ExecutorService pool) throws IOException, ExecutionException, InterruptedException {
+ LockedDriverState lDrvStat = LockedDriverState.getLockedDriverState();
+ List<Path> finalPathsToAdd = new ArrayList<>();
+ try {
+ Map<GetInputPathsCallable, Future<Path>> getPathsCallableToFuture = new LinkedHashMap<>();
+ for (final Path path : pathsToAdd) {
+ if (lDrvStat != null && lDrvStat.driverState == DriverState.INTERRUPT) {
+ throw new IOException("Operation is Canceled.");
+ }
GetInputPathsCallable callable = new GetInputPathsCallable(path, job, work, hiveScratchDir, ctx, skipDummy);
getPathsCallableToFuture.put(callable, pool.submit(callable));
}
- }
+ pool.shutdown();
- if (pool != null) {
for (Map.Entry<GetInputPathsCallable, Future<Path>> future : getPathsCallableToFuture.entrySet()) {
if (lDrvStat != null && lDrvStat.driverState == DriverState.INTERRUPT) {
- throw new IOException("Operation is Canceled. ");
+ throw new IOException("Operation is Canceled.");
}
Path newPath = future.getValue().get();
updatePathForMapWork(newPath, work, future.getKey().path);
finalPathsToAdd.add(newPath);
}
+ } finally {
+ pool.shutdownNow();
}
-
return finalPathsToAdd;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3fc131c7/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
index dcd8f95..1a464c8 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
@@ -26,8 +26,11 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.apache.hadoop.hive.ql.exec.Utilities.getFileExtension;
+import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.File;
@@ -35,12 +38,17 @@ import java.io.IOException;
import java.io.Serializable;
import java.sql.Timestamp;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
@@ -532,6 +540,97 @@ public class TestUtilities {
}
@Test
+ public void testGetInputSummaryPool() throws ExecutionException, InterruptedException, IOException {
+ ExecutorService pool = mock(ExecutorService.class);
+ when(pool.submit(any(Runnable.class))).thenReturn(mock(Future.class));
+
+ Set<Path> pathNeedProcess = new HashSet<>();
+ pathNeedProcess.add(new Path("dummy-path1"));
+ pathNeedProcess.add(new Path("dummy-path2"));
+ pathNeedProcess.add(new Path("dummy-path3"));
+
+ SessionState.start(new HiveConf());
+ JobConf jobConf = new JobConf();
+ Context context = new Context(jobConf);
+
+ Utilities.getInputSummaryWithPool(context, pathNeedProcess, mock(MapWork.class), new long[3], pool);
+ verify(pool, times(3)).submit(any(Runnable.class));
+ verify(pool).shutdown();
+ verify(pool).shutdownNow();
+ }
+
+ @Test
+ public void testGetInputSummaryPoolAndFailure() throws ExecutionException, InterruptedException, IOException {
+ ExecutorService pool = mock(ExecutorService.class);
+ when(pool.submit(any(Runnable.class))).thenReturn(mock(Future.class));
+
+ Set<Path> pathNeedProcess = new HashSet<>();
+ pathNeedProcess.add(new Path("dummy-path1"));
+ pathNeedProcess.add(new Path("dummy-path2"));
+ pathNeedProcess.add(new Path("dummy-path3"));
+
+ SessionState.start(new HiveConf());
+ JobConf jobConf = new JobConf();
+ Context context = new Context(jobConf);
+
+ Utilities.getInputSummaryWithPool(context, pathNeedProcess, mock(MapWork.class), new long[3], pool);
+ verify(pool, times(3)).submit(any(Runnable.class));
+ verify(pool).shutdown();
+ verify(pool).shutdownNow();
+ }
+
+ @Test
+ public void testGetInputPathsPool() throws IOException, ExecutionException, InterruptedException {
+ List<Path> pathsToAdd = new ArrayList<>();
+ Path path = new Path("dummy-path");
+
+ pathsToAdd.add(path);
+ pathsToAdd.add(path);
+ pathsToAdd.add(path);
+
+ ExecutorService pool = mock(ExecutorService.class);
+ Future mockFuture = mock(Future.class);
+
+ when(mockFuture.get()).thenReturn(path);
+ when(pool.submit(any(Callable.class))).thenReturn(mockFuture);
+
+ Utilities.getInputPathsWithPool(mock(JobConf.class), mock(MapWork.class), mock(Path.class), mock(Context.class),
+ false, pathsToAdd, pool);
+
+ verify(pool, times(3)).submit(any(Callable.class));
+ verify(pool).shutdown();
+ verify(pool).shutdownNow();
+ }
+
+ @Test
+ public void testGetInputPathsPoolAndFailure() throws IOException, ExecutionException, InterruptedException {
+ List<Path> pathsToAdd = new ArrayList<>();
+ Path path = new Path("dummy-path");
+
+ pathsToAdd.add(path);
+ pathsToAdd.add(path);
+ pathsToAdd.add(path);
+
+ ExecutorService pool = mock(ExecutorService.class);
+ Future mockFuture = mock(Future.class);
+
+ when(mockFuture.get()).thenThrow(new RuntimeException());
+ when(pool.submit(any(Callable.class))).thenReturn(mockFuture);
+
+ Exception e = null;
+ try {
+ Utilities.getInputPathsWithPool(mock(JobConf.class), mock(MapWork.class), mock(Path.class), mock(Context.class),
+ false, pathsToAdd, pool);
+ } catch (Exception thrownException) {
+ e = thrownException;
+ }
+ assertNotNull(e);
+
+ verify(pool, times(3)).submit(any(Callable.class));
+ verify(pool).shutdownNow();
+ }
+
+ @Test
public void testGetInputSummaryWithASingleThread() throws IOException {
final int NUM_PARTITIONS = 5;
final int BYTES_PER_FILE = 5;
[44/50] [abbrv] hive git commit: HIVE-14671 : merge master into
hive-14535 (Wei Zheng)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
----------------------------------------------------------------------
diff --cc standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
index 0000000,5c09fdd..3e4c754
mode 000000,100644..100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
@@@ -1,0 -1,21817 +1,21819 @@@
+ /**
+ * Autogenerated by Thrift Compiler (0.9.3)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+ #include "hive_metastore_types.h"
+
+ #include <algorithm>
+ #include <ostream>
+
+ #include <thrift/TToString.h>
+
+ namespace Apache { namespace Hadoop { namespace Hive {
+
+ int _kHiveObjectTypeValues[] = {
+ HiveObjectType::GLOBAL,
+ HiveObjectType::DATABASE,
+ HiveObjectType::TABLE,
+ HiveObjectType::PARTITION,
+ HiveObjectType::COLUMN
+ };
+ const char* _kHiveObjectTypeNames[] = {
+ "GLOBAL",
+ "DATABASE",
+ "TABLE",
+ "PARTITION",
+ "COLUMN"
+ };
+ const std::map<int, const char*> _HiveObjectType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(5, _kHiveObjectTypeValues, _kHiveObjectTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+
+ int _kPrincipalTypeValues[] = {
+ PrincipalType::USER,
+ PrincipalType::ROLE,
+ PrincipalType::GROUP
+ };
+ const char* _kPrincipalTypeNames[] = {
+ "USER",
+ "ROLE",
+ "GROUP"
+ };
+ const std::map<int, const char*> _PrincipalType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3, _kPrincipalTypeValues, _kPrincipalTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+
+ int _kPartitionEventTypeValues[] = {
+ PartitionEventType::LOAD_DONE
+ };
+ const char* _kPartitionEventTypeNames[] = {
+ "LOAD_DONE"
+ };
+ const std::map<int, const char*> _PartitionEventType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(1, _kPartitionEventTypeValues, _kPartitionEventTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+
+ int _kTxnStateValues[] = {
+ TxnState::COMMITTED,
+ TxnState::ABORTED,
+ TxnState::OPEN
+ };
+ const char* _kTxnStateNames[] = {
+ "COMMITTED",
+ "ABORTED",
+ "OPEN"
+ };
+ const std::map<int, const char*> _TxnState_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3, _kTxnStateValues, _kTxnStateNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+
+ int _kLockLevelValues[] = {
+ LockLevel::DB,
+ LockLevel::TABLE,
+ LockLevel::PARTITION
+ };
+ const char* _kLockLevelNames[] = {
+ "DB",
+ "TABLE",
+ "PARTITION"
+ };
+ const std::map<int, const char*> _LockLevel_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3, _kLockLevelValues, _kLockLevelNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+
+ int _kLockStateValues[] = {
+ LockState::ACQUIRED,
+ LockState::WAITING,
+ LockState::ABORT,
+ LockState::NOT_ACQUIRED
+ };
+ const char* _kLockStateNames[] = {
+ "ACQUIRED",
+ "WAITING",
+ "ABORT",
+ "NOT_ACQUIRED"
+ };
+ const std::map<int, const char*> _LockState_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(4, _kLockStateValues, _kLockStateNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+
+ int _kLockTypeValues[] = {
+ LockType::SHARED_READ,
+ LockType::SHARED_WRITE,
+ LockType::EXCLUSIVE
+ };
+ const char* _kLockTypeNames[] = {
+ "SHARED_READ",
+ "SHARED_WRITE",
+ "EXCLUSIVE"
+ };
+ const std::map<int, const char*> _LockType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3, _kLockTypeValues, _kLockTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+
+ int _kCompactionTypeValues[] = {
+ CompactionType::MINOR,
+ CompactionType::MAJOR
+ };
+ const char* _kCompactionTypeNames[] = {
+ "MINOR",
+ "MAJOR"
+ };
+ const std::map<int, const char*> _CompactionType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(2, _kCompactionTypeValues, _kCompactionTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+
+ int _kGrantRevokeTypeValues[] = {
+ GrantRevokeType::GRANT,
+ GrantRevokeType::REVOKE
+ };
+ const char* _kGrantRevokeTypeNames[] = {
+ "GRANT",
+ "REVOKE"
+ };
+ const std::map<int, const char*> _GrantRevokeType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(2, _kGrantRevokeTypeValues, _kGrantRevokeTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+
+ int _kDataOperationTypeValues[] = {
+ DataOperationType::SELECT,
+ DataOperationType::INSERT,
+ DataOperationType::UPDATE,
+ DataOperationType::DELETE,
+ DataOperationType::UNSET,
+ DataOperationType::NO_TXN
+ };
+ const char* _kDataOperationTypeNames[] = {
+ "SELECT",
+ "INSERT",
+ "UPDATE",
+ "DELETE",
+ "UNSET",
+ "NO_TXN"
+ };
+ const std::map<int, const char*> _DataOperationType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(6, _kDataOperationTypeValues, _kDataOperationTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+
+ int _kEventRequestTypeValues[] = {
+ EventRequestType::INSERT,
+ EventRequestType::UPDATE,
+ EventRequestType::DELETE
+ };
+ const char* _kEventRequestTypeNames[] = {
+ "INSERT",
+ "UPDATE",
+ "DELETE"
+ };
+ const std::map<int, const char*> _EventRequestType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3, _kEventRequestTypeValues, _kEventRequestTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+
+ int _kFunctionTypeValues[] = {
+ FunctionType::JAVA
+ };
+ const char* _kFunctionTypeNames[] = {
+ "JAVA"
+ };
+ const std::map<int, const char*> _FunctionType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(1, _kFunctionTypeValues, _kFunctionTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+
+ int _kResourceTypeValues[] = {
+ ResourceType::JAR,
+ ResourceType::FILE,
+ ResourceType::ARCHIVE
+ };
+ const char* _kResourceTypeNames[] = {
+ "JAR",
+ "FILE",
+ "ARCHIVE"
+ };
+ const std::map<int, const char*> _ResourceType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3, _kResourceTypeValues, _kResourceTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+
+ int _kFileMetadataExprTypeValues[] = {
+ FileMetadataExprType::ORC_SARG
+ };
+ const char* _kFileMetadataExprTypeNames[] = {
+ "ORC_SARG"
+ };
+ const std::map<int, const char*> _FileMetadataExprType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(1, _kFileMetadataExprTypeValues, _kFileMetadataExprTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+
+ int _kClientCapabilityValues[] = {
- ClientCapability::TEST_CAPABILITY
++ ClientCapability::TEST_CAPABILITY,
++ ClientCapability::INSERT_ONLY_TABLES
+ };
+ const char* _kClientCapabilityNames[] = {
- "TEST_CAPABILITY"
++ "TEST_CAPABILITY",
++ "INSERT_ONLY_TABLES"
+ };
-const std::map<int, const char*> _ClientCapability_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(1, _kClientCapabilityValues, _kClientCapabilityNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
++const std::map<int, const char*> _ClientCapability_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(2, _kClientCapabilityValues, _kClientCapabilityNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+
+
+ Version::~Version() throw() {
+ }
+
+
+ void Version::__set_version(const std::string& val) {
+ this->version = val;
+ }
+
+ void Version::__set_comments(const std::string& val) {
+ this->comments = val;
+ }
+
+ uint32_t Version::read(::apache::thrift::protocol::TProtocol* iprot) {
+
+ apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+ uint32_t xfer = 0;
+ std::string fname;
+ ::apache::thrift::protocol::TType ftype;
+ int16_t fid;
+
+ xfer += iprot->readStructBegin(fname);
+
+ using ::apache::thrift::protocol::TProtocolException;
+
+
+ while (true)
+ {
+ xfer += iprot->readFieldBegin(fname, ftype, fid);
+ if (ftype == ::apache::thrift::protocol::T_STOP) {
+ break;
+ }
+ switch (fid)
+ {
+ case 1:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->version);
+ this->__isset.version = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 2:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->comments);
+ this->__isset.comments = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ default:
+ xfer += iprot->skip(ftype);
+ break;
+ }
+ xfer += iprot->readFieldEnd();
+ }
+
+ xfer += iprot->readStructEnd();
+
+ return xfer;
+ }
+
+ uint32_t Version::write(::apache::thrift::protocol::TProtocol* oprot) const {
+ uint32_t xfer = 0;
+ apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+ xfer += oprot->writeStructBegin("Version");
+
+ xfer += oprot->writeFieldBegin("version", ::apache::thrift::protocol::T_STRING, 1);
+ xfer += oprot->writeString(this->version);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("comments", ::apache::thrift::protocol::T_STRING, 2);
+ xfer += oprot->writeString(this->comments);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldStop();
+ xfer += oprot->writeStructEnd();
+ return xfer;
+ }
+
+ void swap(Version &a, Version &b) {
+ using ::std::swap;
+ swap(a.version, b.version);
+ swap(a.comments, b.comments);
+ swap(a.__isset, b.__isset);
+ }
+
+ Version::Version(const Version& other0) {
+ version = other0.version;
+ comments = other0.comments;
+ __isset = other0.__isset;
+ }
+ Version& Version::operator=(const Version& other1) {
+ version = other1.version;
+ comments = other1.comments;
+ __isset = other1.__isset;
+ return *this;
+ }
+ void Version::printTo(std::ostream& out) const {
+ using ::apache::thrift::to_string;
+ out << "Version(";
+ out << "version=" << to_string(version);
+ out << ", " << "comments=" << to_string(comments);
+ out << ")";
+ }
+
+
+ FieldSchema::~FieldSchema() throw() {
+ }
+
+
+ void FieldSchema::__set_name(const std::string& val) {
+ this->name = val;
+ }
+
+ void FieldSchema::__set_type(const std::string& val) {
+ this->type = val;
+ }
+
+ void FieldSchema::__set_comment(const std::string& val) {
+ this->comment = val;
+ }
+
+ uint32_t FieldSchema::read(::apache::thrift::protocol::TProtocol* iprot) {
+
+ apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+ uint32_t xfer = 0;
+ std::string fname;
+ ::apache::thrift::protocol::TType ftype;
+ int16_t fid;
+
+ xfer += iprot->readStructBegin(fname);
+
+ using ::apache::thrift::protocol::TProtocolException;
+
+
+ while (true)
+ {
+ xfer += iprot->readFieldBegin(fname, ftype, fid);
+ if (ftype == ::apache::thrift::protocol::T_STOP) {
+ break;
+ }
+ switch (fid)
+ {
+ case 1:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->name);
+ this->__isset.name = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 2:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->type);
+ this->__isset.type = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 3:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->comment);
+ this->__isset.comment = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ default:
+ xfer += iprot->skip(ftype);
+ break;
+ }
+ xfer += iprot->readFieldEnd();
+ }
+
+ xfer += iprot->readStructEnd();
+
+ return xfer;
+ }
+
+ uint32_t FieldSchema::write(::apache::thrift::protocol::TProtocol* oprot) const {
+ uint32_t xfer = 0;
+ apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+ xfer += oprot->writeStructBegin("FieldSchema");
+
+ xfer += oprot->writeFieldBegin("name", ::apache::thrift::protocol::T_STRING, 1);
+ xfer += oprot->writeString(this->name);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("type", ::apache::thrift::protocol::T_STRING, 2);
+ xfer += oprot->writeString(this->type);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("comment", ::apache::thrift::protocol::T_STRING, 3);
+ xfer += oprot->writeString(this->comment);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldStop();
+ xfer += oprot->writeStructEnd();
+ return xfer;
+ }
+
+ void swap(FieldSchema &a, FieldSchema &b) {
+ using ::std::swap;
+ swap(a.name, b.name);
+ swap(a.type, b.type);
+ swap(a.comment, b.comment);
+ swap(a.__isset, b.__isset);
+ }
+
+ FieldSchema::FieldSchema(const FieldSchema& other2) {
+ name = other2.name;
+ type = other2.type;
+ comment = other2.comment;
+ __isset = other2.__isset;
+ }
+ FieldSchema& FieldSchema::operator=(const FieldSchema& other3) {
+ name = other3.name;
+ type = other3.type;
+ comment = other3.comment;
+ __isset = other3.__isset;
+ return *this;
+ }
+ void FieldSchema::printTo(std::ostream& out) const {
+ using ::apache::thrift::to_string;
+ out << "FieldSchema(";
+ out << "name=" << to_string(name);
+ out << ", " << "type=" << to_string(type);
+ out << ", " << "comment=" << to_string(comment);
+ out << ")";
+ }
+
+
+ SQLPrimaryKey::~SQLPrimaryKey() throw() {
+ }
+
+
+ void SQLPrimaryKey::__set_table_db(const std::string& val) {
+ this->table_db = val;
+ }
+
+ void SQLPrimaryKey::__set_table_name(const std::string& val) {
+ this->table_name = val;
+ }
+
+ void SQLPrimaryKey::__set_column_name(const std::string& val) {
+ this->column_name = val;
+ }
+
+ void SQLPrimaryKey::__set_key_seq(const int32_t val) {
+ this->key_seq = val;
+ }
+
+ void SQLPrimaryKey::__set_pk_name(const std::string& val) {
+ this->pk_name = val;
+ }
+
+ void SQLPrimaryKey::__set_enable_cstr(const bool val) {
+ this->enable_cstr = val;
+ }
+
+ void SQLPrimaryKey::__set_validate_cstr(const bool val) {
+ this->validate_cstr = val;
+ }
+
+ void SQLPrimaryKey::__set_rely_cstr(const bool val) {
+ this->rely_cstr = val;
+ }
+
+ uint32_t SQLPrimaryKey::read(::apache::thrift::protocol::TProtocol* iprot) {
+
+ apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+ uint32_t xfer = 0;
+ std::string fname;
+ ::apache::thrift::protocol::TType ftype;
+ int16_t fid;
+
+ xfer += iprot->readStructBegin(fname);
+
+ using ::apache::thrift::protocol::TProtocolException;
+
+
+ while (true)
+ {
+ xfer += iprot->readFieldBegin(fname, ftype, fid);
+ if (ftype == ::apache::thrift::protocol::T_STOP) {
+ break;
+ }
+ switch (fid)
+ {
+ case 1:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->table_db);
+ this->__isset.table_db = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 2:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->table_name);
+ this->__isset.table_name = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 3:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->column_name);
+ this->__isset.column_name = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 4:
+ if (ftype == ::apache::thrift::protocol::T_I32) {
+ xfer += iprot->readI32(this->key_seq);
+ this->__isset.key_seq = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 5:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->pk_name);
+ this->__isset.pk_name = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 6:
+ if (ftype == ::apache::thrift::protocol::T_BOOL) {
+ xfer += iprot->readBool(this->enable_cstr);
+ this->__isset.enable_cstr = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 7:
+ if (ftype == ::apache::thrift::protocol::T_BOOL) {
+ xfer += iprot->readBool(this->validate_cstr);
+ this->__isset.validate_cstr = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 8:
+ if (ftype == ::apache::thrift::protocol::T_BOOL) {
+ xfer += iprot->readBool(this->rely_cstr);
+ this->__isset.rely_cstr = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ default:
+ xfer += iprot->skip(ftype);
+ break;
+ }
+ xfer += iprot->readFieldEnd();
+ }
+
+ xfer += iprot->readStructEnd();
+
+ return xfer;
+ }
+
+ uint32_t SQLPrimaryKey::write(::apache::thrift::protocol::TProtocol* oprot) const {
+ uint32_t xfer = 0;
+ apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+ xfer += oprot->writeStructBegin("SQLPrimaryKey");
+
+ xfer += oprot->writeFieldBegin("table_db", ::apache::thrift::protocol::T_STRING, 1);
+ xfer += oprot->writeString(this->table_db);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 2);
+ xfer += oprot->writeString(this->table_name);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("column_name", ::apache::thrift::protocol::T_STRING, 3);
+ xfer += oprot->writeString(this->column_name);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("key_seq", ::apache::thrift::protocol::T_I32, 4);
+ xfer += oprot->writeI32(this->key_seq);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("pk_name", ::apache::thrift::protocol::T_STRING, 5);
+ xfer += oprot->writeString(this->pk_name);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("enable_cstr", ::apache::thrift::protocol::T_BOOL, 6);
+ xfer += oprot->writeBool(this->enable_cstr);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("validate_cstr", ::apache::thrift::protocol::T_BOOL, 7);
+ xfer += oprot->writeBool(this->validate_cstr);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("rely_cstr", ::apache::thrift::protocol::T_BOOL, 8);
+ xfer += oprot->writeBool(this->rely_cstr);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldStop();
+ xfer += oprot->writeStructEnd();
+ return xfer;
+ }
+
+ void swap(SQLPrimaryKey &a, SQLPrimaryKey &b) {
+ using ::std::swap;
+ swap(a.table_db, b.table_db);
+ swap(a.table_name, b.table_name);
+ swap(a.column_name, b.column_name);
+ swap(a.key_seq, b.key_seq);
+ swap(a.pk_name, b.pk_name);
+ swap(a.enable_cstr, b.enable_cstr);
+ swap(a.validate_cstr, b.validate_cstr);
+ swap(a.rely_cstr, b.rely_cstr);
+ swap(a.__isset, b.__isset);
+ }
+
+ SQLPrimaryKey::SQLPrimaryKey(const SQLPrimaryKey& other4) {
+ table_db = other4.table_db;
+ table_name = other4.table_name;
+ column_name = other4.column_name;
+ key_seq = other4.key_seq;
+ pk_name = other4.pk_name;
+ enable_cstr = other4.enable_cstr;
+ validate_cstr = other4.validate_cstr;
+ rely_cstr = other4.rely_cstr;
+ __isset = other4.__isset;
+ }
+ SQLPrimaryKey& SQLPrimaryKey::operator=(const SQLPrimaryKey& other5) {
+ table_db = other5.table_db;
+ table_name = other5.table_name;
+ column_name = other5.column_name;
+ key_seq = other5.key_seq;
+ pk_name = other5.pk_name;
+ enable_cstr = other5.enable_cstr;
+ validate_cstr = other5.validate_cstr;
+ rely_cstr = other5.rely_cstr;
+ __isset = other5.__isset;
+ return *this;
+ }
+ void SQLPrimaryKey::printTo(std::ostream& out) const {
+ using ::apache::thrift::to_string;
+ out << "SQLPrimaryKey(";
+ out << "table_db=" << to_string(table_db);
+ out << ", " << "table_name=" << to_string(table_name);
+ out << ", " << "column_name=" << to_string(column_name);
+ out << ", " << "key_seq=" << to_string(key_seq);
+ out << ", " << "pk_name=" << to_string(pk_name);
+ out << ", " << "enable_cstr=" << to_string(enable_cstr);
+ out << ", " << "validate_cstr=" << to_string(validate_cstr);
+ out << ", " << "rely_cstr=" << to_string(rely_cstr);
+ out << ")";
+ }
+
+
+ SQLForeignKey::~SQLForeignKey() throw() {
+ }
+
+
+ void SQLForeignKey::__set_pktable_db(const std::string& val) {
+ this->pktable_db = val;
+ }
+
+ void SQLForeignKey::__set_pktable_name(const std::string& val) {
+ this->pktable_name = val;
+ }
+
+ void SQLForeignKey::__set_pkcolumn_name(const std::string& val) {
+ this->pkcolumn_name = val;
+ }
+
+ void SQLForeignKey::__set_fktable_db(const std::string& val) {
+ this->fktable_db = val;
+ }
+
+ void SQLForeignKey::__set_fktable_name(const std::string& val) {
+ this->fktable_name = val;
+ }
+
+ void SQLForeignKey::__set_fkcolumn_name(const std::string& val) {
+ this->fkcolumn_name = val;
+ }
+
+ void SQLForeignKey::__set_key_seq(const int32_t val) {
+ this->key_seq = val;
+ }
+
+ void SQLForeignKey::__set_update_rule(const int32_t val) {
+ this->update_rule = val;
+ }
+
+ void SQLForeignKey::__set_delete_rule(const int32_t val) {
+ this->delete_rule = val;
+ }
+
+ void SQLForeignKey::__set_fk_name(const std::string& val) {
+ this->fk_name = val;
+ }
+
+ void SQLForeignKey::__set_pk_name(const std::string& val) {
+ this->pk_name = val;
+ }
+
+ void SQLForeignKey::__set_enable_cstr(const bool val) {
+ this->enable_cstr = val;
+ }
+
+ void SQLForeignKey::__set_validate_cstr(const bool val) {
+ this->validate_cstr = val;
+ }
+
+ void SQLForeignKey::__set_rely_cstr(const bool val) {
+ this->rely_cstr = val;
+ }
+
+ uint32_t SQLForeignKey::read(::apache::thrift::protocol::TProtocol* iprot) {
+
+ apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+ uint32_t xfer = 0;
+ std::string fname;
+ ::apache::thrift::protocol::TType ftype;
+ int16_t fid;
+
+ xfer += iprot->readStructBegin(fname);
+
+ using ::apache::thrift::protocol::TProtocolException;
+
+
+ while (true)
+ {
+ xfer += iprot->readFieldBegin(fname, ftype, fid);
+ if (ftype == ::apache::thrift::protocol::T_STOP) {
+ break;
+ }
+ switch (fid)
+ {
+ case 1:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->pktable_db);
+ this->__isset.pktable_db = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 2:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->pktable_name);
+ this->__isset.pktable_name = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 3:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->pkcolumn_name);
+ this->__isset.pkcolumn_name = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 4:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->fktable_db);
+ this->__isset.fktable_db = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 5:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->fktable_name);
+ this->__isset.fktable_name = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 6:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->fkcolumn_name);
+ this->__isset.fkcolumn_name = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 7:
+ if (ftype == ::apache::thrift::protocol::T_I32) {
+ xfer += iprot->readI32(this->key_seq);
+ this->__isset.key_seq = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 8:
+ if (ftype == ::apache::thrift::protocol::T_I32) {
+ xfer += iprot->readI32(this->update_rule);
+ this->__isset.update_rule = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 9:
+ if (ftype == ::apache::thrift::protocol::T_I32) {
+ xfer += iprot->readI32(this->delete_rule);
+ this->__isset.delete_rule = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 10:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->fk_name);
+ this->__isset.fk_name = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 11:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->pk_name);
+ this->__isset.pk_name = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 12:
+ if (ftype == ::apache::thrift::protocol::T_BOOL) {
+ xfer += iprot->readBool(this->enable_cstr);
+ this->__isset.enable_cstr = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 13:
+ if (ftype == ::apache::thrift::protocol::T_BOOL) {
+ xfer += iprot->readBool(this->validate_cstr);
+ this->__isset.validate_cstr = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 14:
+ if (ftype == ::apache::thrift::protocol::T_BOOL) {
+ xfer += iprot->readBool(this->rely_cstr);
+ this->__isset.rely_cstr = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ default:
+ xfer += iprot->skip(ftype);
+ break;
+ }
+ xfer += iprot->readFieldEnd();
+ }
+
+ xfer += iprot->readStructEnd();
+
+ return xfer;
+ }
+
+ uint32_t SQLForeignKey::write(::apache::thrift::protocol::TProtocol* oprot) const {
+ uint32_t xfer = 0;
+ apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+ xfer += oprot->writeStructBegin("SQLForeignKey");
+
+ xfer += oprot->writeFieldBegin("pktable_db", ::apache::thrift::protocol::T_STRING, 1);
+ xfer += oprot->writeString(this->pktable_db);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("pktable_name", ::apache::thrift::protocol::T_STRING, 2);
+ xfer += oprot->writeString(this->pktable_name);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("pkcolumn_name", ::apache::thrift::protocol::T_STRING, 3);
+ xfer += oprot->writeString(this->pkcolumn_name);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("fktable_db", ::apache::thrift::protocol::T_STRING, 4);
+ xfer += oprot->writeString(this->fktable_db);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("fktable_name", ::apache::thrift::protocol::T_STRING, 5);
+ xfer += oprot->writeString(this->fktable_name);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("fkcolumn_name", ::apache::thrift::protocol::T_STRING, 6);
+ xfer += oprot->writeString(this->fkcolumn_name);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("key_seq", ::apache::thrift::protocol::T_I32, 7);
+ xfer += oprot->writeI32(this->key_seq);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("update_rule", ::apache::thrift::protocol::T_I32, 8);
+ xfer += oprot->writeI32(this->update_rule);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("delete_rule", ::apache::thrift::protocol::T_I32, 9);
+ xfer += oprot->writeI32(this->delete_rule);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("fk_name", ::apache::thrift::protocol::T_STRING, 10);
+ xfer += oprot->writeString(this->fk_name);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("pk_name", ::apache::thrift::protocol::T_STRING, 11);
+ xfer += oprot->writeString(this->pk_name);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("enable_cstr", ::apache::thrift::protocol::T_BOOL, 12);
+ xfer += oprot->writeBool(this->enable_cstr);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("validate_cstr", ::apache::thrift::protocol::T_BOOL, 13);
+ xfer += oprot->writeBool(this->validate_cstr);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("rely_cstr", ::apache::thrift::protocol::T_BOOL, 14);
+ xfer += oprot->writeBool(this->rely_cstr);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldStop();
+ xfer += oprot->writeStructEnd();
+ return xfer;
+ }
+
+ void swap(SQLForeignKey &a, SQLForeignKey &b) {
+ using ::std::swap;
+ swap(a.pktable_db, b.pktable_db);
+ swap(a.pktable_name, b.pktable_name);
+ swap(a.pkcolumn_name, b.pkcolumn_name);
+ swap(a.fktable_db, b.fktable_db);
+ swap(a.fktable_name, b.fktable_name);
+ swap(a.fkcolumn_name, b.fkcolumn_name);
+ swap(a.key_seq, b.key_seq);
+ swap(a.update_rule, b.update_rule);
+ swap(a.delete_rule, b.delete_rule);
+ swap(a.fk_name, b.fk_name);
+ swap(a.pk_name, b.pk_name);
+ swap(a.enable_cstr, b.enable_cstr);
+ swap(a.validate_cstr, b.validate_cstr);
+ swap(a.rely_cstr, b.rely_cstr);
+ swap(a.__isset, b.__isset);
+ }
+
+ SQLForeignKey::SQLForeignKey(const SQLForeignKey& other6) {
+ pktable_db = other6.pktable_db;
+ pktable_name = other6.pktable_name;
+ pkcolumn_name = other6.pkcolumn_name;
+ fktable_db = other6.fktable_db;
+ fktable_name = other6.fktable_name;
+ fkcolumn_name = other6.fkcolumn_name;
+ key_seq = other6.key_seq;
+ update_rule = other6.update_rule;
+ delete_rule = other6.delete_rule;
+ fk_name = other6.fk_name;
+ pk_name = other6.pk_name;
+ enable_cstr = other6.enable_cstr;
+ validate_cstr = other6.validate_cstr;
+ rely_cstr = other6.rely_cstr;
+ __isset = other6.__isset;
+ }
+ SQLForeignKey& SQLForeignKey::operator=(const SQLForeignKey& other7) {
+ pktable_db = other7.pktable_db;
+ pktable_name = other7.pktable_name;
+ pkcolumn_name = other7.pkcolumn_name;
+ fktable_db = other7.fktable_db;
+ fktable_name = other7.fktable_name;
+ fkcolumn_name = other7.fkcolumn_name;
+ key_seq = other7.key_seq;
+ update_rule = other7.update_rule;
+ delete_rule = other7.delete_rule;
+ fk_name = other7.fk_name;
+ pk_name = other7.pk_name;
+ enable_cstr = other7.enable_cstr;
+ validate_cstr = other7.validate_cstr;
+ rely_cstr = other7.rely_cstr;
+ __isset = other7.__isset;
+ return *this;
+ }
+ void SQLForeignKey::printTo(std::ostream& out) const {
+ using ::apache::thrift::to_string;
+ out << "SQLForeignKey(";
+ out << "pktable_db=" << to_string(pktable_db);
+ out << ", " << "pktable_name=" << to_string(pktable_name);
+ out << ", " << "pkcolumn_name=" << to_string(pkcolumn_name);
+ out << ", " << "fktable_db=" << to_string(fktable_db);
+ out << ", " << "fktable_name=" << to_string(fktable_name);
+ out << ", " << "fkcolumn_name=" << to_string(fkcolumn_name);
+ out << ", " << "key_seq=" << to_string(key_seq);
+ out << ", " << "update_rule=" << to_string(update_rule);
+ out << ", " << "delete_rule=" << to_string(delete_rule);
+ out << ", " << "fk_name=" << to_string(fk_name);
+ out << ", " << "pk_name=" << to_string(pk_name);
+ out << ", " << "enable_cstr=" << to_string(enable_cstr);
+ out << ", " << "validate_cstr=" << to_string(validate_cstr);
+ out << ", " << "rely_cstr=" << to_string(rely_cstr);
+ out << ")";
+ }
+
+
+ SQLUniqueConstraint::~SQLUniqueConstraint() throw() {
+ }
+
+
+ void SQLUniqueConstraint::__set_table_db(const std::string& val) {
+ this->table_db = val;
+ }
+
+ void SQLUniqueConstraint::__set_table_name(const std::string& val) {
+ this->table_name = val;
+ }
+
+ void SQLUniqueConstraint::__set_column_name(const std::string& val) {
+ this->column_name = val;
+ }
+
+ void SQLUniqueConstraint::__set_key_seq(const int32_t val) {
+ this->key_seq = val;
+ }
+
+ void SQLUniqueConstraint::__set_uk_name(const std::string& val) {
+ this->uk_name = val;
+ }
+
+ void SQLUniqueConstraint::__set_enable_cstr(const bool val) {
+ this->enable_cstr = val;
+ }
+
+ void SQLUniqueConstraint::__set_validate_cstr(const bool val) {
+ this->validate_cstr = val;
+ }
+
+ void SQLUniqueConstraint::__set_rely_cstr(const bool val) {
+ this->rely_cstr = val;
+ }
+
+ uint32_t SQLUniqueConstraint::read(::apache::thrift::protocol::TProtocol* iprot) {
+
+ apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+ uint32_t xfer = 0;
+ std::string fname;
+ ::apache::thrift::protocol::TType ftype;
+ int16_t fid;
+
+ xfer += iprot->readStructBegin(fname);
+
+ using ::apache::thrift::protocol::TProtocolException;
+
+
+ while (true)
+ {
+ xfer += iprot->readFieldBegin(fname, ftype, fid);
+ if (ftype == ::apache::thrift::protocol::T_STOP) {
+ break;
+ }
+ switch (fid)
+ {
+ case 1:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->table_db);
+ this->__isset.table_db = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 2:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->table_name);
+ this->__isset.table_name = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 3:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->column_name);
+ this->__isset.column_name = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 4:
+ if (ftype == ::apache::thrift::protocol::T_I32) {
+ xfer += iprot->readI32(this->key_seq);
+ this->__isset.key_seq = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 5:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->uk_name);
+ this->__isset.uk_name = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 6:
+ if (ftype == ::apache::thrift::protocol::T_BOOL) {
+ xfer += iprot->readBool(this->enable_cstr);
+ this->__isset.enable_cstr = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 7:
+ if (ftype == ::apache::thrift::protocol::T_BOOL) {
+ xfer += iprot->readBool(this->validate_cstr);
+ this->__isset.validate_cstr = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 8:
+ if (ftype == ::apache::thrift::protocol::T_BOOL) {
+ xfer += iprot->readBool(this->rely_cstr);
+ this->__isset.rely_cstr = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ default:
+ xfer += iprot->skip(ftype);
+ break;
+ }
+ xfer += iprot->readFieldEnd();
+ }
+
+ xfer += iprot->readStructEnd();
+
+ return xfer;
+ }
+
+ uint32_t SQLUniqueConstraint::write(::apache::thrift::protocol::TProtocol* oprot) const {
+ uint32_t xfer = 0;
+ apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+ xfer += oprot->writeStructBegin("SQLUniqueConstraint");
+
+ xfer += oprot->writeFieldBegin("table_db", ::apache::thrift::protocol::T_STRING, 1);
+ xfer += oprot->writeString(this->table_db);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 2);
+ xfer += oprot->writeString(this->table_name);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("column_name", ::apache::thrift::protocol::T_STRING, 3);
+ xfer += oprot->writeString(this->column_name);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("key_seq", ::apache::thrift::protocol::T_I32, 4);
+ xfer += oprot->writeI32(this->key_seq);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("uk_name", ::apache::thrift::protocol::T_STRING, 5);
+ xfer += oprot->writeString(this->uk_name);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("enable_cstr", ::apache::thrift::protocol::T_BOOL, 6);
+ xfer += oprot->writeBool(this->enable_cstr);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("validate_cstr", ::apache::thrift::protocol::T_BOOL, 7);
+ xfer += oprot->writeBool(this->validate_cstr);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("rely_cstr", ::apache::thrift::protocol::T_BOOL, 8);
+ xfer += oprot->writeBool(this->rely_cstr);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldStop();
+ xfer += oprot->writeStructEnd();
+ return xfer;
+ }
+
+ void swap(SQLUniqueConstraint &a, SQLUniqueConstraint &b) {
+ using ::std::swap;
+ swap(a.table_db, b.table_db);
+ swap(a.table_name, b.table_name);
+ swap(a.column_name, b.column_name);
+ swap(a.key_seq, b.key_seq);
+ swap(a.uk_name, b.uk_name);
+ swap(a.enable_cstr, b.enable_cstr);
+ swap(a.validate_cstr, b.validate_cstr);
+ swap(a.rely_cstr, b.rely_cstr);
+ swap(a.__isset, b.__isset);
+ }
+
+ SQLUniqueConstraint::SQLUniqueConstraint(const SQLUniqueConstraint& other8) {
+ table_db = other8.table_db;
+ table_name = other8.table_name;
+ column_name = other8.column_name;
+ key_seq = other8.key_seq;
+ uk_name = other8.uk_name;
+ enable_cstr = other8.enable_cstr;
+ validate_cstr = other8.validate_cstr;
+ rely_cstr = other8.rely_cstr;
+ __isset = other8.__isset;
+ }
+ SQLUniqueConstraint& SQLUniqueConstraint::operator=(const SQLUniqueConstraint& other9) {
+ table_db = other9.table_db;
+ table_name = other9.table_name;
+ column_name = other9.column_name;
+ key_seq = other9.key_seq;
+ uk_name = other9.uk_name;
+ enable_cstr = other9.enable_cstr;
+ validate_cstr = other9.validate_cstr;
+ rely_cstr = other9.rely_cstr;
+ __isset = other9.__isset;
+ return *this;
+ }
+ void SQLUniqueConstraint::printTo(std::ostream& out) const {
+ using ::apache::thrift::to_string;
+ out << "SQLUniqueConstraint(";
+ out << "table_db=" << to_string(table_db);
+ out << ", " << "table_name=" << to_string(table_name);
+ out << ", " << "column_name=" << to_string(column_name);
+ out << ", " << "key_seq=" << to_string(key_seq);
+ out << ", " << "uk_name=" << to_string(uk_name);
+ out << ", " << "enable_cstr=" << to_string(enable_cstr);
+ out << ", " << "validate_cstr=" << to_string(validate_cstr);
+ out << ", " << "rely_cstr=" << to_string(rely_cstr);
+ out << ")";
+ }
+
+
+ SQLNotNullConstraint::~SQLNotNullConstraint() throw() {
+ }
+
+
+ void SQLNotNullConstraint::__set_table_db(const std::string& val) {
+ this->table_db = val;
+ }
+
+ void SQLNotNullConstraint::__set_table_name(const std::string& val) {
+ this->table_name = val;
+ }
+
+ void SQLNotNullConstraint::__set_column_name(const std::string& val) {
+ this->column_name = val;
+ }
+
+ void SQLNotNullConstraint::__set_nn_name(const std::string& val) {
+ this->nn_name = val;
+ }
+
+ void SQLNotNullConstraint::__set_enable_cstr(const bool val) {
+ this->enable_cstr = val;
+ }
+
+ void SQLNotNullConstraint::__set_validate_cstr(const bool val) {
+ this->validate_cstr = val;
+ }
+
+ void SQLNotNullConstraint::__set_rely_cstr(const bool val) {
+ this->rely_cstr = val;
+ }
+
+ uint32_t SQLNotNullConstraint::read(::apache::thrift::protocol::TProtocol* iprot) {
+
+ apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+ uint32_t xfer = 0;
+ std::string fname;
+ ::apache::thrift::protocol::TType ftype;
+ int16_t fid;
+
+ xfer += iprot->readStructBegin(fname);
+
+ using ::apache::thrift::protocol::TProtocolException;
+
+
+ while (true)
+ {
+ xfer += iprot->readFieldBegin(fname, ftype, fid);
+ if (ftype == ::apache::thrift::protocol::T_STOP) {
+ break;
+ }
+ switch (fid)
+ {
+ case 1:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->table_db);
+ this->__isset.table_db = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 2:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->table_name);
+ this->__isset.table_name = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 3:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->column_name);
+ this->__isset.column_name = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 4:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->nn_name);
+ this->__isset.nn_name = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 5:
+ if (ftype == ::apache::thrift::protocol::T_BOOL) {
+ xfer += iprot->readBool(this->enable_cstr);
+ this->__isset.enable_cstr = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 6:
+ if (ftype == ::apache::thrift::protocol::T_BOOL) {
+ xfer += iprot->readBool(this->validate_cstr);
+ this->__isset.validate_cstr = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 7:
+ if (ftype == ::apache::thrift::protocol::T_BOOL) {
+ xfer += iprot->readBool(this->rely_cstr);
+ this->__isset.rely_cstr = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ default:
+ xfer += iprot->skip(ftype);
+ break;
+ }
+ xfer += iprot->readFieldEnd();
+ }
+
+ xfer += iprot->readStructEnd();
+
+ return xfer;
+ }
+
+ uint32_t SQLNotNullConstraint::write(::apache::thrift::protocol::TProtocol* oprot) const {
+ uint32_t xfer = 0;
+ apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+ xfer += oprot->writeStructBegin("SQLNotNullConstraint");
+
+ xfer += oprot->writeFieldBegin("table_db", ::apache::thrift::protocol::T_STRING, 1);
+ xfer += oprot->writeString(this->table_db);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 2);
+ xfer += oprot->writeString(this->table_name);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("column_name", ::apache::thrift::protocol::T_STRING, 3);
+ xfer += oprot->writeString(this->column_name);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("nn_name", ::apache::thrift::protocol::T_STRING, 4);
+ xfer += oprot->writeString(this->nn_name);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("enable_cstr", ::apache::thrift::protocol::T_BOOL, 5);
+ xfer += oprot->writeBool(this->enable_cstr);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("validate_cstr", ::apache::thrift::protocol::T_BOOL, 6);
+ xfer += oprot->writeBool(this->validate_cstr);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("rely_cstr", ::apache::thrift::protocol::T_BOOL, 7);
+ xfer += oprot->writeBool(this->rely_cstr);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldStop();
+ xfer += oprot->writeStructEnd();
+ return xfer;
+ }
+
+ void swap(SQLNotNullConstraint &a, SQLNotNullConstraint &b) {
+ using ::std::swap;
+ swap(a.table_db, b.table_db);
+ swap(a.table_name, b.table_name);
+ swap(a.column_name, b.column_name);
+ swap(a.nn_name, b.nn_name);
+ swap(a.enable_cstr, b.enable_cstr);
+ swap(a.validate_cstr, b.validate_cstr);
+ swap(a.rely_cstr, b.rely_cstr);
+ swap(a.__isset, b.__isset);
+ }
+
+ SQLNotNullConstraint::SQLNotNullConstraint(const SQLNotNullConstraint& other10) {
+ table_db = other10.table_db;
+ table_name = other10.table_name;
+ column_name = other10.column_name;
+ nn_name = other10.nn_name;
+ enable_cstr = other10.enable_cstr;
+ validate_cstr = other10.validate_cstr;
+ rely_cstr = other10.rely_cstr;
+ __isset = other10.__isset;
+ }
+ SQLNotNullConstraint& SQLNotNullConstraint::operator=(const SQLNotNullConstraint& other11) {
+ table_db = other11.table_db;
+ table_name = other11.table_name;
+ column_name = other11.column_name;
+ nn_name = other11.nn_name;
+ enable_cstr = other11.enable_cstr;
+ validate_cstr = other11.validate_cstr;
+ rely_cstr = other11.rely_cstr;
+ __isset = other11.__isset;
+ return *this;
+ }
+ void SQLNotNullConstraint::printTo(std::ostream& out) const {
+ using ::apache::thrift::to_string;
+ out << "SQLNotNullConstraint(";
+ out << "table_db=" << to_string(table_db);
+ out << ", " << "table_name=" << to_string(table_name);
+ out << ", " << "column_name=" << to_string(column_name);
+ out << ", " << "nn_name=" << to_string(nn_name);
+ out << ", " << "enable_cstr=" << to_string(enable_cstr);
+ out << ", " << "validate_cstr=" << to_string(validate_cstr);
+ out << ", " << "rely_cstr=" << to_string(rely_cstr);
+ out << ")";
+ }
+
+
+ Type::~Type() throw() {
+ }
+
+
+ void Type::__set_name(const std::string& val) {
+ this->name = val;
+ }
+
+ void Type::__set_type1(const std::string& val) {
+ this->type1 = val;
+ __isset.type1 = true;
+ }
+
+ void Type::__set_type2(const std::string& val) {
+ this->type2 = val;
+ __isset.type2 = true;
+ }
+
+ void Type::__set_fields(const std::vector<FieldSchema> & val) {
+ this->fields = val;
+ __isset.fields = true;
+ }
+
+ uint32_t Type::read(::apache::thrift::protocol::TProtocol* iprot) {
+
+ apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+ uint32_t xfer = 0;
+ std::string fname;
+ ::apache::thrift::protocol::TType ftype;
+ int16_t fid;
+
+ xfer += iprot->readStructBegin(fname);
+
+ using ::apache::thrift::protocol::TProtocolException;
+
+
+ while (true)
+ {
+ xfer += iprot->readFieldBegin(fname, ftype, fid);
+ if (ftype == ::apache::thrift::protocol::T_STOP) {
+ break;
+ }
+ switch (fid)
+ {
+ case 1:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->name);
+ this->__isset.name = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 2:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->type1);
+ this->__isset.type1 = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 3:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->type2);
+ this->__isset.type2 = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 4:
+ if (ftype == ::apache::thrift::protocol::T_LIST) {
+ {
+ this->fields.clear();
+ uint32_t _size12;
+ ::apache::thrift::protocol::TType _etype15;
+ xfer += iprot->readListBegin(_etype15, _size12);
+ this->fields.resize(_size12);
+ uint32_t _i16;
+ for (_i16 = 0; _i16 < _size12; ++_i16)
+ {
+ xfer += this->fields[_i16].read(iprot);
+ }
+ xfer += iprot->readListEnd();
+ }
+ this->__isset.fields = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ default:
+ xfer += iprot->skip(ftype);
+ break;
+ }
+ xfer += iprot->readFieldEnd();
+ }
+
+ xfer += iprot->readStructEnd();
+
+ return xfer;
+ }
+
+ uint32_t Type::write(::apache::thrift::protocol::TProtocol* oprot) const {
+ uint32_t xfer = 0;
+ apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+ xfer += oprot->writeStructBegin("Type");
+
+ xfer += oprot->writeFieldBegin("name", ::apache::thrift::protocol::T_STRING, 1);
+ xfer += oprot->writeString(this->name);
+ xfer += oprot->writeFieldEnd();
+
+ if (this->__isset.type1) {
+ xfer += oprot->writeFieldBegin("type1", ::apache::thrift::protocol::T_STRING, 2);
+ xfer += oprot->writeString(this->type1);
+ xfer += oprot->writeFieldEnd();
+ }
+ if (this->__isset.type2) {
+ xfer += oprot->writeFieldBegin("type2", ::apache::thrift::protocol::T_STRING, 3);
+ xfer += oprot->writeString(this->type2);
+ xfer += oprot->writeFieldEnd();
+ }
+ if (this->__isset.fields) {
+ xfer += oprot->writeFieldBegin("fields", ::apache::thrift::protocol::T_LIST, 4);
+ {
+ xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(this->fields.size()));
+ std::vector<FieldSchema> ::const_iterator _iter17;
+ for (_iter17 = this->fields.begin(); _iter17 != this->fields.end(); ++_iter17)
+ {
+ xfer += (*_iter17).write(oprot);
+ }
+ xfer += oprot->writeListEnd();
+ }
+ xfer += oprot->writeFieldEnd();
+ }
+ xfer += oprot->writeFieldStop();
+ xfer += oprot->writeStructEnd();
+ return xfer;
+ }
+
+ void swap(Type &a, Type &b) {
+ using ::std::swap;
+ swap(a.name, b.name);
+ swap(a.type1, b.type1);
+ swap(a.type2, b.type2);
+ swap(a.fields, b.fields);
+ swap(a.__isset, b.__isset);
+ }
+
+ Type::Type(const Type& other18) {
+ name = other18.name;
+ type1 = other18.type1;
+ type2 = other18.type2;
+ fields = other18.fields;
+ __isset = other18.__isset;
+ }
+ Type& Type::operator=(const Type& other19) {
+ name = other19.name;
+ type1 = other19.type1;
+ type2 = other19.type2;
+ fields = other19.fields;
+ __isset = other19.__isset;
+ return *this;
+ }
+ void Type::printTo(std::ostream& out) const {
+ using ::apache::thrift::to_string;
+ out << "Type(";
+ out << "name=" << to_string(name);
+ out << ", " << "type1="; (__isset.type1 ? (out << to_string(type1)) : (out << "<null>"));
+ out << ", " << "type2="; (__isset.type2 ? (out << to_string(type2)) : (out << "<null>"));
+ out << ", " << "fields="; (__isset.fields ? (out << to_string(fields)) : (out << "<null>"));
+ out << ")";
+ }
+
+
+ HiveObjectRef::~HiveObjectRef() throw() {
+ }
+
+
+ void HiveObjectRef::__set_objectType(const HiveObjectType::type val) {
+ this->objectType = val;
+ }
+
+ void HiveObjectRef::__set_dbName(const std::string& val) {
+ this->dbName = val;
+ }
+
+ void HiveObjectRef::__set_objectName(const std::string& val) {
+ this->objectName = val;
+ }
+
+ void HiveObjectRef::__set_partValues(const std::vector<std::string> & val) {
+ this->partValues = val;
+ }
+
+ void HiveObjectRef::__set_columnName(const std::string& val) {
+ this->columnName = val;
+ }
+
+ uint32_t HiveObjectRef::read(::apache::thrift::protocol::TProtocol* iprot) {
+
+ apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+ uint32_t xfer = 0;
+ std::string fname;
+ ::apache::thrift::protocol::TType ftype;
+ int16_t fid;
+
+ xfer += iprot->readStructBegin(fname);
+
+ using ::apache::thrift::protocol::TProtocolException;
+
+
+ while (true)
+ {
+ xfer += iprot->readFieldBegin(fname, ftype, fid);
+ if (ftype == ::apache::thrift::protocol::T_STOP) {
+ break;
+ }
+ switch (fid)
+ {
+ case 1:
+ if (ftype == ::apache::thrift::protocol::T_I32) {
+ int32_t ecast20;
+ xfer += iprot->readI32(ecast20);
+ this->objectType = (HiveObjectType::type)ecast20;
+ this->__isset.objectType = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 2:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->dbName);
+ this->__isset.dbName = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 3:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->objectName);
+ this->__isset.objectName = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 4:
+ if (ftype == ::apache::thrift::protocol::T_LIST) {
+ {
+ this->partValues.clear();
+ uint32_t _size21;
+ ::apache::thrift::protocol::TType _etype24;
+ xfer += iprot->readListBegin(_etype24, _size21);
+ this->partValues.resize(_size21);
+ uint32_t _i25;
+ for (_i25 = 0; _i25 < _size21; ++_i25)
+ {
+ xfer += iprot->readString(this->partValues[_i25]);
+ }
+ xfer += iprot->readListEnd();
+ }
+ this->__isset.partValues = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 5:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->columnName);
+ this->__isset.columnName = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ default:
+ xfer += iprot->skip(ftype);
+ break;
+ }
+ xfer += iprot->readFieldEnd();
+ }
+
+ xfer += iprot->readStructEnd();
+
+ return xfer;
+ }
+
+ uint32_t HiveObjectRef::write(::apache::thrift::protocol::TProtocol* oprot) const {
+ uint32_t xfer = 0;
+ apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+ xfer += oprot->writeStructBegin("HiveObjectRef");
+
+ xfer += oprot->writeFieldBegin("objectType", ::apache::thrift::protocol::T_I32, 1);
+ xfer += oprot->writeI32((int32_t)this->objectType);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("dbName", ::apache::thrift::protocol::T_STRING, 2);
+ xfer += oprot->writeString(this->dbName);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("objectName", ::apache::thrift::protocol::T_STRING, 3);
+ xfer += oprot->writeString(this->objectName);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("partValues", ::apache::thrift::protocol::T_LIST, 4);
+ {
+ xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->partValues.size()));
+ std::vector<std::string> ::const_iterator _iter26;
+ for (_iter26 = this->partValues.begin(); _iter26 != this->partValues.end(); ++_iter26)
+ {
+ xfer += oprot->writeString((*_iter26));
+ }
+ xfer += oprot->writeListEnd();
+ }
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("columnName", ::apache::thrift::protocol::T_STRING, 5);
+ xfer += oprot->writeString(this->columnName);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldStop();
+ xfer += oprot->writeStructEnd();
+ return xfer;
+ }
+
+ void swap(HiveObjectRef &a, HiveObjectRef &b) {
+ using ::std::swap;
+ swap(a.objectType, b.objectType);
+ swap(a.dbName, b.dbName);
+ swap(a.objectName, b.objectName);
+ swap(a.partValues, b.partValues);
+ swap(a.columnName, b.columnName);
+ swap(a.__isset, b.__isset);
+ }
+
+ HiveObjectRef::HiveObjectRef(const HiveObjectRef& other27) {
+ objectType = other27.objectType;
+ dbName = other27.dbName;
+ objectName = other27.objectName;
+ partValues = other27.partValues;
+ columnName = other27.columnName;
+ __isset = other27.__isset;
+ }
+ HiveObjectRef& HiveObjectRef::operator=(const HiveObjectRef& other28) {
+ objectType = other28.objectType;
+ dbName = other28.dbName;
+ objectName = other28.objectName;
+ partValues = other28.partValues;
+ columnName = other28.columnName;
+ __isset = other28.__isset;
+ return *this;
+ }
+ void HiveObjectRef::printTo(std::ostream& out) const {
+ using ::apache::thrift::to_string;
+ out << "HiveObjectRef(";
+ out << "objectType=" << to_string(objectType);
+ out << ", " << "dbName=" << to_string(dbName);
+ out << ", " << "objectName=" << to_string(objectName);
+ out << ", " << "partValues=" << to_string(partValues);
+ out << ", " << "columnName=" << to_string(columnName);
+ out << ")";
+ }
+
+
+ PrivilegeGrantInfo::~PrivilegeGrantInfo() throw() {
+ }
+
+
+ void PrivilegeGrantInfo::__set_privilege(const std::string& val) {
+ this->privilege = val;
+ }
+
+ void PrivilegeGrantInfo::__set_createTime(const int32_t val) {
+ this->createTime = val;
+ }
+
+ void PrivilegeGrantInfo::__set_grantor(const std::string& val) {
+ this->grantor = val;
+ }
+
+ void PrivilegeGrantInfo::__set_grantorType(const PrincipalType::type val) {
+ this->grantorType = val;
+ }
+
+ void PrivilegeGrantInfo::__set_grantOption(const bool val) {
+ this->grantOption = val;
+ }
+
+ uint32_t PrivilegeGrantInfo::read(::apache::thrift::protocol::TProtocol* iprot) {
+
+ apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+ uint32_t xfer = 0;
+ std::string fname;
+ ::apache::thrift::protocol::TType ftype;
+ int16_t fid;
+
+ xfer += iprot->readStructBegin(fname);
+
+ using ::apache::thrift::protocol::TProtocolException;
+
+
+ while (true)
+ {
+ xfer += iprot->readFieldBegin(fname, ftype, fid);
+ if (ftype == ::apache::thrift::protocol::T_STOP) {
+ break;
+ }
+ switch (fid)
+ {
+ case 1:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->privilege);
+ this->__isset.privilege = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 2:
+ if (ftype == ::apache::thrift::protocol::T_I32) {
+ xfer += iprot->readI32(this->createTime);
+ this->__isset.createTime = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 3:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->grantor);
+ this->__isset.grantor = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 4:
+ if (ftype == ::apache::thrift::protocol::T_I32) {
+ int32_t ecast29;
+ xfer += iprot->readI32(ecast29);
+ this->grantorType = (PrincipalType::type)ecast29;
+ this->__isset.grantorType = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 5:
+ if (ftype == ::apache::thrift::protocol::T_BOOL) {
+ xfer += iprot->readBool(this->grantOption);
+ this->__isset.grantOption = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ default:
+ xfer += iprot->skip(ftype);
+ break;
+ }
+ xfer += iprot->readFieldEnd();
+ }
+
+ xfer += iprot->readStructEnd();
+
+ return xfer;
+ }
+
+ uint32_t PrivilegeGrantInfo::write(::apache::thrift::protocol::TProtocol* oprot) const {
+ uint32_t xfer = 0;
+ apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+ xfer += oprot->writeStructBegin("PrivilegeGrantInfo");
+
+ xfer += oprot->writeFieldBegin("privilege", ::apache::thrift::protocol::T_STRING, 1);
+ xfer += oprot->writeString(this->privilege);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("createTime", ::apache::thrift::protocol::T_I32, 2);
+ xfer += oprot->writeI32(this->createTime);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("grantor", ::apache::thrift::protocol::T_STRING, 3);
+ xfer += oprot->writeString(this->grantor);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("grantorType", ::apache::thrift::protocol::T_I32, 4);
+ xfer += oprot->writeI32((int32_t)this->grantorType);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("grantOption", ::apache::thrift::protocol::T_BOOL, 5);
+ xfer += oprot->writeBool(this->grantOption);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldStop();
+ xfer += oprot->writeStructEnd();
+ return xfer;
+ }
+
+ void swap(PrivilegeGrantInfo &a, PrivilegeGrantInfo &b) {
+ using ::std::swap;
+ swap(a.privilege, b.privilege);
+ swap(a.createTime, b.createTime);
+ swap(a.grantor, b.grantor);
+ swap(a.grantorType, b.grantorType);
+ swap(a.grantOption, b.grantOption);
+ swap(a.__isset, b.__isset);
+ }
+
+ PrivilegeGrantInfo::PrivilegeGrantInfo(const PrivilegeGrantInfo& other30) {
+ privilege = other30.privilege;
+ createTime = other30.createTime;
+ grantor = other30.grantor;
+ grantorType = other30.grantorType;
+ grantOption = other30.grantOption;
+ __isset = other30.__isset;
+ }
+ PrivilegeGrantInfo& PrivilegeGrantInfo::operator=(const PrivilegeGrantInfo& other31) {
+ privilege = other31.privilege;
+ createTime = other31.createTime;
+ grantor = other31.grantor;
+ grantorType = other31.grantorType;
+ grantOption = other31.grantOption;
+ __isset = other31.__isset;
+ return *this;
+ }
+ void PrivilegeGrantInfo::printTo(std::ostream& out) const {
+ using ::apache::thrift::to_string;
+ out << "PrivilegeGrantInfo(";
+ out << "privilege=" << to_string(privilege);
+ out << ", " << "createTime=" << to_string(createTime);
+ out << ", " << "grantor=" << to_string(grantor);
+ out << ", " << "grantorType=" << to_string(grantorType);
+ out << ", " << "grantOption=" << to_string(grantOption);
+ out << ")";
+ }
+
+
+ HiveObjectPrivilege::~HiveObjectPrivilege() throw() {
+ }
+
+
+ void HiveObjectPrivilege::__set_hiveObject(const HiveObjectRef& val) {
+ this->hiveObject = val;
+ }
+
+ void HiveObjectPrivilege::__set_principalName(const std::string& val) {
+ this->principalName = val;
+ }
+
+ void HiveObjectPrivilege::__set_principalType(const PrincipalType::type val) {
+ this->principalType = val;
+ }
+
+ void HiveObjectPrivilege::__set_grantInfo(const PrivilegeGrantInfo& val) {
+ this->grantInfo = val;
+ }
+
+ uint32_t HiveObjectPrivilege::read(::apache::thrift::protocol::TProtocol* iprot) {
+
+ apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+ uint32_t xfer = 0;
+ std::string fname;
+ ::apache::thrift::protocol::TType ftype;
+ int16_t fid;
+
+ xfer += iprot->readStructBegin(fname);
+
+ using ::apache::thrift::protocol::TProtocolException;
+
+
+ while (true)
+ {
+ xfer += iprot->readFieldBegin(fname, ftype, fid);
+ if (ftype == ::apache::thrift::protocol::T_STOP) {
+ break;
+ }
+ switch (fid)
+ {
+ case 1:
+ if (ftype == ::apache::thrift::protocol::T_STRUCT) {
+ xfer += this->hiveObject.read(iprot);
+ this->__isset.hiveObject = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 2:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->principalName);
+ this->__isset.principalName = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 3:
+ if (ftype == ::apache::thrift::protocol::T_I32) {
+ int32_t ecast32;
+ xfer += iprot->readI32(ecast32);
+ this->principalType = (PrincipalType::type)ecast32;
+ this->__isset.principalType = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 4:
+ if (ftype == ::apache::thrift::protocol::T_STRUCT) {
+ xfer += this->grantInfo.read(iprot);
+ this->__isset.grantInfo = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ default:
+ xfer += iprot->skip(ftype);
+ break;
+ }
+ xfer += iprot->readFieldEnd();
+ }
+
+ xfer += iprot->readStructEnd();
+
+ return xfer;
+ }
+
+ uint32_t HiveObjectPrivilege::write(::apache::thrift::protocol::TProtocol* oprot) const {
+ uint32_t xfer = 0;
+ apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+ xfer += oprot->writeStructBegin("HiveObjectPrivilege");
+
+ xfer += oprot->writeFieldBegin("hiveObject", ::apache::thrift::protocol::T_STRUCT, 1);
+ xfer += this->hiveObject.write(oprot);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("principalName", ::apache::thrift::protocol::T_STRING, 2);
+ xfer += oprot->writeString(this->principalName);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("principalType", ::apache::thrift::protocol::T_I32, 3);
+ xfer += oprot->writeI32((int32_t)this->principalType);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("grantInfo", ::apache::thrift::protocol::T_STRUCT, 4);
+ xfer += this->grantInfo.write(oprot);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldStop();
+ xfer += oprot->writeStructEnd();
+ return xfer;
+ }
+
+ void swap(HiveObjectPrivilege &a, HiveObjectPrivilege &b) {
+ using ::std::swap;
+ swap(a.hiveObject, b.hiveObject);
+ swap(a.principalName, b.principalName);
+ swap(a.principalType, b.principalType);
+ swap(a.grantInfo, b.grantInfo);
+ swap(a.__isset, b.__isset);
+ }
+
+ HiveObjectPrivilege::HiveObjectPrivilege(const HiveObjectPrivilege& other33) {
+ hiveObject = other33.hiveObject;
+ principalName = other33.principalName;
+ principalType = other33.principalType;
+ grantInfo = other33.grantInfo;
+ __isset = other33.__isset;
+ }
+ HiveObjectPrivilege& HiveObjectPrivilege::operator=(const HiveObjectPrivilege& other34) {
+ hiveObject = other34.hiveObject;
+ principalName = other34.principalName;
+ principalType = other34.principalType;
+ grantInfo = other34.grantInfo;
+ __isset = other34.__isset;
+ return *this;
+ }
+ void HiveObjectPrivilege::printTo(std::ostream& out) const {
+ using ::apache::thrift::to_string;
+ out << "HiveObjectPrivilege(";
+ out << "hiveObject=" << to_string(hiveObject);
+ out << ", " << "principalName=" << to_string(principalName);
+ out << ", " << "principalType=" << to_string(principalType);
+ out << ", " << "grantInfo=" << to_string(grantInfo);
+ out << ")";
+ }
+
+
+ PrivilegeBag::~PrivilegeBag() throw() {
+ }
+
+
+ void PrivilegeBag::__set_privileges(const std::vector<HiveObjectPrivilege> & val) {
+ this->privileges = val;
+ }
+
+ uint32_t PrivilegeBag::read(::apache::thrift::protocol::TProtocol* iprot) {
+
+ apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+ uint32_t xfer = 0;
+ std::string fname;
+ ::apache::thrift::protocol::TType ftype;
+ int16_t fid;
+
+ xfer += iprot->readStructBegin(fname);
+
+ using ::apache::thrift::protocol::TProtocolException;
+
+
+ while (true)
+ {
+ xfer += iprot->readFieldBegin(fname, ftype, fid);
+ if (ftype == ::apache::thrift::protocol::T_STOP) {
+ break;
+ }
+ switch (fid)
+ {
+ case 1:
+ if (ftype == ::apache::thrift::protocol::T_LIST) {
+ {
+ this->privileges.clear();
+ uint32_t _size35;
+ ::apache::thrift::protocol::TType _etype38;
+ xfer += iprot->readListBegin(_etype38, _size35);
+ this->privileges.resize(_size35);
+ uint32_t _i39;
+ for (_i39 = 0; _i39 < _size35; ++_i39)
+ {
+ xfer += this->privileges[_i39].read(iprot);
+ }
+ xfer += iprot->readListEnd();
+ }
+ this->__isset.privileges = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ default:
+ xfer += iprot->skip(ftype);
+ break;
+ }
+ xfer += iprot->readFieldEnd();
+ }
+
+ xfer += iprot->readStructEnd();
+
+ return xfer;
+ }
+
+ uint32_t PrivilegeBag::write(::apache::thrift::protocol::TProtocol* oprot) const {
+ uint32_t xfer = 0;
+ apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+ xfer += oprot->writeStructBegin("PrivilegeBag");
+
+ xfer += oprot->writeFieldBegin("privileges", ::apache::thrift::protocol::T_LIST, 1);
+ {
+ xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(this->privileges.size()));
+ std::vector<HiveObjectPrivilege> ::const_iterator _iter40;
+ for (_iter40 = this->privileges.begin(); _iter40 != this->privileges.end(); ++_iter40)
+ {
+ xfer += (*_iter40).write(oprot);
+ }
+ xfer += oprot->writeListEnd();
+ }
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldStop();
+ xfer += oprot->writeStructEnd();
+ return xfer;
+ }
+
+ void swap(PrivilegeBag &a, PrivilegeBag &b) {
+ using ::std::swap;
+ swap(a.privileges, b.privileges);
+ swap(a.__isset, b.__isset);
+ }
+
+ PrivilegeBag::PrivilegeBag(const PrivilegeBag& other41) {
+ privileges = other41.privileges;
+ __isset = other41.__isset;
+ }
+ PrivilegeBag& PrivilegeBag::operator=(const PrivilegeBag& other42) {
+ privileges = other42.privileges;
+ __isset = other42.__isset;
+ return *this;
+ }
+ void PrivilegeBag::printTo(std::ostream& out) const {
+ using ::apache::thrift::to_string;
+ out << "PrivilegeBag(";
+ out << "privileges=" << to_string(privileges);
+ out << ")";
+ }
+
+
+ PrincipalPrivilegeSet::~PrincipalPrivilegeSet() throw() {
+ }
+
+
+ void PrincipalPrivilegeSet::__set_userPrivileges(const std::map<std::string, std::vector<PrivilegeGrantInfo> > & val) {
+ this->userPrivileges = val;
+ }
+
+ void PrincipalPrivilegeSet::__set_groupPrivileges(const std::map<std::string, std::vector<PrivilegeGrantInfo> > & val) {
+ this->groupPrivileges = val;
+ }
+
+ void PrincipalPrivilegeSet::__set_rolePrivileges(const std::map<std::string, std::vector<PrivilegeGrantInfo> > & val) {
+ this->rolePrivileges = val;
+ }
+
+ uint32_t PrincipalPrivilegeSet::read(::apache::thrift::protocol::TProtocol* iprot) {
+
+ apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+ uint32_t xfer = 0;
+ std::string fname;
+ ::apache::thrift::protocol::TType ftype;
+ int16_t fid;
+
+ xfer += iprot->readStructBegin(fname);
+
+ using ::apache::thrift::protocol::TProtocolException;
+
+
+ while (true)
+ {
+ xfer += iprot->readFieldBegin(fname, ftype, fid);
+ if (ftype == ::apache::thrift::protocol::T_STOP) {
+ break;
+ }
+ switch (fid)
+ {
+ case 1:
+ if (ftype == ::apache::thrift::protocol::T_MAP) {
+ {
+ this->userPrivileges.clear();
+ uint32_t _size43;
+ ::apache::thrift::protocol::TType _ktype44;
+ ::apache::thrift::protocol::TType _vtype45;
+ xfer += iprot->readMapBegin(_ktype44, _vtype45, _size43);
+ uint32_t _i47;
+ for (_i47 = 0; _i47 < _size43; ++_i47)
+ {
+ std::string _key48;
+ xfer += iprot->readString(_key48);
+ std::vector<PrivilegeGrantInfo> & _val49 = this->userPrivileges[_key48];
+ {
+ _val49.clear();
+ uint32_t _size50;
+ ::apache::thrift::protocol::TType _etype53;
+ xfer += iprot->readListBegin(_etype53, _size50);
+ _val49.resize(_size50);
+ uint32_t _i54;
+ for (_i54 = 0; _i54 < _size50; ++_i54)
+ {
+ xfer += _val49[_i54].read(iprot);
+ }
+ xfer += iprot->readListEnd();
+ }
+ }
+ xfer += iprot->readMapEnd();
+ }
+ this->__isset.userPrivileges = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 2:
+ if (ftype == ::apache::thrift::protocol::T_MAP) {
+ {
+ this->groupPrivileges.clear();
+ uint32_t _size55;
+ ::apache::thrift::protocol::TType _ktype56;
+ ::apache::thrift::protocol::TType _vtype57;
+ xfer += iprot->readMapBegin(_ktype56, _vtype57, _size55);
+ uint32_t _i59;
+ for (_i59 = 0; _i59 < _size55; ++_i59)
+ {
+ std::string _key60;
+ xfer += iprot->readString(_key60);
+ std::vector<PrivilegeGrantInfo> & _val61 = this->groupPrivileges[_key60];
+ {
+ _val61.clear();
+ uint32_t _size62;
+ ::apache::thrift::protocol::TType _etype65;
+ xfer += iprot->readListBegin(_etype65, _size62);
+ _val61.resize(_size62);
+ uint32_t _i66;
+ for (_i66 = 0; _i66 < _size62; ++_i66)
+ {
+ xfer += _val61[_i66].read(iprot);
+ }
+ xfer += iprot->readListEnd();
+ }
+ }
+ xfer += iprot->readMapEnd();
+ }
+ this->__isset.groupPrivileges = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 3:
+ if (ftype == ::apache::thrift::protocol::T_MAP) {
+ {
+ this->rolePrivileges.clear();
+ uint32_t _size67;
+ ::apache::thrift::protocol::TType _ktype68;
+ ::apache::thrift::protocol::TType _vtype69;
+ xfer += iprot->readMapBegin(_ktype68, _vtype69, _size67);
+ uint32_t _i71;
+ for (_i71 = 0; _i71 < _size67; ++_i71)
+ {
+ std::string _key72;
+ xfer += iprot->readString(_key72);
+ std::vector<PrivilegeGrantInfo> & _val73 = this->rolePrivileges[_key72];
+ {
+ _val73.clear();
+ uint32_t _size74;
+ ::apache::thrift::protocol::TType _etype77;
+ xfer += iprot->readListBegin(_etype77, _size74);
+ _val73.resize(_size74);
+ uint32_t _i78;
+ for (_i78 = 0; _i78 < _size74; ++_i78)
+ {
+ xfer += _val73[_i78].read(iprot);
+ }
+ xfer += iprot->readListEnd();
+ }
+ }
+ xfer += iprot->readMapEnd();
+ }
+ this->__isset.rolePrivileges = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ default:
+ xfer += iprot->skip(ftype);
+ break;
+ }
+ xfer += iprot->readFieldEnd();
+ }
+
+ xfer += iprot->readStructEnd();
+
+ return xfer;
+ }
+
+ uint32_t PrincipalPrivilegeSet::write(::apache::thrift::protocol::TProtocol* oprot) const {
+ uint32_t xfer = 0;
+ apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+ xfer += oprot->writeStructBegin("PrincipalPrivilegeSet");
+
+ xfer += oprot->writeFieldBegin("userPrivileges", ::apache::thrift::protocol::T_MAP, 1);
+ {
+ xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_LIST, static_cast<uint32_t>(this->userPrivileges.size()));
+ std::map<std::string, std::vector<PrivilegeGrantInfo> > ::const_iterator _iter79;
+ for (_iter79 = this->userPrivileges.begin(); _iter79 != this->userPrivileges.end(); ++_iter79)
+ {
+ xfer += oprot->writeString(_iter79->first);
+ {
+ xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(_iter79->second.size()));
+ std::vector<PrivilegeGrantInfo> ::const_iterator _iter80;
+ for (_iter80 = _iter79->second.begin(); _iter80 != _iter79->second.end(); ++_iter80)
+ {
+ xfer += (*_iter80).write(oprot);
+ }
+ xfer += oprot->writeListEnd();
+ }
+ }
+ xfer += oprot->writeMapEnd();
+ }
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("groupPrivileges", ::apache::thrift::protocol::T_MAP, 2);
+ {
+ xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_LIST, static_cast<uint32_t>(this->groupPrivileges.size()));
+ std::map<std::string, std::vector<PrivilegeGrantInfo> > ::const_iterator _iter81;
+ for (_iter81 = this->groupPrivileges.begin(); _iter81 != this->groupPrivileges.end(); ++_iter81)
+ {
+ xfer += oprot->writeString(_iter81->first);
+ {
+ xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(_iter81->second.size()));
+ std::vector<PrivilegeGrantInfo> ::const_iterator _iter82;
+ for (_iter82 = _iter81->second.begin(); _iter82 != _iter81->second.end(); ++_iter82)
+ {
+ xfer += (*_iter82).write(oprot);
+ }
+ xfer += oprot->writeListEnd();
+ }
+ }
+ xfer += oprot->writeMapEnd();
+ }
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("rolePrivileges", ::apache::thrift::protocol::T_MAP, 3);
+ {
+ xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_LIST, static_cast<uint32_t>(this->rolePrivileges.size()));
+ std::map<std::string, std::vector<PrivilegeGrantInfo> > ::const_iterator _iter83;
+ for (_iter83 = this->rolePrivileges.begin(); _iter83 != this->rolePrivileges.end(); ++_iter83)
+ {
+ xfer += oprot->writeString(_iter83->first);
+ {
+ xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(_iter83->second.size()));
+ std::vector<PrivilegeGrantInfo> ::const_iterator _iter84;
+ for (_iter84 = _iter83->second.begin(); _iter84 != _iter83->second.end(); ++_iter84)
+ {
+ xfer += (*_iter84).write(oprot);
+ }
+ xfer += oprot->writeListEnd();
+ }
+ }
+ xfer += oprot->writeMapEnd();
+ }
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldStop();
+ xfer += oprot->writeStructEnd();
+ return xfer;
+ }
+
+ void swap(PrincipalPrivilegeSet &a, PrincipalPrivilegeSet &b) {
+ using ::std::swap;
+ swap(a.userPrivileges, b.userPrivileges);
+ swap(a.groupPrivileges, b.groupPrivileges);
+ swap(a.rolePrivileges, b.rolePrivileges);
+ swap(a.__isset, b.__isset);
+ }
+
+ PrincipalPrivilegeSet::PrincipalPrivilegeSet(const PrincipalPrivilegeSet& other85) {
+ userPrivileges = other85.userPrivileges;
+ groupPrivileges = other85.groupPrivileges;
+ rolePrivileges = other85.rolePrivileges;
+ __isset = other85.__isset;
+ }
+ PrincipalPrivilegeSet& PrincipalPrivilegeSet::operator=(const PrincipalPrivilegeSet& other86) {
+ userPrivileges = other86.userPrivileges;
+ groupPrivileges = other86.groupPrivileges;
+ rolePrivileges = other86.rolePrivileges;
+ __isset = other86.__isset;
+ return *this;
+ }
+ void PrincipalPrivilegeSet::printTo(std::ostream& out) const {
+ using ::apache::thrift::to_string;
+ out << "PrincipalPrivilegeSet(";
+ out << "userPrivileges=" << to_string(userPrivileges);
+ out << ", " << "groupPrivileges=" << to_string(groupPrivileges);
+ out << ", " << "rolePrivileges=" << to_string(rolePrivileges);
+ out << ")";
+ }
+
+
+ GrantRevokePrivilegeRequest::~GrantRevokePrivilegeRequest() throw() {
+ }
+
+
+ void GrantRevokePrivilegeRequest::__set_requestType(const GrantRevokeType::type val) {
+ this->requestType = val;
+ }
+
+ void GrantRevokePrivilegeRequest::__set_privileges(const PrivilegeBag& val) {
+ this->privileges = val;
+ }
+
+ void GrantRevokePrivilegeRequest::__set_revokeGrantOption(const bool val) {
+ this->revokeGrantOption = val;
+ __isset.revokeGrantOption = true;
+ }
+
+ uint32_t GrantRevokePrivilegeRequest::read(::apache::thrift::protocol::TProtocol* iprot) {
+
+ apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+ uint32_t xfer = 0;
+ std::string fname;
+ ::apache::thrift::protocol::TType ftype;
+ int16_t fid;
+
+ xfer += iprot->readStructBegin(fname);
+
+ using ::apache::thrift::protocol::TProtocolException;
+
+
+ while (true)
+ {
+ xfer += iprot->readFieldBegin(fname, ftype, fid);
+ if (ftype == ::apache::thrift::protocol::T_STOP) {
+ break;
+ }
+ switch (fid)
+ {
+ case 1:
+ if (ftype == ::apache::thrift::protocol::T_I32) {
+ int32_t ecast87;
+ xfer += iprot->readI32(ecast87);
+ this->requestType = (GrantRevokeType::type)ecast87;
+ this->__isset.requestType = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 2:
+ if (ftype == ::apache::thrift::protocol::T_STRUCT) {
+ xfer += this->privileges.read(iprot);
+ this->__isset.privileges = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 3:
+ if (ftype == ::apache::thrift::protocol::T_BOOL) {
+ xfer += iprot->readBool(this->revokeGrantOption);
+ this->__isset.revokeGrantOption = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ default:
+ xfer += iprot->skip(ftype);
+ break;
+ }
+ xfer += iprot->readFieldEnd();
+ }
+
+ xfer += iprot->readStructEnd();
+
+ return xfer;
+ }
+
+ uint32_t GrantRevokePrivilegeRequest::write(::apache::thrift::protocol::TProtocol* oprot) const {
+ uint32_t xfer = 0;
+ apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+ xfer += oprot->writeStructBegin("GrantRevokePrivilegeRequest");
+
+ xfer += oprot->writeFieldBegin("requestType", ::apache::thrift::protocol::T_I32, 1);
+ xfer += oprot->writeI32((int32_t)this->requestType);
+ xfer += oprot->writeFieldEnd();
+
+ xfer += oprot->writeFieldBegin("privileges", ::apache::thrift::protocol::T_STRUCT, 2);
+ xfer += this->privileges.write(oprot);
+ xfer += oprot->writeFieldEnd();
+
+ if (this->__isset.revokeGrantOption) {
+ xfer += oprot->writeFieldBegin("revokeGrantOption", ::apache::thrift::protocol::T_BOOL, 3);
+ xfer += oprot->writeBool(this->revokeGrantOption);
+ xfer += oprot->writeFieldEnd();
+ }
+ xfer += oprot->writeFieldStop();
+ xfer += oprot->writeStructEnd();
+ return xfer;
+ }
+
+ void swap(GrantRevokePrivilegeRequest &a, GrantRevokePrivilegeRequest &b) {
+ using ::std::swap;
+ swap(a.requestType, b.requestType);
+ swap(a.privileges, b.privileges);
+ swap(a.revokeGrantOption, b.revokeGrantOption);
+ swap(a.__isset, b.__isset);
+ }
+
+ GrantRevokePrivilegeRequest::GrantRevokePrivilegeRequest(const GrantRevokePrivilegeRequest& other88) {
+ requestType = other88.requestType;
+ privileges = other88.privileges;
+ revokeGrantOption = other88.revokeGrantOption;
+ __isset = other88.__isset;
+ }
+ GrantRevokePrivilegeRequest& GrantRevokePrivilegeRequest::operator=(const GrantRevokePrivilegeRequest& other89) {
+ requestType = other89.requestType;
+ privileges = other89.privileges;
+ revokeGrantOption = other89.revokeGrantOption;
+ __isset = other89.__isset;
+ return *this;
+ }
+ void GrantRevokePrivilegeRequest::printTo(std::ostream& out) const {
+ using ::apache::thrift::to_string;
+ out << "GrantRevokePrivilegeRequest(";
+ out << "requestType=" << to_string(requestType);
+ out << ", " << "privileges=" << to_string(privileges);
+ out << ", " << "revokeGrantOption="; (__isset.revokeGrantOption ? (out << to_string(revokeGrantOption)) : (out << "<null>"));
+ out << ")";
+ }
+
+
+ GrantRevokePrivilegeResponse::~GrantRevokePrivilegeResponse() throw() {
+ }
+
+
+ void GrantRevokePrivilegeResponse::__set_success(const bool val) {
+ this->success = val;
+ __isset.success = true;
+ }
+
+ uint32_t GrantRevokePrivilegeResponse::read(::apache::thrift::protocol::TProtocol* iprot) {
+
+ apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+ uint32_t xfer = 0;
+ std::string fname;
+ ::apache::thrift::protocol::TType ftype;
+ int16_t fid;
+
+ xfer += iprot->readStructBegin(fname);
+
+ using ::apache::thrift::protocol::TProtocolException;
+
+
+ while (true)
+ {
+ xfer += iprot->readFieldBegin(fname, ftype, fid);
+ if (ftype == ::apache::thrift::protocol::T_STOP) {
+ break;
+ }
+ switch (fid)
+ {
+ case 1:
+ if (ftype == ::apache::thrift::protocol::T_BOOL) {
+ xfer += iprot->readBool(this->success);
+ this->__isset.success = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ default:
+ xfer += iprot->skip(ftype);
+ break;
+ }
+ xfer += iprot->readFieldEnd();
+ }
+
+ xfer += iprot->readStructEnd();
+
+ return xfer;
+ }
+
+ uint32_t GrantRevokePrivilegeResponse::write(::apache::thrift::protocol::TProtocol* oprot) const {
+ uint32_t xfer = 0;
+ apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+ xfer += oprot->writeStructBegin("GrantRevokePrivilegeResponse");
+
+ if (this->__isset.success) {
+ xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_BOOL, 1);
+ xfer += oprot->writeBool(this->success);
+ xfer += oprot->writeFieldEnd();
+ }
+ xfer += oprot->writeFieldStop();
+ xfer += oprot->writeStructEnd();
+ return xfer;
+ }
+
+ void swap(GrantRevokePrivilegeResponse &a, GrantRevokePrivilegeResponse &b) {
+ using ::std::swap;
+ swap(a.success, b.success);
+ swap(a.__isset, b.__isset);
+ }
+
+ GrantRevokePrivilegeResponse::GrantRevokePrivilegeResponse(const GrantRevokePrivilegeResponse& other90) {
+ success = other90.success;
+ __isset = other90.__isset;
+ }
+ GrantRevokePrivilegeResponse& GrantRevokePrivilegeResponse::operator=(const GrantRevokePrivilegeResponse& other91) {
+ success = other91.success;
+ __isset = other91.__isset;
+ return *this;
+ }
+ void GrantRevokePrivilegeResponse::printTo(std::ostream& out) const {
+ using ::apache::thrift::to_string;
+ out << "GrantRevokePrivilegeResponse(";
+ out << "success="; (__isset.success ? (out << to_string(success)) : (out << "<null>"));
+ out << ")";
+ }
+
+
+ Role::~Role() throw() {
+ }
+
+
+ void Role::__set_roleName(const std::string& val) {
+ this->roleName = val;
+ }
+
+ void Role::__set_createTime(const int32_t val) {
+ this->createTime = val;
+ }
+
+ void Role::__set_ownerName(const std::string& val) {
+ this->ownerName = val;
+ }
+
+ uint32_t Role::read(::apache::thrift::protocol::TProtocol* iprot) {
+
+ apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+ uint32_t xfer = 0;
+ std::string fname;
+ ::apache::thrift::protocol::TType ftype;
+ int16_t fid;
+
+ xfer += iprot->readStructBegin(fname);
+
+ using ::apache::thrift::protocol::TProtocolException;
+
+
+ while (true)
+ {
+ xfer += iprot->readFieldBegin(fname, ftype, fid);
+ if (ftype == ::apache::thrift::protocol::T_STOP) {
+ break;
+ }
+ switch (fid)
+ {
+ case 1:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->roleName);
+ this->__isset.roleName = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 2:
+ if (ftype == ::apache::thrift::protocol::T_I32) {
+ xfer += iprot->readI32(this->createTime);
+ this->__isset.createTime = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
+ case 3:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readString(this->ownerName);
+ this->__isset.ownerName = true;
+ } else {
+ xfer += ip
<TRUNCATED>
[31/50] [abbrv] hive git commit: HIVE-17006 : LLAP: Parquet caching
v1 (Sergey Shelukhin, reviewed by Gunther Hagleitner)
Posted by we...@apache.org.
HIVE-17006 : LLAP: Parquet caching v1 (Sergey Shelukhin, reviewed by Gunther Hagleitner)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9e673a73
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9e673a73
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9e673a73
Branch: refs/heads/hive-14535
Commit: 9e673a73d28ef047c5d4299b415c306423f68c71
Parents: 642acdf
Author: sergey <se...@apache.org>
Authored: Thu Aug 31 14:21:04 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Thu Aug 31 14:21:04 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/common/FileUtils.java | 35 +
.../test/resources/testconfiguration.properties | 1 +
.../apache/hadoop/hive/llap/io/api/LlapIo.java | 4 +-
.../hive/llap/cache/EvictionDispatcher.java | 13 +-
.../hive/llap/cache/LlapAllocatorBuffer.java | 8 +-
.../hive/llap/io/api/impl/LlapIoImpl.java | 99 ++-
.../io/metadata/ParquetMetadataCacheImpl.java | 353 ++++++++
.../hadoop/hive/llap/LlapCacheAwareFs.java | 422 +++++++++
.../org/apache/hadoop/hive/ql/io/HdfsUtils.java | 3 +-
.../hadoop/hive/ql/io/HiveInputFormat.java | 103 ++-
.../io/LlapCacheOnlyInputFormatInterface.java | 28 +
.../ql/io/parquet/MapredParquetInputFormat.java | 12 +-
.../parquet/VectorizedParquetInputFormat.java | 22 +-
.../vector/ParquetFooterInputFromCache.java | 196 +++++
.../vector/VectorizedParquetRecordReader.java | 169 +++-
.../org/apache/hadoop/hive/ql/plan/MapWork.java | 28 +-
.../clientpositive/parquet_ppd_decimal.q | 1 +
.../clientpositive/parquet_predicate_pushdown.q | 1 +
.../test/queries/clientpositive/parquet_types.q | 1 +
.../parquet_types_vectorization.q | 1 +
.../queries/clientpositive/vectorized_parquet.q | 1 +
.../clientpositive/vectorized_parquet_types.q | 1 +
.../llap/parquet_predicate_pushdown.q.out | 18 +-
.../llap/parquet_types_vectorization.q.out | 849 +++++++++++++++++++
.../llap/vector_partitioned_date_time.q.out | 16 +-
.../llap/vectorized_parquet.q.out | 2 +-
.../llap/vectorized_parquet_types.q.out | 2 +-
.../hive/common/io/FileMetadataCache.java | 51 ++
.../io/encoded/MemoryBufferOrBuffers.java | 24 +
29 files changed, 2360 insertions(+), 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
index e784797..0feff59 100644
--- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
+++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
@@ -18,11 +18,14 @@
package org.apache.hadoop.hive.common;
+import java.io.EOFException;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
+import java.nio.ByteBuffer;
import java.security.AccessControlException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
@@ -1033,4 +1036,36 @@ public final class FileUtils {
return result;
}
+ /**
+ * Reads length bytes of data from the stream into the byte buffer.
+ * @param stream Stream to read from.
+ * @param length The number of bytes to read.
+ * @param bb The buffer to read into; the data is written at current position and then the
+ * position is incremented by length.
+ * @throws EOFException the length bytes cannot be read. The buffer position is not modified.
+ */
+ public static void readFully(InputStream stream, int length, ByteBuffer bb) throws IOException {
+ byte[] b = null;
+ int offset = 0;
+ if (bb.hasArray()) {
+ b = bb.array();
+ offset = bb.arrayOffset() + bb.position();
+ } else {
+ b = new byte[bb.remaining()];
+ }
+ int fullLen = length;
+ while (length > 0) {
+ int result = stream.read(b, offset, length);
+ if (result < 0) {
+ throw new EOFException("Reading " + fullLen + " bytes");
+ }
+ offset += result;
+ length -= result;
+ }
+ if (!bb.hasArray()) {
+ bb.put(b);
+ } else {
+ bb.position(bb.position() + fullLen);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 6f2efa7..f452341 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -75,6 +75,7 @@ minillap.shared.query.files=insert_into1.q,\
orc_merge4.q,\
orc_merge_diff_fs.q,\
parallel_colstats.q,\
+ parquet_types_vectorization.q,\
unionDistinct_1.q,\
union_type_chk.q,\
cte_2.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIo.java
----------------------------------------------------------------------
diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIo.java b/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIo.java
index 42129b7..d2f9e6c 100644
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIo.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIo.java
@@ -23,7 +23,9 @@ import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapred.InputFormat;
public interface LlapIo<T> {
- InputFormat<NullWritable, T> getInputFormat(InputFormat sourceInputFormat, Deserializer serde);
+ InputFormat<NullWritable, T> getInputFormat(
+ InputFormat<?, ?> sourceInputFormat, Deserializer serde);
void close();
String getMemoryInfo();
+ void initCacheOnlyInputFormat(InputFormat<?, ?> inputFormat);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/llap-server/src/java/org/apache/hadoop/hive/llap/cache/EvictionDispatcher.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/EvictionDispatcher.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/EvictionDispatcher.java
index 0cbc8f6..c5248ce 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/EvictionDispatcher.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/EvictionDispatcher.java
@@ -22,6 +22,8 @@ import org.apache.hadoop.hive.llap.io.metadata.OrcFileEstimateErrors;
import org.apache.hadoop.hive.llap.io.metadata.OrcFileMetadata;
import org.apache.hadoop.hive.llap.io.metadata.OrcMetadataCache;
import org.apache.hadoop.hive.llap.io.metadata.OrcStripeMetadata;
+import org.apache.hadoop.hive.llap.io.metadata.ParquetMetadataCacheImpl;
+import org.apache.hadoop.hive.llap.io.metadata.ParquetMetadataCacheImpl.LlapFileMetadataBuffer;
/**
* Eviction dispatcher - uses double dispatch to route eviction notifications to correct caches.
@@ -31,11 +33,15 @@ public final class EvictionDispatcher implements EvictionListener, LlapOomDebugD
private final SerDeLowLevelCacheImpl serdeCache;
private final OrcMetadataCache metadataCache;
private final EvictionAwareAllocator allocator;
+ // TODO# temporary, will be merged with OrcMetadataCache after HIVE-15665.
+ private final ParquetMetadataCacheImpl parquetMetadataCache;
public EvictionDispatcher(LowLevelCache dataCache, SerDeLowLevelCacheImpl serdeCache,
- OrcMetadataCache metadataCache, EvictionAwareAllocator allocator) {
+ OrcMetadataCache metadataCache, EvictionAwareAllocator allocator,
+ ParquetMetadataCacheImpl parquetMetadataCache) {
this.dataCache = dataCache;
this.metadataCache = metadataCache;
+ this.parquetMetadataCache = parquetMetadataCache;
this.serdeCache = serdeCache;
this.allocator = allocator;
}
@@ -45,10 +51,13 @@ public final class EvictionDispatcher implements EvictionListener, LlapOomDebugD
buffer.notifyEvicted(this); // This will call one of the specific notifyEvicted overloads.
}
+ public void notifyEvicted(LlapFileMetadataBuffer buffer) {
+ this.parquetMetadataCache.notifyEvicted(buffer);
+ }
+
public void notifyEvicted(LlapSerDeDataBuffer buffer) {
serdeCache.notifyEvicted(buffer);
allocator.deallocateEvicted(buffer);
-
}
public void notifyEvicted(LlapDataBuffer buffer) {
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LlapAllocatorBuffer.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LlapAllocatorBuffer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LlapAllocatorBuffer.java
index 52144c2..8f57a04 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LlapAllocatorBuffer.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LlapAllocatorBuffer.java
@@ -131,15 +131,17 @@ public abstract class LlapAllocatorBuffer extends LlapCacheableBuffer implements
long newState, oldState;
do {
oldState = state.get();
+ // We have to check it here since invalid decref will overflow.
+ int oldRefCount = State.getRefCount(oldState);
+ if (oldRefCount == 0) {
+ throw new AssertionError("Invalid decRef when refCount is 0: " + this);
+ }
newState = State.decRefCount(oldState);
} while (!state.compareAndSet(oldState, newState));
int newRefCount = State.getRefCount(newState);
if (LlapIoImpl.LOCKING_LOGGER.isTraceEnabled()) {
LlapIoImpl.LOCKING_LOGGER.trace("Unlocked {}; refcount {}", this, newRefCount);
}
- if (newRefCount < 0) {
- throw new AssertionError("Unexpected refCount " + newRefCount + ": " + this);
- }
return newRefCount;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
index 35b9d1f..f42622b 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
@@ -34,11 +34,18 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.io.Allocator;
+import org.apache.hadoop.hive.common.io.DataCache;
+import org.apache.hadoop.hive.common.io.DiskRange;
+import org.apache.hadoop.hive.common.io.DiskRangeList;
+import org.apache.hadoop.hive.common.io.FileMetadataCache;
+import org.apache.hadoop.hive.common.io.Allocator.BufferObjectFactory;
+import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.llap.cache.BuddyAllocator;
import org.apache.hadoop.hive.llap.cache.BufferUsageManager;
import org.apache.hadoop.hive.llap.cache.EvictionDispatcher;
+import org.apache.hadoop.hive.llap.cache.LlapDataBuffer;
import org.apache.hadoop.hive.llap.cache.LlapOomDebugDump;
import org.apache.hadoop.hive.llap.cache.LowLevelCache;
import org.apache.hadoop.hive.llap.cache.LowLevelCacheImpl;
@@ -49,24 +56,30 @@ import org.apache.hadoop.hive.llap.cache.LowLevelLrfuCachePolicy;
import org.apache.hadoop.hive.llap.cache.SerDeLowLevelCacheImpl;
import org.apache.hadoop.hive.llap.cache.SimpleAllocator;
import org.apache.hadoop.hive.llap.cache.SimpleBufferManager;
+import org.apache.hadoop.hive.llap.cache.LowLevelCache.Priority;
import org.apache.hadoop.hive.llap.io.api.LlapIo;
import org.apache.hadoop.hive.llap.io.decode.ColumnVectorProducer;
import org.apache.hadoop.hive.llap.io.decode.GenericColumnVectorProducer;
import org.apache.hadoop.hive.llap.io.decode.OrcColumnVectorProducer;
import org.apache.hadoop.hive.llap.io.metadata.OrcMetadataCache;
+import org.apache.hadoop.hive.llap.io.metadata.ParquetMetadataCacheImpl;
import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics;
import org.apache.hadoop.hive.llap.metrics.LlapDaemonIOMetrics;
import org.apache.hadoop.hive.llap.metrics.MetricsUtils;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.io.LlapCacheOnlyInputFormatInterface;
import org.apache.hadoop.hive.ql.io.orc.encoded.IoTrace;
import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hive.common.util.FixedSizedObjectPool;
+
import com.google.common.primitives.Ints;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
@@ -86,8 +99,13 @@ public class LlapIoImpl implements LlapIo<VectorizedRowBatch> {
private ObjectName buddyAllocatorMXBean;
private final Allocator allocator;
private final LlapOomDebugDump memoryDump;
+ private final FileMetadataCache fileMetadataCache;
+ private final LowLevelCache dataCache;
+ private final BufferUsageManager bufferManager;
+ private final Configuration daemonConf;
private LlapIoImpl(Configuration conf) throws IOException {
+ this.daemonConf = conf;
String ioMode = HiveConf.getVar(conf, HiveConf.ConfVars.LLAP_IO_MEMORY_MODE);
boolean useLowLevelCache = LlapIoImpl.MODE_CACHE.equalsIgnoreCase(ioMode);
LOG.info("Initializing LLAP IO in {} mode", useLowLevelCache ? LlapIoImpl.MODE_CACHE : "none");
@@ -115,7 +133,6 @@ public class LlapIoImpl implements LlapIo<VectorizedRowBatch> {
sessionId);
OrcMetadataCache metadataCache = null;
- LowLevelCache cache = null;
SerDeLowLevelCacheImpl serdeCache = null; // TODO: extract interface when needed
BufferUsageManager bufferManagerOrc = null, bufferManagerGeneric = null;
boolean isEncodeEnabled = HiveConf.getBoolVar(conf, ConfVars.LLAP_IO_ENCODE_ENABLED);
@@ -154,16 +171,22 @@ public class LlapIoImpl implements LlapIo<VectorizedRowBatch> {
this.memoryDump = allocator;
LowLevelCacheImpl cacheImpl = new LowLevelCacheImpl(
cacheMetrics, cachePolicy, allocator, true);
- cache = cacheImpl;
+ dataCache = cacheImpl;
if (isEncodeEnabled) {
SerDeLowLevelCacheImpl serdeCacheImpl = new SerDeLowLevelCacheImpl(
cacheMetrics, cachePolicy, allocator);
serdeCache = serdeCacheImpl;
}
+
boolean useGapCache = HiveConf.getBoolVar(conf, ConfVars.LLAP_CACHE_ENABLE_ORC_GAP_CACHE);
metadataCache = new OrcMetadataCache(metaMemManager, metaCachePolicy, useGapCache);
+ // TODO# temporary, see comments there
+ ParquetMetadataCacheImpl parquetMc = new ParquetMetadataCacheImpl(
+ allocator, memManager, cachePolicy, cacheMetrics);
+ fileMetadataCache = parquetMc;
// And finally cache policy uses cache to notify it of eviction. The cycle is complete!
- EvictionDispatcher e = new EvictionDispatcher(cache, serdeCache, metadataCache, allocator);
+ EvictionDispatcher e = new EvictionDispatcher(
+ dataCache, serdeCache, metadataCache, allocator, parquetMc);
if (isSplitCache) {
metaCachePolicy.setEvictionListener(e);
metaCachePolicy.setParentDebugDumper(e);
@@ -172,14 +195,15 @@ public class LlapIoImpl implements LlapIo<VectorizedRowBatch> {
cachePolicy.setParentDebugDumper(e);
cacheImpl.startThreads(); // Start the cache threads.
- bufferManagerOrc = cacheImpl; // Cache also serves as buffer manager.
+ bufferManager = bufferManagerOrc = cacheImpl; // Cache also serves as buffer manager.
bufferManagerGeneric = serdeCache;
} else {
this.allocator = new SimpleAllocator(conf);
memoryDump = null;
+ fileMetadataCache = null;
SimpleBufferManager sbm = new SimpleBufferManager(allocator, cacheMetrics);
- bufferManagerOrc = bufferManagerGeneric = sbm;
- cache = sbm;
+ bufferManager = bufferManagerOrc = bufferManagerGeneric = sbm;
+ dataCache = sbm;
}
// IO thread pool. Listening is used for unhandled errors for now (TODO: remove?)
int numThreads = HiveConf.getIntVar(conf, HiveConf.ConfVars.LLAP_IO_THREADPOOL_SIZE);
@@ -190,7 +214,7 @@ public class LlapIoImpl implements LlapIo<VectorizedRowBatch> {
FixedSizedObjectPool<IoTrace> tracePool = IoTrace.createTracePool(conf);
// TODO: this should depends on input format and be in a map, or something.
this.orcCvp = new OrcColumnVectorProducer(
- metadataCache, cache, bufferManagerOrc, conf, cacheMetrics, ioMetrics, tracePool);
+ metadataCache, dataCache, bufferManagerOrc, conf, cacheMetrics, ioMetrics, tracePool);
this.genericCvp = isEncodeEnabled ? new GenericColumnVectorProducer(
serdeCache, bufferManagerGeneric, conf, cacheMetrics, ioMetrics, tracePool) : null;
LOG.info("LLAP IO initialized");
@@ -210,10 +234,9 @@ public class LlapIoImpl implements LlapIo<VectorizedRowBatch> {
return sb.toString();
}
- @SuppressWarnings("rawtypes")
@Override
public InputFormat<NullWritable, VectorizedRowBatch> getInputFormat(
- InputFormat sourceInputFormat, Deserializer sourceSerDe) {
+ InputFormat<?, ?> sourceInputFormat, Deserializer sourceSerDe) {
ColumnVectorProducer cvp = genericCvp;
if (sourceInputFormat instanceof OrcInputFormat) {
cvp = orcCvp; // Special-case for ORC.
@@ -233,4 +256,62 @@ public class LlapIoImpl implements LlapIo<VectorizedRowBatch> {
}
executor.shutdownNow();
}
+
+
+ @Override
+ public void initCacheOnlyInputFormat(InputFormat<?, ?> inputFormat) {
+ LlapCacheOnlyInputFormatInterface cacheIf = (LlapCacheOnlyInputFormatInterface)inputFormat;
+ cacheIf.injectCaches(fileMetadataCache,
+ new GenericDataCache(dataCache, bufferManager), daemonConf);
+ }
+
+ private class GenericDataCache implements DataCache, BufferObjectFactory {
+ private final LowLevelCache lowLevelCache;
+ private final BufferUsageManager bufferManager;
+
+ public GenericDataCache(LowLevelCache lowLevelCache, BufferUsageManager bufferManager) {
+ this.lowLevelCache = lowLevelCache;
+ this.bufferManager = bufferManager;
+ }
+
+ @Override
+ public DiskRangeList getFileData(Object fileKey, DiskRangeList range,
+ long baseOffset, DiskRangeListFactory factory, BooleanRef gotAllData) {
+ // TODO: we currently pass null counters because this doesn't use LlapRecordReader.
+ // Create counters for non-elevator-using fragments also?
+ return lowLevelCache.getFileData(fileKey, range, baseOffset, factory, null, gotAllData);
+ }
+
+ @Override
+ public long[] putFileData(Object fileKey, DiskRange[] ranges,
+ MemoryBuffer[] data, long baseOffset) {
+ return lowLevelCache.putFileData(fileKey, ranges, data, baseOffset, Priority.NORMAL, null);
+ }
+
+ @Override
+ public void releaseBuffer(MemoryBuffer buffer) {
+ bufferManager.decRefBuffer(buffer);
+ }
+
+ @Override
+ public void reuseBuffer(MemoryBuffer buffer) {
+ boolean isReused = bufferManager.incRefBuffer(buffer);
+ assert isReused;
+ }
+
+ @Override
+ public Allocator getAllocator() {
+ return bufferManager.getAllocator();
+ }
+
+ @Override
+ public BufferObjectFactory getDataBufferFactory() {
+ return this;
+ }
+
+ @Override
+ public MemoryBuffer create() {
+ return new LlapDataBuffer();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/ParquetMetadataCacheImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/ParquetMetadataCacheImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/ParquetMetadataCacheImpl.java
new file mode 100644
index 0000000..b61a8ca
--- /dev/null
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/ParquetMetadataCacheImpl.java
@@ -0,0 +1,353 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.io.metadata;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.common.io.FileMetadataCache;
+import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer;
+import org.apache.hadoop.hive.common.io.encoded.MemoryBufferOrBuffers;
+import org.apache.hadoop.hive.llap.cache.EvictionAwareAllocator;
+import org.apache.hadoop.hive.llap.cache.EvictionDispatcher;
+import org.apache.hadoop.hive.llap.cache.LlapAllocatorBuffer;
+import org.apache.hadoop.hive.llap.cache.LlapOomDebugDump;
+import org.apache.hadoop.hive.llap.cache.LowLevelCachePolicy;
+import org.apache.hadoop.hive.llap.cache.MemoryManager;
+import org.apache.hadoop.hive.llap.cache.LowLevelCache.Priority;
+import org.apache.hadoop.hive.llap.io.api.impl.LlapIoImpl;
+import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics;
+import org.apache.parquet.io.SeekableInputStream;
+
+// TODO# merge with OrcMetadataCache (and rename) after HIVE-15665. Shares a lot of the code.
+public class ParquetMetadataCacheImpl implements LlapOomDebugDump, FileMetadataCache {
+ private final ConcurrentHashMap<Object, LlapBufferOrBuffers> metadata =
+ new ConcurrentHashMap<>();
+
+ private final MemoryManager memoryManager;
+ private final LowLevelCachePolicy policy;
+ private final EvictionAwareAllocator allocator;
+ private final LlapDaemonCacheMetrics metrics;
+
+ public ParquetMetadataCacheImpl(EvictionAwareAllocator allocator, MemoryManager memoryManager,
+ LowLevelCachePolicy policy, LlapDaemonCacheMetrics metrics) {
+ this.memoryManager = memoryManager;
+ this.allocator = allocator;
+ this.policy = policy;
+ this.metrics = metrics;
+ }
+
+ public void notifyEvicted(LlapFileMetadataBuffer buffer) {
+ LlapBufferOrBuffers removed = metadata.remove(buffer.getFileKey());
+ if (removed == null) return;
+ if (removed.getSingleBuffer() != null) {
+ assert removed.getSingleBuffer() == buffer;
+ return;
+ }
+ discardMultiBuffer(removed);
+ }
+
+ @Override
+ public LlapBufferOrBuffers getFileMetadata(Object fileKey) {
+ LlapBufferOrBuffers result = metadata.get(fileKey);
+ if (result == null) return null;
+ if (!lockBuffer(result, true)) {
+ // No need to discard the buffer we cannot lock - eviction takes care of that.
+ metadata.remove(fileKey, result);
+ return null;
+ }
+ return result;
+ }
+
+ @Override
+ public LlapBufferOrBuffers putFileMetadata(
+ Object fileKey, int length, InputStream is) throws IOException {
+ LlapBufferOrBuffers result = null;
+ while (true) { // Overwhelmingly executes once, or maybe twice (replacing stale value).
+ LlapBufferOrBuffers oldVal = metadata.get(fileKey);
+ if (oldVal == null) {
+ result = wrapBbForFile(result, fileKey, length, is);
+ if (!lockBuffer(result, false)) {
+ throw new AssertionError("Cannot lock a newly created value " + result);
+ }
+ oldVal = metadata.putIfAbsent(fileKey, result);
+ if (oldVal == null) {
+ cacheInPolicy(result); // Cached successfully, add to policy.
+ return result;
+ }
+ }
+ if (lockOldVal(fileKey, result, oldVal)) {
+ return oldVal;
+ }
+ // We found some old value but couldn't incRef it; remove it.
+ metadata.remove(fileKey, oldVal);
+ }
+ }
+
+ private void cacheInPolicy(LlapBufferOrBuffers buffers) {
+ LlapAllocatorBuffer singleBuffer = buffers.getSingleLlapBuffer();
+ if (singleBuffer != null) {
+ policy.cache(singleBuffer, Priority.HIGH);
+ return;
+ }
+ for (LlapAllocatorBuffer buffer : buffers.getMultipleLlapBuffers()) {
+ policy.cache(buffer, Priority.HIGH);
+ }
+ }
+
+ private <T extends LlapBufferOrBuffers> boolean lockOldVal(Object key, T newVal, T oldVal) {
+ if (LlapIoImpl.CACHE_LOGGER.isTraceEnabled()) {
+ LlapIoImpl.CACHE_LOGGER.trace("Trying to cache when metadata is already cached for" +
+ " {}; old {}, new {}", key, oldVal, newVal);
+ }
+ if (LlapIoImpl.LOCKING_LOGGER.isTraceEnabled()) {
+ LlapIoImpl.LOCKING_LOGGER.trace("Locking {} due to cache collision", oldVal);
+ }
+ if (lockBuffer(oldVal, true)) {
+ // We found an old, valid block for this key in the cache.
+ if (LlapIoImpl.LOCKING_LOGGER.isTraceEnabled()) {
+ LlapIoImpl.LOCKING_LOGGER.trace("Unlocking {} due to cache collision with {}",
+ newVal, oldVal);
+ }
+
+ if (newVal != null) {
+ unlockBuffer(newVal, false);
+ }
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public void decRefBuffer(MemoryBufferOrBuffers buffer) {
+ if (!(buffer instanceof LlapBufferOrBuffers)) {
+ throw new AssertionError(buffer.getClass());
+ }
+ unlockBuffer((LlapBufferOrBuffers)buffer, true);
+ }
+
+ private LlapBufferOrBuffers wrapBbForFile(LlapBufferOrBuffers result,
+ Object fileKey, int length, InputStream stream) throws IOException {
+ if (result != null) return result;
+ int maxAlloc = allocator.getMaxAllocation();
+ LlapFileMetadataBuffer[] largeBuffers = null;
+ if (maxAlloc < length) {
+ largeBuffers = new LlapFileMetadataBuffer[length / maxAlloc];
+ for (int i = 0; i < largeBuffers.length; ++i) {
+ largeBuffers[i] = new LlapFileMetadataBuffer(fileKey);
+ }
+ allocator.allocateMultiple(largeBuffers, maxAlloc, null);
+ for (int i = 0; i < largeBuffers.length; ++i) {
+ readIntoCacheBuffer(stream, maxAlloc, largeBuffers[i]);
+ }
+ }
+ int smallSize = length % maxAlloc;
+ if (smallSize == 0) {
+ return new LlapFileMetadataBuffers(largeBuffers);
+ } else {
+ LlapFileMetadataBuffer[] smallBuffer = new LlapFileMetadataBuffer[1];
+ smallBuffer[0] = new LlapFileMetadataBuffer(fileKey);
+ allocator.allocateMultiple(smallBuffer, length, null);
+ readIntoCacheBuffer(stream, smallSize, smallBuffer[0]);
+ if (largeBuffers == null) {
+ return smallBuffer[0];
+ } else {
+ LlapFileMetadataBuffer[] cacheData = new LlapFileMetadataBuffer[largeBuffers.length + 1];
+ System.arraycopy(largeBuffers, 0, cacheData, 0, largeBuffers.length);
+ cacheData[largeBuffers.length] = smallBuffer[0];
+ return new LlapFileMetadataBuffers(largeBuffers);
+ }
+ }
+ }
+
+ private static void readIntoCacheBuffer(
+ InputStream stream, int length, MemoryBuffer dest) throws IOException {
+ ByteBuffer bb = dest.getByteBufferRaw();
+ int pos = bb.position();
+ bb.limit(pos + length);
+ // TODO: SeekableInputStream.readFully eventually calls a Hadoop method that used to be
+ // buggy in 2.7 and also anyway just does a copy for a direct buffer. Do a copy here.
+ // ((SeekableInputStream)stream).readFully(bb);
+ FileUtils.readFully(stream, length, bb);
+ bb.position(pos);
+ }
+
+
+ private boolean lockBuffer(LlapBufferOrBuffers buffers, boolean doNotifyPolicy) {
+ LlapAllocatorBuffer buffer = buffers.getSingleLlapBuffer();
+ if (buffer != null) {
+ return lockOneBuffer(buffer, doNotifyPolicy);
+ }
+ LlapAllocatorBuffer[] bufferArray = buffers.getMultipleLlapBuffers();
+ for (int i = 0; i < bufferArray.length; ++i) {
+ if (lockOneBuffer(bufferArray[i], doNotifyPolicy)) continue;
+ for (int j = 0; j < i; ++j) {
+ unlockSingleBuffer(buffer, true);
+ }
+ discardMultiBuffer(buffers);
+ return false;
+ }
+ return true;
+ }
+
+ private void discardMultiBuffer(LlapBufferOrBuffers removed) {
+ long memoryFreed = 0;
+ for (LlapAllocatorBuffer buf : removed.getMultipleLlapBuffers()) {
+ long memUsage = buf.getMemoryUsage();
+ // We cannot just deallocate the buffer, as it can hypothetically have users.
+ int result = buf.invalidate();
+ switch (result) {
+ case LlapAllocatorBuffer.INVALIDATE_ALREADY_INVALID: continue; // Nothing to do.
+ case LlapAllocatorBuffer.INVALIDATE_FAILED: {
+ // Someone is using this buffer; eventually, it will be evicted.
+ continue;
+ }
+ case LlapAllocatorBuffer.INVALIDATE_OK: {
+ memoryFreed += memUsage;
+ allocator.deallocateEvicted(buf);
+ break;
+ }
+ default: throw new AssertionError(result);
+ }
+ }
+ memoryManager.releaseMemory(memoryFreed);
+ }
+
+ private boolean lockOneBuffer(LlapAllocatorBuffer buffer, boolean doNotifyPolicy) {
+ int rc = buffer.incRef();
+ if (rc > 0) {
+ metrics.incrCacheNumLockedBuffers();
+ }
+ if (doNotifyPolicy && rc == 1) {
+ // We have just locked a buffer that wasn't previously locked.
+ policy.notifyLock(buffer);
+ }
+ return rc > 0;
+ }
+
+ private void unlockBuffer(LlapBufferOrBuffers buffers, boolean isCached) {
+ LlapAllocatorBuffer singleBuffer = buffers.getSingleLlapBuffer();
+ if (singleBuffer != null) {
+ unlockSingleBuffer(singleBuffer, isCached);
+ return;
+ }
+ for (LlapAllocatorBuffer buffer : buffers.getMultipleLlapBuffers()) {
+ unlockSingleBuffer(buffer, isCached);
+ }
+ }
+
+ private void unlockSingleBuffer(LlapAllocatorBuffer buffer, boolean isCached) {
+ boolean isLastDecref = (buffer.decRef() == 0);
+ if (isLastDecref) {
+ if (isCached) {
+ policy.notifyUnlock(buffer);
+ } else {
+ allocator.deallocate(buffer);
+ }
+ }
+ metrics.decrCacheNumLockedBuffers();
+ }
+
+
+ public static interface LlapBufferOrBuffers extends MemoryBufferOrBuffers {
+ LlapAllocatorBuffer getSingleLlapBuffer();
+ LlapAllocatorBuffer[] getMultipleLlapBuffers();
+ }
+
+ public final static class LlapFileMetadataBuffer
+ extends LlapAllocatorBuffer implements LlapBufferOrBuffers {
+ private final Object fileKey;
+
+ public LlapFileMetadataBuffer(Object fileKey) {
+ this.fileKey = fileKey;
+ }
+
+ @Override
+ public void notifyEvicted(EvictionDispatcher evictionDispatcher) {
+ evictionDispatcher.notifyEvicted(this);
+ }
+
+ public Object getFileKey() {
+ return fileKey;
+ }
+
+ @Override
+ public LlapAllocatorBuffer getSingleLlapBuffer() {
+ return this;
+ }
+
+ @Override
+ public LlapAllocatorBuffer[] getMultipleLlapBuffers() {
+ return null;
+ }
+
+ @Override
+ public MemoryBuffer getSingleBuffer() {
+ return this;
+ }
+
+ @Override
+ public MemoryBuffer[] getMultipleBuffers() {
+ return null;
+ }
+ }
+
+ public final static class LlapFileMetadataBuffers implements LlapBufferOrBuffers {
+ private final LlapFileMetadataBuffer[] buffers;
+
+ public LlapFileMetadataBuffers(LlapFileMetadataBuffer[] buffers) {
+ this.buffers = buffers;
+ }
+
+ @Override
+ public LlapAllocatorBuffer getSingleLlapBuffer() {
+ return null;
+ }
+
+ @Override
+ public LlapAllocatorBuffer[] getMultipleLlapBuffers() {
+ return buffers;
+ }
+
+ @Override
+ public MemoryBuffer getSingleBuffer() {
+ return null;
+ }
+
+ @Override
+ public MemoryBuffer[] getMultipleBuffers() {
+ return buffers;
+ }
+ }
+
+ @Override
+ public String debugDumpForOom() {
+ // TODO: nothing, will be merged with ORC cache
+ return null;
+ }
+
+ @Override
+ public void debugDumpShort(StringBuilder sb) {
+ // TODO: nothing, will be merged with ORC cache
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/ql/src/java/org/apache/hadoop/hive/llap/LlapCacheAwareFs.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/llap/LlapCacheAwareFs.java b/ql/src/java/org/apache/hadoop/hive/llap/LlapCacheAwareFs.java
new file mode 100644
index 0000000..5c1eed3
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/llap/LlapCacheAwareFs.java
@@ -0,0 +1,422 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap;
+
+import java.io.EOFException;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.nio.ByteBuffer;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PositionedReadable;
+import org.apache.hadoop.fs.Seekable;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hive.common.io.Allocator;
+import org.apache.hadoop.hive.common.io.DataCache;
+import org.apache.hadoop.hive.common.io.DiskRange;
+import org.apache.hadoop.hive.common.io.DiskRangeList;
+import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer;
+import org.apache.hadoop.hive.ql.io.orc.encoded.CacheChunk;
+import org.apache.hadoop.util.Progressable;
+import org.apache.orc.impl.RecordReaderUtils;
+
+/**
+ * This is currently only used by Parquet; however, a generally applicable approach is used -
+ * you pass in a set of offset pairs for a file, and the file is cached with these boundaries.
+ * Don't add anything format specific here.
+ */
+public class LlapCacheAwareFs extends FileSystem {
+ public static final String SCHEME = "llapcache";
+ private static AtomicLong currentSplitId = new AtomicLong(-1);
+ private URI uri;
+
+ // We store the chunk indices by split file; that way if several callers are reading
+ // the same file they can separately store and remove the relevant parts of the index.
+ private static final ConcurrentHashMap<Long, CacheAwareInputStream> files =
+ new ConcurrentHashMap<>();
+
+ public static Path registerFile(DataCache cache, Path path, Object fileKey,
+ TreeMap<Long, Long> index, Configuration conf) throws IOException {
+ long splitId = currentSplitId.incrementAndGet();
+ CacheAwareInputStream stream = new CacheAwareInputStream(
+ cache, conf, index, path, fileKey, -1);
+ if (files.putIfAbsent(splitId, stream) != null) {
+ throw new IOException("Record already exists for " + splitId);
+ }
+ conf.set("fs." + LlapCacheAwareFs.SCHEME + ".impl", LlapCacheAwareFs.class.getCanonicalName());
+ return new Path(SCHEME + "://" + SCHEME + "/" + splitId);
+ }
+
+ public static void unregisterFile(Path cachePath) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Unregistering " + cachePath);
+ }
+ files.remove(extractSplitId(cachePath));
+ }
+
+ @Override
+ public void initialize(URI uri, Configuration conf) throws IOException {
+ super.initialize(uri, conf);
+ this.uri = URI.create(SCHEME + "://" + SCHEME);
+ }
+
+ @Override
+ public FSDataInputStream open(Path path, int bufferSize) throws IOException {
+ return new FSDataInputStream(getCtx(path).cloneWithBufferSize(bufferSize));
+ }
+
+ private LlapCacheAwareFs.CacheAwareInputStream getCtx(Path path) {
+ return files.get(extractSplitId(path));
+ }
+
+ private static long extractSplitId(Path path) {
+ String pathOnly = path.toUri().getPath();
+ if (pathOnly.startsWith("/")) {
+ pathOnly = pathOnly.substring(1);
+ }
+ return Long.parseLong(pathOnly);
+ }
+
+ @Override
+ public URI getUri() {
+ return uri;
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setWorkingDirectory(Path arg0) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public FSDataOutputStream append(Path arg0, int arg1, Progressable arg2) throws IOException {
+ LlapCacheAwareFs.CacheAwareInputStream ctx = getCtx(arg0);
+ return ctx.getFs().append(ctx.path, arg1, arg2);
+ }
+
+ @Override
+ public FSDataOutputStream create(Path arg0, FsPermission arg1, boolean arg2, int arg3,
+ short arg4, long arg5, Progressable arg6) throws IOException {
+ LlapCacheAwareFs.CacheAwareInputStream ctx = getCtx(arg0);
+ return ctx.getFs().create(ctx.path, arg1, arg2, arg3, arg4, arg5, arg6);
+ }
+
+ @Override
+ public boolean delete(Path arg0, boolean arg1) throws IOException {
+ LlapCacheAwareFs.CacheAwareInputStream ctx = getCtx(arg0);
+ return ctx.getFs().delete(ctx.path, arg1);
+
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path arg0) throws IOException {
+ LlapCacheAwareFs.CacheAwareInputStream ctx = getCtx(arg0);
+ return ctx.getFs().getFileStatus(ctx.path);
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path arg0) throws FileNotFoundException, IOException {
+ LlapCacheAwareFs.CacheAwareInputStream ctx = getCtx(arg0);
+ return ctx.getFs().listStatus(ctx.path);
+ }
+
+ @Override
+ public boolean mkdirs(Path arg0, FsPermission arg1) throws IOException {
+ LlapCacheAwareFs.CacheAwareInputStream ctx = getCtx(arg0);
+ return ctx.getFs().mkdirs(ctx.path, arg1);
+ }
+
+ @Override
+ public boolean rename(Path arg0, Path arg1) throws IOException {
+ LlapCacheAwareFs.CacheAwareInputStream ctx1 = getCtx(arg0), ctx2 = getCtx(arg1);
+ return ctx1.getFs().rename(ctx1.path, ctx2.path);
+ }
+
+ private static class CacheAwareInputStream extends InputStream
+ implements Seekable, PositionedReadable {
+ private final TreeMap<Long, Long> chunkIndex;
+ private final Path path;
+ private final Object fileKey;
+ private final Configuration conf;
+ private final DataCache cache;
+ private final int bufferSize;
+ private long position = 0;
+
+ public CacheAwareInputStream(DataCache cache, Configuration conf,
+ TreeMap<Long, Long> chunkIndex, Path path, Object fileKey, int bufferSize) {
+ this.cache = cache;
+ this.fileKey = fileKey;
+ this.chunkIndex = chunkIndex;
+ this.path = path;
+ this.conf = conf;
+ this.bufferSize = bufferSize;
+ }
+
+ public LlapCacheAwareFs.CacheAwareInputStream cloneWithBufferSize(int bufferSize) {
+ return new CacheAwareInputStream(cache, conf, chunkIndex, path, fileKey, bufferSize);
+ }
+
+ @Override
+ public int read() throws IOException {
+ // This is not called by ConsecutiveChunk stuff in Parquet.
+ // If this were used, it might make sense to make it faster.
+ byte[] theByte = new byte[1];
+ int result = read(theByte, 0, 1);
+ if (result != 1) throw new EOFException();
+ return theByte[0] & 0xFF;
+ }
+
+ @Override
+ public int read(byte[] array, final int arrayOffset, final int len) throws IOException {
+ long readStartPos = position;
+ DiskRangeList drl = new DiskRangeList(readStartPos, readStartPos + len);
+ DataCache.BooleanRef gotAllData = new DataCache.BooleanRef();
+ drl = cache.getFileData(fileKey, drl, 0, new DataCache.DiskRangeListFactory() {
+ @Override
+ public DiskRangeList createCacheChunk(
+ MemoryBuffer buffer, long startOffset, long endOffset) {
+ CacheChunk result = new CacheChunk(); // TODO: pool?
+ result.init(buffer, startOffset, endOffset);
+ return result;
+ }
+ }, gotAllData);
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Buffers after cache " + RecordReaderUtils.stringifyDiskRanges(drl));
+ }
+ if (gotAllData.value) {
+ long sizeRead = 0;
+ while (drl != null) {
+ assert drl.hasData();
+ long from = drl.getOffset(), to = drl.getEnd();
+ int offsetFromReadStart = (int)(from - readStartPos), candidateSize = (int)(to - from);
+ ByteBuffer data = drl.getData().duplicate();
+ data.get(array, arrayOffset + offsetFromReadStart, candidateSize);
+ sizeRead += candidateSize;
+ drl = drl.next;
+ }
+ validateAndUpdatePosition(len, sizeRead);
+ return len;
+ }
+ int maxAlloc = cache.getAllocator().getMaxAllocation();
+ // We have some disk data. Separate it by column chunk and put into cache.
+
+ // We started with a single DRL, so we assume there will be no consecutive missing blocks
+ // after the cache has inserted cache data. We also assume all the missing parts will
+ // represent one or several column chunks, since we always cache on column chunk boundaries.
+ DiskRangeList current = drl;
+ FileSystem fs = path.getFileSystem(conf);
+ FSDataInputStream is = fs.open(path, bufferSize);
+ Allocator allocator = cache.getAllocator();
+ long sizeRead = 0;
+ while (current != null) {
+ DiskRangeList candidate = current;
+ current = current.next;
+ long from = candidate.getOffset(), to = candidate.getEnd();
+ // The offset in the destination array for the beginning of this missing range.
+ int offsetFromReadStart = (int)(from - readStartPos), candidateSize = (int)(to - from);
+ if (candidate.hasData()) {
+ ByteBuffer data = candidate.getData().duplicate();
+ data.get(array, arrayOffset + offsetFromReadStart, candidateSize);
+ sizeRead += candidateSize;
+ continue;
+ }
+ // The data is not in cache.
+
+ // Account for potential partial chunks.
+ SortedMap<Long, Long> chunksInThisRead = getAndValidateMissingChunks(maxAlloc, from, to);
+
+ is.seek(from);
+ is.readFully(array, arrayOffset + offsetFromReadStart, candidateSize);
+ sizeRead += candidateSize;
+ // Now copy missing chunks (and parts of chunks) into cache buffers.
+ if (fileKey == null || cache == null) continue;
+ int extraDiskDataOffset = 0;
+ // TODO: should we try to make a giant array for one cache call to avoid overhead?
+ for (Map.Entry<Long, Long> missingChunk : chunksInThisRead.entrySet()) {
+ long chunkFrom = Math.max(from, missingChunk.getKey()),
+ chunkTo = Math.min(to, missingChunk.getValue()),
+ chunkLength = chunkTo - chunkFrom;
+ MemoryBuffer[] largeBuffers = null, smallBuffer = null, newCacheData = null;
+ try {
+ int largeBufCount = (int) (chunkLength / maxAlloc);
+ int smallSize = (int) (chunkLength % maxAlloc);
+ int chunkPartCount = largeBufCount + ((smallSize > 0) ? 1 : 0);
+ DiskRange[] cacheRanges = new DiskRange[chunkPartCount];
+ int extraOffsetInChunk = 0;
+ if (maxAlloc < chunkLength) {
+ largeBuffers = new MemoryBuffer[largeBufCount];
+ allocator.allocateMultiple(largeBuffers, maxAlloc, cache.getDataBufferFactory());
+ for (int i = 0; i < largeBuffers.length; ++i) {
+ // By definition here we copy up to the limit of the buffer.
+ ByteBuffer bb = largeBuffers[i].getByteBufferRaw();
+ int remaining = bb.remaining();
+ assert remaining == maxAlloc;
+ copyDiskDataToCacheBuffer(array,
+ arrayOffset + offsetFromReadStart + extraDiskDataOffset,
+ remaining, bb, cacheRanges, i, chunkFrom + extraOffsetInChunk);
+ extraDiskDataOffset += remaining;
+ extraOffsetInChunk += remaining;
+ }
+ }
+ newCacheData = largeBuffers;
+ largeBuffers = null;
+ if (smallSize > 0) {
+ smallBuffer = new MemoryBuffer[1];
+ allocator.allocateMultiple(smallBuffer, smallSize, cache.getDataBufferFactory());
+ ByteBuffer bb = smallBuffer[0].getByteBufferRaw();
+ copyDiskDataToCacheBuffer(array,
+ arrayOffset + offsetFromReadStart + extraDiskDataOffset,
+ smallSize, bb, cacheRanges, largeBufCount, chunkFrom + extraOffsetInChunk);
+ extraDiskDataOffset += smallSize;
+ extraOffsetInChunk += smallSize; // Not strictly necessary, noone will look at it.
+ if (newCacheData == null) {
+ newCacheData = smallBuffer;
+ } else {
+ // TODO: add allocate overload with an offset and length
+ MemoryBuffer[] combinedCacheData = new MemoryBuffer[largeBufCount + 1];
+ System.arraycopy(newCacheData, 0, combinedCacheData, 0, largeBufCount);
+ newCacheData = combinedCacheData;
+ newCacheData[largeBufCount] = smallBuffer[0];
+ }
+ smallBuffer = null;
+ }
+ cache.putFileData(fileKey, cacheRanges, newCacheData, 0);
+ } finally {
+ // We do not use the new cache buffers for the actual read, given the way read() API is.
+ // Therefore, we don't need to handle cache collisions - just decref all the buffers.
+ if (newCacheData != null) {
+ for (MemoryBuffer buffer : newCacheData) {
+ if (buffer == null) continue;
+ cache.releaseBuffer(buffer);
+ }
+ }
+ // If we have failed before building newCacheData, deallocate other the allocated.
+ if (largeBuffers != null) {
+ for (MemoryBuffer buffer : largeBuffers) {
+ if (buffer == null) continue;
+ allocator.deallocate(buffer);
+ }
+ }
+ if (smallBuffer != null && smallBuffer[0] != null) {
+ allocator.deallocate(smallBuffer[0]);
+ }
+ }
+ }
+ }
+ validateAndUpdatePosition(len, sizeRead);
+ return len;
+ }
+
+ private void validateAndUpdatePosition(int len, long sizeRead) {
+ if (sizeRead != len) {
+ throw new AssertionError("Reading at " + position + " for " + len + ": "
+ + sizeRead + " bytes copied");
+ }
+ position += len;
+ }
+
+ private void copyDiskDataToCacheBuffer(byte[] diskData, int offsetInDiskData, int sizeToCopy,
+ ByteBuffer cacheBuffer, DiskRange[] cacheRanges, int cacheRangeIx, long cacheRangeStart) {
+ int bbPos = cacheBuffer.position();
+ long cacheRangeEnd = cacheRangeStart + sizeToCopy;
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Caching [" + cacheRangeStart + ", " + cacheRangeEnd + ")");
+ }
+ cacheRanges[cacheRangeIx] = new DiskRange(cacheRangeStart, cacheRangeEnd);
+ cacheBuffer.put(diskData, offsetInDiskData, sizeToCopy);
+ cacheBuffer.position(bbPos);
+ }
+
+ private SortedMap<Long, Long> getAndValidateMissingChunks(
+ int maxAlloc, long from, long to) {
+ Map.Entry<Long, Long> firstMissing = chunkIndex.floorEntry(from);
+ if (firstMissing == null) {
+ throw new AssertionError("No lower bound for offset " + from);
+ }
+ if (firstMissing.getValue() <= from
+ || ((from - firstMissing.getKey()) % maxAlloc) != 0) {
+ // The data does not belong to a recognized chunk, or is split wrong.
+ throw new AssertionError("Lower bound for offset " + from + " is ["
+ + firstMissing.getKey() + ", " + firstMissing.getValue() + ")");
+ }
+ SortedMap<Long, Long> missingChunks = chunkIndex.subMap(firstMissing.getKey(), to);
+ if (missingChunks.isEmpty()) {
+ throw new AssertionError("No chunks for [" + from + ", " + to + ")");
+ }
+ long lastMissingOffset = missingChunks.lastKey(),
+ lastMissingEnd = missingChunks.get(lastMissingOffset);
+ if (lastMissingEnd < to
+ || (to != lastMissingEnd && ((to - lastMissingOffset) % maxAlloc) != 0)) {
+ // The data does not belong to a recognized chunk, or is split wrong.
+ throw new AssertionError("Lower bound for offset " + to + " is ["
+ + lastMissingOffset + ", " + lastMissingEnd + ")");
+ }
+ return missingChunks;
+ }
+
+ public FileSystem getFs() throws IOException {
+ return path.getFileSystem(conf);
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ return position;
+ }
+
+ @Override
+ public void seek(long position) throws IOException {
+ this.position = position;
+ }
+
+ @Override
+ @Private
+ public boolean seekToNewSource(long arg0) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int read(long arg0, byte[] arg1, int arg2, int arg3) throws IOException {
+ seek(arg0);
+ return read(arg1, arg2, arg3);
+ }
+
+ @Override
+ public void readFully(long arg0, byte[] arg1) throws IOException {
+ read(arg0, arg1, 0, arg1.length);
+ }
+
+ @Override
+ public void readFully(long arg0, byte[] arg1, int arg2, int arg3) throws IOException {
+ read(arg0, arg1, 0, arg1.length);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java
index fa7f59d..102150a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java
@@ -49,7 +49,8 @@ public class HdfsUtils {
if (fileSystem instanceof DistributedFileSystem) {
DistributedFileSystem dfs = (DistributedFileSystem) fileSystem;
if ((!checkDefaultFs) || isDefaultFs(dfs)) {
- return SHIMS.getFileId(dfs, path.toUri().getPath());
+ Object result = SHIMS.getFileId(dfs, path.toUri().getPath());
+ if (result != null) return result;
}
}
if (!allowSynthetic) {
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
index 442c921..5c9d289 100755
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
@@ -214,6 +214,7 @@ public class HiveInputFormat<K extends WritableComparable, V extends Writable>
}
String ifName = inputFormat.getClass().getCanonicalName();
boolean isSupported = inputFormat instanceof LlapWrappableInputFormatInterface;
+ boolean isCacheOnly = inputFormat instanceof LlapCacheOnlyInputFormatInterface;
boolean isVectorized = Utilities.getUseVectorizedInputFileFormat(conf);
if (!isVectorized) {
// Pretend it's vectorized if the non-vector wrapped is enabled.
@@ -224,33 +225,17 @@ public class HiveInputFormat<K extends WritableComparable, V extends Writable>
if (isVectorized && !isSupported
&& HiveConf.getBoolVar(conf, ConfVars.LLAP_IO_ENCODE_ENABLED)) {
// See if we can use re-encoding to read the format thru IO elevator.
- String formatList = HiveConf.getVar(conf, ConfVars.LLAP_IO_ENCODE_FORMATS);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Checking " + ifName + " against " + formatList);
- }
- String[] formats = StringUtils.getStrings(formatList);
- if (formats != null) {
- for (String format : formats) {
- // TODO: should we check isAssignableFrom?
- if (ifName.equals(format)) {
- if (LOG.isInfoEnabled()) {
- LOG.info("Using SerDe-based LLAP reader for " + ifName);
- }
- isSupported = isSerdeBased = true;
- break;
- }
- }
- }
+ isSupported = isSerdeBased = checkInputFormatForLlapEncode(conf, ifName);
}
- if (!isSupported || !isVectorized) {
+ if ((!isSupported || !isVectorized) && !isCacheOnly) {
if (LOG.isInfoEnabled()) {
LOG.info("Not using llap for " + ifName + ": supported = "
- + isSupported + ", vectorized = " + isVectorized);
+ + isSupported + ", vectorized = " + isVectorized + ", cache only = " + isCacheOnly);
}
return inputFormat;
}
if (LOG.isDebugEnabled()) {
- LOG.debug("Wrapping " + ifName);
+ LOG.debug("Processing " + ifName);
}
@SuppressWarnings("unchecked")
@@ -264,43 +249,85 @@ public class HiveInputFormat<K extends WritableComparable, V extends Writable>
Deserializer serde = null;
if (isSerdeBased) {
if (part == null) {
- if (LOG.isInfoEnabled()) {
+ if (isCacheOnly) {
+ LOG.info("Using cache only because there's no partition spec for SerDe-based IF");
+ injectLlapCaches(inputFormat, llapIo);
+ } else {
LOG.info("Not using LLAP IO because there's no partition spec for SerDe-based IF");
}
return inputFormat;
}
- VectorPartitionDesc vpart = part.getVectorPartitionDesc();
- if (vpart != null) {
- VectorMapOperatorReadType old = vpart.getVectorMapOperatorReadType();
- if (old != VectorMapOperatorReadType.VECTORIZED_INPUT_FILE_FORMAT) {
+ serde = findSerDeForLlapSerDeIf(conf, part);
+ }
+ if (isSupported && isVectorized) {
+ InputFormat<?, ?> wrappedIf = llapIo.getInputFormat(inputFormat, serde);
+ // null means we cannot wrap; the cause is logged inside.
+ if (wrappedIf != null) {
+ return castInputFormat(wrappedIf);
+ }
+ }
+ if (isCacheOnly) {
+ injectLlapCaches(inputFormat, llapIo);
+ }
+ return inputFormat;
+ }
+
+ private static boolean checkInputFormatForLlapEncode(Configuration conf, String ifName) {
+ String formatList = HiveConf.getVar(conf, ConfVars.LLAP_IO_ENCODE_FORMATS);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Checking " + ifName + " against " + formatList);
+ }
+ String[] formats = StringUtils.getStrings(formatList);
+ if (formats != null) {
+ for (String format : formats) {
+ // TODO: should we check isAssignableFrom?
+ if (ifName.equals(format)) {
if (LOG.isInfoEnabled()) {
- LOG.info("Resetting VectorMapOperatorReadType from " + old + " for partition "
- + part.getTableName() + " " + part.getPartSpec());
+ LOG.info("Using SerDe-based LLAP reader for " + ifName);
}
- vpart.setVectorMapOperatorReadType(
- VectorMapOperatorReadType.VECTORIZED_INPUT_FILE_FORMAT);
+ return true;
}
}
- try {
- serde = part.getDeserializer(conf);
- } catch (Exception e) {
- throw new HiveException("Error creating SerDe for LLAP IO", e);
+ }
+ return false;
+ }
+
+ private static Deserializer findSerDeForLlapSerDeIf(
+ Configuration conf, PartitionDesc part) throws HiveException {
+ VectorPartitionDesc vpart = part.getVectorPartitionDesc();
+ if (vpart != null) {
+ VectorMapOperatorReadType old = vpart.getVectorMapOperatorReadType();
+ if (old != VectorMapOperatorReadType.VECTORIZED_INPUT_FILE_FORMAT) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Resetting VectorMapOperatorReadType from " + old + " for partition "
+ + part.getTableName() + " " + part.getPartSpec());
+ }
+ vpart.setVectorMapOperatorReadType(
+ VectorMapOperatorReadType.VECTORIZED_INPUT_FILE_FORMAT);
}
}
- InputFormat<?, ?> wrappedIf = llapIo.getInputFormat(inputFormat, serde);
- if (wrappedIf == null) {
- return inputFormat; // We cannot wrap; the cause is logged inside.
+ try {
+ return part.getDeserializer(conf);
+ } catch (Exception e) {
+ throw new HiveException("Error creating SerDe for LLAP IO", e);
}
- return castInputFormat(wrappedIf);
}
-
+ public static void injectLlapCaches(InputFormat<WritableComparable, Writable> inputFormat,
+ LlapIo<VectorizedRowBatch> llapIo) {
+ LOG.info("Injecting LLAP caches into " + inputFormat.getClass().getCanonicalName());
+ llapIo.initCacheOnlyInputFormat(inputFormat);
+ }
public static boolean canWrapForLlap(Class<? extends InputFormat> clazz, boolean checkVector) {
return LlapWrappableInputFormatInterface.class.isAssignableFrom(clazz) &&
(!checkVector || BatchToRowInputFormat.class.isAssignableFrom(clazz));
}
+ public static boolean canInjectCaches(Class<? extends InputFormat> clazz) {
+ return LlapCacheOnlyInputFormatInterface.class.isAssignableFrom(clazz);
+ }
+
@SuppressWarnings("unchecked")
private static <T, U, V, W> InputFormat<T, U> castInputFormat(InputFormat<V, W> from) {
// This is ugly in two ways...
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/ql/src/java/org/apache/hadoop/hive/ql/io/LlapCacheOnlyInputFormatInterface.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/LlapCacheOnlyInputFormatInterface.java b/ql/src/java/org/apache/hadoop/hive/ql/io/LlapCacheOnlyInputFormatInterface.java
new file mode 100644
index 0000000..13d594c
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/LlapCacheOnlyInputFormatInterface.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.io.DataCache;
+import org.apache.hadoop.hive.common.io.FileMetadataCache;
+
+/** Marker interface for LLAP IO. */
+public interface LlapCacheOnlyInputFormatInterface {
+ void injectCaches(FileMetadataCache metadataCache, DataCache dataCache, Configuration cacheConf);
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
index f4fadbb..38aaeed 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
@@ -15,7 +15,11 @@ package org.apache.hadoop.hive.ql.io.parquet;
import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.io.DataCache;
+import org.apache.hadoop.hive.common.io.FileMetadataCache;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface;
+import org.apache.hadoop.hive.ql.io.LlapCacheOnlyInputFormatInterface;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -37,7 +41,7 @@ import org.apache.parquet.hadoop.ParquetInputFormat;
* are not currently supported. Removing the interface turns off vectorization.
*/
public class MapredParquetInputFormat extends FileInputFormat<NullWritable, ArrayWritable>
- implements VectorizedInputFormatInterface {
+ implements VectorizedInputFormatInterface, LlapCacheOnlyInputFormatInterface {
private static final Logger LOG = LoggerFactory.getLogger(MapredParquetInputFormat.class);
@@ -78,4 +82,10 @@ public class MapredParquetInputFormat extends FileInputFormat<NullWritable, Arra
throw new RuntimeException("Cannot create a RecordReaderWrapper", e);
}
}
+
+ @Override
+ public void injectCaches(
+ FileMetadataCache metadataCache, DataCache dataCache, Configuration cacheConf) {
+ vectorizedSelf.injectCaches(metadataCache, dataCache, cacheConf);
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/VectorizedParquetInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/VectorizedParquetInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/VectorizedParquetInputFormat.java
index 322178a..0cc580a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/VectorizedParquetInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/VectorizedParquetInputFormat.java
@@ -15,6 +15,10 @@ package org.apache.hadoop.hive.ql.io.parquet;
import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.io.DataCache;
+import org.apache.hadoop.hive.common.io.FileMetadataCache;
+import org.apache.hadoop.hive.ql.io.LlapCacheOnlyInputFormatInterface;
import org.apache.hadoop.hive.ql.io.parquet.vector.VectorizedParquetRecordReader;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
@@ -28,7 +32,12 @@ import org.apache.hadoop.mapred.Reporter;
* Vectorized input format for Parquet files
*/
public class VectorizedParquetInputFormat
- extends FileInputFormat<NullWritable, VectorizedRowBatch> {
+ extends FileInputFormat<NullWritable, VectorizedRowBatch>
+ implements LlapCacheOnlyInputFormatInterface {
+
+ private FileMetadataCache metadataCache = null;
+ private DataCache dataCache = null;
+ private Configuration cacheConf = null;
public VectorizedParquetInputFormat() {
}
@@ -38,6 +47,15 @@ public class VectorizedParquetInputFormat
InputSplit inputSplit,
JobConf jobConf,
Reporter reporter) throws IOException {
- return new VectorizedParquetRecordReader(inputSplit, jobConf);
+ return new VectorizedParquetRecordReader(
+ inputSplit, jobConf, metadataCache, dataCache, cacheConf);
+ }
+
+ @Override
+ public void injectCaches(
+ FileMetadataCache metadataCache, DataCache dataCache, Configuration cacheConf) {
+ this.metadataCache = metadataCache;
+ this.dataCache = dataCache;
+ this.cacheConf = cacheConf;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetFooterInputFromCache.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetFooterInputFromCache.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetFooterInputFromCache.java
new file mode 100644
index 0000000..2a6e052
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetFooterInputFromCache.java
@@ -0,0 +1,196 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io.parquet.vector;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer;
+import org.apache.hadoop.hive.common.io.encoded.MemoryBufferOrBuffers;
+import org.apache.parquet.hadoop.ParquetFileWriter;
+import org.apache.parquet.io.InputFile;
+import org.apache.parquet.io.SeekableInputStream;
+
+/**
+ * The Parquet InputFile implementation that allows the reader to
+ * read the footer from cache without being aware of the latter.
+ * This implements both InputFile and the InputStream that the reader gets from InputFile.
+ */
+final class ParquetFooterInputFromCache
+ extends SeekableInputStream implements InputFile {
+ final static int FOOTER_LENGTH_SIZE = 4; // For the file size check.
+ private static final int TAIL_LENGTH = ParquetFileWriter.MAGIC.length + FOOTER_LENGTH_SIZE;
+ private static final int FAKE_PREFIX_LENGTH = ParquetFileWriter.MAGIC.length;
+ private final int length, footerLength;
+ private int position = 0, bufferIx = 0, bufferPos = 0;
+ private final MemoryBuffer[] cacheData;
+ private final int[] positions;
+
+ public ParquetFooterInputFromCache(MemoryBufferOrBuffers footerData) {
+ MemoryBuffer oneBuffer = footerData.getSingleBuffer();
+ if (oneBuffer != null) {
+ cacheData = new MemoryBuffer[2];
+ cacheData[0] = oneBuffer;
+ } else {
+ MemoryBuffer[] bufs = footerData.getMultipleBuffers();
+ cacheData = new MemoryBuffer[bufs.length + 1];
+ System.arraycopy(bufs, 0, cacheData, 0, bufs.length);
+ }
+ int footerLength = 0;
+ positions = new int[cacheData.length];
+ for (int i = 0; i < cacheData.length - 1; ++i) {
+ positions[i] = footerLength;
+ int dataLen = cacheData[i].getByteBufferRaw().remaining();
+ assert dataLen > 0;
+ footerLength += dataLen;
+ }
+ positions[cacheData.length - 1] = footerLength;
+ cacheData[cacheData.length - 1] = new FooterEndBuffer(footerLength);
+ this.footerLength = footerLength;
+ this.length = footerLength + FAKE_PREFIX_LENGTH + TAIL_LENGTH;
+ }
+
+ @Override
+ public long getLength() throws IOException {
+ return length;
+ }
+
+ @Override
+ public SeekableInputStream newStream() throws IOException {
+ // Note: this doesn't maintain proper newStream semantics (if any).
+ // We could either clone this instead or enforce that this is only called once.
+ return this;
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ return position;
+ }
+
+ @Override
+ public void seek(long targetPos) throws IOException {
+ this.position = (int)targetPos;
+ targetPos -= FAKE_PREFIX_LENGTH;
+ // Not efficient, but we don't expect this to be called frequently.
+ for (int i = 1; i <= positions.length; ++i) {
+ int endPos = (i == positions.length) ? (length - FAKE_PREFIX_LENGTH) : positions[i];
+ if (endPos > targetPos) {
+ bufferIx = i - 1;
+ bufferPos = (int) (targetPos - positions[i - 1]);
+ return;
+ }
+ }
+ throw new IOException("Incorrect seek " + targetPos + "; footer length " + footerLength
+ + Arrays.toString(positions));
+ }
+
+ @Override
+ public void readFully(byte[] b, int offset, int len) throws IOException {
+ if (readInternal(b, offset, len) == len) return;
+ throw new EOFException();
+ }
+
+ public int readInternal(byte[] b, int offset, int len) {
+ if (position >= length) return -1;
+ int argPos = offset, argEnd = offset + len;
+ while (argPos < argEnd) {
+ if (bufferIx == cacheData.length) return (argPos - offset);
+ ByteBuffer data = cacheData[bufferIx].getByteBufferDup();
+ int toConsume = Math.min(argEnd - argPos, data.remaining() - bufferPos);
+ data.position(data.position() + bufferPos);
+ data.get(b, argPos, toConsume);
+ if (data.remaining() == 0) {
+ ++bufferIx;
+ bufferPos = 0;
+ } else {
+ bufferPos += toConsume;
+ }
+ argPos += toConsume;
+ }
+ return len;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (position >= length) return -1;
+ ++position;
+ ByteBuffer data = cacheData[bufferIx].getByteBufferRaw();
+ int bp = bufferPos;
+ ++bufferPos;
+ if (bufferPos == data.remaining()) {
+ ++bufferIx; // The first line check should handle the OOB.
+ bufferPos = 0;
+ }
+ return data.get(data.position() + bp) & 0xFF;
+ }
+
+ @Override
+ public int read(ByteBuffer bb) throws IOException {
+ // Simple implementation for now - currently Parquet uses heap buffers.
+ int result = -1;
+ if (bb.hasArray()) {
+ result = readInternal(bb.array(), bb.arrayOffset(), result);
+ if (result > 0) {
+ bb.position(bb.position() + result);
+ }
+ } else {
+ byte[] b = new byte[bb.remaining()];
+ result = readInternal(b, 0, result);
+ bb.put(b, 0, result);
+ }
+ return result;
+ }
+
+ @Override
+ public void readFully(byte[] arg0) throws IOException {
+ readFully(arg0, 0, arg0.length);
+ }
+
+ @Override
+ public void readFully(ByteBuffer arg0) throws IOException {
+ read(arg0);
+ }
+
+ /**
+ * The fake buffer that emulates end of file, with footer length and magic. Given that these
+ * can be generated based on the footer buffer itself, we don't cache them.
+ */
+ private final static class FooterEndBuffer implements MemoryBuffer {
+ private final ByteBuffer bb;
+ public FooterEndBuffer(int footerLength) {
+ byte[] b = new byte[8];
+ b[0] = (byte) ((footerLength >>> 0) & 0xFF);
+ b[1] = (byte) ((footerLength >>> 8) & 0xFF);
+ b[2] = (byte) ((footerLength >>> 16) & 0xFF);
+ b[3] = (byte) ((footerLength >>> 24) & 0xFF);
+ for (int i = 0; i < ParquetFileWriter.MAGIC.length; ++i) {
+ b[4 + i] = ParquetFileWriter.MAGIC[i];
+ }
+ bb = ByteBuffer.wrap(b);
+ }
+
+ @Override
+ public ByteBuffer getByteBufferRaw() {
+ return bb;
+ }
+
+ @Override
+ public ByteBuffer getByteBufferDup() {
+ return bb.duplicate();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java
index 6a7a219..0977759 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java
@@ -14,9 +14,22 @@
package org.apache.hadoop.hive.ql.io.parquet.vector;
import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.io.DataCache;
+import org.apache.hadoop.hive.common.io.FileMetadataCache;
+import org.apache.hadoop.hive.common.io.encoded.MemoryBufferOrBuffers;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.llap.LlapCacheAwareFs;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx;
+import org.apache.hadoop.hive.ql.io.HdfsUtils;
import org.apache.hadoop.hive.ql.io.IOConstants;
import org.apache.hadoop.hive.ql.io.parquet.ParquetRecordReaderBase;
import org.apache.hadoop.hive.ql.io.parquet.ProjectionPusher;
@@ -31,13 +44,21 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.parquet.ParquetRuntimeException;
+import org.apache.parquet.bytes.BytesUtils;
import org.apache.parquet.column.ColumnDescriptor;
import org.apache.parquet.column.page.PageReadStore;
import org.apache.parquet.filter2.compat.FilterCompat;
+import org.apache.parquet.format.converter.ParquetMetadataConverter.MetadataFilter;
import org.apache.parquet.hadoop.ParquetFileReader;
+import org.apache.parquet.hadoop.ParquetFileWriter;
import org.apache.parquet.hadoop.ParquetInputSplit;
import org.apache.parquet.hadoop.metadata.BlockMetaData;
+import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
+import org.apache.parquet.hadoop.metadata.ColumnPath;
import org.apache.parquet.hadoop.metadata.ParquetMetadata;
+import org.apache.parquet.hadoop.util.HadoopStreams;
+import org.apache.parquet.io.InputFile;
+import org.apache.parquet.io.SeekableInputStream;
import org.apache.parquet.schema.InvalidSchemaException;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.Type;
@@ -50,11 +71,11 @@ import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.TreeMap;
import static org.apache.parquet.filter2.compat.RowGroupFilter.filterRowGroups;
import static org.apache.parquet.format.converter.ParquetMetadataConverter.NO_FILTER;
import static org.apache.parquet.format.converter.ParquetMetadataConverter.range;
-import static org.apache.parquet.hadoop.ParquetFileReader.readFooter;
import static org.apache.parquet.hadoop.ParquetInputFormat.getFilter;
/**
@@ -74,6 +95,7 @@ public class VectorizedParquetRecordReader extends ParquetRecordReaderBase
private VectorizedRowBatchCtx rbCtx;
private List<Integer> indexColumnsWanted;
private Object[] partitionValues;
+ private Path cacheFsPath;
/**
* For each request column, the reader to read this column. This is NULL if this column
@@ -114,9 +136,17 @@ public class VectorizedParquetRecordReader extends ParquetRecordReaderBase
}
public VectorizedParquetRecordReader(
- org.apache.hadoop.mapred.InputSplit oldInputSplit,
- JobConf conf) {
+ org.apache.hadoop.mapred.InputSplit oldInputSplit, JobConf conf) {
+ this(oldInputSplit, conf, null, null, null);
+ }
+
+ public VectorizedParquetRecordReader(
+ org.apache.hadoop.mapred.InputSplit oldInputSplit, JobConf conf,
+ FileMetadataCache metadataCache, DataCache dataCache, Configuration cacheConf) {
try {
+ this.metadataCache = metadataCache;
+ this.cache = dataCache;
+ this.cacheConf = cacheConf;
serDeStats = new SerDeStats();
projectionPusher = new ProjectionPusher();
ParquetInputSplit inputSplit = getSplit(oldInputSplit, conf);
@@ -132,16 +162,17 @@ public class VectorizedParquetRecordReader extends ParquetRecordReaderBase
}
}
- private void initPartitionValues(FileSplit fileSplit, JobConf conf) throws IOException {
- int partitionColumnCount = rbCtx.getPartitionColumnCount();
- if (partitionColumnCount > 0) {
- partitionValues = new Object[partitionColumnCount];
- rbCtx.getPartitionValues(rbCtx, conf, fileSplit, partitionValues);
- } else {
- partitionValues = null;
- }
- }
+ private void initPartitionValues(FileSplit fileSplit, JobConf conf) throws IOException {
+ int partitionColumnCount = rbCtx.getPartitionColumnCount();
+ if (partitionColumnCount > 0) {
+ partitionValues = new Object[partitionColumnCount];
+ VectorizedRowBatchCtx.getPartitionValues(rbCtx, conf, fileSplit, partitionValues);
+ } else {
+ partitionValues = null;
+ }
+ }
+ @SuppressWarnings("deprecation")
public void initialize(
InputSplit oldSplit,
JobConf configuration) throws IOException, InterruptedException {
@@ -164,16 +195,33 @@ public class VectorizedParquetRecordReader extends ParquetRecordReaderBase
columnTypesList = DataWritableReadSupport.getColumnTypes(columnTypes);
// if task.side.metadata is set, rowGroupOffsets is null
+ Object cacheKey = null;
+ // TODO: also support fileKey in splits, like OrcSplit does
+ if (metadataCache != null) {
+ cacheKey = HdfsUtils.getFileId(file.getFileSystem(configuration), file,
+ HiveConf.getBoolVar(cacheConf, ConfVars.LLAP_CACHE_ALLOW_SYNTHETIC_FILEID),
+ HiveConf.getBoolVar(cacheConf, ConfVars.LLAP_CACHE_DEFAULT_FS_FILE_ID));
+ }
+ if (cacheKey != null) {
+ // If we are going to use cache, change the path to depend on file ID for extra consistency.
+ FileSystem fs = file.getFileSystem(configuration);
+ if (cacheKey instanceof Long && HiveConf.getBoolVar(
+ cacheConf, ConfVars.LLAP_IO_USE_FILEID_PATH)) {
+ file = HdfsUtils.getFileIdPath(fs, file, (long)cacheKey);
+ }
+ }
+
if (rowGroupOffsets == null) {
//TODO check whether rowGroupOffSets can be null
// then we need to apply the predicate push down filter
- footer = readFooter(configuration, file, range(split.getStart(), split.getEnd()));
+ footer = readSplitFooter(
+ configuration, file, cacheKey, range(split.getStart(), split.getEnd()));
MessageType fileSchema = footer.getFileMetaData().getSchema();
FilterCompat.Filter filter = getFilter(configuration);
blocks = filterRowGroups(filter, footer.getBlocks(), fileSchema);
} else {
// otherwise we find the row groups that were selected on the client
- footer = readFooter(configuration, file, NO_FILTER);
+ footer = readSplitFooter(configuration, file, cacheKey, NO_FILTER);
Set<Long> offsets = new HashSet<>();
for (long offset : rowGroupOffsets) {
offsets.add(offset);
@@ -230,10 +278,98 @@ public class VectorizedParquetRecordReader extends ParquetRecordReaderBase
requestedSchema = fileSchema;
}
+ Path path = wrapPathForCache(file, cacheKey, configuration, blocks);
this.reader = new ParquetFileReader(
- configuration, footer.getFileMetaData(), file, blocks, requestedSchema.getColumns());
+ configuration, footer.getFileMetaData(), path, blocks, requestedSchema.getColumns());
+ }
+
+ private Path wrapPathForCache(Path path, Object fileKey, JobConf configuration,
+ List<BlockMetaData> blocks) throws IOException {
+ if (fileKey == null || cache == null) {
+ return path;
+ }
+ HashSet<ColumnPath> includedCols = new HashSet<>();
+ for (ColumnDescriptor col : requestedSchema.getColumns()) {
+ includedCols.add(ColumnPath.get(col.getPath()));
+ }
+ // We could make some assumptions given how the reader currently does the work (consecutive
+ // chunks, etc.; blocks and columns stored in offset order in the lists), but we won't -
+ // just save all the chunk boundaries and lengths for now.
+ TreeMap<Long, Long> chunkIndex = new TreeMap<>();
+ for (BlockMetaData block : blocks) {
+ for (ColumnChunkMetaData mc : block.getColumns()) {
+ if (!includedCols.contains(mc.getPath())) continue;
+ chunkIndex.put(mc.getStartingPos(), mc.getStartingPos() + mc.getTotalSize());
+ }
+ }
+ // Register the cache-aware path so that Parquet reader would go thru it.
+ configuration.set("fs." + LlapCacheAwareFs.SCHEME + ".impl",
+ LlapCacheAwareFs.class.getCanonicalName());
+ path = LlapCacheAwareFs.registerFile(cache, path, fileKey, chunkIndex, configuration);
+ this.cacheFsPath = path;
+ return path;
+ }
+
+ private ParquetMetadata readSplitFooter(
+ JobConf configuration, final Path file, Object cacheKey, MetadataFilter filter) throws IOException {
+ MemoryBufferOrBuffers footerData = (cacheKey == null || metadataCache == null) ? null
+ : metadataCache.getFileMetadata(cacheKey);
+ if (footerData != null) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Found the footer in cache for " + cacheKey);
+ }
+ try {
+ return ParquetFileReader.readFooter(new ParquetFooterInputFromCache(footerData), filter);
+ } finally {
+ metadataCache.decRefBuffer(footerData);
+ }
+ }
+ final FileSystem fs = file.getFileSystem(configuration);
+ final FileStatus stat = fs.getFileStatus(file);
+ if (cacheKey == null || metadataCache == null) {
+ return readFooterFromFile(file, fs, stat, filter);
+ }
+
+ // To avoid reading the footer twice, we will cache it first and then read from cache.
+ // Parquet calls protobuf methods directly on the stream and we can't get bytes after the fact.
+ try (SeekableInputStream stream = HadoopStreams.wrap(fs.open(file))) {
+ long footerLengthIndex = stat.getLen()
+ - ParquetFooterInputFromCache.FOOTER_LENGTH_SIZE - ParquetFileWriter.MAGIC.length;
+ stream.seek(footerLengthIndex);
+ int footerLength = BytesUtils.readIntLittleEndian(stream);
+ stream.seek(footerLengthIndex - footerLength);
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Caching the footer of length " + footerLength + " for " + cacheKey);
+ }
+ footerData = metadataCache.putFileMetadata(cacheKey, footerLength, stream);
+ try {
+ return ParquetFileReader.readFooter(new ParquetFooterInputFromCache(footerData), filter);
+ } finally {
+ metadataCache.decRefBuffer(footerData);
+ }
+ }
}
+ private ParquetMetadata readFooterFromFile(final Path file, final FileSystem fs,
+ final FileStatus stat, MetadataFilter filter) throws IOException {
+ InputFile inputFile = new InputFile() {
+ @Override
+ public SeekableInputStream newStream() throws IOException {
+ return HadoopStreams.wrap(fs.open(file));
+ }
+ @Override
+ public long getLength() throws IOException {
+ return stat.getLen();
+ }
+ };
+ return ParquetFileReader.readFooter(inputFile, filter);
+ }
+
+
+ private FileMetadataCache metadataCache;
+ private DataCache cache;
+ private Configuration cacheConf;
+
@Override
public boolean next(
NullWritable nullWritable,
@@ -259,6 +395,9 @@ public class VectorizedParquetRecordReader extends ParquetRecordReaderBase
@Override
public void close() throws IOException {
+ if (cacheFsPath != null) {
+ LlapCacheAwareFs.unregisterFile(cacheFsPath);
+ }
if (reader != null) {
reader.close();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
index 8b99ae0..2e63260 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
@@ -258,7 +258,7 @@ public class MapWork extends BaseWork {
}
public void deriveLlap(Configuration conf, boolean isExecDriver) {
- boolean hasLlap = false, hasNonLlap = false, hasAcid = false;
+ boolean hasLlap = false, hasNonLlap = false, hasAcid = false, hasCacheOnly = false;
// Assume the IO is enabled on the daemon by default. We cannot reasonably check it here.
boolean isLlapOn = HiveConf.getBoolVar(conf, ConfVars.LLAP_IO_ENABLED, llapMode);
boolean canWrapAny = false, doCheckIfs = false;
@@ -272,10 +272,9 @@ public class MapWork extends BaseWork {
}
}
boolean hasPathToPartInfo = (pathToPartitionInfo != null && !pathToPartitionInfo.isEmpty());
- if (canWrapAny && hasPathToPartInfo) {
- assert isLlapOn;
+ if (hasPathToPartInfo) {
for (PartitionDesc part : pathToPartitionInfo.values()) {
- boolean isUsingLlapIo = HiveInputFormat.canWrapForLlap(
+ boolean isUsingLlapIo = canWrapAny && HiveInputFormat.canWrapForLlap(
part.getInputFileFormatClass(), doCheckIfs);
if (isUsingLlapIo) {
if (part.getTableDesc() != null &&
@@ -284,28 +283,31 @@ public class MapWork extends BaseWork {
} else {
hasLlap = true;
}
+ } else if (isLlapOn && HiveInputFormat.canInjectCaches(part.getInputFileFormatClass())) {
+ hasCacheOnly = true;
} else {
hasNonLlap = true;
}
}
}
- // check if the column types that are read are supported by LLAP IO
- if (hasLlap) {
- // TODO: no need for now hasLlap = checkVectorizerSupportedTypes();
- }
-
llapIoDesc = deriveLlapIoDescString(
- isLlapOn, canWrapAny, hasPathToPartInfo, hasLlap, hasNonLlap, hasAcid);
+ isLlapOn, canWrapAny, hasPathToPartInfo, hasLlap, hasNonLlap, hasAcid, hasCacheOnly);
}
private static String deriveLlapIoDescString(boolean isLlapOn, boolean canWrapAny,
- boolean hasPathToPartInfo, boolean hasLlap, boolean hasNonLlap, boolean hasAcid) {
+ boolean hasPathToPartInfo, boolean hasLlap, boolean hasNonLlap, boolean hasAcid,
+ boolean hasCacheOnly) {
if (!isLlapOn) return null; // LLAP IO is off, don't output.
- if (!canWrapAny) return "no inputs"; // Cannot use with input formats.
+ if (!canWrapAny && !hasCacheOnly) return "no inputs"; // Cannot use with input formats.
if (!hasPathToPartInfo) return "unknown"; // No information to judge.
+ int varieties = (hasAcid ? 1 : 0) + (hasLlap ? 1 : 0)
+ + (hasCacheOnly ? 1 : 0) + (hasNonLlap ? 1 : 0);
+ if (varieties > 1) return "some inputs"; // Will probably never actually happen.
if (hasAcid) return "may be used (ACID table)";
- return (hasLlap ? (hasNonLlap ? "some inputs" : "all inputs") : "no inputs");
+ if (hasLlap) return "all inputs";
+ if (hasCacheOnly) return "all inputs (cache only)";
+ return "no inputs";
}
public void internTable(Interner<TableDesc> interner) {
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/ql/src/test/queries/clientpositive/parquet_ppd_decimal.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/parquet_ppd_decimal.q b/ql/src/test/queries/clientpositive/parquet_ppd_decimal.q
index dfca486..ff883db 100644
--- a/ql/src/test/queries/clientpositive/parquet_ppd_decimal.q
+++ b/ql/src/test/queries/clientpositive/parquet_ppd_decimal.q
@@ -1,6 +1,7 @@
SET hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
SET mapred.min.split.size=1000;
SET mapred.max.split.size=5000;
+set hive.llap.cache.allow.synthetic.fileid=true;
create table newtypestbl(c char(10), v varchar(10), d decimal(5,3), da date) stored as parquet;
http://git-wip-us.apache.org/repos/asf/hive/blob/9e673a73/ql/src/test/queries/clientpositive/parquet_predicate_pushdown.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/parquet_predicate_pushdown.q b/ql/src/test/queries/clientpositive/parquet_predicate_pushdown.q
index a38cdbe..0e8dd04 100644
--- a/ql/src/test/queries/clientpositive/parquet_predicate_pushdown.q
+++ b/ql/src/test/queries/clientpositive/parquet_predicate_pushdown.q
@@ -1,5 +1,6 @@
set hive.mapred.mode=nonstrict;
SET hive.optimize.ppd=true;
+set hive.llap.cache.allow.synthetic.fileid=true;
-- SORT_QUERY_RESULTS
CREATE TABLE tbl_pred(t tinyint,
[46/50] [abbrv] hive git commit: HIVE-14671 : merge master into
hive-14535 (Wei Zheng)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/test/results/clientpositive/mm_all.q.out
----------------------------------------------------------------------
diff --cc ql/src/test/results/clientpositive/mm_all.q.out
index e08f1a8,0000000..ea60414
mode 100644,000000..100644
--- a/ql/src/test/results/clientpositive/mm_all.q.out
+++ b/ql/src/test/results/clientpositive/mm_all.q.out
@@@ -1,2443 -1,0 +1,2147 @@@
+PREHOOK: query: drop table intermediate
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table intermediate
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table intermediate(key int) partitioned by (p int) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@intermediate
+POSTHOOK: query: create table intermediate(key int) partitioned by (p int) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@intermediate
+PREHOOK: query: insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@intermediate@p=455
+POSTHOOK: query: insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@intermediate@p=455
+POSTHOOK: Lineage: intermediate PARTITION(p=455).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+PREHOOK: query: insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@intermediate@p=456
+POSTHOOK: query: insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@intermediate@p=456
+POSTHOOK: Lineage: intermediate PARTITION(p=456).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+PREHOOK: query: insert into table intermediate partition(p='457') select distinct key from src where key >= 100 order by key asc limit 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@intermediate@p=457
+POSTHOOK: query: insert into table intermediate partition(p='457') select distinct key from src where key >= 100 order by key asc limit 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@intermediate@p=457
+POSTHOOK: Lineage: intermediate PARTITION(p=457).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+PREHOOK: query: drop table part_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table part_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table part_mm(key int) partitioned by (key_mm int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@part_mm
+POSTHOOK: query: create table part_mm(key int) partitioned by (key_mm int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@part_mm
+PREHOOK: query: explain insert into table part_mm partition(key_mm=455) select key from intermediate
+PREHOOK: type: QUERY
+POSTHOOK: query: explain insert into table part_mm partition(key_mm=455) select key from intermediate
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
+ Stage-4
+ Stage-0 depends on stages: Stage-4, Stage-3, Stage-6
+ Stage-2 depends on stages: Stage-0
+ Stage-3
+ Stage-5
+ Stage-6 depends on stages: Stage-5
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: intermediate
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.part_mm
+ Write Type: INSERT
+
+ Stage: Stage-7
+ Conditional Operator
+
+ Stage: Stage-4
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ key_mm 455
+ replace: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.part_mm
+ micromanaged table: true
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+ Stage: Stage-3
+ Merge File Operator
+ Map Operator Tree:
+ ORC File Merge Operator
+ merge level: stripe
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+
+ Stage: Stage-5
+ Merge File Operator
+ Map Operator Tree:
+ ORC File Merge Operator
+ merge level: stripe
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+
+ Stage: Stage-6
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+PREHOOK: query: insert into table part_mm partition(key_mm=455) select key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@part_mm@key_mm=455
+POSTHOOK: query: insert into table part_mm partition(key_mm=455) select key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@part_mm@key_mm=455
+POSTHOOK: Lineage: part_mm PARTITION(key_mm=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: insert into table part_mm partition(key_mm=456) select key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@part_mm@key_mm=456
+POSTHOOK: query: insert into table part_mm partition(key_mm=456) select key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@part_mm@key_mm=456
+POSTHOOK: Lineage: part_mm PARTITION(key_mm=456).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: insert into table part_mm partition(key_mm=455) select key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@part_mm@key_mm=455
+POSTHOOK: query: insert into table part_mm partition(key_mm=455) select key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@part_mm@key_mm=455
+POSTHOOK: Lineage: part_mm PARTITION(key_mm=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from part_mm order by key, key_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part_mm
+PREHOOK: Input: default@part_mm@key_mm=455
+PREHOOK: Input: default@part_mm@key_mm=456
+#### A masked pattern was here ####
+POSTHOOK: query: select * from part_mm order by key, key_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part_mm
+POSTHOOK: Input: default@part_mm@key_mm=455
+POSTHOOK: Input: default@part_mm@key_mm=456
+#### A masked pattern was here ####
+0 455
+0 455
+0 456
+10 455
+10 455
+10 456
+97 455
+97 455
+97 456
+98 455
+98 455
+98 456
+100 455
+100 455
+100 456
+103 455
+103 455
+103 456
+PREHOOK: query: select * from part_mm order by key, key_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part_mm
+PREHOOK: Input: default@part_mm@key_mm=455
+PREHOOK: Input: default@part_mm@key_mm=456
+#### A masked pattern was here ####
+POSTHOOK: query: select * from part_mm order by key, key_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part_mm
+POSTHOOK: Input: default@part_mm@key_mm=455
+POSTHOOK: Input: default@part_mm@key_mm=456
+#### A masked pattern was here ####
+0 455
+0 455
+0 456
+10 455
+10 455
+10 456
+97 455
+97 455
+97 456
+98 455
+98 455
+98 456
+100 455
+100 455
+100 456
+103 455
+103 455
+103 456
+PREHOOK: query: truncate table part_mm
+PREHOOK: type: TRUNCATETABLE
+PREHOOK: Output: default@part_mm@key_mm=455
+PREHOOK: Output: default@part_mm@key_mm=456
+POSTHOOK: query: truncate table part_mm
+POSTHOOK: type: TRUNCATETABLE
+POSTHOOK: Output: default@part_mm@key_mm=455
+POSTHOOK: Output: default@part_mm@key_mm=456
+PREHOOK: query: select * from part_mm order by key, key_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part_mm
+PREHOOK: Input: default@part_mm@key_mm=455
+PREHOOK: Input: default@part_mm@key_mm=456
+#### A masked pattern was here ####
+POSTHOOK: query: select * from part_mm order by key, key_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part_mm
+POSTHOOK: Input: default@part_mm@key_mm=455
+POSTHOOK: Input: default@part_mm@key_mm=456
+#### A masked pattern was here ####
+PREHOOK: query: drop table part_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@part_mm
+PREHOOK: Output: default@part_mm
+POSTHOOK: query: drop table part_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@part_mm
+POSTHOOK: Output: default@part_mm
+PREHOOK: query: drop table simple_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table simple_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table simple_mm(key int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@simple_mm
+POSTHOOK: query: create table simple_mm(key int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@simple_mm
+PREHOOK: query: insert into table simple_mm select key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@simple_mm
+POSTHOOK: query: insert into table simple_mm select key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@simple_mm
+POSTHOOK: Lineage: simple_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from simple_mm order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@simple_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from simple_mm order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@simple_mm
+#### A masked pattern was here ####
+0
+10
+97
+98
+100
+103
+PREHOOK: query: insert into table simple_mm select key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@simple_mm
+POSTHOOK: query: insert into table simple_mm select key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@simple_mm
+POSTHOOK: Lineage: simple_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from simple_mm order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@simple_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from simple_mm order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@simple_mm
+#### A masked pattern was here ####
+0
+0
+10
+10
+97
+97
+98
+98
+100
+100
+103
+103
+PREHOOK: query: truncate table simple_mm
+PREHOOK: type: TRUNCATETABLE
+PREHOOK: Output: default@simple_mm
+POSTHOOK: query: truncate table simple_mm
+POSTHOOK: type: TRUNCATETABLE
+POSTHOOK: Output: default@simple_mm
+PREHOOK: query: select * from simple_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@simple_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from simple_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@simple_mm
+#### A masked pattern was here ####
+PREHOOK: query: drop table simple_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@simple_mm
+PREHOOK: Output: default@simple_mm
+POSTHOOK: query: drop table simple_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@simple_mm
+POSTHOOK: Output: default@simple_mm
+PREHOOK: query: drop table dp_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table dp_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table dp_mm (key int) partitioned by (key1 string, key2 int) stored as orc
+ tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@dp_mm
+POSTHOOK: query: create table dp_mm (key int) partitioned by (key1 string, key2 int) stored as orc
+ tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@dp_mm
+PREHOOK: query: insert into table dp_mm partition (key1='123', key2) select key, key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@dp_mm@key1=123
+POSTHOOK: query: insert into table dp_mm partition (key1='123', key2) select key, key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@dp_mm@key1=123/key2=0
+POSTHOOK: Output: default@dp_mm@key1=123/key2=10
+POSTHOOK: Output: default@dp_mm@key1=123/key2=100
+POSTHOOK: Output: default@dp_mm@key1=123/key2=103
+POSTHOOK: Output: default@dp_mm@key1=123/key2=97
+POSTHOOK: Output: default@dp_mm@key1=123/key2=98
+POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=0).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=100).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=103).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=10).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=97).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=98).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from dp_mm order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dp_mm
+PREHOOK: Input: default@dp_mm@key1=123/key2=0
+PREHOOK: Input: default@dp_mm@key1=123/key2=10
+PREHOOK: Input: default@dp_mm@key1=123/key2=100
+PREHOOK: Input: default@dp_mm@key1=123/key2=103
+PREHOOK: Input: default@dp_mm@key1=123/key2=97
+PREHOOK: Input: default@dp_mm@key1=123/key2=98
+#### A masked pattern was here ####
+POSTHOOK: query: select * from dp_mm order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dp_mm
+POSTHOOK: Input: default@dp_mm@key1=123/key2=0
+POSTHOOK: Input: default@dp_mm@key1=123/key2=10
+POSTHOOK: Input: default@dp_mm@key1=123/key2=100
+POSTHOOK: Input: default@dp_mm@key1=123/key2=103
+POSTHOOK: Input: default@dp_mm@key1=123/key2=97
+POSTHOOK: Input: default@dp_mm@key1=123/key2=98
+#### A masked pattern was here ####
+0 123 0
+10 123 10
+97 123 97
+98 123 98
+100 123 100
+103 123 103
+PREHOOK: query: drop table dp_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@dp_mm
+PREHOOK: Output: default@dp_mm
+POSTHOOK: query: drop table dp_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@dp_mm
+POSTHOOK: Output: default@dp_mm
+PREHOOK: query: create table union_mm(id int) tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@union_mm
+POSTHOOK: query: create table union_mm(id int) tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@union_mm
+PREHOOK: query: insert into table union_mm
+select temps.p from (
+select key as p from intermediate
+union all
+select key + 1 as p from intermediate ) temps
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@union_mm
+POSTHOOK: query: insert into table union_mm
+select temps.p from (
+select key as p from intermediate
+union all
+select key + 1 as p from intermediate ) temps
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@union_mm
+POSTHOOK: Lineage: union_mm.id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from union_mm order by id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@union_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from union_mm order by id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@union_mm
+#### A masked pattern was here ####
+0
+1
+10
+11
+97
+98
+98
+99
+100
+101
+103
+104
+PREHOOK: query: insert into table union_mm
+select p from
+(
+select key + 1 as p from intermediate
+union all
+select key from intermediate
+) tab group by p
+union all
+select key + 2 as p from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@union_mm
+POSTHOOK: query: insert into table union_mm
+select p from
+(
+select key + 1 as p from intermediate
+union all
+select key from intermediate
+) tab group by p
+union all
+select key + 2 as p from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@union_mm
+POSTHOOK: Lineage: union_mm.id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from union_mm order by id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@union_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from union_mm order by id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@union_mm
+#### A masked pattern was here ####
+0
+0
+1
+1
+2
+10
+10
+11
+11
+12
+97
+97
+98
+98
+98
+99
+99
+99
+100
+100
+100
+101
+101
+102
+103
+103
+104
+104
+105
+PREHOOK: query: insert into table union_mm
+SELECT p FROM
+(
+ SELECT key + 1 as p FROM intermediate
+ UNION ALL
+ SELECT key as p FROM (
+ SELECT distinct key FROM (
+ SELECT key FROM (
+ SELECT key + 2 as key FROM intermediate
+ UNION ALL
+ SELECT key FROM intermediate
+ )t1
+ group by key)t2
+ )t3
+)t4
+group by p
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@union_mm
+POSTHOOK: query: insert into table union_mm
+SELECT p FROM
+(
+ SELECT key + 1 as p FROM intermediate
+ UNION ALL
+ SELECT key as p FROM (
+ SELECT distinct key FROM (
+ SELECT key FROM (
+ SELECT key + 2 as key FROM intermediate
+ UNION ALL
+ SELECT key FROM intermediate
+ )t1
+ group by key)t2
+ )t3
+)t4
+group by p
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@union_mm
+POSTHOOK: Lineage: union_mm.id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from union_mm order by id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@union_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from union_mm order by id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@union_mm
+#### A masked pattern was here ####
+0
+0
+0
+1
+1
+1
+2
+2
+10
+10
+10
+11
+11
+11
+12
+12
+97
+97
+97
+98
+98
+98
+98
+99
+99
+99
+99
+100
+100
+100
+100
+101
+101
+101
+102
+102
+103
+103
+103
+104
+104
+104
+105
+105
+PREHOOK: query: drop table union_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@union_mm
+PREHOOK: Output: default@union_mm
+POSTHOOK: query: drop table union_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@union_mm
+POSTHOOK: Output: default@union_mm
+PREHOOK: query: create table partunion_mm(id int) partitioned by (key int) tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@partunion_mm
+POSTHOOK: query: create table partunion_mm(id int) partitioned by (key int) tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@partunion_mm
+PREHOOK: query: insert into table partunion_mm partition(key)
+select temps.* from (
+select key as p, key from intermediate
+union all
+select key + 1 as p, key + 1 from intermediate ) temps
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@partunion_mm
+POSTHOOK: query: insert into table partunion_mm partition(key)
+select temps.* from (
+select key as p, key from intermediate
+union all
+select key + 1 as p, key + 1 from intermediate ) temps
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@partunion_mm@key=0
+POSTHOOK: Output: default@partunion_mm@key=1
+POSTHOOK: Output: default@partunion_mm@key=10
+POSTHOOK: Output: default@partunion_mm@key=100
+POSTHOOK: Output: default@partunion_mm@key=101
+POSTHOOK: Output: default@partunion_mm@key=103
+POSTHOOK: Output: default@partunion_mm@key=104
+POSTHOOK: Output: default@partunion_mm@key=11
+POSTHOOK: Output: default@partunion_mm@key=97
+POSTHOOK: Output: default@partunion_mm@key=98
+POSTHOOK: Output: default@partunion_mm@key=99
+POSTHOOK: Lineage: partunion_mm PARTITION(key=0).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: partunion_mm PARTITION(key=100).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: partunion_mm PARTITION(key=101).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: partunion_mm PARTITION(key=103).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: partunion_mm PARTITION(key=104).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: partunion_mm PARTITION(key=10).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: partunion_mm PARTITION(key=11).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: partunion_mm PARTITION(key=1).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: partunion_mm PARTITION(key=97).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: partunion_mm PARTITION(key=98).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: partunion_mm PARTITION(key=99).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from partunion_mm order by id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@partunion_mm
+PREHOOK: Input: default@partunion_mm@key=0
+PREHOOK: Input: default@partunion_mm@key=1
+PREHOOK: Input: default@partunion_mm@key=10
+PREHOOK: Input: default@partunion_mm@key=100
+PREHOOK: Input: default@partunion_mm@key=101
+PREHOOK: Input: default@partunion_mm@key=103
+PREHOOK: Input: default@partunion_mm@key=104
+PREHOOK: Input: default@partunion_mm@key=11
+PREHOOK: Input: default@partunion_mm@key=97
+PREHOOK: Input: default@partunion_mm@key=98
+PREHOOK: Input: default@partunion_mm@key=99
+#### A masked pattern was here ####
+POSTHOOK: query: select * from partunion_mm order by id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@partunion_mm
+POSTHOOK: Input: default@partunion_mm@key=0
+POSTHOOK: Input: default@partunion_mm@key=1
+POSTHOOK: Input: default@partunion_mm@key=10
+POSTHOOK: Input: default@partunion_mm@key=100
+POSTHOOK: Input: default@partunion_mm@key=101
+POSTHOOK: Input: default@partunion_mm@key=103
+POSTHOOK: Input: default@partunion_mm@key=104
+POSTHOOK: Input: default@partunion_mm@key=11
+POSTHOOK: Input: default@partunion_mm@key=97
+POSTHOOK: Input: default@partunion_mm@key=98
+POSTHOOK: Input: default@partunion_mm@key=99
+#### A masked pattern was here ####
+0 0
+1 1
+10 10
+11 11
+97 97
+98 98
+98 98
+99 99
+100 100
+101 101
+103 103
+104 104
+PREHOOK: query: drop table partunion_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@partunion_mm
+PREHOOK: Output: default@partunion_mm
+POSTHOOK: query: drop table partunion_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@partunion_mm
+POSTHOOK: Output: default@partunion_mm
+PREHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3))
+ stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@skew_mm
+POSTHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3))
+ stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@skew_mm
+PREHOOK: query: insert into table skew_mm
+select key, key, key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@skew_mm
+POSTHOOK: query: insert into table skew_mm
+select key, key, key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@skew_mm
+POSTHOOK: Lineage: skew_mm.k1 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_mm.k2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_mm.k4 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from skew_mm order by k2, k1, k4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@skew_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from skew_mm order by k2, k1, k4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@skew_mm
+#### A masked pattern was here ####
+0 0 0
+10 10 10
+97 97 97
+98 98 98
+100 100 100
+103 103 103
+PREHOOK: query: drop table skew_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@skew_mm
+PREHOOK: Output: default@skew_mm
+POSTHOOK: query: drop table skew_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@skew_mm
+POSTHOOK: Output: default@skew_mm
+PREHOOK: query: create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int)
+skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@skew_dp_union_mm
+POSTHOOK: query: create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int)
+skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@skew_dp_union_mm
+PREHOOK: query: insert into table skew_dp_union_mm partition (k3)
+select key as i, key as j, key as k, key as l from intermediate
+union all
+select key +1 as i, key +2 as j, key +3 as k, key +4 as l from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@skew_dp_union_mm
+POSTHOOK: query: insert into table skew_dp_union_mm partition (k3)
+select key as i, key as j, key as k, key as l from intermediate
+union all
+select key +1 as i, key +2 as j, key +3 as k, key +4 as l from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@skew_dp_union_mm@k3=0
+POSTHOOK: Output: default@skew_dp_union_mm@k3=10
+POSTHOOK: Output: default@skew_dp_union_mm@k3=100
+POSTHOOK: Output: default@skew_dp_union_mm@k3=101
+POSTHOOK: Output: default@skew_dp_union_mm@k3=102
+POSTHOOK: Output: default@skew_dp_union_mm@k3=103
+POSTHOOK: Output: default@skew_dp_union_mm@k3=104
+POSTHOOK: Output: default@skew_dp_union_mm@k3=107
+POSTHOOK: Output: default@skew_dp_union_mm@k3=14
+POSTHOOK: Output: default@skew_dp_union_mm@k3=4
+POSTHOOK: Output: default@skew_dp_union_mm@k3=97
+POSTHOOK: Output: default@skew_dp_union_mm@k3=98
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=0).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=0).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=0).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=100).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=100).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=100).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=101).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=101).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=101).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=102).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=102).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=102).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=103).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=103).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=103).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=104).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=104).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=104).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=107).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=107).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=107).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=10).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=10).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=10).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=14).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=14).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=14).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=4).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=4).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=4).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=97).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=97).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=97).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=98).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=98).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=98).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from skew_dp_union_mm order by k2, k1, k4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@skew_dp_union_mm
+PREHOOK: Input: default@skew_dp_union_mm@k3=0
+PREHOOK: Input: default@skew_dp_union_mm@k3=10
+PREHOOK: Input: default@skew_dp_union_mm@k3=100
+PREHOOK: Input: default@skew_dp_union_mm@k3=101
+PREHOOK: Input: default@skew_dp_union_mm@k3=102
+PREHOOK: Input: default@skew_dp_union_mm@k3=103
+PREHOOK: Input: default@skew_dp_union_mm@k3=104
+PREHOOK: Input: default@skew_dp_union_mm@k3=107
+PREHOOK: Input: default@skew_dp_union_mm@k3=14
+PREHOOK: Input: default@skew_dp_union_mm@k3=4
+PREHOOK: Input: default@skew_dp_union_mm@k3=97
+PREHOOK: Input: default@skew_dp_union_mm@k3=98
+#### A masked pattern was here ####
+POSTHOOK: query: select * from skew_dp_union_mm order by k2, k1, k4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@skew_dp_union_mm
+POSTHOOK: Input: default@skew_dp_union_mm@k3=0
+POSTHOOK: Input: default@skew_dp_union_mm@k3=10
+POSTHOOK: Input: default@skew_dp_union_mm@k3=100
+POSTHOOK: Input: default@skew_dp_union_mm@k3=101
+POSTHOOK: Input: default@skew_dp_union_mm@k3=102
+POSTHOOK: Input: default@skew_dp_union_mm@k3=103
+POSTHOOK: Input: default@skew_dp_union_mm@k3=104
+POSTHOOK: Input: default@skew_dp_union_mm@k3=107
+POSTHOOK: Input: default@skew_dp_union_mm@k3=14
+POSTHOOK: Input: default@skew_dp_union_mm@k3=4
+POSTHOOK: Input: default@skew_dp_union_mm@k3=97
+POSTHOOK: Input: default@skew_dp_union_mm@k3=98
+#### A masked pattern was here ####
+0 0 0 0
+1 2 3 4
+10 10 10 10
+11 12 13 14
+97 97 97 97
+98 98 98 98
+98 99 100 101
+99 100 101 102
+100 100 100 100
+101 102 103 104
+103 103 103 103
+104 105 106 107
+PREHOOK: query: drop table skew_dp_union_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@skew_dp_union_mm
+PREHOOK: Output: default@skew_dp_union_mm
+POSTHOOK: query: drop table skew_dp_union_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@skew_dp_union_mm
+POSTHOOK: Output: default@skew_dp_union_mm
+PREHOOK: query: create table merge0_mm (id int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@merge0_mm
+POSTHOOK: query: create table merge0_mm (id int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@merge0_mm
+PREHOOK: query: insert into table merge0_mm select key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@merge0_mm
+POSTHOOK: query: insert into table merge0_mm select key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@merge0_mm
+POSTHOOK: Lineage: merge0_mm.id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from merge0_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@merge0_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from merge0_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@merge0_mm
+#### A masked pattern was here ####
+98
+97
+0
+10
+100
+103
+PREHOOK: query: insert into table merge0_mm select key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@merge0_mm
+POSTHOOK: query: insert into table merge0_mm select key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@merge0_mm
+POSTHOOK: Lineage: merge0_mm.id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from merge0_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@merge0_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from merge0_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@merge0_mm
+#### A masked pattern was here ####
+98
+97
+0
+10
+100
+103
+98
+97
+0
+10
+100
+103
+PREHOOK: query: drop table merge0_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@merge0_mm
+PREHOOK: Output: default@merge0_mm
+POSTHOOK: query: drop table merge0_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@merge0_mm
+POSTHOOK: Output: default@merge0_mm
+PREHOOK: query: create table merge2_mm (id int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@merge2_mm
+POSTHOOK: query: create table merge2_mm (id int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@merge2_mm
+PREHOOK: query: insert into table merge2_mm select key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@merge2_mm
+POSTHOOK: query: insert into table merge2_mm select key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@merge2_mm
+POSTHOOK: Lineage: merge2_mm.id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from merge2_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@merge2_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from merge2_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@merge2_mm
+#### A masked pattern was here ####
+98
+97
+0
+10
+100
+103
+PREHOOK: query: insert into table merge2_mm select key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@merge2_mm
+POSTHOOK: query: insert into table merge2_mm select key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@merge2_mm
+POSTHOOK: Lineage: merge2_mm.id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from merge2_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@merge2_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from merge2_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@merge2_mm
+#### A masked pattern was here ####
+98
+97
+0
+10
+100
+103
+98
+97
+0
+10
+100
+103
+PREHOOK: query: drop table merge2_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@merge2_mm
+PREHOOK: Output: default@merge2_mm
+POSTHOOK: query: drop table merge2_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@merge2_mm
+POSTHOOK: Output: default@merge2_mm
+PREHOOK: query: create table merge1_mm (id int) partitioned by (key int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@merge1_mm
+POSTHOOK: query: create table merge1_mm (id int) partitioned by (key int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@merge1_mm
+PREHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@merge1_mm
+POSTHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@merge1_mm@key=0
+POSTHOOK: Output: default@merge1_mm@key=10
+POSTHOOK: Output: default@merge1_mm@key=100
+POSTHOOK: Output: default@merge1_mm@key=103
+POSTHOOK: Output: default@merge1_mm@key=97
+POSTHOOK: Output: default@merge1_mm@key=98
+POSTHOOK: Lineage: merge1_mm PARTITION(key=0).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: merge1_mm PARTITION(key=100).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: merge1_mm PARTITION(key=103).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: merge1_mm PARTITION(key=10).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: merge1_mm PARTITION(key=97).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: merge1_mm PARTITION(key=98).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from merge1_mm order by id, key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@merge1_mm
+PREHOOK: Input: default@merge1_mm@key=0
+PREHOOK: Input: default@merge1_mm@key=10
+PREHOOK: Input: default@merge1_mm@key=100
+PREHOOK: Input: default@merge1_mm@key=103
+PREHOOK: Input: default@merge1_mm@key=97
+PREHOOK: Input: default@merge1_mm@key=98
+#### A masked pattern was here ####
+POSTHOOK: query: select * from merge1_mm order by id, key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@merge1_mm
+POSTHOOK: Input: default@merge1_mm@key=0
+POSTHOOK: Input: default@merge1_mm@key=10
+POSTHOOK: Input: default@merge1_mm@key=100
+POSTHOOK: Input: default@merge1_mm@key=103
+POSTHOOK: Input: default@merge1_mm@key=97
+POSTHOOK: Input: default@merge1_mm@key=98
+#### A masked pattern was here ####
+0 0
+10 10
+97 97
+98 98
+100 100
+103 103
+PREHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@merge1_mm
+POSTHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@merge1_mm@key=0
+POSTHOOK: Output: default@merge1_mm@key=10
+POSTHOOK: Output: default@merge1_mm@key=100
+POSTHOOK: Output: default@merge1_mm@key=103
+POSTHOOK: Output: default@merge1_mm@key=97
+POSTHOOK: Output: default@merge1_mm@key=98
+POSTHOOK: Lineage: merge1_mm PARTITION(key=0).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: merge1_mm PARTITION(key=100).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: merge1_mm PARTITION(key=103).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: merge1_mm PARTITION(key=10).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: merge1_mm PARTITION(key=97).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: merge1_mm PARTITION(key=98).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from merge1_mm order by id, key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@merge1_mm
+PREHOOK: Input: default@merge1_mm@key=0
+PREHOOK: Input: default@merge1_mm@key=10
+PREHOOK: Input: default@merge1_mm@key=100
+PREHOOK: Input: default@merge1_mm@key=103
+PREHOOK: Input: default@merge1_mm@key=97
+PREHOOK: Input: default@merge1_mm@key=98
+#### A masked pattern was here ####
+POSTHOOK: query: select * from merge1_mm order by id, key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@merge1_mm
+POSTHOOK: Input: default@merge1_mm@key=0
+POSTHOOK: Input: default@merge1_mm@key=10
+POSTHOOK: Input: default@merge1_mm@key=100
+POSTHOOK: Input: default@merge1_mm@key=103
+POSTHOOK: Input: default@merge1_mm@key=97
+POSTHOOK: Input: default@merge1_mm@key=98
+#### A masked pattern was here ####
+0 0
+0 0
+10 10
+10 10
+97 97
+97 97
+98 98
+98 98
+100 100
+100 100
+103 103
+103 103
+PREHOOK: query: drop table merge1_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@merge1_mm
+PREHOOK: Output: default@merge1_mm
+POSTHOOK: query: drop table merge1_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@merge1_mm
+POSTHOOK: Output: default@merge1_mm
+PREHOOK: query: drop table ctas0_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table ctas0_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table ctas0_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as select * from intermediate
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ctas0_mm
+POSTHOOK: query: create table ctas0_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as select * from intermediate
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ctas0_mm
+POSTHOOK: Lineage: ctas0_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: ctas0_mm.p SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+PREHOOK: query: select * from ctas0_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ctas0_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from ctas0_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ctas0_mm
+#### A masked pattern was here ####
+98 455
+97 455
+0 456
+10 456
+100 457
+103 457
+PREHOOK: query: drop table ctas0_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@ctas0_mm
+PREHOOK: Output: default@ctas0_mm
+POSTHOOK: query: drop table ctas0_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@ctas0_mm
+POSTHOOK: Output: default@ctas0_mm
+PREHOOK: query: drop table ctas1_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table ctas1_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table ctas1_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as
+ select * from intermediate union all select * from intermediate
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ctas1_mm
+POSTHOOK: query: create table ctas1_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as
+ select * from intermediate union all select * from intermediate
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ctas1_mm
+POSTHOOK: Lineage: ctas1_mm.key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: ctas1_mm.p EXPRESSION [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+PREHOOK: query: select * from ctas1_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ctas1_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from ctas1_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ctas1_mm
+#### A masked pattern was here ####
+98 455
+98 455
+97 455
+97 455
+0 456
+0 456
+10 456
+10 456
+100 457
+100 457
+103 457
+103 457
+PREHOOK: query: drop table ctas1_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@ctas1_mm
+PREHOOK: Output: default@ctas1_mm
+POSTHOOK: query: drop table ctas1_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@ctas1_mm
+POSTHOOK: Output: default@ctas1_mm
- PREHOOK: query: drop table load0_mm
- PREHOOK: type: DROPTABLE
- POSTHOOK: query: drop table load0_mm
- POSTHOOK: type: DROPTABLE
- PREHOOK: query: create table load0_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
- PREHOOK: type: CREATETABLE
- PREHOOK: Output: database:default
- PREHOOK: Output: default@load0_mm
- POSTHOOK: query: create table load0_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
- POSTHOOK: type: CREATETABLE
- POSTHOOK: Output: database:default
- POSTHOOK: Output: default@load0_mm
- PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table load0_mm
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@load0_mm
- POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table load0_mm
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@load0_mm
- PREHOOK: query: select count(1) from load0_mm
- PREHOOK: type: QUERY
- PREHOOK: Input: default@load0_mm
- #### A masked pattern was here ####
- POSTHOOK: query: select count(1) from load0_mm
- POSTHOOK: type: QUERY
- POSTHOOK: Input: default@load0_mm
- #### A masked pattern was here ####
- 500
- PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table load0_mm
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@load0_mm
- POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table load0_mm
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@load0_mm
- PREHOOK: query: select count(1) from load0_mm
- PREHOOK: type: QUERY
- PREHOOK: Input: default@load0_mm
- #### A masked pattern was here ####
- POSTHOOK: query: select count(1) from load0_mm
- POSTHOOK: type: QUERY
- POSTHOOK: Input: default@load0_mm
- #### A masked pattern was here ####
- 1000
- PREHOOK: query: load data local inpath '../../data/files/kv2.txt' overwrite into table load0_mm
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@load0_mm
- POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' overwrite into table load0_mm
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@load0_mm
- PREHOOK: query: select count(1) from load0_mm
- PREHOOK: type: QUERY
- PREHOOK: Input: default@load0_mm
- #### A masked pattern was here ####
- POSTHOOK: query: select count(1) from load0_mm
- POSTHOOK: type: QUERY
- POSTHOOK: Input: default@load0_mm
- #### A masked pattern was here ####
- 500
- PREHOOK: query: drop table load0_mm
- PREHOOK: type: DROPTABLE
- PREHOOK: Input: default@load0_mm
- PREHOOK: Output: default@load0_mm
- POSTHOOK: query: drop table load0_mm
- POSTHOOK: type: DROPTABLE
- POSTHOOK: Input: default@load0_mm
- POSTHOOK: Output: default@load0_mm
- PREHOOK: query: drop table intermediate2
- PREHOOK: type: DROPTABLE
- POSTHOOK: query: drop table intermediate2
- POSTHOOK: type: DROPTABLE
- PREHOOK: query: create table intermediate2 (key string, value string) stored as textfile
- #### A masked pattern was here ####
- PREHOOK: type: CREATETABLE
- #### A masked pattern was here ####
- PREHOOK: Output: database:default
- PREHOOK: Output: default@intermediate2
- POSTHOOK: query: create table intermediate2 (key string, value string) stored as textfile
- #### A masked pattern was here ####
- POSTHOOK: type: CREATETABLE
- #### A masked pattern was here ####
- POSTHOOK: Output: database:default
- POSTHOOK: Output: default@intermediate2
- PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@intermediate2
- POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@intermediate2
- PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@intermediate2
- POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@intermediate2
- PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@intermediate2
- POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@intermediate2
- PREHOOK: query: drop table load1_mm
- PREHOOK: type: DROPTABLE
- POSTHOOK: query: drop table load1_mm
- POSTHOOK: type: DROPTABLE
- PREHOOK: query: create table load1_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
- PREHOOK: type: CREATETABLE
- PREHOOK: Output: database:default
- PREHOOK: Output: default@load1_mm
- POSTHOOK: query: create table load1_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
- POSTHOOK: type: CREATETABLE
- POSTHOOK: Output: database:default
- POSTHOOK: Output: default@load1_mm
- #### A masked pattern was here ####
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@load1_mm
- #### A masked pattern was here ####
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@load1_mm
- #### A masked pattern was here ####
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@load1_mm
- #### A masked pattern was here ####
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@load1_mm
- PREHOOK: query: select count(1) from load1_mm
- PREHOOK: type: QUERY
- PREHOOK: Input: default@load1_mm
- #### A masked pattern was here ####
- POSTHOOK: query: select count(1) from load1_mm
- POSTHOOK: type: QUERY
- POSTHOOK: Input: default@load1_mm
- #### A masked pattern was here ####
- 1000
- PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@intermediate2
- POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@intermediate2
- PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@intermediate2
- POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@intermediate2
- PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@intermediate2
- POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@intermediate2
- #### A masked pattern was here ####
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@load1_mm
- #### A masked pattern was here ####
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@load1_mm
- PREHOOK: query: select count(1) from load1_mm
- PREHOOK: type: QUERY
- PREHOOK: Input: default@load1_mm
- #### A masked pattern was here ####
- POSTHOOK: query: select count(1) from load1_mm
- POSTHOOK: type: QUERY
- POSTHOOK: Input: default@load1_mm
- #### A masked pattern was here ####
- 1050
- PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@intermediate2
- POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@intermediate2
- #### A masked pattern was here ####
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@load1_mm
- #### A masked pattern was here ####
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@load1_mm
- PREHOOK: query: select count(1) from load1_mm
- PREHOOK: type: QUERY
- PREHOOK: Input: default@load1_mm
- #### A masked pattern was here ####
- POSTHOOK: query: select count(1) from load1_mm
- POSTHOOK: type: QUERY
- POSTHOOK: Input: default@load1_mm
- #### A masked pattern was here ####
- 500
- PREHOOK: query: drop table load1_mm
- PREHOOK: type: DROPTABLE
- PREHOOK: Input: default@load1_mm
- PREHOOK: Output: default@load1_mm
- POSTHOOK: query: drop table load1_mm
- POSTHOOK: type: DROPTABLE
- POSTHOOK: Input: default@load1_mm
- POSTHOOK: Output: default@load1_mm
- PREHOOK: query: drop table load2_mm
- PREHOOK: type: DROPTABLE
- POSTHOOK: query: drop table load2_mm
- POSTHOOK: type: DROPTABLE
- PREHOOK: query: create table load2_mm (key string, value string)
- partitioned by (k int, l int) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
- PREHOOK: type: CREATETABLE
- PREHOOK: Output: database:default
- PREHOOK: Output: default@load2_mm
- POSTHOOK: query: create table load2_mm (key string, value string)
- partitioned by (k int, l int) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
- POSTHOOK: type: CREATETABLE
- POSTHOOK: Output: database:default
- POSTHOOK: Output: default@load2_mm
- PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@intermediate2
- POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@intermediate2
- PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@intermediate2
- POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@intermediate2
- PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@intermediate2
- POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@intermediate2
- #### A masked pattern was here ####
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@load2_mm
- #### A masked pattern was here ####
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@load2_mm
- POSTHOOK: Output: default@load2_mm@k=5/l=5
- PREHOOK: query: select count(1) from load2_mm
- PREHOOK: type: QUERY
- PREHOOK: Input: default@load2_mm
- PREHOOK: Input: default@load2_mm@k=5/l=5
- #### A masked pattern was here ####
- POSTHOOK: query: select count(1) from load2_mm
- POSTHOOK: type: QUERY
- POSTHOOK: Input: default@load2_mm
- POSTHOOK: Input: default@load2_mm@k=5/l=5
- #### A masked pattern was here ####
- 1025
- PREHOOK: query: drop table load2_mm
- PREHOOK: type: DROPTABLE
- PREHOOK: Input: default@load2_mm
- PREHOOK: Output: default@load2_mm
- POSTHOOK: query: drop table load2_mm
- POSTHOOK: type: DROPTABLE
- POSTHOOK: Input: default@load2_mm
- POSTHOOK: Output: default@load2_mm
- PREHOOK: query: drop table intermediate2
- PREHOOK: type: DROPTABLE
- PREHOOK: Input: default@intermediate2
- PREHOOK: Output: default@intermediate2
- POSTHOOK: query: drop table intermediate2
- POSTHOOK: type: DROPTABLE
- POSTHOOK: Input: default@intermediate2
- POSTHOOK: Output: default@intermediate2
+PREHOOK: query: drop table multi0_1_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table multi0_1_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table multi0_2_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table multi0_2_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table multi0_1_mm (key int, key2 int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@multi0_1_mm
+POSTHOOK: query: create table multi0_1_mm (key int, key2 int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@multi0_1_mm
+PREHOOK: query: create table multi0_2_mm (key int, key2 int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@multi0_2_mm
+POSTHOOK: query: create table multi0_2_mm (key int, key2 int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@multi0_2_mm
+PREHOOK: query: insert into table multi0_1_mm select key, p from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@multi0_1_mm
+POSTHOOK: query: insert into table multi0_1_mm select key, p from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@multi0_1_mm
+POSTHOOK: Lineage: multi0_1_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: multi0_1_mm.key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+PREHOOK: query: insert into table multi0_2_mm select p, key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@multi0_2_mm
+POSTHOOK: query: insert into table multi0_2_mm select p, key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@multi0_2_mm
+POSTHOOK: Lineage: multi0_2_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+POSTHOOK: Lineage: multi0_2_mm.key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from multi0_1_mm order by key, key2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@multi0_1_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from multi0_1_mm order by key, key2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@multi0_1_mm
+#### A masked pattern was here ####
+0 456
+10 456
+97 455
+98 455
+100 457
+103 457
+PREHOOK: query: select * from multi0_2_mm order by key, key2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@multi0_2_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from multi0_2_mm order by key, key2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@multi0_2_mm
+#### A masked pattern was here ####
+455 97
+455 98
+456 0
+456 10
+457 100
+457 103
+PREHOOK: query: insert into table multi0_1_mm select p, key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@multi0_1_mm
+POSTHOOK: query: insert into table multi0_1_mm select p, key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@multi0_1_mm
+POSTHOOK: Lineage: multi0_1_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+POSTHOOK: Lineage: multi0_1_mm.key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: insert into table multi0_2_mm select key, p from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@multi0_2_mm
+POSTHOOK: query: insert into table multi0_2_mm select key, p from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@multi0_2_mm
+POSTHOOK: Lineage: multi0_2_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: multi0_2_mm.key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+PREHOOK: query: select * from multi0_1_mm order by key, key2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@multi0_1_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from multi0_1_mm order by key, key2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@multi0_1_mm
+#### A masked pattern was here ####
+0 456
+10 456
+97 455
+98 455
+100 457
+103 457
+455 97
+455 98
+456 0
+456 10
+457 100
+457 103
+PREHOOK: query: select * from multi0_2_mm order by key, key2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@multi0_2_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from multi0_2_mm order by key, key2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@multi0_2_mm
+#### A masked pattern was here ####
+0 456
+10 456
+97 455
+98 455
+100 457
+103 457
+455 97
+455 98
+456 0
+456 10
+457 100
+457 103
+PREHOOK: query: drop table multi0_1_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@multi0_1_mm
+PREHOOK: Output: default@multi0_1_mm
+POSTHOOK: query: drop table multi0_1_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@multi0_1_mm
+POSTHOOK: Output: default@multi0_1_mm
+PREHOOK: query: drop table multi0_2_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@multi0_2_mm
+PREHOOK: Output: default@multi0_2_mm
+POSTHOOK: query: drop table multi0_2_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@multi0_2_mm
+POSTHOOK: Output: default@multi0_2_mm
+PREHOOK: query: drop table multi1_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table multi1_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table multi1_mm (key int, key2 int) partitioned by (p int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@multi1_mm
+POSTHOOK: query: create table multi1_mm (key int, key2 int) partitioned by (p int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@multi1_mm
+PREHOOK: query: from intermediate
+insert into table multi1_mm partition(p=1) select p, key
+insert into table multi1_mm partition(p=2) select key, p
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@multi1_mm@p=1
+PREHOOK: Output: default@multi1_mm@p=2
+POSTHOOK: query: from intermediate
+insert into table multi1_mm partition(p=1) select p, key
+insert into table multi1_mm partition(p=2) select key, p
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@multi1_mm@p=1
+POSTHOOK: Output: default@multi1_mm@p=2
+POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=2).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=2).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+PREHOOK: query: select * from multi1_mm order by key, key2, p
+PREHOOK: type: QUERY
+PREHOOK: Input: default@multi1_mm
+PREHOOK: Input: default@multi1_mm@p=1
+PREHOOK: Input: default@multi1_mm@p=2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from multi1_mm order by key, key2, p
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@multi1_mm
+POSTHOOK: Input: default@multi1_mm@p=1
+POSTHOOK: Input: default@multi1_mm@p=2
+#### A masked pattern was here ####
+0 456 2
+10 456 2
+97 455 2
+98 455 2
+100 457 2
+103 457 2
+455 97 1
+455 98 1
+456 0 1
+456 10 1
+457 100 1
+457 103 1
+PREHOOK: query: insert into table multi1_mm partition(p=2) select p, key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@multi1_mm@p=2
+POSTHOOK: query: insert into table multi1_mm partition(p=2) select p, key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@multi1_mm@p=2
+POSTHOOK: Lineage: multi1_mm PARTITION(p=2).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=2).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: insert into table multi1_mm partition(p=1) select key, p from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@multi1_mm@p=1
+POSTHOOK: query: insert into table multi1_mm partition(p=1) select key, p from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@multi1_mm@p=1
+POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+PREHOOK: query: select * from multi1_mm order by key, key2, p
+PREHOOK: type: QUERY
+PREHOOK: Input: default@multi1_mm
+PREHOOK: Input: default@multi1_mm@p=1
+PREHOOK: Input: default@multi1_mm@p=2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from multi1_mm order by key, key2, p
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@multi1_mm
+POSTHOOK: Input: default@multi1_mm@p=1
+POSTHOOK: Input: default@multi1_mm@p=2
+#### A masked pattern was here ####
+0 456 1
+0 456 2
+10 456 1
+10 456 2
+97 455 1
+97 455 2
+98 455 1
+98 455 2
+100 457 1
+100 457 2
+103 457 1
+103 457 2
+455 97 1
+455 97 2
+455 98 1
+455 98 2
+456 0 1
+456 0 2
+456 10 1
+456 10 2
+457 100 1
+457 100 2
+457 103 1
+457 103 2
+PREHOOK: query: from intermediate
+insert into table multi1_mm partition(p) select p, key, p
+insert into table multi1_mm partition(p=1) select key, p
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@multi1_mm
+PREHOOK: Output: default@multi1_mm@p=1
+POSTHOOK: query: from intermediate
+insert into table multi1_mm partition(p) select p, key, p
+insert into table multi1_mm partition(p=1) select key, p
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@multi1_mm@p=1
+POSTHOOK: Output: default@multi1_mm@p=455
+POSTHOOK: Output: default@multi1_mm@p=456
+POSTHOOK: Output: default@multi1_mm@p=457
+POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=455).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=456).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=456).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=457).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=457).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
+PREHOOK: type: QUERY
+PREHOOK: Input: default@multi1_mm
+PREHOOK: Input: default@multi1_mm@p=1
+PREHOOK: Input: default@multi1_mm@p=2
+PREHOOK: Input: default@multi1_mm@p=455
+PREHOOK: Input: default@multi1_mm@p=456
+PREHOOK: Input: default@multi1_mm@p=457
+#### A masked pattern was here ####
+POSTHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@multi1_mm
+POSTHOOK: Input: default@multi1_mm@p=1
+POSTHOOK: Input: default@multi1_mm@p=2
+POSTHOOK: Input: default@multi1_mm@p=455
+POSTHOOK: Input: default@multi1_mm@p=456
+POSTHOOK: Input: default@multi1_mm@p=457
+#### A masked pattern was here ####
+0 456 1
+0 456 1
+0 456 2
+10 456 1
+10 456 1
+10 456 2
+97 455 1
+97 455 1
+97 455 2
+98 455 1
+98 455 1
+98 455 2
+100 457 1
+100 457 1
+100 457 2
+103 457 1
+103 457 1
+103 457 2
+455 97 1
+455 97 2
+455 97 455
+455 98 1
+455 98 2
+455 98 455
+456 0 1
+456 0 2
+456 0 456
+456 10 1
+456 10 2
+456 10 456
+457 100 1
+457 100 2
+457 100 457
+457 103 1
+457 103 2
+457 103 457
+PREHOOK: query: from intermediate
+insert into table multi1_mm partition(p) select p, key, 1
+insert into table multi1_mm partition(p=1) select key, p
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@multi1_mm
+PREHOOK: Output: default@multi1_mm@p=1
+POSTHOOK: query: from intermediate
+insert into table multi1_mm partition(p) select p, key, 1
+insert into table multi1_mm partition(p=1) select key, p
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@multi1_mm@p=1
+POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
+PREHOOK: type: QUERY
+PREHOOK: Input: default@multi1_mm
+PREHOOK: Input: default@multi1_mm@p=1
+PREHOOK: Input: default@multi1_mm@p=2
+PREHOOK: Input: default@multi1_mm@p=455
+PREHOOK: Input: default@multi1_mm@p=456
+PREHOOK: Input: default@multi1_mm@p=457
+#### A masked pattern was here ####
+POSTHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@multi1_mm
+POSTHOOK: Input: default@multi1_mm@p=1
+POSTHOOK: Input: default@multi1_mm@p=2
+POSTHOOK: Input: default@multi1_mm@p=455
+POSTHOOK: Input: default@multi1_mm@p=456
+POSTHOOK: Input: default@multi1_mm@p=457
+#### A masked pattern was here ####
+0 456 1
+0 456 1
+0 456 1
+0 456 2
+10 456 1
+10 456 1
+10 456 1
+10 456 2
+97 455 1
+97 455 1
+97 455 1
+97 455 2
+98 455 1
+98 455 1
+98 455 1
+98 455 2
+100 457 1
+100 457 1
+100 457 1
+100 457 2
+103 457 1
+103 457 1
+103 457 1
+103 457 2
+455 97 1
+455 97 1
+455 97 2
+455 97 455
+455 98 1
+455 98 1
+455 98 2
+455 98 455
+456 0 1
+456 0 1
+456 0 2
+456 0 456
+456 10 1
+456 10 1
+456 10 2
+456 10 456
+457 100 1
+457 100 1
+457 100 2
+457 100 457
+457 103 1
+457 103 1
+457 103 2
+457 103 457
+PREHOOK: query: drop table multi1_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@multi1_mm
+PREHOOK: Output: default@multi1_mm
+POSTHOOK: query: drop table multi1_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@multi1_mm
+POSTHOOK: Output: default@multi1_mm
+PREHOOK: query: drop table stats_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table stats_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table stats_mm(key int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@stats_mm
+POSTHOOK: query: create table stats_mm(key int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@stats_mm
+PREHOOK: query: insert into table stats_mm select key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@stats_mm
+POSTHOOK: query: insert into table stats_mm select key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@stats_mm
+POSTHOOK: Lineage: stats_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: desc formatted stats_mm
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@stats_mm
+POSTHOOK: query: desc formatted stats_mm
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@stats_mm
+# col_name data_type comment
+
+key int
+
+# Detailed Table Information
+Database: default
+#### A masked p
<TRUNCATED>
[02/50] [abbrv] hive git commit: HIVE-17330 : refactor
TezSessionPoolManager to separate its multiple functions (Sergey Shelukhin,
reviewed by Gunther Hagleitner)
Posted by we...@apache.org.
HIVE-17330 : refactor TezSessionPoolManager to separate its multiple functions (Sergey Shelukhin, reviewed by Gunther Hagleitner)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/733bc5f0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/733bc5f0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/733bc5f0
Branch: refs/heads/hive-14535
Commit: 733bc5f0280f0f8145b953d67bb50e34af713e04
Parents: 6b10382
Author: sergey <se...@apache.org>
Authored: Fri Aug 25 11:52:09 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Fri Aug 25 11:53:05 2017 -0700
----------------------------------------------------------------------
.../ql/exec/tez/RestrictedConfigChecker.java | 79 +++
.../ql/exec/tez/SessionExpirationTracker.java | 235 +++++++++
.../hadoop/hive/ql/exec/tez/TezSessionPool.java | 165 ++++++
.../hive/ql/exec/tez/TezSessionPoolManager.java | 516 +++----------------
.../hive/ql/exec/tez/TezSessionPoolSession.java | 158 ++++++
.../hive/ql/exec/tez/SampleTezSessionState.java | 3 +-
.../hive/ql/exec/tez/TestTezSessionPool.java | 2 +-
7 files changed, 704 insertions(+), 454 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/733bc5f0/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RestrictedConfigChecker.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RestrictedConfigChecker.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RestrictedConfigChecker.java
new file mode 100644
index 0000000..f6b1c1d
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RestrictedConfigChecker.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.tez;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class RestrictedConfigChecker {
+ private final static Logger LOG = LoggerFactory.getLogger(RestrictedConfigChecker.class);
+ private final List<ConfVars> restrictedHiveConf = new ArrayList<>();
+ private final List<String> restrictedNonHiveConf = new ArrayList<>();
+ private final HiveConf initConf;
+
+ RestrictedConfigChecker(HiveConf initConf) {
+ this.initConf = initConf;
+ String[] restrictedConfigs = HiveConf.getTrimmedStringsVar(initConf,
+ ConfVars.HIVE_SERVER2_TEZ_SESSION_RESTRICTED_CONFIGS);
+ if (restrictedConfigs == null || restrictedConfigs.length == 0) return;
+ HashMap<String, ConfVars> confVars = HiveConf.getOrCreateReverseMap();
+ for (String confName : restrictedConfigs) {
+ if (confName == null || confName.isEmpty()) continue;
+ confName = confName.toLowerCase();
+ ConfVars cv = confVars.get(confName);
+ if (cv != null) {
+ restrictedHiveConf.add(cv);
+ } else {
+ LOG.warn("A restricted config " + confName + " is not recognized as a Hive setting.");
+ restrictedNonHiveConf.add(confName);
+ }
+ }
+ }
+
+ public void validate(HiveConf conf) throws HiveException {
+ for (ConfVars var : restrictedHiveConf) {
+ String userValue = HiveConf.getVarWithoutType(conf, var),
+ serverValue = HiveConf.getVarWithoutType(initConf, var);
+ // Note: with some trickery, we could add logic for each type in ConfVars; for now the
+ // potential spurious mismatches (e.g. 0 and 0.0 for float) should be easy to work around.
+ validateRestrictedConfigValues(var.varname, userValue, serverValue);
+ }
+ for (String var : restrictedNonHiveConf) {
+ String userValue = conf.get(var), serverValue = initConf.get(var);
+ validateRestrictedConfigValues(var, userValue, serverValue);
+ }
+ }
+
+ private void validateRestrictedConfigValues(
+ String var, String userValue, String serverValue) throws HiveException {
+ if ((userValue == null) != (serverValue == null)
+ || (userValue != null && !userValue.equals(serverValue))) {
+ String logValue = initConf.isHiddenConfig(var) ? "(hidden)" : serverValue;
+ throw new HiveException(var + " is restricted from being set; server is configured"
+ + " to use " + logValue + ", but the query configuration specifies " + userValue);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/733bc5f0/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SessionExpirationTracker.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SessionExpirationTracker.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SessionExpirationTracker.java
new file mode 100644
index 0000000..8bee77e
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SessionExpirationTracker.java
@@ -0,0 +1,235 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.tez;
+
+import java.util.Comparator;
+import java.util.Random;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.PriorityBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class SessionExpirationTracker {
+ private static final Logger LOG = LoggerFactory.getLogger(SessionExpirationTracker.class);
+ private static final Random rdm = new Random();
+
+ /** Priority queue sorted by expiration time of live sessions that could be expired. */
+ private final PriorityBlockingQueue<TezSessionPoolSession> expirationQueue;
+ /** The background restart queue that is populated when expiration is triggered by a foreground
+ * thread (i.e. getting or returning a session), to avoid delaying it. */
+ private final BlockingQueue<TezSessionPoolSession> restartQueue;
+ private final Thread expirationThread;
+ private final Thread restartThread;
+ private final long sessionLifetimeMs;
+ private final long sessionLifetimeJitterMs;
+ private final RestartImpl sessionRestartImpl;
+
+ interface RestartImpl {
+ void closeAndReopenPoolSession(TezSessionPoolSession session) throws Exception;
+ }
+
+ public static SessionExpirationTracker create(HiveConf conf, RestartImpl restartImpl) {
+ long sessionLifetimeMs = conf.getTimeVar(
+ ConfVars.HIVE_SERVER2_TEZ_SESSION_LIFETIME, TimeUnit.MILLISECONDS);
+ if (sessionLifetimeMs == 0) return null;
+ return new SessionExpirationTracker(sessionLifetimeMs, conf.getTimeVar(
+ ConfVars.HIVE_SERVER2_TEZ_SESSION_LIFETIME_JITTER, TimeUnit.MILLISECONDS), restartImpl);
+ }
+
+ private SessionExpirationTracker(
+ long sessionLifetimeMs, long sessionLifetimeJitterMs, RestartImpl restartImpl) {
+ this.sessionRestartImpl = restartImpl;
+ this.sessionLifetimeMs = sessionLifetimeMs;
+ this.sessionLifetimeJitterMs = sessionLifetimeJitterMs;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Session expiration is enabled; session lifetime is "
+ + sessionLifetimeMs + " + [0, " + sessionLifetimeJitterMs + ") ms");
+ }
+ final SessionState initSessionState = SessionState.get();
+ expirationQueue = new PriorityBlockingQueue<>(11, new Comparator<TezSessionPoolSession>() {
+ @Override
+ public int compare(TezSessionPoolSession o1, TezSessionPoolSession o2) {
+ assert o1.getExpirationNs() != null && o2.getExpirationNs() != null;
+ return o1.getExpirationNs().compareTo(o2.getExpirationNs());
+ }
+ });
+ restartQueue = new LinkedBlockingQueue<>();
+
+ expirationThread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ SessionState.setCurrentSessionState(initSessionState);
+ runExpirationThread();
+ } catch (Exception e) {
+ LOG.warn("Exception in TezSessionPool-expiration thread. Thread will shut down", e);
+ } finally {
+ LOG.info("TezSessionPool-expiration thread exiting");
+ }
+ }
+ }, "TezSessionPool-expiration");
+ restartThread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ SessionState.setCurrentSessionState(initSessionState);
+ runRestartThread();
+ } catch (Exception e) {
+ LOG.warn("Exception in TezSessionPool-cleanup thread. Thread will shut down", e);
+ } finally {
+ LOG.info("TezSessionPool-cleanup thread exiting");
+ }
+ }
+ }, "TezSessionPool-cleanup");
+ }
+
+
+ /** Logic for the thread that restarts the sessions expired during foreground operations. */
+ private void runRestartThread() {
+ try {
+ while (true) {
+ TezSessionPoolSession next = restartQueue.take();
+ LOG.info("Restarting the expired session [" + next + "]");
+ try {
+ sessionRestartImpl.closeAndReopenPoolSession(next);
+ } catch (InterruptedException ie) {
+ throw ie;
+ } catch (Exception e) {
+ LOG.error("Failed to close or restart a session, ignoring", e);
+ }
+ }
+ } catch (InterruptedException e) {
+ LOG.info("Restart thread is exiting due to an interruption");
+ }
+ }
+
+ /** Logic for the thread that tracks session expiration and restarts sessions in background. */
+ private void runExpirationThread() {
+ try {
+ while (true) {
+ TezSessionPoolSession nextToExpire = null;
+ while (true) {
+ // Restart the sessions until one of them refuses to restart.
+ nextToExpire = expirationQueue.take();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Seeing if we can expire [" + nextToExpire + "]");
+ }
+ try {
+ if (!nextToExpire.tryExpire(false)) break;
+ } catch (Exception e) {
+ // Reopen happens even when close fails, so there's not much to do here.
+ LOG.error("Failed to expire session " + nextToExpire + "; ignoring", e);
+ nextToExpire = null;
+ break; // Not strictly necessary; do the whole queue check again.
+ }
+ LOG.info("Tez session [" + nextToExpire + "] has expired");
+ }
+ if (nextToExpire != null && LOG.isDebugEnabled()) {
+ LOG.debug("[" + nextToExpire + "] is not ready to expire; adding it back");
+ }
+
+ // See addToExpirationQueue for why we re-check the queue.
+ synchronized (expirationQueue) {
+ // Add back the non-expired session. No need to notify, we are the only ones waiting.
+ if (nextToExpire != null) {
+ expirationQueue.add(nextToExpire);
+ }
+ nextToExpire = expirationQueue.peek();
+ if (nextToExpire != null) {
+ // Add some margin to the wait to avoid rechecking close to the boundary.
+ long timeToWaitMs = (nextToExpire.getExpirationNs() - System.nanoTime()) / 1000000L;
+ timeToWaitMs = Math.max(1, timeToWaitMs + 10);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Waiting for ~" + timeToWaitMs + "ms to expire [" + nextToExpire + "]");
+ }
+ expirationQueue.wait(timeToWaitMs);
+ } else if (LOG.isDebugEnabled()) {
+ // Don't wait if empty - go to take() above, that will wait for us.
+ LOG.debug("Expiration queue is empty");
+ }
+ }
+ }
+ } catch (InterruptedException e) {
+ LOG.info("Expiration thread is exiting due to an interruption");
+ }
+ }
+
+
+ public void start() {
+ expirationThread.start();
+ restartThread.start();
+ }
+
+
+ public void stop() {
+ if (expirationThread != null) {
+ expirationThread.interrupt();
+ }
+ if (restartThread != null) {
+ restartThread.interrupt();
+ }
+ }
+
+
+ public void addToExpirationQueue(TezSessionPoolSession session) {
+ long jitterModMs = (long)(sessionLifetimeJitterMs * rdm.nextFloat());
+ session.setExpirationNs(System.nanoTime() + (sessionLifetimeMs + jitterModMs) * 1000000L);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding a pool session [" + this + "] to expiration queue");
+ }
+ // Expiration queue is synchronized and notified upon when adding elements. Without jitter, we
+ // wouldn't need this, and could simple look at the first element and sleep for the wait time.
+ // However, when many things are added at once, it may happen that we will see the one that
+ // expires later first, and will sleep past the earlier expiration times. When we wake up we
+ // may kill many sessions at once. To avoid this, we will add to queue under lock and recheck
+ // time before we wait. We don't have to worry about removals; at worst we'd wake up in vain.
+ // Example: expirations of 1:03:00, 1:00:00, 1:02:00 are added (in this order due to jitter).
+ // If the expiration threads sees that 1:03 first, it will sleep for 1:03, then wake up and
+ // kill all 3 sessions at once because they all have expired, removing any effect from jitter.
+ // Instead, expiration thread rechecks the first queue item and waits on the queue. If nothing
+ // is added to the queue, the item examined is still the earliest to be expired. If someone
+ // adds to the queue while it is waiting, it will notify the thread and it would wake up and
+ // recheck the queue.
+ synchronized (expirationQueue) {
+ expirationQueue.add(session);
+ expirationQueue.notifyAll();
+ }
+ }
+
+
+ public void removeFromExpirationQueue(TezSessionPoolSession session) {
+ expirationQueue.remove(session);
+ }
+
+ public void closeAndRestartExpiredSession(
+ TezSessionPoolSession session, boolean isAsync) throws Exception {
+ if (isAsync) {
+ restartQueue.add(session);
+ } else {
+ sessionRestartImpl.closeAndReopenPoolSession(session);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/733bc5f0/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPool.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPool.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPool.java
new file mode 100644
index 0000000..4f58565
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPool.java
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.exec.tez;
+
+import java.io.IOException;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Distinct from TezSessionPool manager in that it implements a session pool, and nothing else.
+ */
+class TezSessionPool {
+ private static final Logger LOG = LoggerFactory.getLogger(TezSessionPool.class);
+
+ /** A queue for initial sessions that have not been started yet. */
+ private final Queue<TezSessionPoolSession> initialSessions =
+ new ConcurrentLinkedQueue<TezSessionPoolSession>();
+
+ private final HiveConf initConf;
+ private final BlockingDeque<TezSessionPoolSession> defaultQueuePool;
+
+ TezSessionPool(HiveConf initConf, int numSessionsTotal) {
+ this.initConf = initConf;
+ assert numSessionsTotal > 0;
+ defaultQueuePool = new LinkedBlockingDeque<TezSessionPoolSession>(numSessionsTotal);
+ }
+
+ void startInitialSessions() throws Exception {
+ if (initialSessions.isEmpty()) return;
+ int threadCount = Math.min(initialSessions.size(),
+ HiveConf.getIntVar(initConf, ConfVars.HIVE_SERVER2_TEZ_SESSION_MAX_INIT_THREADS));
+ Preconditions.checkArgument(threadCount > 0);
+ if (threadCount == 1) {
+ while (true) {
+ TezSessionPoolSession session = initialSessions.poll();
+ if (session == null) break;
+ startInitialSession(session);
+ }
+ } else {
+ final SessionState parentSessionState = SessionState.get();
+ // The runnable has no mutable state, so each thread can run the same thing.
+ final AtomicReference<Exception> firstError = new AtomicReference<>(null);
+ Runnable runnable = new Runnable() {
+ public void run() {
+ if (parentSessionState != null) {
+ SessionState.setCurrentSessionState(parentSessionState);
+ }
+ while (true) {
+ TezSessionPoolSession session = initialSessions.poll();
+ if (session == null) break;
+ if (firstError.get() != null) break; // Best-effort.
+ try {
+ startInitialSession(session);
+ } catch (Exception e) {
+ if (!firstError.compareAndSet(null, e)) {
+ LOG.error("Failed to start session; ignoring due to previous error", e);
+ }
+ break;
+ }
+ }
+ }
+ };
+ Thread[] threads = new Thread[threadCount - 1];
+ for (int i = 0; i < threads.length; ++i) {
+ threads[i] = new Thread(runnable, "Tez session init " + i);
+ threads[i].start();
+ }
+ runnable.run();
+ for (int i = 0; i < threads.length; ++i) {
+ threads[i].join();
+ }
+ Exception ex = firstError.get();
+ if (ex != null) {
+ throw ex;
+ }
+ }
+ }
+
+ void addInitialSession(TezSessionPoolSession session) {
+ initialSessions.add(session);
+ }
+
+ TezSessionState getSession() throws Exception {
+ while (true) {
+ TezSessionPoolSession result = defaultQueuePool.take();
+ if (result.tryUse()) return result;
+ LOG.info("Couldn't use a session [" + result + "]; attempting another one");
+ }
+ }
+
+ void returnSession(TezSessionPoolSession session) throws Exception {
+ // TODO: should this be in pool, or pool manager? Probably common to all the use cases.
+ SessionState sessionState = SessionState.get();
+ if (sessionState != null) {
+ sessionState.setTezSession(null);
+ }
+ if (session.returnAfterUse()) {
+ defaultQueuePool.putFirst(session);
+ }
+ }
+
+ void replaceSession(
+ TezSessionPoolSession oldSession, TezSessionPoolSession newSession) throws Exception {
+ // Retain the stuff from the old session.
+ // Re-setting the queue config is an old hack that we may remove in future.
+ Path scratchDir = oldSession.getTezScratchDir();
+ Set<String> additionalFiles = oldSession.getAdditionalFilesNotFromConf();
+ HiveConf conf = oldSession.getConf();
+ String queueName = oldSession.getQueueName();
+ try {
+ oldSession.close(false);
+ boolean wasRemoved = defaultQueuePool.remove(oldSession);
+ if (!wasRemoved) {
+ LOG.error("Old session was closed but it was not in the pool", oldSession);
+ }
+ } finally {
+ // There's some bogus code that can modify the queue name. Force-set it for pool sessions.
+ // TODO: this might only be applicable to TezSessionPoolManager; try moving it there?
+ conf.set(TezConfiguration.TEZ_QUEUE_NAME, queueName);
+ newSession.open(conf, additionalFiles, scratchDir);
+ defaultQueuePool.put(newSession);
+ }
+ }
+
+ private void startInitialSession(TezSessionPoolSession sessionState) throws Exception {
+ HiveConf newConf = new HiveConf(initConf);
+ // Makes no senses for it to be mixed up like this.
+ boolean isUsable = sessionState.tryUse();
+ if (!isUsable) throw new IOException(sessionState + " is not usable at pool startup");
+ newConf.set(TezConfiguration.TEZ_QUEUE_NAME, sessionState.getQueueName());
+ sessionState.open(newConf);
+ if (sessionState.returnAfterUse()) {
+ defaultQueuePool.put(sessionState);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/733bc5f0/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
index 7c94002..1f4705c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
@@ -21,45 +21,23 @@ package org.apache.hadoop.hive.ql.exec.tez;
import org.apache.hadoop.conf.Configuration;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingDeque;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.LinkedBlockingDeque;
-import java.util.concurrent.PriorityBlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
+
import java.util.concurrent.Semaphore;
-import java.util.concurrent.TimeUnit;
import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
-import java.util.Queue;
import java.util.Random;
import java.util.Set;
-import javax.security.auth.login.LoginException;
-
import org.apache.tez.dag.api.TezConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolSession.OpenSessionTracker;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
import org.apache.hadoop.hive.shims.Utils;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.tez.dag.api.TezException;
/**
* This class is for managing multiple tez sessions particularly when
@@ -68,7 +46,8 @@ import org.apache.tez.dag.api.TezException;
* In case the user specifies a queue explicitly, a new session is created
* on that queue and assigned to the session state.
*/
-public class TezSessionPoolManager {
+public class TezSessionPoolManager
+ implements SessionExpirationTracker.RestartImpl, OpenSessionTracker {
private enum CustomQueueAllowed {
TRUE,
@@ -77,127 +56,55 @@ public class TezSessionPoolManager {
}
private static final Logger LOG = LoggerFactory.getLogger(TezSessionPoolManager.class);
- private static final Random rdm = new Random();
-
- private volatile SessionState initSessionState;
- private BlockingDeque<TezSessionPoolSession> defaultQueuePool;
-
- /** Priority queue sorted by expiration time of live sessions that could be expired. */
- private PriorityBlockingQueue<TezSessionPoolSession> expirationQueue;
- /** The background restart queue that is populated when expiration is triggered by a foreground
- * thread (i.e. getting or returning a session), to avoid delaying it. */
- private BlockingQueue<TezSessionPoolSession> restartQueue;
- private Thread expirationThread;
- private Thread restartThread;
+ static final Random rdm = new Random();
private Semaphore llapQueue;
private HiveConf initConf = null;
// Config settings.
private int numConcurrentLlapQueries = -1;
- private long sessionLifetimeMs = 0;
- private long sessionLifetimeJitterMs = 0;
private CustomQueueAllowed customQueueAllowed = CustomQueueAllowed.TRUE;
- private List<ConfVars> restrictedHiveConf = new ArrayList<>();
- private List<String> restrictedNonHiveConf = new ArrayList<>();
- /** A queue for initial sessions that have not been started yet. */
- private Queue<TezSessionPoolSession> initialSessions =
- new ConcurrentLinkedQueue<TezSessionPoolSession>();
+ private TezSessionPool defaultSessionPool;
+ private SessionExpirationTracker expirationTracker;
+ private RestrictedConfigChecker restrictedConfig;
+
/**
* Indicates whether we should try to use defaultSessionPool.
* We assume that setupPool is either called before any activity, or not called at all.
*/
private volatile boolean hasInitialSessions = false;
- private static TezSessionPoolManager sessionPool = null;
+ private static TezSessionPoolManager instance = null;
- private static final List<TezSessionPoolSession> openSessions
- = new LinkedList<TezSessionPoolSession>();
+ /** This is used to close non-default sessions, and also all sessions when stopping. */
+ private final List<TezSessionPoolSession> openSessions = new LinkedList<>();
- public static TezSessionPoolManager getInstance()
- throws Exception {
- if (sessionPool == null) {
- sessionPool = new TezSessionPoolManager();
+ /** Note: this is not thread-safe. */
+ public static TezSessionPoolManager getInstance() throws Exception {
+ TezSessionPoolManager local = instance;
+ if (local == null) {
+ instance = local = new TezSessionPoolManager();
}
- return sessionPool;
+ return local;
}
protected TezSessionPoolManager() {
}
- private void startInitialSession(TezSessionPoolSession sessionState) throws Exception {
- HiveConf newConf = new HiveConf(initConf); // TODO Why is this configuration management not happening inside TezSessionPool.
- // Makes no senses for it to be mixed up like this.
- boolean isUsable = sessionState.tryUse();
- if (!isUsable) throw new IOException(sessionState + " is not usable at pool startup");
- newConf.set(TezConfiguration.TEZ_QUEUE_NAME, sessionState.getQueueName());
- sessionState.open(newConf);
- if (sessionState.returnAfterUse()) {
- defaultQueuePool.put(sessionState);
- }
- }
-
public void startPool() throws Exception {
- if (initialSessions.isEmpty()) return;
- // Hive SessionState available at this point.
- initSessionState = SessionState.get();
- int threadCount = Math.min(initialSessions.size(),
- HiveConf.getIntVar(initConf, ConfVars.HIVE_SERVER2_TEZ_SESSION_MAX_INIT_THREADS));
- Preconditions.checkArgument(threadCount > 0);
- if (threadCount == 1) {
- while (true) {
- TezSessionPoolSession session = initialSessions.poll();
- if (session == null) break;
- startInitialSession(session);
- }
- } else {
- // TODO What is this doing now ?
- final SessionState parentSessionState = SessionState.get();
- // The runnable has no mutable state, so each thread can run the same thing.
- final AtomicReference<Exception> firstError = new AtomicReference<>(null);
- Runnable runnable = new Runnable() {
- public void run() {
- if (parentSessionState != null) {
- SessionState.setCurrentSessionState(parentSessionState);
- }
- while (true) {
- TezSessionPoolSession session = initialSessions.poll();
- if (session == null) break;
- try {
- startInitialSession(session);
- } catch (Exception e) {
- if (!firstError.compareAndSet(null, e)) {
- LOG.error("Failed to start session; ignoring due to previous error", e);
- // TODO Why even continue after this. We're already in a state where things are messed up ?
- }
- }
- }
- }
- };
- Thread[] threads = new Thread[threadCount - 1];
- for (int i = 0; i < threads.length; ++i) {
- threads[i] = new Thread(runnable, "Tez session init " + i);
- threads[i].start();
- }
- runnable.run();
- for (int i = 0; i < threads.length; ++i) {
- threads[i].join();
- }
- Exception ex = firstError.get();
- if (ex != null) {
- throw ex;
- }
+ if (defaultSessionPool != null) {
+ defaultSessionPool.startInitialSessions();
}
- if (expirationThread != null) {
- expirationThread.start();
- restartThread.start();
+ if (expirationTracker != null) {
+ expirationTracker.start();
}
}
public void setupPool(HiveConf conf) throws InterruptedException {
String[] defaultQueueList = HiveConf.getTrimmedStringsVar(
conf, HiveConf.ConfVars.HIVE_SERVER2_TEZ_DEFAULT_QUEUES);
+ this.initConf = conf;
int emptyNames = 0; // We don't create sessions for empty entries.
for (String queueName : defaultQueueList) {
if (queueName.isEmpty()) {
@@ -207,13 +114,12 @@ public class TezSessionPoolManager {
int numSessions = conf.getIntVar(ConfVars.HIVE_SERVER2_TEZ_SESSIONS_PER_DEFAULT_QUEUE);
int numSessionsTotal = numSessions * (defaultQueueList.length - emptyNames);
if (numSessionsTotal > 0) {
- defaultQueuePool = new LinkedBlockingDeque<TezSessionPoolSession>(numSessionsTotal);
+ defaultSessionPool = new TezSessionPool(initConf, numSessionsTotal);
}
numConcurrentLlapQueries = conf.getIntVar(ConfVars.HIVE_SERVER2_LLAP_CONCURRENT_QUERIES);
llapQueue = new Semaphore(numConcurrentLlapQueries, true);
- this.initConf = conf;
String queueAllowedStr = HiveConf.getVar(initConf,
ConfVars.HIVE_SERVER2_TEZ_SESSION_CUSTOM_QUEUE_ALLOWED);
try {
@@ -222,73 +128,18 @@ public class TezSessionPoolManager {
throw new RuntimeException("Invalid value '" + queueAllowedStr + "' for " +
ConfVars.HIVE_SERVER2_TEZ_SESSION_CUSTOM_QUEUE_ALLOWED.varname);
}
- String[] restrictedConfigs = HiveConf.getTrimmedStringsVar(initConf,
- ConfVars.HIVE_SERVER2_TEZ_SESSION_RESTRICTED_CONFIGS);
- if (restrictedConfigs != null && restrictedConfigs.length > 0) {
- HashMap<String, ConfVars> confVars = HiveConf.getOrCreateReverseMap();
- for (String confName : restrictedConfigs) {
- if (confName == null || confName.isEmpty()) continue;
- confName = confName.toLowerCase();
- ConfVars cv = confVars.get(confName);
- if (cv != null) {
- restrictedHiveConf.add(cv);
- } else {
- LOG.warn("A restricted config " + confName + " is not recognized as a Hive setting.");
- restrictedNonHiveConf.add(confName);
- }
- }
- }
- sessionLifetimeMs = conf.getTimeVar(
- ConfVars.HIVE_SERVER2_TEZ_SESSION_LIFETIME, TimeUnit.MILLISECONDS);
- if (sessionLifetimeMs != 0) {
- sessionLifetimeJitterMs = conf.getTimeVar(
- ConfVars.HIVE_SERVER2_TEZ_SESSION_LIFETIME_JITTER, TimeUnit.MILLISECONDS);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Session expiration is enabled; session lifetime is "
- + sessionLifetimeMs + " + [0, " + sessionLifetimeJitterMs + ") ms");
- }
- expirationQueue = new PriorityBlockingQueue<>(11, new Comparator<TezSessionPoolSession>() {
- @Override
- public int compare(TezSessionPoolSession o1, TezSessionPoolSession o2) {
- assert o1.expirationNs != null && o2.expirationNs != null;
- return o1.expirationNs.compareTo(o2.expirationNs);
- }
- });
- restartQueue = new LinkedBlockingQueue<>();
- }
- this.hasInitialSessions = numSessionsTotal > 0;
- // From this point on, session creation will wait for the default pool (if # of sessions > 0).
+ restrictedConfig = new RestrictedConfigChecker(conf);
+ // Only creates the expiration tracker if expiration is configured.
+ expirationTracker = SessionExpirationTracker.create(conf, this);
- if (sessionLifetimeMs != 0) {
- expirationThread = new Thread(new Runnable() {
- @Override
- public void run() {
- try {
- SessionState.setCurrentSessionState(initSessionState);
- runExpirationThread();
- } catch (Exception e) {
- LOG.warn("Exception in TezSessionPool-expiration thread. Thread will shut down", e);
- } finally {
- LOG.info("TezSessionPool-expiration thread exiting");
- }
- }
- }, "TezSessionPool-expiration");
- restartThread = new Thread(new Runnable() {
- @Override
- public void run() {
- try {
- SessionState.setCurrentSessionState(initSessionState);
- runRestartThread();
- } catch (Exception e) {
- LOG.warn("Exception in TezSessionPool-cleanup thread. Thread will shut down", e);
- } finally {
- LOG.info("TezSessionPool-cleanup thread exiting");
- }
- }
- }, "TezSessionPool-cleanup");
+ // From this point on, session creation will wait for the default pool (if # of sessions > 0).
+ this.hasInitialSessions = numSessionsTotal > 0;
+ if (!hasInitialSessions) {
+ return;
}
+
/*
* In a single-threaded init case, with this the ordering of sessions in the queue will be
* (with 2 sessions 3 queues) s1q1, s1q2, s1q3, s2q1, s2q2, s2q3 there by ensuring uniform
@@ -301,7 +152,7 @@ public class TezSessionPoolManager {
if (queueName.isEmpty()) {
continue;
}
- initialSessions.add(createAndInitSession(queueName, true));
+ defaultSessionPool.addInitialSession(createAndInitSession(queueName, true));
}
}
}
@@ -322,8 +173,10 @@ public class TezSessionPoolManager {
return sessionState;
}
- private TezSessionState getSession(HiveConf conf, boolean doOpen)
- throws Exception {
+ private TezSessionState getSession(HiveConf conf, boolean doOpen) throws Exception {
+ // NOTE: this can be called outside of HS2, without calling setupPool. Basically it should be
+ // able to handle not being initialized. Perhaps we should get rid of the instance and
+ // move the setupPool code to ctor. For now, at least hasInitialSessions will be false.
String queueName = conf.get(TezConfiguration.TEZ_QUEUE_NAME);
boolean hasQueue = (queueName != null) && !queueName.isEmpty();
if (hasQueue) {
@@ -340,16 +193,8 @@ public class TezSessionPoolManager {
}
// Check the restricted configs that the users cannot set.
- for (ConfVars var : restrictedHiveConf) {
- String userValue = HiveConf.getVarWithoutType(conf, var),
- serverValue = HiveConf.getVarWithoutType(initConf, var);
- // Note: with some trickery, we could add logic for each type in ConfVars; for now the
- // potential spurious mismatches (e.g. 0 and 0.0 for float) should be easy to work around.
- validateRestrictedConfigValues(var.varname, userValue, serverValue);
- }
- for (String var : restrictedNonHiveConf) {
- String userValue = conf.get(var), serverValue = initConf.get(var);
- validateRestrictedConfigValues(var, userValue, serverValue);
+ if (restrictedConfig != null) {
+ restrictedConfig.validate(conf);
}
// Propagate this value from HS2; don't allow users to set it.
@@ -371,26 +216,12 @@ public class TezSessionPoolManager {
*/
if (nonDefaultUser || !hasInitialSessions || hasQueue) {
LOG.info("QueueName: {} nonDefaultUser: {} defaultQueuePool: {} hasInitialSessions: {}",
- queueName, nonDefaultUser, defaultQueuePool, hasInitialSessions);
+ queueName, nonDefaultUser, defaultSessionPool, hasInitialSessions);
return getNewSessionState(conf, queueName, doOpen);
}
LOG.info("Choosing a session from the defaultQueuePool");
- while (true) {
- TezSessionPoolSession result = defaultQueuePool.take();
- if (result.tryUse()) return result;
- LOG.info("Couldn't use a session [" + result + "]; attempting another one");
- }
- }
-
- private void validateRestrictedConfigValues(
- String var, String userValue, String serverValue) throws HiveException {
- if ((userValue == null) != (serverValue == null)
- || (userValue != null && !userValue.equals(serverValue))) {
- String logValue = initConf.isHiddenConfig(var) ? "(hidden)" : serverValue;
- throw new HiveException(var + " is restricted from being set; server is configured"
- + " to use " + logValue + ", but the query configuration specifies " + userValue);
- }
+ return defaultSessionPool.getSession();
}
/**
@@ -431,15 +262,7 @@ public class TezSessionPoolManager {
tezSessionState instanceof TezSessionPoolSession) {
LOG.info("The session " + tezSessionState.getSessionId()
+ " belongs to the pool. Put it back in");
- SessionState sessionState = SessionState.get();
- if (sessionState != null) {
- sessionState.setTezSession(null);
- }
- TezSessionPoolSession poolSession =
- (TezSessionPoolSession) tezSessionState;
- if (poolSession.returnAfterUse()) {
- defaultQueuePool.putFirst(poolSession);
- }
+ defaultSessionPool.returnSession((TezSessionPoolSession)tezSessionState);
}
// non default session nothing changes. The user can continue to use the existing
// session in the SessionState
@@ -460,7 +283,7 @@ public class TezSessionPoolManager {
}
public void stop() throws Exception {
- if ((sessionPool == null) || !this.hasInitialSessions) {
+ if ((instance == null) || !this.hasInitialSessions) {
return;
}
@@ -474,12 +297,8 @@ public class TezSessionPoolManager {
sessionState.close(false);
}
}
-
- if (expirationThread != null) {
- expirationThread.interrupt();
- }
- if (restartThread != null) {
- restartThread.interrupt();
+ if (expirationTracker != null) {
+ expirationTracker.stop();
}
}
@@ -498,7 +317,7 @@ public class TezSessionPoolManager {
}
protected TezSessionPoolSession createSession(String sessionId) {
- return new TezSessionPoolSession(sessionId, this);
+ return new TezSessionPoolSession(sessionId, this, expirationTracker);
}
/*
@@ -597,242 +416,37 @@ public class TezSessionPoolManager {
}
/** Closes a running (expired) pool session and reopens it. */
- private void closeAndReopenPoolSession(TezSessionPoolSession oldSession) throws Exception {
+ @Override
+ public void closeAndReopenPoolSession(TezSessionPoolSession oldSession) throws Exception {
String queueName = oldSession.getQueueName();
if (queueName == null) {
LOG.warn("Pool session has a null queue: " + oldSession);
}
- HiveConf conf = oldSession.getConf();
- Path scratchDir = oldSession.getTezScratchDir();
- boolean isDefault = oldSession.isDefault();
- Set<String> additionalFiles = oldSession.getAdditionalFilesNotFromConf();
- try {
- oldSession.close(false);
- defaultQueuePool.remove(oldSession); // Make sure it's removed.
- } finally {
- TezSessionPoolSession newSession = createAndInitSession(queueName, isDefault);
- // There's some bogus code that can modify the queue name. Force-set it for pool sessions.
- conf.set(TezConfiguration.TEZ_QUEUE_NAME, queueName);
- newSession.open(conf, additionalFiles, scratchDir);
- defaultQueuePool.put(newSession);
- }
- }
-
- /** Logic for the thread that restarts the sessions expired during foreground operations. */
- private void runRestartThread() {
- try {
- while (true) {
- TezSessionPoolSession next = restartQueue.take();
- LOG.info("Restarting the expired session [" + next + "]");
- try {
- closeAndReopenPoolSession(next);
- } catch (InterruptedException ie) {
- throw ie;
- } catch (Exception e) {
- LOG.error("Failed to close or restart a session, ignoring", e);
- }
- }
- } catch (InterruptedException e) {
- LOG.info("Restart thread is exiting due to an interruption");
- }
+ TezSessionPoolSession newSession = createAndInitSession(queueName, oldSession.isDefault());
+ defaultSessionPool.replaceSession(oldSession, newSession);
}
- /** Logic for the thread that tracks session expiration and restarts sessions in background. */
- private void runExpirationThread() {
- try {
- while (true) {
- TezSessionPoolSession nextToExpire = null;
- while (true) {
- // Restart the sessions until one of them refuses to restart.
- nextToExpire = expirationQueue.take();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Seeing if we can expire [" + nextToExpire + "]");
- }
- try {
- if (!nextToExpire.tryExpire(false)) break;
- } catch (Exception e) {
- // Reopen happens even when close fails, so there's not much to do here.
- LOG.error("Failed to expire session " + nextToExpire + "; ignoring", e);
- nextToExpire = null;
- break; // Not strictly necessary; do the whole queue check again.
- }
- LOG.info("Tez session [" + nextToExpire + "] has expired");
- }
- if (nextToExpire != null && LOG.isDebugEnabled()) {
- LOG.debug("[" + nextToExpire + "] is not ready to expire; adding it back");
- }
-
- // See addToExpirationQueue for why we re-check the queue.
- synchronized (expirationQueue) {
- // Add back the non-expired session. No need to notify, we are the only ones waiting.
- if (nextToExpire != null) {
- expirationQueue.add(nextToExpire);
- }
- nextToExpire = expirationQueue.peek();
- if (nextToExpire != null) {
- // Add some margin to the wait to avoid rechecking close to the boundary.
- long timeToWaitMs = 10 + (nextToExpire.expirationNs - System.nanoTime()) / 1000000L;
- timeToWaitMs = Math.max(1, timeToWaitMs);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Waiting for ~" + timeToWaitMs + "ms to expire [" + nextToExpire + "]");
- }
- expirationQueue.wait(timeToWaitMs);
- } else if (LOG.isDebugEnabled()) {
- // Don't wait if empty - go to take() above, that will wait for us.
- LOG.debug("Expiration queue is empty");
- }
- }
- }
- } catch (InterruptedException e) {
- LOG.info("Expiration thread is exiting due to an interruption");
+ /** Called by TezSessionPoolSession when opened. */
+ @Override
+ public void registerOpenSession(TezSessionPoolSession session) {
+ synchronized (openSessions) {
+ openSessions.add(session);
}
}
- /**
- * TezSession that keeps track of expiration and use.
- * It has 3 states - not in use, in use, and expired. When in the pool, it is not in use;
- * use and expiration may compete to take the session out of the pool and change it to the
- * corresponding states. When someone tries to get a session, they check for expiration time;
- * if it's time, the expiration is triggered; in that case, or if it was already triggered, the
- * caller gets a different session. When the session is in use when it expires, the expiration
- * thread ignores it and lets the return to the pool take care of the expiration.
- */
- @VisibleForTesting
- static class TezSessionPoolSession extends TezSessionState {
- private static final int STATE_NONE = 0, STATE_IN_USE = 1, STATE_EXPIRED = 2;
-
- private final AtomicInteger sessionState = new AtomicInteger(STATE_NONE);
- private Long expirationNs;
- private final TezSessionPoolManager parent; // Static class allows us to be used in tests.
-
- public TezSessionPoolSession(String sessionId, TezSessionPoolManager parent) {
- super(sessionId);
- this.parent = parent;
- }
-
- @Override
- public void close(boolean keepTmpDir) throws Exception {
- try {
- super.close(keepTmpDir);
- } finally {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Closed a pool session [" + this + "]");
- }
- synchronized (openSessions) {
- openSessions.remove(this);
- }
- if (parent.expirationQueue != null) {
- parent.expirationQueue.remove(this);
- }
- }
- }
-
- @Override
- protected void openInternal(HiveConf conf, Collection<String> additionalFiles,
- boolean isAsync, LogHelper console, Path scratchDir)
- throws IOException, LoginException, URISyntaxException, TezException {
- super.openInternal(conf, additionalFiles, isAsync, console, scratchDir);
- synchronized (openSessions) {
- openSessions.add(this);
- }
- if (parent.expirationQueue != null) {
- long jitterModMs = (long)(parent.sessionLifetimeJitterMs * rdm.nextFloat());
- expirationNs = System.nanoTime() + (parent.sessionLifetimeMs + jitterModMs) * 1000000L;
- if (LOG.isDebugEnabled()) {
- LOG.debug("Adding a pool session [" + this + "] to expiration queue");
- }
- parent.addToExpirationQueue(this);
- }
+ /** Called by TezSessionPoolSession when closed. */
+ @Override
+ public void unregisterOpenSession(TezSessionPoolSession session) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Closed a pool session [" + this + "]");
}
-
- @Override
- public String toString() {
- if (expirationNs == null) return super.toString();
- long expiresInMs = (expirationNs - System.nanoTime()) / 1000000L;
- return super.toString() + ", expires in " + expiresInMs + "ms";
- }
-
- /**
- * Tries to use this session. When the session is in use, it will not expire.
- * @return true if the session can be used; false if it has already expired.
- */
- public boolean tryUse() throws Exception {
- while (true) {
- int oldValue = sessionState.get();
- if (oldValue == STATE_IN_USE) throw new AssertionError(this + " is already in use");
- if (oldValue == STATE_EXPIRED) return false;
- int finalState = shouldExpire() ? STATE_EXPIRED : STATE_IN_USE;
- if (sessionState.compareAndSet(STATE_NONE, finalState)) {
- if (finalState == STATE_IN_USE) return true;
- closeAndRestartExpiredSession(true); // Restart asynchronously, don't block the caller.
- return false;
- }
- }
- }
-
- /**
- * Notifies the session that it's no longer in use. If the session has expired while in use,
- * this method will take care of the expiration.
- * @return True if the session was returned, false if it was restarted.
- */
- public boolean returnAfterUse() throws Exception {
- int finalState = shouldExpire() ? STATE_EXPIRED : STATE_NONE;
- if (!sessionState.compareAndSet(STATE_IN_USE, finalState)) {
- throw new AssertionError("Unexpected state change; currently " + sessionState.get());
- }
- if (finalState == STATE_NONE) return true;
- closeAndRestartExpiredSession(true);
- return false;
- }
-
- /**
- * Tries to expire and restart the session.
- * @param isAsync Whether the restart should happen asynchronously.
- * @return True if the session was, or will be restarted.
- */
- public boolean tryExpire(boolean isAsync) throws Exception {
- if (expirationNs == null) return true;
- if (!shouldExpire()) return false;
- // Try to expire the session if it's not in use; if in use, bail.
- while (true) {
- if (sessionState.get() != STATE_NONE) return true; // returnAfterUse will take care of this
- if (sessionState.compareAndSet(STATE_NONE, STATE_EXPIRED)) {
- closeAndRestartExpiredSession(isAsync);
- return true;
- }
- }
- }
-
- private void closeAndRestartExpiredSession(boolean async) throws Exception {
- if (async) {
- parent.restartQueue.add(this);
- } else {
- parent.closeAndReopenPoolSession(this);
- }
- }
-
- private boolean shouldExpire() {
- return expirationNs != null && (System.nanoTime() - expirationNs) >= 0;
+ synchronized (openSessions) {
+ openSessions.remove(session);
}
}
- private void addToExpirationQueue(TezSessionPoolSession session) {
- // Expiration queue is synchronized and notified upon when adding elements. Without jitter, we
- // wouldn't need this, and could simple look at the first element and sleep for the wait time.
- // However, when many things are added at once, it may happen that we will see the one that
- // expires later first, and will sleep past the earlier expiration times. When we wake up we
- // may kill many sessions at once. To avoid this, we will add to queue under lock and recheck
- // time before we wait. We don't have to worry about removals; at worst we'd wake up in vain.
- // Example: expirations of 1:03:00, 1:00:00, 1:02:00 are added (in this order due to jitter).
- // If the expiration threads sees that 1:03 first, it will sleep for 1:03, then wake up and
- // kill all 3 sessions at once because they all have expired, removing any effect from jitter.
- // Instead, expiration thread rechecks the first queue item and waits on the queue. If nothing
- // is added to the queue, the item examined is still the earliest to be expired. If someone
- // adds to the queue while it is waiting, it will notify the thread and it would wake up and
- // recheck the queue.
- synchronized (expirationQueue) {
- expirationQueue.add(session);
- expirationQueue.notifyAll();
- }
+ @VisibleForTesting
+ public SessionExpirationTracker getExpirationTracker() {
+ return expirationTracker;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/733bc5f0/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolSession.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolSession.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolSession.java
new file mode 100644
index 0000000..005eeed
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolSession.java
@@ -0,0 +1,158 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.tez;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.Collection;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.security.auth.login.LoginException;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
+import org.apache.tez.dag.api.TezException;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * TezSession that is aware of the session pool, and also keeps track of expiration and use.
+ * It has 3 states - not in use, in use, and expired. When in the pool, it is not in use;
+ * use and expiration may compete to take the session out of the pool and change it to the
+ * corresponding states. When someone tries to get a session, they check for expiration time;
+ * if it's time, the expiration is triggered; in that case, or if it was already triggered, the
+ * caller gets a different session. When the session is in use when it expires, the expiration
+ * thread ignores it and lets the return to the pool take care of the expiration.
+ */
+@VisibleForTesting
+class TezSessionPoolSession extends TezSessionState {
+ private static final int STATE_NONE = 0, STATE_IN_USE = 1, STATE_EXPIRED = 2;
+
+ interface OpenSessionTracker {
+ void registerOpenSession(TezSessionPoolSession session);
+ void unregisterOpenSession(TezSessionPoolSession session);
+ }
+
+ private final AtomicInteger sessionState = new AtomicInteger(STATE_NONE);
+ private Long expirationNs;
+ private final OpenSessionTracker parent;
+ private final SessionExpirationTracker expirationTracker;
+
+ public TezSessionPoolSession(String sessionId, OpenSessionTracker parent,
+ SessionExpirationTracker expirationTracker) {
+ super(sessionId);
+ this.parent = parent;
+ this.expirationTracker = expirationTracker;
+ }
+
+ void setExpirationNs(long expirationNs) {
+ this.expirationNs = expirationNs;
+ }
+
+ Long getExpirationNs() {
+ return expirationNs;
+ }
+
+ @Override
+ public void close(boolean keepTmpDir) throws Exception {
+ try {
+ super.close(keepTmpDir);
+ } finally {
+ parent.unregisterOpenSession(this);
+ if (expirationTracker != null) {
+ expirationTracker.removeFromExpirationQueue(this);
+ }
+ }
+ }
+
+ @Override
+ protected void openInternal(HiveConf conf, Collection<String> additionalFiles,
+ boolean isAsync, LogHelper console, Path scratchDir)
+ throws IOException, LoginException, URISyntaxException, TezException {
+ super.openInternal(conf, additionalFiles, isAsync, console, scratchDir);
+ parent.registerOpenSession(this);
+ if (expirationTracker != null) {
+ expirationTracker.addToExpirationQueue(this);
+ }
+ }
+
+ @Override
+ public String toString() {
+ if (expirationNs == null) return super.toString();
+ long expiresInMs = (expirationNs - System.nanoTime()) / 1000000L;
+ return super.toString() + ", expires in " + expiresInMs + "ms";
+ }
+
+ /**
+ * Tries to use this session. When the session is in use, it will not expire.
+ * @return true if the session can be used; false if it has already expired.
+ */
+ public boolean tryUse() throws Exception {
+ while (true) {
+ int oldValue = sessionState.get();
+ if (oldValue == STATE_IN_USE) throw new AssertionError(this + " is already in use");
+ if (oldValue == STATE_EXPIRED) return false;
+ int finalState = shouldExpire() ? STATE_EXPIRED : STATE_IN_USE;
+ if (sessionState.compareAndSet(STATE_NONE, finalState)) {
+ if (finalState == STATE_IN_USE) return true;
+ // Restart asynchronously, don't block the caller.
+ expirationTracker.closeAndRestartExpiredSession(this, true);
+ return false;
+ }
+ }
+ }
+
+ /**
+ * Notifies the session that it's no longer in use. If the session has expired while in use,
+ * this method will take care of the expiration.
+ * @return True if the session was returned, false if it was restarted.
+ */
+ public boolean returnAfterUse() throws Exception {
+ int finalState = shouldExpire() ? STATE_EXPIRED : STATE_NONE;
+ if (!sessionState.compareAndSet(STATE_IN_USE, finalState)) {
+ throw new AssertionError("Unexpected state change; currently " + sessionState.get());
+ }
+ if (finalState == STATE_NONE) return true;
+ expirationTracker.closeAndRestartExpiredSession(this, true);
+ return false;
+ }
+
+ /**
+ * Tries to expire and restart the session.
+ * @param isAsync Whether the restart should happen asynchronously.
+ * @return True if the session was, or will be restarted.
+ */
+ public boolean tryExpire(boolean isAsync) throws Exception {
+ if (expirationNs == null) return true;
+ if (!shouldExpire()) return false;
+ // Try to expire the session if it's not in use; if in use, bail.
+ while (true) {
+ if (sessionState.get() != STATE_NONE) return true; // returnAfterUse will take care of this
+ if (sessionState.compareAndSet(STATE_NONE, STATE_EXPIRED)) {
+ expirationTracker.closeAndRestartExpiredSession(this, isAsync);
+ return true;
+ }
+ }
+ }
+
+ private boolean shouldExpire() {
+ return expirationNs != null && (System.nanoTime() - expirationNs) >= 0;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/733bc5f0/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/SampleTezSessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/SampleTezSessionState.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/SampleTezSessionState.java
index 2d1c687..973c0cc 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/SampleTezSessionState.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/SampleTezSessionState.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.ql.exec.tez;
-import org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager.TezSessionPoolSession;
import java.io.IOException;
import java.net.URISyntaxException;
@@ -45,7 +44,7 @@ public class SampleTezSessionState extends TezSessionPoolSession {
private boolean doAsEnabled;
public SampleTezSessionState(String sessionId, TezSessionPoolManager parent) {
- super(sessionId, parent);
+ super(sessionId, parent, parent.getExpirationTracker());
this.sessionId = sessionId;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/733bc5f0/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java
index 88c8122..d2b98c4 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java
@@ -49,7 +49,7 @@ public class TestTezSessionPool {
}
@Override
- public TezSessionPoolManager.TezSessionPoolSession createSession(String sessionId) {
+ public TezSessionPoolSession createSession(String sessionId) {
return new SampleTezSessionState(sessionId, this);
}
}