You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by we...@apache.org on 2017/05/16 22:53:23 UTC
[18/18] hive git commit: HIVE-14879 : integrate MM tables into ACID:
replace MM metastore calls and structures with ACID ones (Wei Zheng)
HIVE-14879 : integrate MM tables into ACID: replace MM metastore calls and structures with ACID ones (Wei Zheng)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/77511070
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/77511070
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/77511070
Branch: refs/heads/hive-14535
Commit: 77511070dd8b7176e98454b9a7010eedfbd5d981
Parents: 1ceaf35
Author: Wei Zheng <we...@apache.org>
Authored: Tue May 16 15:52:03 2017 -0700
Committer: Wei Zheng <we...@apache.org>
Committed: Tue May 16 15:52:03 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/common/JavaUtils.java | 67 +
.../hadoop/hive/common/ValidWriteIds.java | 218 -
.../org/apache/hadoop/hive/conf/HiveConf.java | 23 +-
.../listener/DummyRawStoreFailEvent.java | 48 -
.../hadoop/hive/ql/history/TestHiveHistory.java | 2 +-
metastore/if/hive_metastore.thrift | 44 -
.../upgrade/derby/038-HIVE-14637.derby.sql | 6 -
.../upgrade/derby/hive-schema-2.2.0.derby.sql | 11 +-
.../derby/upgrade-2.1.0-to-2.2.0.derby.sql | 2 -
.../upgrade/mssql/023-HIVE-14637.mssql.sql | 15 -
.../upgrade/mssql/hive-schema-2.2.0.mssql.sql | 21 +-
.../mssql/upgrade-2.1.0-to-2.2.0.mssql.sql | 1 -
.../upgrade/mysql/038-HIVE-14637.mysql.sql | 15 -
.../upgrade/mysql/hive-schema-2.2.0.mysql.sql | 16 -
.../mysql/upgrade-2.1.0-to-2.2.0.mysql.sql | 1 -
.../upgrade/oracle/038-HIVE-14637.oracle.sql | 15 -
.../upgrade/oracle/hive-schema-2.2.0.oracle.sql | 20 +-
.../oracle/upgrade-2.1.0-to-2.2.0.oracle.sql | 1 -
.../postgres/037-HIVE-14637.postgres.sql | 16 -
.../postgres/hive-schema-2.2.0.postgres.sql | 22 +-
.../upgrade-2.1.0-to-2.2.0.postgres.sql | 1 -
.../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 6078 +++++--------
.../gen/thrift/gen-cpp/ThriftHiveMetastore.h | 504 -
.../ThriftHiveMetastore_server.skeleton.cpp | 20 -
.../gen/thrift/gen-cpp/hive_metastore_types.cpp | 1645 +---
.../gen/thrift/gen-cpp/hive_metastore_types.h | 407 +-
.../hive/metastore/api/ClientCapabilities.java | 32 +-
.../metastore/api/FinalizeWriteIdRequest.java | 684 --
.../metastore/api/FinalizeWriteIdResult.java | 283 -
.../metastore/api/GetAllFunctionsResponse.java | 36 +-
.../metastore/api/GetNextWriteIdRequest.java | 490 -
.../metastore/api/GetNextWriteIdResult.java | 387 -
.../hive/metastore/api/GetTablesRequest.java | 32 +-
.../hive/metastore/api/GetTablesResult.java | 36 +-
.../metastore/api/GetValidWriteIdsRequest.java | 490 -
.../metastore/api/GetValidWriteIdsResult.java | 740 --
.../metastore/api/HeartbeatWriteIdRequest.java | 589 --
.../metastore/api/HeartbeatWriteIdResult.java | 283 -
.../apache/hadoop/hive/metastore/api/Table.java | 206 +-
.../hive/metastore/api/ThriftHiveMetastore.java | 8610 ++++++------------
.../gen-php/metastore/ThriftHiveMetastore.php | 2132 ++---
.../src/gen/thrift/gen-php/metastore/Types.php | 1216 +--
.../hive_metastore/ThriftHiveMetastore-remote | 28 -
.../hive_metastore/ThriftHiveMetastore.py | 2502 ++---
.../gen/thrift/gen-py/hive_metastore/ttypes.py | 918 +-
.../gen/thrift/gen-rb/hive_metastore_types.rb | 166 +-
.../gen/thrift/gen-rb/thrift_hive_metastore.rb | 216 -
.../hadoop/hive/metastore/HiveMetaStore.java | 228 +-
.../hive/metastore/HiveMetaStoreClient.java | 23 -
.../hadoop/hive/metastore/IMetaStoreClient.java | 10 -
.../hadoop/hive/metastore/MmCleanerThread.java | 397 -
.../hadoop/hive/metastore/ObjectStore.java | 283 +-
.../apache/hadoop/hive/metastore/RawStore.java | 43 -
.../hive/metastore/cache/CachedStore.java | 46 -
.../hadoop/hive/metastore/hbase/HBaseStore.java | 74 +-
.../hadoop/hive/metastore/model/MTable.java | 23 +-
.../hive/metastore/model/MTableWrite.java | 77 -
metastore/src/model/package.jdo | 33 -
.../DummyRawStoreControlledCommit.java | 51 -
.../DummyRawStoreForJdoConnection.java | 50 -
.../hadoop/hive/metastore/TestObjectStore.java | 153 -
.../java/org/apache/hadoop/hive/ql/Driver.java | 55 -
.../hive/ql/exec/AbstractFileMergeOperator.java | 9 +-
.../apache/hadoop/hive/ql/exec/CopyTask.java | 7 +-
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 58 +-
.../hadoop/hive/ql/exec/FetchOperator.java | 27 +-
.../apache/hadoop/hive/ql/exec/FetchTask.java | 5 -
.../hadoop/hive/ql/exec/FileSinkOperator.java | 32 +-
.../hadoop/hive/ql/exec/ImportCommitTask.java | 7 +-
.../hadoop/hive/ql/exec/ImportCommitWork.java | 16 +-
.../apache/hadoop/hive/ql/exec/MoveTask.java | 29 +-
.../apache/hadoop/hive/ql/exec/Utilities.java | 115 +-
.../org/apache/hadoop/hive/ql/io/AcidUtils.java | 3 +-
.../hadoop/hive/ql/io/HiveInputFormat.java | 58 +-
.../apache/hadoop/hive/ql/metadata/Hive.java | 107 +-
.../apache/hadoop/hive/ql/metadata/Table.java | 4 -
.../hive/ql/optimizer/GenMapRedUtils.java | 4 +-
.../ql/optimizer/physical/SkewJoinResolver.java | 2 +-
.../hive/ql/parse/BaseSemanticAnalyzer.java | 3 +-
.../hive/ql/parse/ExportSemanticAnalyzer.java | 25 +-
.../hive/ql/parse/ImportSemanticAnalyzer.java | 89 +-
.../hadoop/hive/ql/parse/IndexUpdater.java | 29 +-
.../hive/ql/parse/LoadSemanticAnalyzer.java | 15 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 68 +-
.../hadoop/hive/ql/parse/TaskCompiler.java | 18 +-
.../hadoop/hive/ql/plan/FileMergeDesc.java | 19 +-
.../hadoop/hive/ql/plan/FileSinkDesc.java | 4 +-
.../hadoop/hive/ql/plan/LoadMultiFilesDesc.java | 8 +
.../hadoop/hive/ql/plan/LoadTableDesc.java | 49 +-
.../hadoop/hive/ql/exec/TestExecDriver.java | 2 +-
ql/src/test/queries/clientpositive/mm_all.q | 205 -
.../queries/clientpositive/mm_conversions.q | 87 +-
ql/src/test/queries/clientpositive/mm_exim.q | 98 +
.../queries/clientpositive/mm_insertonly_acid.q | 16 -
.../results/clientpositive/llap/mm_all.q.out | 1982 +---
.../clientpositive/llap/mm_conversions.q.out | 797 +-
ql/src/test/results/clientpositive/mm_all.q.out | 1570 +---
.../results/clientpositive/mm_conversions.q.out | 586 +-
.../test/results/clientpositive/mm_exim.q.out | 457 +
.../clientpositive/mm_insertonly_acid.q.out | 115 -
100 files changed, 8686 insertions(+), 28880 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/77511070/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java b/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
index 3916fe3..28490e2 100644
--- a/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
+++ b/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
@@ -28,6 +28,8 @@ import java.net.URLClassLoader;
import java.util.Arrays;
import java.util.List;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,6 +39,10 @@ import org.slf4j.LoggerFactory;
*/
public final class JavaUtils {
+ public static final String DELTA_PREFIX = "delta";
+ public static final String DELTA_DIGITS = "%07d";
+ public static final int DELTA_DIGITS_LEN = 7;
+ public static final String STATEMENT_DIGITS = "%04d";
private static final Logger LOG = LoggerFactory.getLogger(JavaUtils.class);
private static final Method SUN_MISC_UTIL_RELEASE;
@@ -158,4 +164,65 @@ public final class JavaUtils {
private JavaUtils() {
// prevent instantiation
}
+
+ public static Long extractTxnId(Path file) {
+ String fileName = file.getName();
+ String[] parts = fileName.split("_", 4); // e.g. delta_0000001_0000001_0000
+ if (parts.length < 4 || !DELTA_PREFIX.equals(parts[0])) {
+ LOG.debug("Cannot extract transaction ID for a MM table: " + file
+ + " (" + Arrays.toString(parts) + ")");
+ return null;
+ }
+ long writeId = -1;
+ try {
+ writeId = Long.parseLong(parts[1]);
+ } catch (NumberFormatException ex) {
+ LOG.debug("Cannot extract transaction ID for a MM table: " + file
+ + "; parsing " + parts[1] + " got " + ex.getMessage());
+ return null;
+ }
+ return writeId;
+ }
+
+ public static class IdPathFilter implements PathFilter {
+ private final String mmDirName;
+ private final boolean isMatch, isIgnoreTemp;
+ public IdPathFilter(long writeId, int stmtId, boolean isMatch) {
+ this(writeId, stmtId, isMatch, false);
+ }
+ public IdPathFilter(long writeId, int stmtId, boolean isMatch, boolean isIgnoreTemp) {
+ this.mmDirName = DELTA_PREFIX + "_" + String.format(DELTA_DIGITS, writeId) + "_" +
+ String.format(DELTA_DIGITS, writeId) + "_" + String.format(STATEMENT_DIGITS, stmtId);
+ this.isMatch = isMatch;
+ this.isIgnoreTemp = isIgnoreTemp;
+ }
+
+ @Override
+ public boolean accept(Path path) {
+ String name = path.getName();
+ if (name.equals(mmDirName)) {
+ return isMatch;
+ }
+ if (isIgnoreTemp && name.length() > 0) {
+ char c = name.charAt(0);
+ if (c == '.' || c == '_') return false; // Regardless of isMatch, ignore this.
+ }
+ return !isMatch;
+ }
+ }
+
+ public static class AnyIdDirFilter implements PathFilter {
+ @Override
+ public boolean accept(Path path) {
+ String name = path.getName();
+ if (!name.startsWith(DELTA_PREFIX + "_")) return false;
+ String idStr = name.substring(DELTA_PREFIX.length() + 1, DELTA_PREFIX.length() + 1 + DELTA_DIGITS_LEN);
+ try {
+ Long.parseLong(idStr);
+ } catch (NumberFormatException ex) {
+ return false;
+ }
+ return true;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/77511070/common/src/java/org/apache/hadoop/hive/common/ValidWriteIds.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/ValidWriteIds.java b/common/src/java/org/apache/hadoop/hive/common/ValidWriteIds.java
deleted file mode 100644
index 4cbeb89..0000000
--- a/common/src/java/org/apache/hadoop/hive/common/ValidWriteIds.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.common;
-
-import java.util.Arrays;
-import java.util.HashSet;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ValidWriteIds {
- public static final ValidWriteIds NO_WRITE_IDS = new ValidWriteIds(-1, -1, false, null);
-
- public static final String MM_PREFIX = "mm";
- private static final String CURRENT_SUFFIX = ".current";
-
- private final static Logger LOG = LoggerFactory.getLogger(ValidWriteIds.class);
-
- private static final String VALID_WRITEIDS_PREFIX = "hive.valid.write.ids.";
- private final long lowWatermark, highWatermark;
- private final boolean areIdsValid;
- private final HashSet<Long> ids;
- private String source = null;
-
- public ValidWriteIds(
- long lowWatermark, long highWatermark, boolean areIdsValid, HashSet<Long> ids) {
- this.lowWatermark = lowWatermark;
- this.highWatermark = highWatermark;
- this.areIdsValid = areIdsValid;
- this.ids = ids;
- }
-
- public static ValidWriteIds createFromConf(Configuration conf, String dbName, String tblName) {
- return createFromConf(conf, dbName + "." + tblName);
- }
-
- public static ValidWriteIds createFromConf(Configuration conf, String fullTblName) {
- String key = createConfKey(fullTblName);
- String idStr = conf.get(key, null);
- String current = conf.get(key + CURRENT_SUFFIX, null);
- if (idStr == null || idStr.isEmpty()) return null;
- return new ValidWriteIds(idStr, current);
- }
-
- private static String createConfKey(String dbName, String tblName) {
- return createConfKey(dbName + "." + tblName);
- }
-
- private static String createConfKey(String fullName) {
- return VALID_WRITEIDS_PREFIX + fullName;
- }
-
- private ValidWriteIds(String src, String current) {
- // TODO: lifted from ACID config implementation... optimize if needed? e.g. ranges, base64
- String[] values = src.split(":");
- highWatermark = Long.parseLong(values[0]);
- lowWatermark = Long.parseLong(values[1]);
- if (values.length > 2) {
- areIdsValid = Long.parseLong(values[2]) > 0;
- ids = new HashSet<Long>();
- for(int i = 3; i < values.length; ++i) {
- ids.add(Long.parseLong(values[i]));
- }
- if (current != null) {
- long currentId = Long.parseLong(current);
- if (areIdsValid) {
- ids.add(currentId);
- } else {
- ids.remove(currentId);
- }
- }
- } else if (current != null) {
- long currentId = Long.parseLong(current);
- areIdsValid = true;
- ids = new HashSet<Long>();
- ids.add(currentId);
- } else {
- areIdsValid = false;
- ids = null;
- }
- }
-
- public static void addCurrentToConf(
- Configuration conf, String dbName, String tblName, long mmWriteId) {
- String key = createConfKey(dbName, tblName) + CURRENT_SUFFIX;
- if (LOG.isDebugEnabled()) {
- LOG.debug("Setting " + key + " => " + mmWriteId);
- }
- conf.set(key, Long.toString(mmWriteId));
- }
-
- public void addToConf(Configuration conf, String dbName, String tblName) {
- if (source == null) {
- source = toString();
- }
- String key = createConfKey(dbName, tblName);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Setting " + key + " => " + source
- + " (old value was " + conf.get(key, null) + ")");
- }
- conf.set(key, source);
- }
-
- public static void clearConf(Configuration conf, String dbName, String tblName) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Unsetting " + createConfKey(dbName, tblName));
- }
- conf.unset(createConfKey(dbName, tblName));
- }
-
- public String toString() {
- // TODO: lifted from ACID config implementation... optimize if needed? e.g. ranges, base64
- StringBuilder buf = new StringBuilder();
- buf.append(highWatermark);
- buf.append(':');
- buf.append(lowWatermark);
- if (ids != null) {
- buf.append(':');
- buf.append(areIdsValid ? 1 : 0);
- for (long id : ids) {
- buf.append(':');
- buf.append(id);
- }
- }
- return buf.toString();
- }
-
- public boolean isValid(long writeId) {
- if (writeId < 0) throw new RuntimeException("Incorrect write ID " + writeId);
- if (writeId <= lowWatermark) return true;
- if (writeId >= highWatermark) return false;
- return ids != null && (areIdsValid == ids.contains(writeId));
- }
-
- public static String getMmFilePrefix(long mmWriteId) {
- return MM_PREFIX + "_" + mmWriteId;
- }
-
-
- public static class IdPathFilter implements PathFilter {
- private final String mmDirName;
- private final boolean isMatch, isIgnoreTemp;
- public IdPathFilter(long writeId, boolean isMatch) {
- this(writeId, isMatch, false);
- }
- public IdPathFilter(long writeId, boolean isMatch, boolean isIgnoreTemp) {
- this.mmDirName = ValidWriteIds.getMmFilePrefix(writeId);
- this.isMatch = isMatch;
- this.isIgnoreTemp = isIgnoreTemp;
- }
-
- @Override
- public boolean accept(Path path) {
- String name = path.getName();
- if (name.equals(mmDirName)) {
- return isMatch;
- }
- if (isIgnoreTemp && name.length() > 0) {
- char c = name.charAt(0);
- if (c == '.' || c == '_') return false; // Regardless of isMatch, ignore this.
- }
- return !isMatch;
- }
- }
-
- public static class AnyIdDirFilter implements PathFilter {
- @Override
- public boolean accept(Path path) {
- String name = path.getName();
- if (!name.startsWith(MM_PREFIX + "_")) return false;
- String idStr = name.substring(MM_PREFIX.length() + 1);
- try {
- Long.parseLong(idStr);
- } catch (NumberFormatException ex) {
- return false;
- }
- return true;
- }
- }
- public static Long extractWriteId(Path file) {
- String fileName = file.getName();
- String[] parts = fileName.split("_", 3);
- if (parts.length < 2 || !MM_PREFIX.equals(parts[0])) {
- LOG.info("Cannot extract write ID for a MM table: " + file
- + " (" + Arrays.toString(parts) + ")");
- return null;
- }
- long writeId = -1;
- try {
- writeId = Long.parseLong(parts[1]);
- } catch (NumberFormatException ex) {
- LOG.info("Cannot extract write ID for a MM table: " + file
- + "; parsing " + parts[1] + " got " + ex.getMessage());
- return null;
- }
- return writeId;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/77511070/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index d32f1e5..a49f667 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -300,10 +300,7 @@ public class HiveConf extends Configuration {
HiveConf.ConfVars.METASTORE_HBASE_AGGR_STATS_MEMORY_TTL,
HiveConf.ConfVars.METASTORE_HBASE_AGGR_STATS_INVALIDATOR_FREQUENCY,
HiveConf.ConfVars.METASTORE_HBASE_AGGR_STATS_HBASE_TTL,
- HiveConf.ConfVars.METASTORE_HBASE_FILE_METADATA_THREADS,
- HiveConf.ConfVars.HIVE_METASTORE_MM_THREAD_SCAN_INTERVAL,
- HiveConf.ConfVars.HIVE_METASTORE_MM_HEARTBEAT_TIMEOUT,
- HiveConf.ConfVars.HIVE_METASTORE_MM_ABSOLUTE_TIMEOUT
+ HiveConf.ConfVars.METASTORE_HBASE_FILE_METADATA_THREADS
};
/**
@@ -3385,24 +3382,6 @@ public class HiveConf extends Configuration {
"Log tracing id that can be used by upstream clients for tracking respective logs. " +
"Truncated to " + LOG_PREFIX_LENGTH + " characters. Defaults to use auto-generated session id."),
- HIVE_METASTORE_MM_THREAD_SCAN_INTERVAL("hive.metastore.mm.thread.scan.interval", "900s",
- new TimeValidator(TimeUnit.SECONDS),
- "MM table housekeeping thread interval in this metastore instance. 0 to disable."),
-
- HIVE_METASTORE_MM_HEARTBEAT_TIMEOUT("hive.metastore.mm.heartbeat.timeout", "1800s",
- new TimeValidator(TimeUnit.SECONDS),
- "MM write ID times out after this long if a heartbeat is not send. Currently disabled."),
-
- HIVE_METASTORE_MM_ABSOLUTE_TIMEOUT("hive.metastore.mm.absolute.timeout", "7d",
- new TimeValidator(TimeUnit.SECONDS),
- "MM write ID cannot be outstanding for more than this long."),
-
- HIVE_METASTORE_MM_ABORTED_GRACE_PERIOD("hive.metastore.mm.aborted.grace.period", "1d",
- new TimeValidator(TimeUnit.SECONDS),
- "MM write ID will not be removed up for that long after it has been aborted;\n" +
- "this is to work around potential races e.g. with FS visibility, when deleting files."),
-
-
HIVE_MM_AVOID_GLOBSTATUS_ON_S3("hive.mm.avoid.s3.globstatus", true,
"Whether to use listFiles (optimized on S3) instead of globStatus when on S3."),
http://git-wip-us.apache.org/repos/asf/hive/blob/77511070/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
----------------------------------------------------------------------
diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
index 93ff498..309dbac 100644
--- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
+++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
@@ -63,7 +63,6 @@ import org.apache.hadoop.hive.metastore.api.Type;
import org.apache.hadoop.hive.metastore.api.UnknownDBException;
import org.apache.hadoop.hive.metastore.api.UnknownPartitionException;
import org.apache.hadoop.hive.metastore.api.UnknownTableException;
-import org.apache.hadoop.hive.metastore.model.MTableWrite;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.thrift.TException;
@@ -920,51 +919,4 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable {
String tableName) throws MetaException, NoSuchObjectException {
return objectStore.getAggrColStatsForTablePartitions(dbName, tableName);
}
-
- @Override
- @CanNotRetry
- public Boolean commitTransactionExpectDeadlock() {
- return null;
- }
-
- @Override
- public void createTableWrite(Table arg0, long arg1, char arg2, long arg3) {
- }
-
- @Override
- public void deleteTableWrites(String arg0, String arg1, long arg2, long arg3)
- throws MetaException {
- }
-
- @Override
- public List<FullTableName> getAllMmTablesForCleanup() throws MetaException {
- return null;
- }
-
- @Override
- public Collection<String> getAllPartitionLocations(String arg0, String arg1) {
- return null;
- }
-
- @Override
- public MTableWrite getTableWrite(String arg0, String arg1, long arg2)
- throws MetaException {
- return null;
- }
-
- @Override
- public List<Long> getTableWriteIds(String arg0, String arg1, long arg2,
- long arg3, char arg4) throws MetaException {
- return null;
- }
-
- @Override
- public List<MTableWrite> getTableWrites(String arg0, String arg1, long arg2,
- long arg3) throws MetaException {
- return null;
- }
-
- @Override
- public void updateTableWrite(MTableWrite arg0) {
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/77511070/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java
index 0c51a68..c70925a 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java
@@ -103,7 +103,7 @@ public class TestHiveHistory extends TestCase {
db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, src, true, true);
db.createTable(src, cols, null, TextInputFormat.class,
IgnoreKeyTextOutputFormat.class);
- db.loadTable(hadoopDataFile[i], src, false, false, false, false, false, null);
+ db.loadTable(hadoopDataFile[i], src, false, false, false, false, false, null, 0);
i++;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/77511070/metastore/if/hive_metastore.thrift
----------------------------------------------------------------------
diff --git a/metastore/if/hive_metastore.thrift b/metastore/if/hive_metastore.thrift
index 2800e23..64c782e 100755
--- a/metastore/if/hive_metastore.thrift
+++ b/metastore/if/hive_metastore.thrift
@@ -306,8 +306,6 @@ struct Table {
13: optional PrincipalPrivilegeSet privileges,
14: optional bool temporary=false,
15: optional bool rewriteEnabled, // rewrite enabled or not
- 16: optional i64 mmNextWriteId,
- 17: optional i64 mmWatermarkWriteId
}
struct Partition {
@@ -907,44 +905,6 @@ struct CacheFileMetadataRequest {
4: optional bool isAllParts
}
-
-struct GetNextWriteIdRequest {
- 1: required string dbName,
- 2: required string tblName
-}
-struct GetNextWriteIdResult {
- 1: required i64 writeId
-}
-
-struct FinalizeWriteIdRequest {
- 1: required string dbName,
- 2: required string tblName,
- 3: required i64 writeId,
- 4: required bool commit
-}
-struct FinalizeWriteIdResult {
-}
-
-struct HeartbeatWriteIdRequest {
- 1: required string dbName,
- 2: required string tblName,
- 3: required i64 writeId
-}
-struct HeartbeatWriteIdResult {
-}
-
-struct GetValidWriteIdsRequest {
- 1: required string dbName,
- 2: required string tblName
-}
-struct GetValidWriteIdsResult {
- 1: required i64 lowWatermarkId,
- 2: required i64 highWatermarkId,
- 3: optional bool areIdsValid,
- 4: optional list<i64> ids
-}
-
-
struct GetAllFunctionsResponse {
1: optional list<Function> functions
}
@@ -1532,10 +1492,6 @@ service ThriftHiveMetastore extends fb303.FacebookService
ClearFileMetadataResult clear_file_metadata(1:ClearFileMetadataRequest req)
CacheFileMetadataResult cache_file_metadata(1:CacheFileMetadataRequest req)
- GetNextWriteIdResult get_next_write_id(1:GetNextWriteIdRequest req)
- FinalizeWriteIdResult finalize_write_id(1:FinalizeWriteIdRequest req)
- HeartbeatWriteIdResult heartbeat_write_id(1:HeartbeatWriteIdRequest req)
- GetValidWriteIdsResult get_valid_write_ids(1:GetValidWriteIdsRequest req)
}
// * Note about the DDL_TIME: When creating or altering a table or a partition,
http://git-wip-us.apache.org/repos/asf/hive/blob/77511070/metastore/scripts/upgrade/derby/038-HIVE-14637.derby.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/derby/038-HIVE-14637.derby.sql b/metastore/scripts/upgrade/derby/038-HIVE-14637.derby.sql
deleted file mode 100644
index cb6e5f6..0000000
--- a/metastore/scripts/upgrade/derby/038-HIVE-14637.derby.sql
+++ /dev/null
@@ -1,6 +0,0 @@
-ALTER TABLE "TBLS" ADD "MM_WATERMARK_WRITE_ID" BIGINT DEFAULT -1;
-ALTER TABLE "TBLS" ADD "MM_NEXT_WRITE_ID" BIGINT DEFAULT 0;
-CREATE TABLE "APP"."TBL_WRITES" ("TW_ID" BIGINT NOT NULL, "TBL_ID" BIGINT NOT NULL, "WRITE_ID" BIGINT NOT NULL, "STATE" CHAR(1) NOT NULL, "CREATED" BIGINT NOT NULL, "LAST_HEARTBEAT" BIGINT NOT NULL);
-ALTER TABLE "APP"."TBL_WRITES" ADD CONSTRAINT "TBL_WRITES_PK" PRIMARY KEY ("TW_ID");
-ALTER TABLE "APP"."TBL_WRITES" ADD CONSTRAINT "TBL_WRITES_FK1" FOREIGN KEY ("TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION;
-CREATE UNIQUE INDEX "APP"."UNIQUEWRITE" ON "APP"."TBL_WRITES" ("TBL_ID", "WRITE_ID");
http://git-wip-us.apache.org/repos/asf/hive/blob/77511070/metastore/scripts/upgrade/derby/hive-schema-2.2.0.derby.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/derby/hive-schema-2.2.0.derby.sql b/metastore/scripts/upgrade/derby/hive-schema-2.2.0.derby.sql
index a4977b6..6dd3dee 100644
--- a/metastore/scripts/upgrade/derby/hive-schema-2.2.0.derby.sql
+++ b/metastore/scripts/upgrade/derby/hive-schema-2.2.0.derby.sql
@@ -60,7 +60,7 @@ CREATE TABLE "APP"."COLUMNS" ("SD_ID" BIGINT NOT NULL, "COMMENT" VARCHAR(256), "
CREATE TABLE "APP"."ROLES" ("ROLE_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "OWNER_NAME" VARCHAR(128), "ROLE_NAME" VARCHAR(128));
-CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER" VARCHAR(767), "RETENTION" INTEGER NOT NULL, "SD_ID" BIGINT, "TBL_NAME" VARCHAR(256), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT" LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR, "IS_REWRITE_ENABLED" CHAR(1) NOT NULL, "MM_WATERMARK_WRITE_ID" BIGINT DEFAULT -1, "MM_NEXT_WRITE_ID" BIGINT DEFAULT 0);
+CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER" VARCHAR(767), "RETENTION" INTEGER NOT NULL, "SD_ID" BIGINT, "TBL_NAME" VARCHAR(256), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT" LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR, "IS_REWRITE_ENABLED" CHAR(1) NOT NULL);
CREATE TABLE "APP"."PARTITION_KEYS" ("TBL_ID" BIGINT NOT NULL, "PKEY_COMMENT" VARCHAR(4000), "PKEY_NAME" VARCHAR(128) NOT NULL, "PKEY_TYPE" VARCHAR(767) NOT NULL, "INTEGER_IDX" INTEGER NOT NULL);
@@ -112,15 +112,6 @@ ALTER TABLE "APP"."KEY_CONSTRAINTS" ADD CONSTRAINT "CONSTRAINTS_PK" PRIMARY KEY
CREATE INDEX "APP"."CONSTRAINTS_PARENT_TBL_ID_INDEX" ON "APP"."KEY_CONSTRAINTS"("PARENT_TBL_ID");
-CREATE TABLE "APP"."TBL_WRITES" ("TW_ID" BIGINT NOT NULL, "TBL_ID" BIGINT NOT NULL, "WRITE_ID" BIGINT NOT NULL, "STATE" CHAR(1) NOT NULL, "CREATED" BIGINT NOT NULL, "LAST_HEARTBEAT" BIGINT NOT NULL);
-
-ALTER TABLE "APP"."TBL_WRITES" ADD CONSTRAINT "TBL_WRITES_PK" PRIMARY KEY ("TW_ID");
-
-ALTER TABLE "APP"."TBL_WRITES" ADD CONSTRAINT "TBL_WRITES_FK1" FOREIGN KEY ("TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION;
-
-CREATE UNIQUE INDEX "APP"."UNIQUEWRITE" ON "APP"."TBL_WRITES" ("TBL_ID", "WRITE_ID");
-
-
-- ----------------------------------------------
-- DDL Statements for indexes
http://git-wip-us.apache.org/repos/asf/hive/blob/77511070/metastore/scripts/upgrade/derby/upgrade-2.1.0-to-2.2.0.derby.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/derby/upgrade-2.1.0-to-2.2.0.derby.sql b/metastore/scripts/upgrade/derby/upgrade-2.1.0-to-2.2.0.derby.sql
index 3e87091..b05942f 100644
--- a/metastore/scripts/upgrade/derby/upgrade-2.1.0-to-2.2.0.derby.sql
+++ b/metastore/scripts/upgrade/derby/upgrade-2.1.0-to-2.2.0.derby.sql
@@ -3,6 +3,4 @@ RUN '037-HIVE-14496.derby.sql';
RUN '038-HIVE-10562.derby.sql';
RUN '039-HIVE-12274.derby.sql';
-RUN '037-HIVE-14637.derby.sql';
-
UPDATE "APP".VERSION SET SCHEMA_VERSION='2.2.0', VERSION_COMMENT='Hive release version 2.2.0' where VER_ID=1;
http://git-wip-us.apache.org/repos/asf/hive/blob/77511070/metastore/scripts/upgrade/mssql/023-HIVE-14637.mssql.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/mssql/023-HIVE-14637.mssql.sql b/metastore/scripts/upgrade/mssql/023-HIVE-14637.mssql.sql
deleted file mode 100644
index 9666d2b..0000000
--- a/metastore/scripts/upgrade/mssql/023-HIVE-14637.mssql.sql
+++ /dev/null
@@ -1,15 +0,0 @@
-ALTER TABLE TBLS ADD MM_WATERMARK_WRITE_ID BIGINT DEFAULT -1;
-ALTER TABLE TBLS ADD MM_NEXT_WRITE_ID BIGINT DEFAULT 0;
-
-CREATE TABLE TBL_WRITES
-(
- TW_ID BIGINT NOT NULL,
- TBL_ID BIGINT NOT NULL,
- WRITE_ID BIGINT NOT NULL,
- STATE CHAR(1) NOT NULL,
- CREATED BIGINT NOT NULL,
- LAST_HEARTBEAT BIGINT NOT NULL
-);
-ALTER TABLE TBL_WRITES ADD CONSTRAINT TBL_WRITES_PK PRIMARY KEY (TW_ID);
-ALTER TABLE TBL_WRITES ADD CONSTRAINT TBL_WRITES_FK1 FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) ;
-CREATE UNIQUE INDEX UNIQUEWRITE ON TBL_WRITES (TBL_ID, WRITE_ID);
http://git-wip-us.apache.org/repos/asf/hive/blob/77511070/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql b/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql
index 3621ef6..b6fdc7b 100644
--- a/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql
+++ b/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql
@@ -359,9 +359,7 @@ CREATE TABLE TBLS
TBL_TYPE nvarchar(128) NULL,
VIEW_EXPANDED_TEXT text NULL,
VIEW_ORIGINAL_TEXT text NULL,
- IS_REWRITE_ENABLED bit NOT NULL,
- MM_WATERMARK_WRITE_ID BIGINT NULL DEFAULT -1,
- MM_NEXT_WRITE_ID BIGINT NULL DEFAULT 0
+ IS_REWRITE_ENABLED bit NOT NULL
);
ALTER TABLE TBLS ADD CONSTRAINT TBLS_PK PRIMARY KEY (TBL_ID);
@@ -596,23 +594,6 @@ CREATE TABLE NOTIFICATION_SEQUENCE
ALTER TABLE NOTIFICATION_SEQUENCE ADD CONSTRAINT NOTIFICATION_SEQUENCE_PK PRIMARY KEY (NNI_ID);
-CREATE TABLE TBL_WRITES
-(
- TW_ID BIGINT NOT NULL,
- TBL_ID BIGINT NOT NULL,
- WRITE_ID BIGINT NOT NULL,
- STATE CHAR(1) NOT NULL,
- CREATED BIGINT NOT NULL,
- LAST_HEARTBEAT BIGINT NOT NULL
-);
-
-ALTER TABLE TBL_WRITES ADD CONSTRAINT TBL_WRITES_PK PRIMARY KEY (TW_ID);
-
-ALTER TABLE TBL_WRITES ADD CONSTRAINT TBL_WRITES_FK1 FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) ;
-
-CREATE UNIQUE INDEX UNIQUEWRITE ON TBL_WRITES (TBL_ID, WRITE_ID);
-
-
-- Constraints for table MASTER_KEYS for class(es) [org.apache.hadoop.hive.metastore.model.MMasterKey]
-- Constraints for table IDXS for class(es) [org.apache.hadoop.hive.metastore.model.MIndex]
http://git-wip-us.apache.org/repos/asf/hive/blob/77511070/metastore/scripts/upgrade/mssql/upgrade-2.1.0-to-2.2.0.mssql.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/mssql/upgrade-2.1.0-to-2.2.0.mssql.sql b/metastore/scripts/upgrade/mssql/upgrade-2.1.0-to-2.2.0.mssql.sql
index b786b16..4995349 100644
--- a/metastore/scripts/upgrade/mssql/upgrade-2.1.0-to-2.2.0.mssql.sql
+++ b/metastore/scripts/upgrade/mssql/upgrade-2.1.0-to-2.2.0.mssql.sql
@@ -1,7 +1,6 @@
SELECT 'Upgrading MetaStore schema from 2.1.0 to 2.2.0' AS MESSAGE;
:r 022-HIVE-14496.mssql.sql
-:r 023-HIVE-14637.mssql.sql
:r 023-HIVE-10562.mssql.sql
:r 024-HIVE-12274.mssql.sql
http://git-wip-us.apache.org/repos/asf/hive/blob/77511070/metastore/scripts/upgrade/mysql/038-HIVE-14637.mysql.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/mysql/038-HIVE-14637.mysql.sql b/metastore/scripts/upgrade/mysql/038-HIVE-14637.mysql.sql
deleted file mode 100644
index 9e34db2..0000000
--- a/metastore/scripts/upgrade/mysql/038-HIVE-14637.mysql.sql
+++ /dev/null
@@ -1,15 +0,0 @@
-alter table `TBLS` ADD COLUMN `MM_WATERMARK_WRITE_ID` bigint(20) DEFAULT -1;
-alter table `TBLS` ADD COLUMN `MM_NEXT_WRITE_ID` bigint(20) DEFAULT 0;
-
-CREATE TABLE IF NOT EXISTS `TBL_WRITES`
-(
- `TW_ID` BIGINT NOT NULL,
- `TBL_ID` BIGINT NOT NULL,
- `WRITE_ID` BIGINT NOT NULL,
- `STATE` CHAR(1) NOT NULL,
- `CREATED` BIGINT NOT NULL,
- `LAST_HEARTBEAT` BIGINT NOT NULL,
- PRIMARY KEY (`TW_ID`),
- UNIQUE KEY `UNIQUEWRITE` (`TBL_ID`,`WRITE_ID`),
- CONSTRAINT `TBL_WRITES_FK1` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`)
-) ENGINE=InnoDB DEFAULT CHARSET=latin1;
http://git-wip-us.apache.org/repos/asf/hive/blob/77511070/metastore/scripts/upgrade/mysql/hive-schema-2.2.0.mysql.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/mysql/hive-schema-2.2.0.mysql.sql b/metastore/scripts/upgrade/mysql/hive-schema-2.2.0.mysql.sql
index 20cfbc4..d1852df 100644
--- a/metastore/scripts/upgrade/mysql/hive-schema-2.2.0.mysql.sql
+++ b/metastore/scripts/upgrade/mysql/hive-schema-2.2.0.mysql.sql
@@ -588,8 +588,6 @@ CREATE TABLE IF NOT EXISTS `TBLS` (
`VIEW_EXPANDED_TEXT` mediumtext,
`VIEW_ORIGINAL_TEXT` mediumtext,
`IS_REWRITE_ENABLED` bit(1) NOT NULL,
- `MM_WATERMARK_WRITE_ID` bigint(20) DEFAULT -1,
- `MM_NEXT_WRITE_ID` bigint(20) DEFAULT 0,
PRIMARY KEY (`TBL_ID`),
UNIQUE KEY `UNIQUETABLE` (`TBL_NAME`,`DB_ID`),
KEY `TBLS_N50` (`SD_ID`),
@@ -831,20 +829,6 @@ CREATE TABLE IF NOT EXISTS `KEY_CONSTRAINTS`
CREATE INDEX `CONSTRAINTS_PARENT_TABLE_ID_INDEX` ON KEY_CONSTRAINTS (`PARENT_TBL_ID`) USING BTREE;
-CREATE TABLE IF NOT EXISTS `TBL_WRITES`
-(
- `TW_ID` BIGINT NOT NULL,
- `TBL_ID` BIGINT NOT NULL,
- `WRITE_ID` BIGINT NOT NULL,
- `STATE` CHAR(1) NOT NULL,
- `CREATED` BIGINT NOT NULL,
- `LAST_HEARTBEAT` BIGINT NOT NULL,
- PRIMARY KEY (`TW_ID`),
- UNIQUE KEY `UNIQUEWRITE` (`TBL_ID`,`WRITE_ID`),
- CONSTRAINT `TBL_WRITES_FK1` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`)
-) ENGINE=InnoDB DEFAULT CHARSET=latin1;
-
-
-- ----------------------------
-- Transaction and Lock Tables
-- ----------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/77511070/metastore/scripts/upgrade/mysql/upgrade-2.1.0-to-2.2.0.mysql.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/mysql/upgrade-2.1.0-to-2.2.0.mysql.sql b/metastore/scripts/upgrade/mysql/upgrade-2.1.0-to-2.2.0.mysql.sql
index f4c69a5..e221439 100644
--- a/metastore/scripts/upgrade/mysql/upgrade-2.1.0-to-2.2.0.mysql.sql
+++ b/metastore/scripts/upgrade/mysql/upgrade-2.1.0-to-2.2.0.mysql.sql
@@ -1,7 +1,6 @@
SELECT 'Upgrading MetaStore schema from 2.1.0 to 2.2.0' AS ' ';
SOURCE 037-HIVE-14496.mysql.sql;
-SOURCE 038-HIVE-14637.mysql.sql;
SOURCE 038-HIVE-10562.mysql.sql;
SOURCE 039-HIVE-12274.mysql.sql;
http://git-wip-us.apache.org/repos/asf/hive/blob/77511070/metastore/scripts/upgrade/oracle/038-HIVE-14637.oracle.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/oracle/038-HIVE-14637.oracle.sql b/metastore/scripts/upgrade/oracle/038-HIVE-14637.oracle.sql
deleted file mode 100644
index 218eefe..0000000
--- a/metastore/scripts/upgrade/oracle/038-HIVE-14637.oracle.sql
+++ /dev/null
@@ -1,15 +0,0 @@
-ALTER TABLE TBLS ADD MM_WATERMARK_WRITE_ID NUMBER DEFAULT -1;
-ALTER TABLE TBLS ADD MM_NEXT_WRITE_ID NUMBER DEFAULT 0;
-
-CREATE TABLE TBL_WRITES
-(
- TW_ID NUMBER NOT NULL,
- TBL_ID NUMBER NOT NULL,
- WRITE_ID NUMBER NOT NULL,
- STATE CHAR(1) NOT NULL,
- CREATED NUMBER NOT NULL,
- LAST_HEARTBEAT NUMBER NOT NULL
-);
-ALTER TABLE TBL_WRITES ADD CONSTRAINT TBL_WRITES_PK PRIMARY KEY (TW_ID);
-ALTER TABLE TBL_WRITES ADD CONSTRAINT TBL_WRITES_FK1 FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) INITIALLY DEFERRED ;
-CREATE UNIQUE INDEX UNIQUEWRITE ON TBL_WRITES (TBL_ID, WRITE_ID);
http://git-wip-us.apache.org/repos/asf/hive/blob/77511070/metastore/scripts/upgrade/oracle/hive-schema-2.2.0.oracle.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/oracle/hive-schema-2.2.0.oracle.sql b/metastore/scripts/upgrade/oracle/hive-schema-2.2.0.oracle.sql
index c9b1aeb..4aaa5e7 100644
--- a/metastore/scripts/upgrade/oracle/hive-schema-2.2.0.oracle.sql
+++ b/metastore/scripts/upgrade/oracle/hive-schema-2.2.0.oracle.sql
@@ -376,9 +376,7 @@ CREATE TABLE TBLS
TBL_TYPE VARCHAR2(128) NULL,
VIEW_EXPANDED_TEXT CLOB NULL,
VIEW_ORIGINAL_TEXT CLOB NULL,
- IS_REWRITE_ENABLED NUMBER(1) NOT NULL CHECK (IS_REWRITE_ENABLED IN (1,0)),
- MM_WATERMARK_WRITE_ID NUMBER DEFAULT -1,
- MM_NEXT_WRITE_ID NUMBER DEFAULT 0
+ IS_REWRITE_ENABLED NUMBER(1) NOT NULL CHECK (IS_REWRITE_ENABLED IN (1,0))
);
ALTER TABLE TBLS ADD CONSTRAINT TBLS_PK PRIMARY KEY (TBL_ID);
@@ -801,22 +799,6 @@ ALTER TABLE KEY_CONSTRAINTS ADD CONSTRAINT CONSTRAINTS_PK PRIMARY KEY (CONSTRAIN
CREATE INDEX CONSTRAINTS_PT_INDEX ON KEY_CONSTRAINTS(PARENT_TBL_ID);
-CREATE TABLE TBL_WRITES
-(
- TW_ID NUMBER NOT NULL,
- TBL_ID NUMBER NOT NULL,
- WRITE_ID NUMBER NOT NULL,
- STATE CHAR(1) NOT NULL,
- CREATED NUMBER NOT NULL,
- LAST_HEARTBEAT NUMBER NOT NULL
-);
-
-ALTER TABLE TBL_WRITES ADD CONSTRAINT TBL_WRITES_PK PRIMARY KEY (TW_ID);
-
-ALTER TABLE TBL_WRITES ADD CONSTRAINT TBL_WRITES_FK1 FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) INITIALLY DEFERRED ;
-
-CREATE UNIQUE INDEX UNIQUEWRITE ON TBL_WRITES (TBL_ID, WRITE_ID);
-
------------------------------
-- Transaction and lock tables
------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/77511070/metastore/scripts/upgrade/oracle/upgrade-2.1.0-to-2.2.0.oracle.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/oracle/upgrade-2.1.0-to-2.2.0.oracle.sql b/metastore/scripts/upgrade/oracle/upgrade-2.1.0-to-2.2.0.oracle.sql
index b2f35de..53ec681 100644
--- a/metastore/scripts/upgrade/oracle/upgrade-2.1.0-to-2.2.0.oracle.sql
+++ b/metastore/scripts/upgrade/oracle/upgrade-2.1.0-to-2.2.0.oracle.sql
@@ -1,7 +1,6 @@
SELECT 'Upgrading MetaStore schema from 2.1.0 to 2.2.0' AS Status from dual;
@037-HIVE-14496.oracle.sql;
-@038-HIVE-14637.oracle.sql;
@038-HIVE-10562.oracle.sql;
@039-HIVE-12274.oracle.sql;
http://git-wip-us.apache.org/repos/asf/hive/blob/77511070/metastore/scripts/upgrade/postgres/037-HIVE-14637.postgres.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/postgres/037-HIVE-14637.postgres.sql b/metastore/scripts/upgrade/postgres/037-HIVE-14637.postgres.sql
deleted file mode 100644
index 310f51e..0000000
--- a/metastore/scripts/upgrade/postgres/037-HIVE-14637.postgres.sql
+++ /dev/null
@@ -1,16 +0,0 @@
-
-ALTER TABLE "TBLS" ADD COLUMN "MM_WATERMARK_WRITE_ID" bigint DEFAULT -1;
-ALTER TABLE "TBLS" ADD COLUMN "MM_NEXT_WRITE_ID" bigint DEFAULT 0;
-
-CREATE TABLE "TBL_WRITES"
-(
- "TW_ID" BIGINT NOT NULL,
- "TBL_ID" BIGINT NOT NULL,
- "WRITE_ID" BIGINT NOT NULL,
- "STATE" CHAR(1) NOT NULL,
- "CREATED" BIGINT NOT NULL,
- "LAST_HEARTBEAT" BIGINT NOT NULL
-);
-ALTER TABLE ONLY "TBL_WRITES" ADD CONSTRAINT "TBL_WRITES_PK" PRIMARY KEY ("TW_ID");
-ALTER TABLE ONLY "TBL_WRITES" ADD CONSTRAINT "TBL_WRITES_FK1" FOREIGN KEY ("TBL_ID") REFERENCES "TBLS" ("TBL_ID") DEFERRABLE;
-ALTER TABLE ONLY "TBL_WRITES" ADD CONSTRAINT "UNIQUEWRITE" UNIQUE ("TBL_ID", "WRITE_ID");
http://git-wip-us.apache.org/repos/asf/hive/blob/77511070/metastore/scripts/upgrade/postgres/hive-schema-2.2.0.postgres.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/postgres/hive-schema-2.2.0.postgres.sql b/metastore/scripts/upgrade/postgres/hive-schema-2.2.0.postgres.sql
index 424c6a1..5feab4e 100644
--- a/metastore/scripts/upgrade/postgres/hive-schema-2.2.0.postgres.sql
+++ b/metastore/scripts/upgrade/postgres/hive-schema-2.2.0.postgres.sql
@@ -373,9 +373,7 @@ CREATE TABLE "TBLS" (
"TBL_TYPE" character varying(128) DEFAULT NULL::character varying,
"VIEW_EXPANDED_TEXT" text,
"VIEW_ORIGINAL_TEXT" text,
- "IS_REWRITE_ENABLED" boolean NOT NULL,
- "MM_WATERMARK_WRITE_ID" bigint DEFAULT -1,
- "MM_NEXT_WRITE_ID" bigint DEFAULT 0
+ "IS_REWRITE_ENABLED" boolean NOT NULL
);
@@ -609,24 +607,6 @@ CREATE TABLE "KEY_CONSTRAINTS"
CREATE INDEX "CONSTRAINTS_PARENT_TBLID_INDEX" ON "KEY_CONSTRAINTS" USING BTREE ("PARENT_TBL_ID");
-
-CREATE TABLE "TBL_WRITES"
-(
- "TW_ID" BIGINT NOT NULL,
- "TBL_ID" BIGINT NOT NULL,
- "WRITE_ID" BIGINT NOT NULL,
- "STATE" CHAR(1) NOT NULL,
- "CREATED" BIGINT NOT NULL,
- "LAST_HEARTBEAT" BIGINT NOT NULL
-);
-
-ALTER TABLE ONLY "TBL_WRITES" ADD CONSTRAINT "TBL_WRITES_PK" PRIMARY KEY ("TW_ID");
-
-ALTER TABLE ONLY "TBL_WRITES" ADD CONSTRAINT "TBL_WRITES_FK1" FOREIGN KEY ("TBL_ID") REFERENCES "TBLS" ("TBL_ID") DEFERRABLE;
-
-ALTER TABLE ONLY "TBL_WRITES" ADD CONSTRAINT "UNIQUEWRITE" UNIQUE ("TBL_ID", "WRITE_ID");
-
-
--
-- Name: BUCKETING_COLS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace:
--
http://git-wip-us.apache.org/repos/asf/hive/blob/77511070/metastore/scripts/upgrade/postgres/upgrade-2.1.0-to-2.2.0.postgres.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/postgres/upgrade-2.1.0-to-2.2.0.postgres.sql b/metastore/scripts/upgrade/postgres/upgrade-2.1.0-to-2.2.0.postgres.sql
index e6daeca..732e184 100644
--- a/metastore/scripts/upgrade/postgres/upgrade-2.1.0-to-2.2.0.postgres.sql
+++ b/metastore/scripts/upgrade/postgres/upgrade-2.1.0-to-2.2.0.postgres.sql
@@ -1,7 +1,6 @@
SELECT 'Upgrading MetaStore schema from 2.1.0 to 2.2.0';
\i 036-HIVE-14496.postgres.sql;
-\i 037-HIVE-14637.postgres.sql;
\i 037-HIVE-10562.postgres.sql;
\i 038-HIVE-12274.postgres.sql;