You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2017/02/02 02:03:48 UTC
[49/50] [abbrv] hive git commit: HIVE-14671 : merge master into
hive-14535 (Sergey Shelukhin) UNVERIFIED (build only)
http://git-wip-us.apache.org/repos/asf/hive/blob/748c1bd2/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/748c1bd2/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/748c1bd2/metastore/if/hive_metastore.thrift
----------------------------------------------------------------------
diff --cc metastore/if/hive_metastore.thrift
index 6260dad,bf80455..4a86313
--- a/metastore/if/hive_metastore.thrift
+++ b/metastore/if/hive_metastore.thrift
@@@ -302,11 -302,10 +302,12 @@@ struct Table
9: map<string, string> parameters, // to store comments or any other user level parameters
10: string viewOriginalText, // original view text, null for non-view
11: string viewExpandedText, // expanded view text, null for non-view
- 12: string tableType, // table type enum, e.g. EXTERNAL_TABLE
+ 12: string tableType, // table type enum, e.g. EXTERNAL_TABLE
13: optional PrincipalPrivilegeSet privileges,
14: optional bool temporary=false,
- 15: optional i64 mmNextWriteId,
- 16: optional i64 mmWatermarkWriteId
- 15: optional bool rewriteEnabled // rewrite enabled or not
++ 15: optional bool rewriteEnabled, // rewrite enabled or not
++ 16: optional i64 mmNextWriteId,
++ 17: optional i64 mmWatermarkWriteId
}
struct Partition {
http://git-wip-us.apache.org/repos/asf/hive/blob/748c1bd2/metastore/scripts/upgrade/derby/038-HIVE-14637.derby.sql
----------------------------------------------------------------------
diff --cc metastore/scripts/upgrade/derby/038-HIVE-14637.derby.sql
index 0000000,0000000..cb6e5f6
new file mode 100644
--- /dev/null
+++ b/metastore/scripts/upgrade/derby/038-HIVE-14637.derby.sql
@@@ -1,0 -1,0 +1,6 @@@
++ALTER TABLE "TBLS" ADD "MM_WATERMARK_WRITE_ID" BIGINT DEFAULT -1;
++ALTER TABLE "TBLS" ADD "MM_NEXT_WRITE_ID" BIGINT DEFAULT 0;
++CREATE TABLE "APP"."TBL_WRITES" ("TW_ID" BIGINT NOT NULL, "TBL_ID" BIGINT NOT NULL, "WRITE_ID" BIGINT NOT NULL, "STATE" CHAR(1) NOT NULL, "CREATED" BIGINT NOT NULL, "LAST_HEARTBEAT" BIGINT NOT NULL);
++ALTER TABLE "APP"."TBL_WRITES" ADD CONSTRAINT "TBL_WRITES_PK" PRIMARY KEY ("TW_ID");
++ALTER TABLE "APP"."TBL_WRITES" ADD CONSTRAINT "TBL_WRITES_FK1" FOREIGN KEY ("TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION;
++CREATE UNIQUE INDEX "APP"."UNIQUEWRITE" ON "APP"."TBL_WRITES" ("TBL_ID", "WRITE_ID");
http://git-wip-us.apache.org/repos/asf/hive/blob/748c1bd2/metastore/scripts/upgrade/derby/hive-schema-2.2.0.derby.sql
----------------------------------------------------------------------
diff --cc metastore/scripts/upgrade/derby/hive-schema-2.2.0.derby.sql
index 9da1703,fe18089..7666fbc
--- a/metastore/scripts/upgrade/derby/hive-schema-2.2.0.derby.sql
+++ b/metastore/scripts/upgrade/derby/hive-schema-2.2.0.derby.sql
@@@ -60,7 -60,7 +60,7 @@@ CREATE TABLE "APP"."COLUMNS" ("SD_ID" B
CREATE TABLE "APP"."ROLES" ("ROLE_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "OWNER_NAME" VARCHAR(128), "ROLE_NAME" VARCHAR(128));
- CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER" VARCHAR(767), "RETENTION" INTEGER NOT NULL, "SD_ID" BIGINT, "TBL_NAME" VARCHAR(128), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT" LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR, "MM_WATERMARK_WRITE_ID" BIGINT DEFAULT -1, "MM_NEXT_WRITE_ID" BIGINT DEFAULT 0);
-CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER" VARCHAR(767), "RETENTION" INTEGER NOT NULL, "SD_ID" BIGINT, "TBL_NAME" VARCHAR(128), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT" LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR, "IS_REWRITE_ENABLED" CHAR(1) NOT NULL);
++CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER" VARCHAR(767), "RETENTION" INTEGER NOT NULL, "SD_ID" BIGINT, "TBL_NAME" VARCHAR(128), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT" LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR, "IS_REWRITE_ENABLED" CHAR(1) NOT NULL, "MM_WATERMARK_WRITE_ID" BIGINT DEFAULT -1, "MM_NEXT_WRITE_ID" BIGINT DEFAULT 0);
CREATE TABLE "APP"."PARTITION_KEYS" ("TBL_ID" BIGINT NOT NULL, "PKEY_COMMENT" VARCHAR(4000), "PKEY_NAME" VARCHAR(128) NOT NULL, "PKEY_TYPE" VARCHAR(767) NOT NULL, "INTEGER_IDX" INTEGER NOT NULL);
http://git-wip-us.apache.org/repos/asf/hive/blob/748c1bd2/metastore/scripts/upgrade/derby/upgrade-2.1.0-to-2.2.0.derby.sql
----------------------------------------------------------------------
diff --cc metastore/scripts/upgrade/derby/upgrade-2.1.0-to-2.2.0.derby.sql
index 67750a6,699a619..f59b37f
--- a/metastore/scripts/upgrade/derby/upgrade-2.1.0-to-2.2.0.derby.sql
+++ b/metastore/scripts/upgrade/derby/upgrade-2.1.0-to-2.2.0.derby.sql
@@@ -1,5 -1,4 +1,6 @@@
-- Upgrade MetaStore schema from 2.1.0 to 2.2.0
+ RUN '037-HIVE-14496.derby.sql';
+RUN '037-HIVE-14637.derby.sql';
+
UPDATE "APP".VERSION SET SCHEMA_VERSION='2.2.0', VERSION_COMMENT='Hive release version 2.2.0' where VER_ID=1;
http://git-wip-us.apache.org/repos/asf/hive/blob/748c1bd2/metastore/scripts/upgrade/mssql/023-HIVE-14637.mssql.sql
----------------------------------------------------------------------
diff --cc metastore/scripts/upgrade/mssql/023-HIVE-14637.mssql.sql
index 0000000,0000000..9666d2b
new file mode 100644
--- /dev/null
+++ b/metastore/scripts/upgrade/mssql/023-HIVE-14637.mssql.sql
@@@ -1,0 -1,0 +1,15 @@@
++ALTER TABLE TBLS ADD MM_WATERMARK_WRITE_ID BIGINT DEFAULT -1;
++ALTER TABLE TBLS ADD MM_NEXT_WRITE_ID BIGINT DEFAULT 0;
++
++CREATE TABLE TBL_WRITES
++(
++ TW_ID BIGINT NOT NULL,
++ TBL_ID BIGINT NOT NULL,
++ WRITE_ID BIGINT NOT NULL,
++ STATE CHAR(1) NOT NULL,
++ CREATED BIGINT NOT NULL,
++ LAST_HEARTBEAT BIGINT NOT NULL
++);
++ALTER TABLE TBL_WRITES ADD CONSTRAINT TBL_WRITES_PK PRIMARY KEY (TW_ID);
++ALTER TABLE TBL_WRITES ADD CONSTRAINT TBL_WRITES_FK1 FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) ;
++CREATE UNIQUE INDEX UNIQUEWRITE ON TBL_WRITES (TBL_ID, WRITE_ID);
http://git-wip-us.apache.org/repos/asf/hive/blob/748c1bd2/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql
----------------------------------------------------------------------
diff --cc metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql
index 31016e2,7ff881c..2606de5
--- a/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql
+++ b/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql
@@@ -359,8 -359,7 +359,9 @@@ CREATE TABLE TBL
TBL_TYPE nvarchar(128) NULL,
VIEW_EXPANDED_TEXT text NULL,
VIEW_ORIGINAL_TEXT text NULL,
- IS_REWRITE_ENABLED bit NOT NULL
++ IS_REWRITE_ENABLED bit NOT NULL,
+ MM_WATERMARK_WRITE_ID BIGINT NULL DEFAULT -1,
+ MM_NEXT_WRITE_ID BIGINT NULL DEFAULT 0
);
ALTER TABLE TBLS ADD CONSTRAINT TBLS_PK PRIMARY KEY (TBL_ID);
http://git-wip-us.apache.org/repos/asf/hive/blob/748c1bd2/metastore/scripts/upgrade/mssql/upgrade-2.1.0-to-2.2.0.mssql.sql
----------------------------------------------------------------------
diff --cc metastore/scripts/upgrade/mssql/upgrade-2.1.0-to-2.2.0.mssql.sql
index 2e6f394,55d8e9b..2dcdedc
--- a/metastore/scripts/upgrade/mssql/upgrade-2.1.0-to-2.2.0.mssql.sql
+++ b/metastore/scripts/upgrade/mssql/upgrade-2.1.0-to-2.2.0.mssql.sql
@@@ -1,6 -1,6 +1,7 @@@
SELECT 'Upgrading MetaStore schema from 2.1.0 to 2.2.0' AS MESSAGE;
- :r 022-HIVE-14637.mssql.sql
+ :r 022-HIVE-14496.mssql.sql
++:r 023-HIVE-14637.mssql.sql
UPDATE VERSION SET SCHEMA_VERSION='2.2.0', VERSION_COMMENT='Hive release version 2.2.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 2.1.0 to 2.2.0' AS MESSAGE;
http://git-wip-us.apache.org/repos/asf/hive/blob/748c1bd2/metastore/scripts/upgrade/mysql/038-HIVE-14637.mysql.sql
----------------------------------------------------------------------
diff --cc metastore/scripts/upgrade/mysql/038-HIVE-14637.mysql.sql
index 0000000,0000000..9e34db2
new file mode 100644
--- /dev/null
+++ b/metastore/scripts/upgrade/mysql/038-HIVE-14637.mysql.sql
@@@ -1,0 -1,0 +1,15 @@@
++alter table `TBLS` ADD COLUMN `MM_WATERMARK_WRITE_ID` bigint(20) DEFAULT -1;
++alter table `TBLS` ADD COLUMN `MM_NEXT_WRITE_ID` bigint(20) DEFAULT 0;
++
++CREATE TABLE IF NOT EXISTS `TBL_WRITES`
++(
++ `TW_ID` BIGINT NOT NULL,
++ `TBL_ID` BIGINT NOT NULL,
++ `WRITE_ID` BIGINT NOT NULL,
++ `STATE` CHAR(1) NOT NULL,
++ `CREATED` BIGINT NOT NULL,
++ `LAST_HEARTBEAT` BIGINT NOT NULL,
++ PRIMARY KEY (`TW_ID`),
++ UNIQUE KEY `UNIQUEWRITE` (`TBL_ID`,`WRITE_ID`),
++ CONSTRAINT `TBL_WRITES_FK1` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`)
++) ENGINE=InnoDB DEFAULT CHARSET=latin1;
http://git-wip-us.apache.org/repos/asf/hive/blob/748c1bd2/metastore/scripts/upgrade/mysql/hive-schema-2.2.0.mysql.sql
----------------------------------------------------------------------
diff --cc metastore/scripts/upgrade/mysql/hive-schema-2.2.0.mysql.sql
index 3e73008,2009f1f..7696a62
--- a/metastore/scripts/upgrade/mysql/hive-schema-2.2.0.mysql.sql
+++ b/metastore/scripts/upgrade/mysql/hive-schema-2.2.0.mysql.sql
@@@ -587,8 -587,7 +587,9 @@@ CREATE TABLE IF NOT EXISTS `TBLS`
`TBL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
`VIEW_EXPANDED_TEXT` mediumtext,
`VIEW_ORIGINAL_TEXT` mediumtext,
+ `IS_REWRITE_ENABLED` bit(1) NOT NULL,
+ `MM_WATERMARK_WRITE_ID` bigint(20) DEFAULT -1,
+ `MM_NEXT_WRITE_ID` bigint(20) DEFAULT 0,
PRIMARY KEY (`TBL_ID`),
UNIQUE KEY `UNIQUETABLE` (`TBL_NAME`,`DB_ID`),
KEY `TBLS_N50` (`SD_ID`),
http://git-wip-us.apache.org/repos/asf/hive/blob/748c1bd2/metastore/scripts/upgrade/mysql/upgrade-2.1.0-to-2.2.0.mysql.sql
----------------------------------------------------------------------
diff --cc metastore/scripts/upgrade/mysql/upgrade-2.1.0-to-2.2.0.mysql.sql
index 6ac1b89,07a002f..afe17f9
--- a/metastore/scripts/upgrade/mysql/upgrade-2.1.0-to-2.2.0.mysql.sql
+++ b/metastore/scripts/upgrade/mysql/upgrade-2.1.0-to-2.2.0.mysql.sql
@@@ -1,6 -1,6 +1,7 @@@
SELECT 'Upgrading MetaStore schema from 2.1.0 to 2.2.0' AS ' ';
- SOURCE 037-HIVE-14637.mysql.sql;
+ SOURCE 037-HIVE-14496.mysql.sql;
++SOURCE 038-HIVE-14637.mysql.sql;
UPDATE VERSION SET SCHEMA_VERSION='2.2.0', VERSION_COMMENT='Hive release version 2.2.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 2.1.0 to 2.2.0' AS ' ';
http://git-wip-us.apache.org/repos/asf/hive/blob/748c1bd2/metastore/scripts/upgrade/oracle/038-HIVE-14637.oracle.sql
----------------------------------------------------------------------
diff --cc metastore/scripts/upgrade/oracle/038-HIVE-14637.oracle.sql
index 0000000,0000000..218eefe
new file mode 100644
--- /dev/null
+++ b/metastore/scripts/upgrade/oracle/038-HIVE-14637.oracle.sql
@@@ -1,0 -1,0 +1,15 @@@
++ALTER TABLE TBLS ADD MM_WATERMARK_WRITE_ID NUMBER DEFAULT -1;
++ALTER TABLE TBLS ADD MM_NEXT_WRITE_ID NUMBER DEFAULT 0;
++
++CREATE TABLE TBL_WRITES
++(
++ TW_ID NUMBER NOT NULL,
++ TBL_ID NUMBER NOT NULL,
++ WRITE_ID NUMBER NOT NULL,
++ STATE CHAR(1) NOT NULL,
++ CREATED NUMBER NOT NULL,
++ LAST_HEARTBEAT NUMBER NOT NULL
++);
++ALTER TABLE TBL_WRITES ADD CONSTRAINT TBL_WRITES_PK PRIMARY KEY (TW_ID);
++ALTER TABLE TBL_WRITES ADD CONSTRAINT TBL_WRITES_FK1 FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) INITIALLY DEFERRED ;
++CREATE UNIQUE INDEX UNIQUEWRITE ON TBL_WRITES (TBL_ID, WRITE_ID);
http://git-wip-us.apache.org/repos/asf/hive/blob/748c1bd2/metastore/scripts/upgrade/oracle/hive-schema-2.2.0.oracle.sql
----------------------------------------------------------------------
diff --cc metastore/scripts/upgrade/oracle/hive-schema-2.2.0.oracle.sql
index 5479712,bb5a934..122aa9a
--- a/metastore/scripts/upgrade/oracle/hive-schema-2.2.0.oracle.sql
+++ b/metastore/scripts/upgrade/oracle/hive-schema-2.2.0.oracle.sql
@@@ -376,8 -376,7 +376,9 @@@ CREATE TABLE TBL
TBL_TYPE VARCHAR2(128) NULL,
VIEW_EXPANDED_TEXT CLOB NULL,
VIEW_ORIGINAL_TEXT CLOB NULL,
- IS_REWRITE_ENABLED NUMBER(1) NOT NULL CHECK (IS_REWRITE_ENABLED IN (1,0))
++ IS_REWRITE_ENABLED NUMBER(1) NOT NULL CHECK (IS_REWRITE_ENABLED IN (1,0)),
+ MM_WATERMARK_WRITE_ID NUMBER DEFAULT -1,
+ MM_NEXT_WRITE_ID NUMBER DEFAULT 0
);
ALTER TABLE TBLS ADD CONSTRAINT TBLS_PK PRIMARY KEY (TBL_ID);
http://git-wip-us.apache.org/repos/asf/hive/blob/748c1bd2/metastore/scripts/upgrade/oracle/upgrade-2.1.0-to-2.2.0.oracle.sql
----------------------------------------------------------------------
diff --cc metastore/scripts/upgrade/oracle/upgrade-2.1.0-to-2.2.0.oracle.sql
index 8d841d6,b5e65b9..dd1b97c
--- a/metastore/scripts/upgrade/oracle/upgrade-2.1.0-to-2.2.0.oracle.sql
+++ b/metastore/scripts/upgrade/oracle/upgrade-2.1.0-to-2.2.0.oracle.sql
@@@ -1,6 -1,6 +1,7 @@@
SELECT 'Upgrading MetaStore schema from 2.1.0 to 2.2.0' AS Status from dual;
- @037-HIVE-14637.oracle.sql;
+ @037-HIVE-14496.oracle.sql;
++@038-HIVE-14637.oracle.sql;
UPDATE VERSION SET SCHEMA_VERSION='2.2.0', VERSION_COMMENT='Hive release version 2.2.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 2.1.0 to 2.2.0' AS Status from dual;
http://git-wip-us.apache.org/repos/asf/hive/blob/748c1bd2/metastore/scripts/upgrade/postgres/037-HIVE-14637.postgres.sql
----------------------------------------------------------------------
diff --cc metastore/scripts/upgrade/postgres/037-HIVE-14637.postgres.sql
index 0000000,0000000..310f51e
new file mode 100644
--- /dev/null
+++ b/metastore/scripts/upgrade/postgres/037-HIVE-14637.postgres.sql
@@@ -1,0 -1,0 +1,16 @@@
++
++ALTER TABLE "TBLS" ADD COLUMN "MM_WATERMARK_WRITE_ID" bigint DEFAULT -1;
++ALTER TABLE "TBLS" ADD COLUMN "MM_NEXT_WRITE_ID" bigint DEFAULT 0;
++
++CREATE TABLE "TBL_WRITES"
++(
++ "TW_ID" BIGINT NOT NULL,
++ "TBL_ID" BIGINT NOT NULL,
++ "WRITE_ID" BIGINT NOT NULL,
++ "STATE" CHAR(1) NOT NULL,
++ "CREATED" BIGINT NOT NULL,
++ "LAST_HEARTBEAT" BIGINT NOT NULL
++);
++ALTER TABLE ONLY "TBL_WRITES" ADD CONSTRAINT "TBL_WRITES_PK" PRIMARY KEY ("TW_ID");
++ALTER TABLE ONLY "TBL_WRITES" ADD CONSTRAINT "TBL_WRITES_FK1" FOREIGN KEY ("TBL_ID") REFERENCES "TBLS" ("TBL_ID") DEFERRABLE;
++ALTER TABLE ONLY "TBL_WRITES" ADD CONSTRAINT "UNIQUEWRITE" UNIQUE ("TBL_ID", "WRITE_ID");
http://git-wip-us.apache.org/repos/asf/hive/blob/748c1bd2/metastore/scripts/upgrade/postgres/hive-schema-2.2.0.postgres.sql
----------------------------------------------------------------------
diff --cc metastore/scripts/upgrade/postgres/hive-schema-2.2.0.postgres.sql
index bc865ed,0021df0..f36a111
--- a/metastore/scripts/upgrade/postgres/hive-schema-2.2.0.postgres.sql
+++ b/metastore/scripts/upgrade/postgres/hive-schema-2.2.0.postgres.sql
@@@ -373,8 -373,7 +373,9 @@@ CREATE TABLE "TBLS"
"TBL_TYPE" character varying(128) DEFAULT NULL::character varying,
"VIEW_EXPANDED_TEXT" text,
"VIEW_ORIGINAL_TEXT" text,
- "IS_REWRITE_ENABLED" boolean NOT NULL
++ "IS_REWRITE_ENABLED" boolean NOT NULL,
+ "MM_WATERMARK_WRITE_ID" bigint DEFAULT -1,
+ "MM_NEXT_WRITE_ID" bigint DEFAULT 0
);
http://git-wip-us.apache.org/repos/asf/hive/blob/748c1bd2/metastore/scripts/upgrade/postgres/upgrade-2.1.0-to-2.2.0.postgres.sql
----------------------------------------------------------------------
diff --cc metastore/scripts/upgrade/postgres/upgrade-2.1.0-to-2.2.0.postgres.sql
index 70542b8,0f7139a..a316383
--- a/metastore/scripts/upgrade/postgres/upgrade-2.1.0-to-2.2.0.postgres.sql
+++ b/metastore/scripts/upgrade/postgres/upgrade-2.1.0-to-2.2.0.postgres.sql
@@@ -1,6 -1,6 +1,7 @@@
SELECT 'Upgrading MetaStore schema from 2.1.0 to 2.2.0';
- \i 036-HIVE-14637.postgres.sql;
+ \i 036-HIVE-14496.postgres.sql;
++\i 037-HIVE-14637.postgres.sql;
UPDATE "VERSION" SET "SCHEMA_VERSION"='2.2.0', "VERSION_COMMENT"='Hive release version 2.2.0' where "VER_ID"=1;
SELECT 'Finished upgrading MetaStore schema from 2.1.0 to 2.2.0';