You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by da...@apache.org on 2017/08/09 19:36:23 UTC
[1/4] hive git commit: HIVE-15705: Event replication for constraints
(Daniel Dai, reviewed by Sankar Hariappan, Thejas Nair)
Repository: hive
Updated Branches:
refs/heads/master 16bfb9c94 -> 90a92b746
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
index 0b63a20..fdb2866 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
@@ -868,12 +868,13 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable {
}
@Override
- public void createTableWithConstraints(Table tbl,
+ public List<String> createTableWithConstraints(Table tbl,
List<SQLPrimaryKey> primaryKeys, List<SQLForeignKey> foreignKeys,
List<SQLUniqueConstraint> uniqueConstraints,
List<SQLNotNullConstraint> notNullConstraints)
throws InvalidObjectException, MetaException {
// TODO Auto-generated method stub
+ return null;
}
@Override
@@ -883,27 +884,29 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable {
}
@Override
- public void addPrimaryKeys(List<SQLPrimaryKey> pks)
+ public List<String> addPrimaryKeys(List<SQLPrimaryKey> pks)
throws InvalidObjectException, MetaException {
- // TODO Auto-generated method stub
+ return null;
}
@Override
- public void addForeignKeys(List<SQLForeignKey> fks)
+ public List<String> addForeignKeys(List<SQLForeignKey> fks)
throws InvalidObjectException, MetaException {
- // TODO Auto-generated method stub
+ return null;
}
@Override
- public void addUniqueConstraints(List<SQLUniqueConstraint> uks)
+ public List<String> addUniqueConstraints(List<SQLUniqueConstraint> uks)
throws InvalidObjectException, MetaException {
// TODO Auto-generated method stub
+ return null;
}
@Override
- public void addNotNullConstraints(List<SQLNotNullConstraint> nns)
+ public List<String> addNotNullConstraints(List<SQLNotNullConstraint> nns)
throws InvalidObjectException, MetaException {
// TODO Auto-generated method stub
+ return null;
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
index 5348011..f422c4e 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
@@ -884,12 +884,13 @@ public class DummyRawStoreForJdoConnection implements RawStore {
}
@Override
- public void createTableWithConstraints(Table tbl,
+ public List<String> createTableWithConstraints(Table tbl,
List<SQLPrimaryKey> primaryKeys, List<SQLForeignKey> foreignKeys,
List<SQLUniqueConstraint> uniqueConstraints,
List<SQLNotNullConstraint> notNullConstraints)
throws InvalidObjectException, MetaException {
// TODO Auto-generated method stub
+ return null;
}
@Override
@@ -899,26 +900,30 @@ public class DummyRawStoreForJdoConnection implements RawStore {
}
@Override
- public void addPrimaryKeys(List<SQLPrimaryKey> pks)
+ public List<String> addPrimaryKeys(List<SQLPrimaryKey> pks)
throws InvalidObjectException, MetaException {
// TODO Auto-generated method stub
+ return null;
}
@Override
- public void addForeignKeys(List<SQLForeignKey> fks)
+ public List<String> addForeignKeys(List<SQLForeignKey> fks)
throws InvalidObjectException, MetaException {
// TODO Auto-generated method stub
+ return null;
}
- public void addUniqueConstraints(List<SQLUniqueConstraint> uks)
+ public List<String> addUniqueConstraints(List<SQLUniqueConstraint> uks)
throws InvalidObjectException, MetaException {
// TODO Auto-generated method stub
+ return null;
}
@Override
- public void addNotNullConstraints(List<SQLNotNullConstraint> nns)
+ public List<String> addNotNullConstraints(List<SQLNotNullConstraint> nns)
throws InvalidObjectException, MetaException {
// TODO Auto-generated method stub
+ return null;
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 16c440f..185ac1d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -385,6 +385,12 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
AlterTableDesc alterTbl = work.getAlterTblDesc();
if (alterTbl != null) {
+ if (!allowOperationInReplicationScope(db, alterTbl.getOldName(), null, alterTbl.getReplicationSpec())) {
+ // no alter, the table is missing either due to drop/rename which follows the alter.
+ // or the existing table is newer than our update.
+ LOG.debug("DDLTask: Alter Table is skipped as table {} is newer than update", alterTbl.getOldName());
+ return 0;
+ }
if (alterTbl.getOp() == AlterTableTypes.DROPCONSTRAINT ) {
return dropConstraint(db, alterTbl);
} else if (alterTbl.getOp() == AlterTableTypes.ADDCONSTRAINT) {
@@ -3571,13 +3577,6 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
* Throws this exception if an unexpected error occurs.
*/
private int alterTable(Hive db, AlterTableDesc alterTbl) throws HiveException {
- if (!allowOperationInReplicationScope(db, alterTbl.getOldName(), null, alterTbl.getReplicationSpec())) {
- // no alter, the table is missing either due to drop/rename which follows the alter.
- // or the existing table is newer than our update.
- LOG.debug("DDLTask: Alter Table is skipped as table {} is newer than update", alterTbl.getOldName());
- return 0;
- }
-
// alter the table
Table tbl = db.getTable(alterTbl.getOldName());
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index 5b7fc25..d595de4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@ -1782,7 +1782,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
private void analyzeAlterTableDropConstraint(ASTNode ast, String tableName)
throws SemanticException {
String dropConstraintName = unescapeIdentifier(ast.getChild(0).getText());
- AlterTableDesc alterTblDesc = new AlterTableDesc(tableName, dropConstraintName);
+ AlterTableDesc alterTblDesc = new AlterTableDesc(tableName, dropConstraintName, (ReplicationSpec)null);
rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
alterTblDesc), conf));
@@ -1815,7 +1815,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
child.getToken().getText()));
}
AlterTableDesc alterTblDesc = new AlterTableDesc(tableName, primaryKeys, foreignKeys,
- uniqueConstraints);
+ uniqueConstraints, null);
rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
alterTblDesc), conf));
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/DumpType.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/DumpType.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/DumpType.java
index 0580546..e982603 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/DumpType.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/DumpType.java
@@ -17,8 +17,13 @@
*/
package org.apache.hadoop.hive.ql.parse.repl;
+import org.apache.hadoop.hive.ql.parse.repl.load.message.AddNotNullConstraintHandler;
+import org.apache.hadoop.hive.ql.parse.repl.load.message.AddForeignKeyHandler;
+import org.apache.hadoop.hive.ql.parse.repl.load.message.AddPrimaryKeyHandler;
+import org.apache.hadoop.hive.ql.parse.repl.load.message.AddUniqueConstraintHandler;
import org.apache.hadoop.hive.ql.parse.repl.load.message.CreateFunctionHandler;
import org.apache.hadoop.hive.ql.parse.repl.load.message.DefaultHandler;
+import org.apache.hadoop.hive.ql.parse.repl.load.message.DropConstraintHandler;
import org.apache.hadoop.hive.ql.parse.repl.load.message.DropFunctionHandler;
import org.apache.hadoop.hive.ql.parse.repl.load.message.DropPartitionHandler;
import org.apache.hadoop.hive.ql.parse.repl.load.message.DropTableHandler;
@@ -104,6 +109,36 @@ public enum DumpType {
return new InsertHandler();
}
},
+ EVENT_ADD_PRIMARYKEY("EVENT_ADD_PRIMARYKEY") {
+ @Override
+ public MessageHandler handler() {
+ return new AddPrimaryKeyHandler();
+ }
+ },
+ EVENT_ADD_FOREIGNKEY("EVENT_ADD_FOREIGNKEY") {
+ @Override
+ public MessageHandler handler() {
+ return new AddForeignKeyHandler();
+ }
+ },
+ EVENT_ADD_UNIQUECONSTRAINT("EVENT_ADD_UNIQUECONSTRAINT") {
+ @Override
+ public MessageHandler handler() {
+ return new AddUniqueConstraintHandler();
+ }
+ },
+ EVENT_ADD_NOTNULLCONSTRAINT("EVENT_ADD_NOTNULLCONSTRAINT") {
+ @Override
+ public MessageHandler handler() {
+ return new AddNotNullConstraintHandler();
+ }
+ },
+ EVENT_DROP_CONSTRAINT("EVENT_DROP_CONSTRAINT") {
+ @Override
+ public MessageHandler handler() {
+ return new DropConstraintHandler();
+ }
+ },
EVENT_CREATE_FUNCTION("EVENT_CREATE_FUNCTION") {
@Override
public MessageHandler handler() {
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddForeignKeyHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddForeignKeyHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddForeignKeyHandler.java
new file mode 100644
index 0000000..ccd95fe
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddForeignKeyHandler.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.events;
+
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
+
+public class AddForeignKeyHandler extends AbstractEventHandler {
+ AddForeignKeyHandler(NotificationEvent event) {
+ super(event);
+ }
+
+ @Override
+ public void handle(Context withinContext) throws Exception {
+ LOG.info("Processing#{} ADD_FOREIGNKEY_MESSAGE message : {}", fromEventId(), event.getMessage());
+ DumpMetaData dmd = withinContext.createDmd(this);
+ dmd.setPayload(event.getMessage());
+ dmd.write();
+ }
+
+ @Override
+ public DumpType dumpType() {
+ return DumpType.EVENT_ADD_FOREIGNKEY;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddNotNullConstraintHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddNotNullConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddNotNullConstraintHandler.java
new file mode 100644
index 0000000..7973eb3
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddNotNullConstraintHandler.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.events;
+
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
+
+public class AddNotNullConstraintHandler extends AbstractEventHandler {
+ AddNotNullConstraintHandler(NotificationEvent event) {
+ super(event);
+ }
+
+ @Override
+ public void handle(Context withinContext) throws Exception {
+ LOG.info("Processing#{} ADD_NOTNULLCONSTRAINT_MESSAGE message : {}", fromEventId(), event.getMessage());
+ DumpMetaData dmd = withinContext.createDmd(this);
+ dmd.setPayload(event.getMessage());
+ dmd.write();
+ }
+
+ @Override
+ public DumpType dumpType() {
+ return DumpType.EVENT_ADD_NOTNULLCONSTRAINT;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPrimaryKeyHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPrimaryKeyHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPrimaryKeyHandler.java
new file mode 100644
index 0000000..3fb3e8b
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPrimaryKeyHandler.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.events;
+
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
+
+public class AddPrimaryKeyHandler extends AbstractEventHandler {
+ AddPrimaryKeyHandler(NotificationEvent event) {
+ super(event);
+ }
+
+ @Override
+ public void handle(Context withinContext) throws Exception {
+ LOG.info("Processing#{} ADD_PRIMARYKEY_MESSAGE message : {}", fromEventId(), event.getMessage());
+ DumpMetaData dmd = withinContext.createDmd(this);
+ dmd.setPayload(event.getMessage());
+ dmd.write();
+ }
+
+ @Override
+ public DumpType dumpType() {
+ return DumpType.EVENT_ADD_PRIMARYKEY;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddUniqueConstraintHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddUniqueConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddUniqueConstraintHandler.java
new file mode 100644
index 0000000..e4f5415
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddUniqueConstraintHandler.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.events;
+
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
+
+public class AddUniqueConstraintHandler extends AbstractEventHandler {
+ AddUniqueConstraintHandler(NotificationEvent event) {
+ super(event);
+ }
+
+ @Override
+ public void handle(Context withinContext) throws Exception {
+ LOG.info("Processing#{} ADD_UNIQUECONSTRAINT_MESSAGE message : {}", fromEventId(), event.getMessage());
+ DumpMetaData dmd = withinContext.createDmd(this);
+ dmd.setPayload(event.getMessage());
+ dmd.write();
+ }
+
+ @Override
+ public DumpType dumpType() {
+ return DumpType.EVENT_ADD_UNIQUECONSTRAINT;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/DropConstraintHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/DropConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/DropConstraintHandler.java
new file mode 100644
index 0000000..6b709a6
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/DropConstraintHandler.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.events;
+
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
+
+public class DropConstraintHandler extends AbstractEventHandler {
+ DropConstraintHandler(NotificationEvent event) {
+ super(event);
+ }
+
+ @Override
+ public void handle(Context withinContext) throws Exception {
+ LOG.info("Processing#{} DROP_CONSTRAINT_MESSAGE message : {}", fromEventId(), event.getMessage());
+ DumpMetaData dmd = withinContext.createDmd(this);
+ dmd.setPayload(event.getMessage());
+ dmd.write();
+ }
+
+ @Override
+ public DumpType dumpType() {
+ return DumpType.EVENT_DROP_CONSTRAINT;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/EventHandlerFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/EventHandlerFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/EventHandlerFactory.java
index 7e655fa..4a68235 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/EventHandlerFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/EventHandlerFactory.java
@@ -42,6 +42,11 @@ public class EventHandlerFactory {
register(MessageFactory.DROP_TABLE_EVENT, DropTableHandler.class);
register(MessageFactory.INSERT_EVENT, InsertHandler.class);
register(MessageFactory.DROP_FUNCTION_EVENT, DropFunctionHandler.class);
+ register(MessageFactory.ADD_PRIMARYKEY_EVENT, AddPrimaryKeyHandler.class);
+ register(MessageFactory.ADD_FOREIGNKEY_EVENT, AddForeignKeyHandler.class);
+ register(MessageFactory.ADD_UNIQUECONSTRAINT_EVENT, AddUniqueConstraintHandler.class);
+ register(MessageFactory.ADD_NOTNULLCONSTRAINT_EVENT, AddNotNullConstraintHandler.class);
+ register(MessageFactory.DROP_CONSTRAINT_EVENT, DropConstraintHandler.class);
}
static void register(String event, Class<? extends EventHandler> handlerClazz) {
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java
index f12bd13..24906a9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java
@@ -66,9 +66,11 @@ public class DumpMetaData {
}
private void loadDumpFromFile() throws SemanticException {
- try (FileSystem fs = dumpFile.getFileSystem(hiveConf); BufferedReader br =
- new BufferedReader(new InputStreamReader(fs.open(dumpFile)))) {
+ BufferedReader br = null;
+ try {
// read from dumpfile and instantiate self
+ FileSystem fs = dumpFile.getFileSystem(hiveConf);
+ br = new BufferedReader(new InputStreamReader(fs.open(dumpFile)));
String line = null;
if ((line = br.readLine()) != null) {
String[] lineContents = line.split("\t", 5);
@@ -83,6 +85,14 @@ public class DumpMetaData {
}
} catch (IOException ioe) {
throw new SemanticException(ioe);
+ } finally {
+ if (br != null) {
+ try {
+ br.close();
+ } catch (IOException e) {
+ throw new SemanticException(e);
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddForeignKeyHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddForeignKeyHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddForeignKeyHandler.java
new file mode 100644
index 0000000..39697bb
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddForeignKeyHandler.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load.message;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
+import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
+import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
+import org.apache.hadoop.hive.metastore.messaging.AddForeignKeyMessage;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.TaskFactory;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.AlterTableDesc;
+import org.apache.hadoop.hive.ql.plan.DDLWork;
+
+public class AddForeignKeyHandler extends AbstractMessageHandler {
+ @Override
+ public List<Task<? extends Serializable>> handle(Context context)
+ throws SemanticException {
+ AddForeignKeyMessage msg = deserializer.getAddForeignKeyMessage(context.dmd.getPayload());
+
+ List<SQLForeignKey> fks = null;
+ try {
+ fks = msg.getForeignKeys();
+ } catch (Exception e) {
+ if (!(e instanceof SemanticException)){
+ throw new SemanticException("Error reading message members", e);
+ } else {
+ throw (SemanticException)e;
+ }
+ }
+
+ String actualDbName = context.isDbNameEmpty() ? fks.get(0).getFktable_db() : context.dbName;
+ String actualTblName = context.isTableNameEmpty() ? fks.get(0).getPktable_name() : context.tableName;
+
+ for (SQLForeignKey fk : fks) {
+ fk.setPktable_db(actualDbName);
+ fk.setPktable_name(actualTblName);
+ fk.setFktable_db(actualDbName);
+ }
+
+ AlterTableDesc addConstraintsDesc = new AlterTableDesc(actualDbName + "." + actualTblName, new ArrayList<SQLPrimaryKey>(), fks,
+ new ArrayList<SQLUniqueConstraint>(), context.eventOnlyReplicationSpec());
+ Task<DDLWork> addConstraintsTask = TaskFactory.get(new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc), context.hiveConf);
+ List<Task<? extends Serializable>> tasks = new ArrayList<Task<? extends Serializable>>();
+ tasks.add(addConstraintsTask);
+ context.log.debug("Added add constrains task : {}:{}", addConstraintsTask.getId(), actualTblName);
+ databasesUpdated.put(actualDbName, context.dmd.getEventTo());
+ tablesUpdated.put(actualDbName + "." + actualTblName, context.dmd.getEventTo());
+ return Collections.singletonList(addConstraintsTask);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddNotNullConstraintHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddNotNullConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddNotNullConstraintHandler.java
new file mode 100644
index 0000000..e2c1d1d
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddNotNullConstraintHandler.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load.message;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
+import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
+import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
+import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
+import org.apache.hadoop.hive.metastore.messaging.AddNotNullConstraintMessage;
+import org.apache.hadoop.hive.metastore.messaging.AddUniqueConstraintMessage;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.TaskFactory;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.AlterTableDesc;
+import org.apache.hadoop.hive.ql.plan.DDLWork;
+
+public class AddNotNullConstraintHandler extends AbstractMessageHandler {
+ @Override
+ public List<Task<? extends Serializable>> handle(Context context)
+ throws SemanticException {
+ AddNotNullConstraintMessage msg = deserializer.getAddNotNullConstraintMessage(context.dmd.getPayload());
+
+ List<SQLNotNullConstraint> nns = null;
+ try {
+ nns = msg.getNotNullConstraints();
+ } catch (Exception e) {
+ if (!(e instanceof SemanticException)){
+ throw new SemanticException("Error reading message members", e);
+ } else {
+ throw (SemanticException)e;
+ }
+ }
+
+ String actualDbName = context.isDbNameEmpty() ? nns.get(0).getTable_db() : context.dbName;
+ String actualTblName = context.isTableNameEmpty() ? nns.get(0).getTable_name() : context.tableName;
+
+ for (SQLNotNullConstraint nn : nns) {
+ nn.setTable_db(actualDbName);
+ nn.setTable_name(actualTblName);
+ }
+
+ AlterTableDesc addConstraintsDesc = new AlterTableDesc(actualDbName + "." + actualTblName, new ArrayList<SQLPrimaryKey>(), new ArrayList<SQLForeignKey>(),
+ new ArrayList<SQLUniqueConstraint>(), nns, context.eventOnlyReplicationSpec());
+ Task<DDLWork> addConstraintsTask = TaskFactory.get(new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc), context.hiveConf);
+ List<Task<? extends Serializable>> tasks = new ArrayList<Task<? extends Serializable>>();
+ tasks.add(addConstraintsTask);
+ context.log.debug("Added add constrains task : {}:{}", addConstraintsTask.getId(), actualTblName);
+ databasesUpdated.put(actualDbName, context.dmd.getEventTo());
+ tablesUpdated.put(actualDbName + "." + actualTblName, context.dmd.getEventTo());
+ return Collections.singletonList(addConstraintsTask);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddPrimaryKeyHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddPrimaryKeyHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddPrimaryKeyHandler.java
new file mode 100644
index 0000000..7babb6a
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddPrimaryKeyHandler.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load.message;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
+import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
+import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
+import org.apache.hadoop.hive.metastore.messaging.AddPrimaryKeyMessage;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.TaskFactory;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.AlterTableDesc;
+import org.apache.hadoop.hive.ql.plan.DDLWork;
+
+public class AddPrimaryKeyHandler extends AbstractMessageHandler {
+ @Override
+ public List<Task<? extends Serializable>> handle(Context context)
+ throws SemanticException {
+ AddPrimaryKeyMessage msg = deserializer.getAddPrimaryKeyMessage(context.dmd.getPayload());
+
+ List<SQLPrimaryKey> pks = null;
+ try {
+ pks = msg.getPrimaryKeys();
+ } catch (Exception e) {
+ if (!(e instanceof SemanticException)){
+ throw new SemanticException("Error reading message members", e);
+ } else {
+ throw (SemanticException)e;
+ }
+ }
+ String actualDbName = context.isDbNameEmpty() ? pks.get(0).getTable_db() : context.dbName;
+ String actualTblName = context.isTableNameEmpty() ? pks.get(0).getTable_name() : context.tableName;
+
+ for (SQLPrimaryKey pk : pks) {
+ pk.setTable_db(actualDbName);
+ pk.setTable_name(actualTblName);
+ }
+
+ AlterTableDesc addConstraintsDesc = new AlterTableDesc(actualDbName + "." + actualTblName, pks, new ArrayList<SQLForeignKey>(),
+ new ArrayList<SQLUniqueConstraint>(), context.eventOnlyReplicationSpec());
+ Task<DDLWork> addConstraintsTask = TaskFactory.get(new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc), context.hiveConf);
+ List<Task<? extends Serializable>> tasks = new ArrayList<Task<? extends Serializable>>();
+ tasks.add(addConstraintsTask);
+ context.log.debug("Added add constrains task : {}:{}", addConstraintsTask.getId(), actualTblName);
+ databasesUpdated.put(actualDbName, context.dmd.getEventTo());
+ tablesUpdated.put(actualDbName + "." + actualTblName, context.dmd.getEventTo());
+ return Collections.singletonList(addConstraintsTask);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddUniqueConstraintHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddUniqueConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddUniqueConstraintHandler.java
new file mode 100644
index 0000000..e7b404a
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddUniqueConstraintHandler.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load.message;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
+import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
+import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
+import org.apache.hadoop.hive.metastore.messaging.AddUniqueConstraintMessage;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.TaskFactory;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.AlterTableDesc;
+import org.apache.hadoop.hive.ql.plan.DDLWork;
+
+public class AddUniqueConstraintHandler extends AbstractMessageHandler {
+ @Override
+ public List<Task<? extends Serializable>> handle(Context context)
+ throws SemanticException {
+ AddUniqueConstraintMessage msg = deserializer.getAddUniqueConstraintMessage(context.dmd.getPayload());
+
+ List<SQLUniqueConstraint> uks = null;
+ try {
+ uks = msg.getUniqueConstraints();
+ } catch (Exception e) {
+ if (!(e instanceof SemanticException)){
+ throw new SemanticException("Error reading message members", e);
+ } else {
+ throw (SemanticException)e;
+ }
+ }
+
+ String actualDbName = context.isDbNameEmpty() ? uks.get(0).getTable_db() : context.dbName;
+ String actualTblName = context.isTableNameEmpty() ? uks.get(0).getTable_name() : context.tableName;
+
+ for (SQLUniqueConstraint uk : uks) {
+ uk.setTable_db(actualDbName);
+ uk.setTable_name(actualTblName);
+ }
+
+ AlterTableDesc addConstraintsDesc = new AlterTableDesc(actualDbName + "." + actualTblName, new ArrayList<SQLPrimaryKey>(), new ArrayList<SQLForeignKey>(),
+ uks, context.eventOnlyReplicationSpec());
+ Task<DDLWork> addConstraintsTask = TaskFactory.get(new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc), context.hiveConf);
+ List<Task<? extends Serializable>> tasks = new ArrayList<Task<? extends Serializable>>();
+ tasks.add(addConstraintsTask);
+ context.log.debug("Added add constrains task : {}:{}", addConstraintsTask.getId(), actualTblName);
+ databasesUpdated.put(actualDbName, context.dmd.getEventTo());
+ tablesUpdated.put(actualDbName + "." + actualTblName, context.dmd.getEventTo());
+ return Collections.singletonList(addConstraintsTask);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropConstraintHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropConstraintHandler.java
new file mode 100644
index 0000000..58aa214
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropConstraintHandler.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load.message;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.messaging.DropConstraintMessage;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.TaskFactory;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.AlterTableDesc;
+import org.apache.hadoop.hive.ql.plan.DDLWork;
+
+public class DropConstraintHandler extends AbstractMessageHandler {
+ @Override
+ public List<Task<? extends Serializable>> handle(Context context)
+ throws SemanticException {
+ DropConstraintMessage msg = deserializer.getDropConstraintMessage(context.dmd.getPayload());
+ String actualDbName = context.isDbNameEmpty() ? msg.getDB() : context.dbName;
+ String actualTblName = context.isTableNameEmpty() ? msg.getTable() : context.tableName;
+ String constraintName = msg.getConstraint();
+
+ AlterTableDesc dropConstraintsDesc = new AlterTableDesc(actualDbName + "." + actualTblName, constraintName,
+ context.eventOnlyReplicationSpec());
+ Task<DDLWork> dropConstraintsTask = TaskFactory.get(new DDLWork(readEntitySet, writeEntitySet, dropConstraintsDesc), context.hiveConf);
+ List<Task<? extends Serializable>> tasks = new ArrayList<Task<? extends Serializable>>();
+ tasks.add(dropConstraintsTask);
+ context.log.debug("Added drop constrain task : {}:{}", dropConstraintsTask.getId(), actualTblName);
+ databasesUpdated.put(actualDbName, context.dmd.getEventTo());
+ tablesUpdated.put(actualDbName + "." + actualTblName, context.dmd.getEventTo());
+ return Collections.singletonList(dropConstraintsTask);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
index 2691faa..65e375f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
@@ -322,18 +322,33 @@ public class AlterTableDesc extends DDLDesc implements Serializable {
this.numberBuckets = numBuckets;
}
- public AlterTableDesc(String tableName, String dropConstraintName) {
+ public AlterTableDesc(String tableName, String dropConstraintName, ReplicationSpec replicationSpec) {
this.oldName = tableName;
this.dropConstraintName = dropConstraintName;
+ this.replicationSpec = replicationSpec;
op = AlterTableTypes.DROPCONSTRAINT;
}
public AlterTableDesc(String tableName, List<SQLPrimaryKey> primaryKeyCols,
- List<SQLForeignKey> foreignKeyCols, List<SQLUniqueConstraint> uniqueConstraintCols) {
+ List<SQLForeignKey> foreignKeyCols, List<SQLUniqueConstraint> uniqueConstraintCols,
+ ReplicationSpec replicationSpec) {
this.oldName = tableName;
this.primaryKeyCols = primaryKeyCols;
this.foreignKeyCols = foreignKeyCols;
this.uniqueConstraintCols = uniqueConstraintCols;
+ this.replicationSpec = replicationSpec;
+ op = AlterTableTypes.ADDCONSTRAINT;
+ }
+
+ public AlterTableDesc(String tableName, List<SQLPrimaryKey> primaryKeyCols,
+ List<SQLForeignKey> foreignKeyCols, List<SQLUniqueConstraint> uniqueConstraintCols,
+ List<SQLNotNullConstraint> notNullConstraintCols, ReplicationSpec replicationSpec) {
+ this.oldName = tableName;
+ this.primaryKeyCols = primaryKeyCols;
+ this.foreignKeyCols = foreignKeyCols;
+ this.uniqueConstraintCols = uniqueConstraintCols;
+ this.notNullConstraintCols = notNullConstraintCols;
+ this.replicationSpec = replicationSpec;
op = AlterTableTypes.ADDCONSTRAINT;
}
[3/4] hive git commit: HIVE-15705: Event replication for constraints
(Daniel Dai, reviewed by Sankar Hariappan, Thejas Nair)
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
index cf2b517..b020351 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hive.ql.parse;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.cli.CliSessionState;
@@ -26,11 +27,20 @@ import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore;
import org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore.BehaviourInjection;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.ObjectStore;
+import org.apache.hadoop.hive.metastore.api.ForeignKeysRequest;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.NotNullConstraintsRequest;
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PrimaryKeysRequest;
+import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
+import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
+import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
+import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.UniqueConstraintsRequest;
import org.apache.hadoop.hive.metastore.messaging.MessageFactory;
import org.apache.hadoop.hive.metastore.messaging.event.filters.AndFilter;
import org.apache.hadoop.hive.metastore.messaging.event.filters.DatabaseAndTableFilter;
@@ -79,15 +89,18 @@ public class TestReplicationScenarios {
"org.apache.hive.hcatalog.listener.DbNotificationListener";
// FIXME : replace with hive copy once that is copied
private final static String tid =
- TestReplicationScenarios.class.getCanonicalName().replace('.','_') + "_" + System.currentTimeMillis();
+ TestReplicationScenarios.class.getCanonicalName().toLowerCase().replace('.','_') + "_" + System.currentTimeMillis();
private final static String TEST_PATH =
System.getProperty("test.warehouse.dir", "/tmp") + Path.SEPARATOR + tid;
private static HiveConf hconf;
- private static boolean useExternalMS = false;
private static int msPort;
private static Driver driver;
private static HiveMetaStoreClient metaStoreClient;
+ static HiveConf hconfMirror;
+ static int msPortMirror;
+ static Driver driverMirror;
+ static HiveMetaStoreClient metaStoreClientMirror;
@Rule
public TestRule replV1BackwardCompatibleRule =
@@ -111,7 +124,6 @@ public class TestReplicationScenarios {
String metastoreUri = System.getProperty("test."+HiveConf.ConfVars.METASTOREURIS.varname);
if (metastoreUri != null) {
hconf.setVar(HiveConf.ConfVars.METASTOREURIS, metastoreUri);
- useExternalMS = true;
return;
}
@@ -140,6 +152,18 @@ public class TestReplicationScenarios {
driver = new Driver(hconf);
SessionState.start(new CliSessionState(hconf));
metaStoreClient = new HiveMetaStoreClient(hconf);
+
+ FileUtils.deleteDirectory(new File("metastore_db2"));
+ HiveConf hconfMirrorServer = new HiveConf();
+ hconfMirrorServer.set(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, "jdbc:derby:;databaseName=metastore_db2;create=true");
+ msPortMirror = MetaStoreUtils.startMetaStore(hconfMirrorServer);
+ hconfMirror = new HiveConf(hconf);
+ hconfMirror.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:"
+ + msPortMirror);
+ driverMirror = new Driver(hconfMirror);
+ metaStoreClientMirror = new HiveMetaStoreClient(hconfMirror);
+
+ ObjectStore.setTwoMetastoreTesting(true);
}
@AfterClass
@@ -150,6 +174,7 @@ public class TestReplicationScenarios {
@Before
public void setUp(){
// before each test
+ SessionState.get().setCurrentDatabase("default");
}
@After
@@ -199,18 +224,18 @@ public class TestReplicationScenarios {
if (null != limit) {
dumpCmd = dumpCmd + " LIMIT " + limit;
}
- run(dumpCmd);
- String dumpLocation = getResult(0, 0);
- String lastReplId = getResult(0, 1, true);
+ run(dumpCmd, driver);
+ String dumpLocation = getResult(0, 0, driver);
+ String lastReplId = getResult(0, 1, true, driver);
LOG.info("Dumped to {} with id {} for command: {}", dumpLocation, lastReplId, dumpCmd);
return new Tuple(dumpLocation, lastReplId);
}
private void loadAndVerify(String replDbName, String dumpLocation, String lastReplId) throws IOException {
- run("EXPLAIN REPL LOAD " + replDbName + " FROM '" + dumpLocation + "'");
- printOutput();
- run("REPL LOAD " + replDbName + " FROM '" + dumpLocation + "'");
- verifyRun("REPL STATUS " + replDbName, lastReplId);
+ run("EXPLAIN REPL LOAD " + replDbName + " FROM '" + dumpLocation + "'", driverMirror);
+ printOutput(driverMirror);
+ run("REPL LOAD " + replDbName + " FROM '" + dumpLocation + "'", driverMirror);
+ verifyRun("REPL STATUS " + replDbName, lastReplId, driverMirror);
return;
}
@@ -223,11 +248,11 @@ public class TestReplicationScenarios {
@Test
public void testBasic() throws IOException {
String name = testName.getMethodName();
- String dbName = createDB(name);
- run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE");
- run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE");
- run("CREATE TABLE " + dbName + ".unptned_empty(a string) STORED AS TEXTFILE");
- run("CREATE TABLE " + dbName + ".ptned_empty(a string) partitioned by (b int) STORED AS TEXTFILE");
+ String dbName = createDB(name, driver);
+ run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver);
+ run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE", driver);
+ run("CREATE TABLE " + dbName + ".unptned_empty(a string) STORED AS TEXTFILE", driver);
+ run("CREATE TABLE " + dbName + ".ptned_empty(a string) partitioned by (b int) STORED AS TEXTFILE", driver);
String[] unptn_data = new String[]{ "eleven" , "twelve" };
String[] ptn_data_1 = new String[]{ "thirteen", "fourteen", "fifteen"};
@@ -242,33 +267,33 @@ public class TestReplicationScenarios {
createTestDataFile(ptn_locn_1, ptn_data_1);
createTestDataFile(ptn_locn_2, ptn_data_2);
- run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + dbName + ".unptned");
- verifySetup("SELECT * from " + dbName + ".unptned", unptn_data);
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=1)");
- verifySetup("SELECT a from " + dbName + ".ptned WHERE b=1", ptn_data_1);
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=2)");
- verifySetup("SELECT a from " + dbName + ".ptned WHERE b=2", ptn_data_2);
- verifySetup("SELECT a from " + dbName + ".ptned_empty", empty);
- verifySetup("SELECT * from " + dbName + ".unptned_empty", empty);
+ run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + dbName + ".unptned", driver);
+ verifySetup("SELECT * from " + dbName + ".unptned", unptn_data, driver);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=1)", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=1", ptn_data_1, driver);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=2)", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=2", ptn_data_2, driver);
+ verifySetup("SELECT a from " + dbName + ".ptned_empty", empty, driver);
+ verifySetup("SELECT * from " + dbName + ".unptned_empty", empty, driver);
String replicatedDbName = dbName + "_dupe";
bootstrapLoadAndVerify(dbName, replicatedDbName);
- verifyRun("SELECT * from " + replicatedDbName + ".unptned", unptn_data);
- verifyRun("SELECT a from " + replicatedDbName + ".ptned WHERE b=1", ptn_data_1);
- verifyRun("SELECT a from " + replicatedDbName + ".ptned WHERE b=2", ptn_data_2);
- verifyRun("SELECT a from " + dbName + ".ptned_empty", empty);
- verifyRun("SELECT * from " + dbName + ".unptned_empty", empty);
+ verifyRun("SELECT * from " + replicatedDbName + ".unptned", unptn_data, driverMirror);
+ verifyRun("SELECT a from " + replicatedDbName + ".ptned WHERE b=1", ptn_data_1, driverMirror);
+ verifyRun("SELECT a from " + replicatedDbName + ".ptned WHERE b=2", ptn_data_2, driverMirror);
+ verifyRun("SELECT a from " + dbName + ".ptned_empty", empty, driverMirror);
+ verifyRun("SELECT * from " + dbName + ".unptned_empty", empty, driverMirror);
}
@Test
public void testBasicWithCM() throws Exception {
String name = testName.getMethodName();
- String dbName = createDB(name);
- run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE");
- run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE");
- run("CREATE TABLE " + dbName + ".unptned_empty(a string) STORED AS TEXTFILE");
- run("CREATE TABLE " + dbName + ".ptned_empty(a string) partitioned by (b int) STORED AS TEXTFILE");
+ String dbName = createDB(name, driver);
+ run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver);
+ run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE", driver);
+ run("CREATE TABLE " + dbName + ".unptned_empty(a string) STORED AS TEXTFILE", driver);
+ run("CREATE TABLE " + dbName + ".ptned_empty(a string) partitioned by (b int) STORED AS TEXTFILE", driver);
String[] unptn_data = new String[]{ "eleven" , "twelve" };
String[] ptn_data_1 = new String[]{ "thirteen", "fourteen", "fifteen"};
@@ -286,29 +311,29 @@ public class TestReplicationScenarios {
createTestDataFile(ptn_locn_2, ptn_data_2);
createTestDataFile(ptn_locn_2_later, ptn_data_2_later);
- run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + dbName + ".unptned");
- run("SELECT * from " + dbName + ".unptned");
- verifyResults(unptn_data);
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=1)");
- run("SELECT a from " + dbName + ".ptned WHERE b=1");
- verifyResults(ptn_data_1);
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=2)");
- run("SELECT a from " + dbName + ".ptned WHERE b=2");
- verifyResults(ptn_data_2);
- run("SELECT a from " + dbName + ".ptned_empty");
- verifyResults(empty);
- run("SELECT * from " + dbName + ".unptned_empty");
- verifyResults(empty);
-
- advanceDumpDir();
- run("REPL DUMP " + dbName);
- String replDumpLocn = getResult(0,0);
- String replDumpId = getResult(0,1,true);
+ run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + dbName + ".unptned", driver);
+ run("SELECT * from " + dbName + ".unptned", driver);
+ verifyResults(unptn_data, driver);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=1)", driver);
+ run("SELECT a from " + dbName + ".ptned WHERE b=1", driver);
+ verifyResults(ptn_data_1, driver);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=2)", driver);
+ run("SELECT a from " + dbName + ".ptned WHERE b=2", driver);
+ verifyResults(ptn_data_2, driver);
+ run("SELECT a from " + dbName + ".ptned_empty", driver);
+ verifyResults(empty, driver);
+ run("SELECT * from " + dbName + ".unptned_empty", driver);
+ verifyResults(empty, driver);
+
+ advanceDumpDir();
+ run("REPL DUMP " + dbName, driver);
+ String replDumpLocn = getResult(0,0, driver);
+ String replDumpId = getResult(0,1,true, driver);
// Table dropped after "repl dump"
- run("DROP TABLE " + dbName + ".unptned");
+ run("DROP TABLE " + dbName + ".unptned", driver);
// Partition droppped after "repl dump"
- run("ALTER TABLE " + dbName + ".ptned " + "DROP PARTITION(b=1)");
+ run("ALTER TABLE " + dbName + ".ptned " + "DROP PARTITION(b=1)", driver);
// File changed after "repl dump"
Partition p = metaStoreClient.getPartition(dbName, "ptned", "b=2");
Path loc = new Path(p.getSd().getLocation());
@@ -317,25 +342,25 @@ public class TestReplicationScenarios {
fs.delete(file, false);
fs.copyFromLocalFile(new Path(ptn_locn_2_later), file);
- run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'");
- printOutput();
- run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'");
+ run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'", driverMirror);
+ printOutput(driverMirror);
+ run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'", driverMirror);
- run("REPL STATUS " + dbName + "_dupe");
- verifyResults(new String[] {replDumpId});
+ run("REPL STATUS " + dbName + "_dupe", driverMirror);
+ verifyResults(new String[] {replDumpId}, driverMirror);
- run("SELECT * from " + dbName + "_dupe.unptned");
- verifyResults(unptn_data);
- run("SELECT a from " + dbName + "_dupe.ptned WHERE b=1");
- verifyResults(ptn_data_1);
+ run("SELECT * from " + dbName + "_dupe.unptned", driverMirror);
+ verifyResults(unptn_data, driverMirror);
+ run("SELECT a from " + dbName + "_dupe.ptned WHERE b=1", driverMirror);
+ verifyResults(ptn_data_1, driverMirror);
// Since partition(b=2) changed manually, Hive cannot find
// it in original location and cmroot, thus empty
- run("SELECT a from " + dbName + "_dupe.ptned WHERE b=2");
- verifyResults(empty);
- run("SELECT a from " + dbName + ".ptned_empty");
- verifyResults(empty);
- run("SELECT * from " + dbName + ".unptned_empty");
- verifyResults(empty);
+ run("SELECT a from " + dbName + "_dupe.ptned WHERE b=2", driverMirror);
+ verifyResults(empty, driverMirror);
+ run("SELECT a from " + dbName + ".ptned_empty", driverMirror);
+ verifyResults(empty, driverMirror);
+ run("SELECT * from " + dbName + ".unptned_empty", driverMirror);
+ verifyResults(empty, driverMirror);
}
@Test
@@ -344,58 +369,58 @@ public class TestReplicationScenarios {
LOG.info("Testing "+testName);
String dbName = testName + "_" + tid;
- run("CREATE DATABASE " + dbName);
- run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE");
+ run("CREATE DATABASE " + dbName, driver);
+ run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver);
String[] unptn_data = new String[]{ "eleven" , "twelve" };
String unptn_locn = new Path(TEST_PATH , testName + "_unptn").toUri().getPath();
createTestDataFile(unptn_locn, unptn_data);
- run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + dbName + ".unptned");
- verifySetup("SELECT * from " + dbName + ".unptned ORDER BY a", unptn_data);
+ run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + dbName + ".unptned", driver);
+ verifySetup("SELECT * from " + dbName + ".unptned ORDER BY a", unptn_data, driver);
// Create an empty database to load
- run("CREATE DATABASE " + dbName + "_empty");
+ run("CREATE DATABASE " + dbName + "_empty", driverMirror);
advanceDumpDir();
- run("REPL DUMP " + dbName);
- String replDumpLocn = getResult(0,0);
- String replDumpId = getResult(0,1,true);
+ run("REPL DUMP " + dbName, driver);
+ String replDumpLocn = getResult(0,0,driver);
+ String replDumpId = getResult(0,1,true,driver);
// Load to an empty database
- run("REPL LOAD " + dbName + "_empty FROM '" + replDumpLocn + "'");
+ run("REPL LOAD " + dbName + "_empty FROM '" + replDumpLocn + "'", driverMirror);
// REPL STATUS should return same repl ID as dump
- verifyRun("REPL STATUS " + dbName + "_empty", replDumpId);
- verifyRun("SELECT * from " + dbName + "_empty.unptned", unptn_data);
+ verifyRun("REPL STATUS " + dbName + "_empty", replDumpId, driverMirror);
+ verifyRun("SELECT * from " + dbName + "_empty.unptned", unptn_data, driverMirror);
String[] nullReplId = new String[]{ "NULL" };
// Create a database with a table
- run("CREATE DATABASE " + dbName + "_withtable");
- run("CREATE TABLE " + dbName + "_withtable.unptned(a string) STORED AS TEXTFILE");
+ run("CREATE DATABASE " + dbName + "_withtable", driverMirror);
+ run("CREATE TABLE " + dbName + "_withtable.unptned(a string) STORED AS TEXTFILE", driverMirror);
// Load using same dump to a DB with table. It should fail as DB is not empty.
- verifyFail("REPL LOAD " + dbName + "_withtable FROM '" + replDumpLocn + "'");
+ verifyFail("REPL LOAD " + dbName + "_withtable FROM '" + replDumpLocn + "'", driverMirror);
// REPL STATUS should return NULL
- verifyRun("REPL STATUS " + dbName + "_withtable", nullReplId);
+ verifyRun("REPL STATUS " + dbName + "_withtable", nullReplId, driverMirror);
// Create a database with a view
- run("CREATE DATABASE " + dbName + "_withview");
- run("CREATE TABLE " + dbName + "_withview.unptned(a string) STORED AS TEXTFILE");
- run("CREATE VIEW " + dbName + "_withview.view AS SELECT * FROM " + dbName + "_withview.unptned");
+ run("CREATE DATABASE " + dbName + "_withview", driverMirror);
+ run("CREATE TABLE " + dbName + "_withview.unptned(a string) STORED AS TEXTFILE", driverMirror);
+ run("CREATE VIEW " + dbName + "_withview.view AS SELECT * FROM " + dbName + "_withview.unptned", driverMirror);
// Load using same dump to a DB with view. It should fail as DB is not empty.
- verifyFail("REPL LOAD " + dbName + "_withview FROM '" + replDumpLocn + "'");
+ verifyFail("REPL LOAD " + dbName + "_withview FROM '" + replDumpLocn + "'", driverMirror);
// REPL STATUS should return NULL
- verifyRun("REPL STATUS " + dbName + "_withview", nullReplId);
+ verifyRun("REPL STATUS " + dbName + "_withview", nullReplId, driverMirror);
}
@Test
public void testBootstrapWithConcurrentDropTable() throws IOException {
String name = testName.getMethodName();
- String dbName = createDB(name);
- run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE");
- run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE");
+ String dbName = createDB(name, driver);
+ run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver);
+ run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE", driver);
String[] unptn_data = new String[]{ "eleven" , "twelve" };
String[] ptn_data_1 = new String[]{ "thirteen", "fourteen", "fifteen"};
@@ -410,12 +435,12 @@ public class TestReplicationScenarios {
createTestDataFile(ptn_locn_1, ptn_data_1);
createTestDataFile(ptn_locn_2, ptn_data_2);
- run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + dbName + ".unptned");
- verifySetup("SELECT * from " + dbName + ".unptned", unptn_data);
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=1)");
- verifySetup("SELECT a from " + dbName + ".ptned WHERE b=1", ptn_data_1);
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=2)");
- verifySetup("SELECT a from " + dbName + ".ptned WHERE b=2", ptn_data_2);
+ run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + dbName + ".unptned", driver);
+ verifySetup("SELECT * from " + dbName + ".unptned", unptn_data, driver);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=1)", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=1", ptn_data_1, driver);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=2)", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=2", ptn_data_2, driver);
advanceDumpDir();
@@ -435,41 +460,41 @@ public class TestReplicationScenarios {
InjectableBehaviourObjectStore.setGetTableBehaviour(ptnedTableNuller);
// The ptned table will not be dumped as getTable will return null
- run("REPL DUMP " + dbName);
+ run("REPL DUMP " + dbName, driver);
ptnedTableNuller.assertInjectionsPerformed(true,true);
InjectableBehaviourObjectStore.resetGetTableBehaviour(); // reset the behaviour
- String replDumpLocn = getResult(0, 0);
- String replDumpId = getResult(0, 1, true);
+ String replDumpLocn = getResult(0, 0, driver);
+ String replDumpId = getResult(0, 1, true, driver);
LOG.info("Bootstrap-Dump: Dumped to {} with id {}", replDumpLocn, replDumpId);
- run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'");
+ run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'", driverMirror);
// The ptned table should miss in target as the table was marked virtually as dropped
- verifyRun("SELECT * from " + dbName + "_dupe.unptned", unptn_data);
- verifyFail("SELECT a from " + dbName + "_dupe.ptned WHERE b=1");
- verifyIfTableNotExist(dbName + "_dupe", "ptned");
+ verifyRun("SELECT * from " + dbName + "_dupe.unptned", unptn_data, driverMirror);
+ verifyFail("SELECT a from " + dbName + "_dupe.ptned WHERE b=1", driverMirror);
+ verifyIfTableNotExist(dbName + "_dupe", "ptned", metaStoreClient);
// Verify if Drop table on a non-existing table is idempotent
- run("DROP TABLE " + dbName + ".ptned");
- verifyIfTableNotExist(dbName, "ptned");
+ run("DROP TABLE " + dbName + ".ptned", driver);
+ verifyIfTableNotExist(dbName, "ptned", metaStoreClient);
advanceDumpDir();
- run("REPL DUMP " + dbName + " FROM " + replDumpId);
- String postDropReplDumpLocn = getResult(0,0);
- String postDropReplDumpId = getResult(0,1,true);
+ run("REPL DUMP " + dbName + " FROM " + replDumpId, driver);
+ String postDropReplDumpLocn = getResult(0,0, driver);
+ String postDropReplDumpId = getResult(0,1,true,driver);
LOG.info("Dumped to {} with id {}->{}", postDropReplDumpLocn, replDumpId, postDropReplDumpId);
- assert(run("REPL LOAD " + dbName + "_dupe FROM '" + postDropReplDumpLocn + "'", true));
+ assert(run("REPL LOAD " + dbName + "_dupe FROM '" + postDropReplDumpLocn + "'", true, driverMirror));
- verifyRun("SELECT * from " + dbName + "_dupe.unptned", unptn_data);
- verifyIfTableNotExist(dbName + "_dupe", "ptned");
- verifyFail("SELECT a from " + dbName + "_dupe.ptned WHERE b=1");
+ verifyRun("SELECT * from " + dbName + "_dupe.unptned", unptn_data, driverMirror);
+ verifyIfTableNotExist(dbName + "_dupe", "ptned", metaStoreClientMirror);
+ verifyFail("SELECT a from " + dbName + "_dupe.ptned WHERE b=1", driverMirror);
}
@Test
public void testBootstrapWithConcurrentDropPartition() throws IOException {
String name = testName.getMethodName();
- String dbName = createDB(name);
- run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE");
+ String dbName = createDB(name, driver);
+ run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE", driver);
String[] ptn_data_1 = new String[]{ "thirteen", "fourteen", "fifteen"};
String[] ptn_data_2 = new String[]{ "fifteen", "sixteen", "seventeen"};
@@ -481,10 +506,10 @@ public class TestReplicationScenarios {
createTestDataFile(ptn_locn_1, ptn_data_1);
createTestDataFile(ptn_locn_2, ptn_data_2);
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=1)");
- verifySetup("SELECT a from " + dbName + ".ptned WHERE b=1", ptn_data_1);
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=2)");
- verifySetup("SELECT a from " + dbName + ".ptned WHERE b=2", ptn_data_2);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=1)", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=1", ptn_data_1, driver);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=2)", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=2", ptn_data_2, driver);
advanceDumpDir();
@@ -500,58 +525,58 @@ public class TestReplicationScenarios {
InjectableBehaviourObjectStore.setListPartitionNamesBehaviour(listPartitionNamesNuller);
// None of the partitions will be dumped as the partitions list was empty
- run("REPL DUMP " + dbName);
+ run("REPL DUMP " + dbName, driver);
listPartitionNamesNuller.assertInjectionsPerformed(true, false);
InjectableBehaviourObjectStore.resetListPartitionNamesBehaviour(); // reset the behaviour
- String replDumpLocn = getResult(0, 0);
- String replDumpId = getResult(0, 1, true);
+ String replDumpLocn = getResult(0, 0, driver);
+ String replDumpId = getResult(0, 1, true, driver);
LOG.info("Bootstrap-Dump: Dumped to {} with id {}", replDumpLocn, replDumpId);
- run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'");
+ run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'", driverMirror);
// All partitions should miss in target as it was marked virtually as dropped
- verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=1", empty);
- verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=2", empty);
- verifyIfPartitionNotExist(dbName + "_dupe", "ptned", new ArrayList<>(Arrays.asList("1")));
- verifyIfPartitionNotExist(dbName + "_dupe", "ptned", new ArrayList<>(Arrays.asList("2")));
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=1", empty, driverMirror);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=2", empty, driverMirror);
+ verifyIfPartitionNotExist(dbName + "_dupe", "ptned", new ArrayList<>(Arrays.asList("1")), metaStoreClientMirror);
+ verifyIfPartitionNotExist(dbName + "_dupe", "ptned", new ArrayList<>(Arrays.asList("2")), metaStoreClientMirror);
// Verify if drop partition on a non-existing partition is idempotent and just a noop.
- run("ALTER TABLE " + dbName + ".ptned DROP PARTITION (b=1)");
- run("ALTER TABLE " + dbName + ".ptned DROP PARTITION (b=2)");
- verifyIfPartitionNotExist(dbName, "ptned", new ArrayList<>(Arrays.asList("1")));
- verifyIfPartitionNotExist(dbName, "ptned", new ArrayList<>(Arrays.asList("2")));
- verifySetup("SELECT a from " + dbName + ".ptned WHERE b=1", empty);
- verifySetup("SELECT a from " + dbName + ".ptned WHERE b=2", empty);
+ run("ALTER TABLE " + dbName + ".ptned DROP PARTITION (b=1)", driver);
+ run("ALTER TABLE " + dbName + ".ptned DROP PARTITION (b=2)", driver);
+ verifyIfPartitionNotExist(dbName, "ptned", new ArrayList<>(Arrays.asList("1")), metaStoreClient);
+ verifyIfPartitionNotExist(dbName, "ptned", new ArrayList<>(Arrays.asList("2")), metaStoreClient);
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=1", empty, driver);
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=2", empty, driver);
advanceDumpDir();
- run("REPL DUMP " + dbName + " FROM " + replDumpId);
- String postDropReplDumpLocn = getResult(0,0);
- String postDropReplDumpId = getResult(0,1,true);
+ run("REPL DUMP " + dbName + " FROM " + replDumpId, driver);
+ String postDropReplDumpLocn = getResult(0,0,driver);
+ String postDropReplDumpId = getResult(0,1,true,driver);
LOG.info("Dumped to {} with id {}->{}", postDropReplDumpLocn, replDumpId, postDropReplDumpId);
- assert(run("REPL LOAD " + dbName + "_dupe FROM '" + postDropReplDumpLocn + "'", true));
+ assert(run("REPL LOAD " + dbName + "_dupe FROM '" + postDropReplDumpLocn + "'", true, driverMirror));
- verifyIfPartitionNotExist(dbName + "_dupe", "ptned", new ArrayList<>(Arrays.asList("1")));
- verifyIfPartitionNotExist(dbName + "_dupe", "ptned", new ArrayList<>(Arrays.asList("2")));
- verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=1", empty);
- verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=2", empty);
+ verifyIfPartitionNotExist(dbName + "_dupe", "ptned", new ArrayList<>(Arrays.asList("1")), metaStoreClientMirror);
+ verifyIfPartitionNotExist(dbName + "_dupe", "ptned", new ArrayList<>(Arrays.asList("2")), metaStoreClientMirror);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=1", empty, driverMirror);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=2", empty, driverMirror);
}
@Test
public void testIncrementalAdds() throws IOException {
String name = testName.getMethodName();
- String dbName = createDB(name);
+ String dbName = createDB(name, driver);
- run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE");
- run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE");
- run("CREATE TABLE " + dbName + ".unptned_empty(a string) STORED AS TEXTFILE");
- run("CREATE TABLE " + dbName + ".ptned_empty(a string) partitioned by (b int) STORED AS TEXTFILE");
+ run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver);
+ run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE", driver);
+ run("CREATE TABLE " + dbName + ".unptned_empty(a string) STORED AS TEXTFILE", driver);
+ run("CREATE TABLE " + dbName + ".ptned_empty(a string) partitioned by (b int) STORED AS TEXTFILE", driver);
advanceDumpDir();
- run("REPL DUMP " + dbName);
- String replDumpLocn = getResult(0,0);
- String replDumpId = getResult(0,1,true);
+ run("REPL DUMP " + dbName, driver);
+ String replDumpLocn = getResult(0,0,driver);
+ String replDumpId = getResult(0,1,true,driver);
LOG.info("Dumped to {} with id {}",replDumpLocn,replDumpId);
- run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'");
+ run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'", driverMirror);
String[] unptn_data = new String[]{ "eleven" , "twelve" };
String[] ptn_data_1 = new String[]{ "thirteen", "fourteen", "fifteen"};
@@ -566,79 +591,79 @@ public class TestReplicationScenarios {
createTestDataFile(ptn_locn_1, ptn_data_1);
createTestDataFile(ptn_locn_2, ptn_data_2);
- verifySetup("SELECT a from " + dbName + ".ptned_empty", empty);
- verifySetup("SELECT * from " + dbName + ".unptned_empty", empty);
+ verifySetup("SELECT a from " + dbName + ".ptned_empty", empty, driverMirror);
+ verifySetup("SELECT * from " + dbName + ".unptned_empty", empty, driverMirror);
// Now, we load data into the tables, and see if an incremental
// repl drop/load can duplicate it.
- run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + dbName + ".unptned");
- verifySetup("SELECT * from " + dbName + ".unptned", unptn_data);
- run("CREATE TABLE " + dbName + ".unptned_late AS SELECT * from " + dbName + ".unptned");
- verifySetup("SELECT * from " + dbName + ".unptned_late", unptn_data);
+ run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + dbName + ".unptned", driver);
+ verifySetup("SELECT * from " + dbName + ".unptned", unptn_data, driver);
+ run("CREATE TABLE " + dbName + ".unptned_late AS SELECT * from " + dbName + ".unptned", driver);
+ verifySetup("SELECT * from " + dbName + ".unptned_late", unptn_data, driver);
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=1)");
- verifySetup("SELECT a from " + dbName + ".ptned WHERE b=1", ptn_data_1);
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=2)");
- verifySetup("SELECT a from " + dbName + ".ptned WHERE b=2", ptn_data_2);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=1)", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=1", ptn_data_1, driver);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=2)", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=2", ptn_data_2, driver);
- run("CREATE TABLE " + dbName + ".ptned_late(a string) PARTITIONED BY (b int) STORED AS TEXTFILE");
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned_late PARTITION(b=1)");
- verifySetup("SELECT a from " + dbName + ".ptned_late WHERE b=1",ptn_data_1);
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned_late PARTITION(b=2)");
- verifySetup("SELECT a from " + dbName + ".ptned_late WHERE b=2", ptn_data_2);
+ run("CREATE TABLE " + dbName + ".ptned_late(a string) PARTITIONED BY (b int) STORED AS TEXTFILE", driver);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned_late PARTITION(b=1)", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned_late WHERE b=1",ptn_data_1, driver);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned_late PARTITION(b=2)", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned_late WHERE b=2", ptn_data_2, driver);
// Perform REPL-DUMP/LOAD
advanceDumpDir();
- run("REPL DUMP " + dbName + " FROM " + replDumpId );
- String incrementalDumpLocn = getResult(0,0);
- String incrementalDumpId = getResult(0,1,true);
+ run("REPL DUMP " + dbName + " FROM " + replDumpId, driver);
+ String incrementalDumpLocn = getResult(0,0,driver);
+ String incrementalDumpId = getResult(0,1,true,driver);
LOG.info("Dumped to {} with id {}", incrementalDumpLocn, incrementalDumpId);
- run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'");
- printOutput();
- run("REPL LOAD " + dbName + "_dupe FROM '"+incrementalDumpLocn+"'");
+ run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'", driverMirror);
+ printOutput(driverMirror);
+ run("REPL LOAD " + dbName + "_dupe FROM '"+incrementalDumpLocn+"'", driverMirror);
- run("REPL STATUS " + dbName + "_dupe");
- verifyResults(new String[] {incrementalDumpId});
+ run("REPL STATUS " + dbName + "_dupe", driverMirror);
+ verifyResults(new String[] {incrementalDumpId}, driverMirror);
// VERIFY tables and partitions on destination for equivalence.
- verifyRun("SELECT * from " + dbName + "_dupe.unptned_empty", empty);
- verifyRun("SELECT a from " + dbName + "_dupe.ptned_empty", empty);
+ verifyRun("SELECT * from " + dbName + "_dupe.unptned_empty", empty, driverMirror);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned_empty", empty, driverMirror);
// verifyRun("SELECT * from " + dbName + "_dupe.unptned", unptn_data);
// TODO :this does not work because LOAD DATA LOCAL INPATH into an unptned table seems
// to use ALTER_TABLE only - it does not emit an INSERT or CREATE - re-enable after
// fixing that.
- verifyRun("SELECT * from " + dbName + "_dupe.unptned_late", unptn_data);
+ verifyRun("SELECT * from " + dbName + "_dupe.unptned_late", unptn_data, driverMirror);
- verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=1", ptn_data_1);
- verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=2", ptn_data_2);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=1", ptn_data_1, driverMirror);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=2", ptn_data_2, driverMirror);
- verifyRun("SELECT a from " + dbName + "_dupe.ptned_late WHERE b=1", ptn_data_1);
- verifyRun("SELECT a from " + dbName + "_dupe.ptned_late WHERE b=2", ptn_data_2);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned_late WHERE b=1", ptn_data_1, driverMirror);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned_late WHERE b=2", ptn_data_2, driverMirror);
}
@Test
public void testIncrementalLoadWithVariableLengthEventId() throws IOException, TException {
String testName = "incrementalLoadWithVariableLengthEventId";
- String dbName = createDB(testName);
- run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE");
- run("INSERT INTO TABLE " + dbName + ".unptned values('ten')");
+ String dbName = createDB(testName, driver);
+ run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver);
+ run("INSERT INTO TABLE " + dbName + ".unptned values('ten')", driver);
advanceDumpDir();
- run("REPL DUMP " + dbName);
- String replDumpLocn = getResult(0, 0);
- String replDumpId = getResult(0, 1, true);
+ run("REPL DUMP " + dbName, driver);
+ String replDumpLocn = getResult(0, 0, driver);
+ String replDumpId = getResult(0, 1, true, driver);
LOG.info("Bootstrap-Dump: Dumped to {} with id {}", replDumpLocn, replDumpId);
- run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'");
+ run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'", driverMirror);
// CREATE_TABLE - TRUNCATE - INSERT - The result is just one record.
// Creating dummy table to control the event ID of TRUNCATE not to be 10 or 100 or 1000...
String[] unptn_data = new String[]{ "eleven" };
- run("CREATE TABLE " + dbName + ".dummy(a string) STORED AS TEXTFILE");
- run("TRUNCATE TABLE " + dbName + ".unptned");
- run("INSERT INTO TABLE " + dbName + ".unptned values('" + unptn_data[0] + "')");
+ run("CREATE TABLE " + dbName + ".dummy(a string) STORED AS TEXTFILE", driver);
+ run("TRUNCATE TABLE " + dbName + ".unptned", driver);
+ run("INSERT INTO TABLE " + dbName + ".unptned values('" + unptn_data[0] + "')", driver);
// Inject a behaviour where all events will get ID less than 100 except TRUNCATE which will get ID 100.
// This enesures variable length of event ID in the incremental dump
@@ -713,28 +738,28 @@ public class TestReplicationScenarios {
+ " TO " + String.valueOf(metaStoreClient.getCurrentNotificationEventId().getEventId()*100);
advanceDumpDir();
- run(cmd);
+ run(cmd, driver);
eventIdModifier.assertInjectionsPerformed(true,false);
InjectableBehaviourObjectStore.resetGetNextNotificationBehaviour(); // reset the behaviour
- String incrementalDumpLocn = getResult(0, 0);
- String incrementalDumpId = getResult(0, 1, true);
+ String incrementalDumpLocn = getResult(0, 0, driver);
+ String incrementalDumpId = getResult(0, 1, true, driver);
LOG.info("Incremental-Dump: Dumped to {} with id {} from {}", incrementalDumpLocn, incrementalDumpId, replDumpId);
- run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'");
- printOutput();
- run("REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'");
- verifyRun("SELECT a from " + dbName + "_dupe.unptned ORDER BY a", unptn_data);
+ run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'", driverMirror);
+ printOutput(driverMirror);
+ run("REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'", driverMirror);
+ verifyRun("SELECT a from " + dbName + "_dupe.unptned ORDER BY a", unptn_data, driverMirror);
}
@Test
public void testDrops() throws IOException {
String name = testName.getMethodName();
- String dbName = createDB(name);
- run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE");
- run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b string) STORED AS TEXTFILE");
- run("CREATE TABLE " + dbName + ".ptned2(a string) partitioned by (b string) STORED AS TEXTFILE");
- run("CREATE TABLE " + dbName + ".ptned3(a string) partitioned by (b int) STORED AS TEXTFILE");
+ String dbName = createDB(name, driver);
+ run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver);
+ run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b string) STORED AS TEXTFILE", driver);
+ run("CREATE TABLE " + dbName + ".ptned2(a string) partitioned by (b string) STORED AS TEXTFILE", driver);
+ run("CREATE TABLE " + dbName + ".ptned3(a string) partitioned by (b int) STORED AS TEXTFILE", driver);
String[] unptn_data = new String[]{ "eleven" , "twelve" };
String[] ptn_data_1 = new String[]{ "thirteen", "fourteen", "fifteen"};
@@ -749,85 +774,85 @@ public class TestReplicationScenarios {
createTestDataFile(ptn_locn_1, ptn_data_1);
createTestDataFile(ptn_locn_2, ptn_data_2);
- run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + dbName + ".unptned");
- verifySetup("SELECT * from " + dbName + ".unptned", unptn_data);
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b='1')");
- verifySetup("SELECT a from " + dbName + ".ptned WHERE b='1'", ptn_data_1);
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b='2')");
- verifySetup("SELECT a from " + dbName + ".ptned WHERE b='2'", ptn_data_2);
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned2 PARTITION(b='1')");
- verifySetup("SELECT a from " + dbName + ".ptned2 WHERE b='1'", ptn_data_1);
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned2 PARTITION(b='2')");
- verifySetup("SELECT a from " + dbName + ".ptned2 WHERE b='2'", ptn_data_2);
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned3 PARTITION(b=1)");
- verifySetup("SELECT a from " + dbName + ".ptned2 WHERE b=1", ptn_data_1);
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned3 PARTITION(b=2)");
- verifySetup("SELECT a from " + dbName + ".ptned2 WHERE b=2", ptn_data_2);
+ run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + dbName + ".unptned", driver);
+ verifySetup("SELECT * from " + dbName + ".unptned", unptn_data, driver);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b='1')", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b='1'", ptn_data_1, driver);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b='2')", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b='2'", ptn_data_2, driver);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned2 PARTITION(b='1')", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned2 WHERE b='1'", ptn_data_1, driver);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned2 PARTITION(b='2')", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned2 WHERE b='2'", ptn_data_2, driver);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned3 PARTITION(b=1)", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned2 WHERE b=1", ptn_data_1, driver);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned3 PARTITION(b=2)", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned2 WHERE b=2", ptn_data_2, driver);
// At this point, we've set up all the tables and ptns we're going to test drops across
// Replicate it first, and then we'll drop it on the source.
advanceDumpDir();
- run("REPL DUMP " + dbName);
- String replDumpLocn = getResult(0,0);
- String replDumpId = getResult(0,1,true);
- run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'");
- printOutput();
- run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'");
- verifySetup("REPL STATUS " + dbName + "_dupe", new String[]{replDumpId});
+ run("REPL DUMP " + dbName, driver);
+ String replDumpLocn = getResult(0,0,driver);
+ String replDumpId = getResult(0,1,true,driver);
+ run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'", driverMirror);
+ printOutput(driverMirror);
+ run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'", driverMirror);
+ verifySetup("REPL STATUS " + dbName + "_dupe", new String[]{replDumpId}, driverMirror);
- verifySetup("SELECT * from " + dbName + "_dupe.unptned", unptn_data);
- verifySetup("SELECT a from " + dbName + "_dupe.ptned WHERE b='1'", ptn_data_1);
- verifySetup("SELECT a from " + dbName + "_dupe.ptned WHERE b='2'", ptn_data_2);
- verifySetup("SELECT a from " + dbName + "_dupe.ptned2 WHERE b='1'", ptn_data_1);
- verifySetup("SELECT a from " + dbName + "_dupe.ptned2 WHERE b='2'", ptn_data_2);
- verifySetup("SELECT a from " + dbName + "_dupe.ptned3 WHERE b=1", ptn_data_1);
- verifySetup("SELECT a from " + dbName + "_dupe.ptned3 WHERE b=2", ptn_data_2);
+ verifySetup("SELECT * from " + dbName + "_dupe.unptned", unptn_data, driverMirror);
+ verifySetup("SELECT a from " + dbName + "_dupe.ptned WHERE b='1'", ptn_data_1, driverMirror);
+ verifySetup("SELECT a from " + dbName + "_dupe.ptned WHERE b='2'", ptn_data_2, driverMirror);
+ verifySetup("SELECT a from " + dbName + "_dupe.ptned2 WHERE b='1'", ptn_data_1, driverMirror);
+ verifySetup("SELECT a from " + dbName + "_dupe.ptned2 WHERE b='2'", ptn_data_2, driverMirror);
+ verifySetup("SELECT a from " + dbName + "_dupe.ptned3 WHERE b=1", ptn_data_1, driverMirror);
+ verifySetup("SELECT a from " + dbName + "_dupe.ptned3 WHERE b=2", ptn_data_2, driverMirror);
// All tables good on destination, drop on source.
- run("DROP TABLE " + dbName + ".unptned");
- run("ALTER TABLE " + dbName + ".ptned DROP PARTITION (b='2')");
- run("DROP TABLE " + dbName + ".ptned2");
- run("ALTER TABLE " + dbName + ".ptned3 DROP PARTITION (b=1)");
- verifySetup("SELECT a from " + dbName + ".ptned WHERE b='2'", empty);
- verifySetup("SELECT a from " + dbName + ".ptned", ptn_data_1);
- verifySetup("SELECT a from " + dbName + ".ptned3 WHERE b=1",empty);
- verifySetup("SELECT a from " + dbName + ".ptned3", ptn_data_2);
+ run("DROP TABLE " + dbName + ".unptned", driver);
+ run("ALTER TABLE " + dbName + ".ptned DROP PARTITION (b='2')", driver);
+ run("DROP TABLE " + dbName + ".ptned2", driver);
+ run("ALTER TABLE " + dbName + ".ptned3 DROP PARTITION (b=1)", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b='2'", empty, driver);
+ verifySetup("SELECT a from " + dbName + ".ptned", ptn_data_1, driver);
+ verifySetup("SELECT a from " + dbName + ".ptned3 WHERE b=1",empty, driver);
+ verifySetup("SELECT a from " + dbName + ".ptned3", ptn_data_2, driver);
// replicate the incremental drops
advanceDumpDir();
- run("REPL DUMP " + dbName + " FROM " + replDumpId);
- String postDropReplDumpLocn = getResult(0,0);
- String postDropReplDumpId = getResult(0,1,true);
+ run("REPL DUMP " + dbName + " FROM " + replDumpId, driver);
+ String postDropReplDumpLocn = getResult(0,0,driver);
+ String postDropReplDumpId = getResult(0,1,true,driver);
LOG.info("Dumped to {} with id {}->{}", postDropReplDumpLocn, replDumpId, postDropReplDumpId);
- run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + postDropReplDumpLocn + "'");
- printOutput();
- run("REPL LOAD " + dbName + "_dupe FROM '" + postDropReplDumpLocn + "'");
+ run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + postDropReplDumpLocn + "'", driverMirror);
+ printOutput(driverMirror);
+ run("REPL LOAD " + dbName + "_dupe FROM '" + postDropReplDumpLocn + "'", driverMirror);
// verify that drops were replicated. This can either be from tables or ptns
// not existing, and thus, throwing a NoSuchObjectException, or returning nulls
// or select * returning empty, depending on what we're testing.
- verifyIfTableNotExist(dbName + "_dupe", "unptned");
+ verifyIfTableNotExist(dbName + "_dupe", "unptned", metaStoreClientMirror);
- verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b='2'", empty);
- verifyRun("SELECT a from " + dbName + "_dupe.ptned", ptn_data_1);
- verifyRun("SELECT a from " + dbName + "_dupe.ptned3 WHERE b=1", empty);
- verifyRun("SELECT a from " + dbName + "_dupe.ptned3", ptn_data_2);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b='2'", empty, driverMirror);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned", ptn_data_1, driverMirror);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned3 WHERE b=1", empty, driverMirror);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned3", ptn_data_2, driverMirror);
- verifyIfTableNotExist(dbName + "_dupe", "ptned2");
+ verifyIfTableNotExist(dbName + "_dupe", "ptned2", metaStoreClientMirror);
}
@Test
public void testDropsWithCM() throws IOException {
String testName = "drops_with_cm";
- String dbName = createDB(testName);
- run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE");
- run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b string) STORED AS TEXTFILE");
- run("CREATE TABLE " + dbName + ".ptned2(a string) partitioned by (b string) STORED AS TEXTFILE");
+ String dbName = createDB(testName, driver);
+ run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver);
+ run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b string) STORED AS TEXTFILE", driver);
+ run("CREATE TABLE " + dbName + ".ptned2(a string) partitioned by (b string) STORED AS TEXTFILE", driver);
String[] unptn_data = new String[]{ "eleven" , "twelve" };
String[] ptn_data_1 = new String[]{ "thirteen", "fourteen", "fifteen"};
@@ -842,79 +867,79 @@ public class TestReplicationScenarios {
createTestDataFile(ptn_locn_1, ptn_data_1);
createTestDataFile(ptn_locn_2, ptn_data_2);
- run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + dbName + ".unptned");
- run("SELECT * from " + dbName + ".unptned");
- verifyResults(unptn_data);
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b='1')");
- run("SELECT a from " + dbName + ".ptned WHERE b='1'");
- verifyResults(ptn_data_1);
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b='2')");
- run("SELECT a from " + dbName + ".ptned WHERE b='2'");
- verifyResults(ptn_data_2);
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned2 PARTITION(b='1')");
- run("SELECT a from " + dbName + ".ptned2 WHERE b='1'");
- verifyResults(ptn_data_1);
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned2 PARTITION(b='2')");
- run("SELECT a from " + dbName + ".ptned2 WHERE b='2'");
- verifyResults(ptn_data_2);
-
- advanceDumpDir();
- run("REPL DUMP " + dbName);
- String replDumpLocn = getResult(0,0);
- String replDumpId = getResult(0,1,true);
- run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'");
- printOutput();
- run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'");
-
- run("REPL STATUS " + dbName + "_dupe");
- verifyResults(new String[] {replDumpId});
-
- run("SELECT * from " + dbName + "_dupe.unptned");
- verifyResults(unptn_data);
- run("SELECT a from " + dbName + "_dupe.ptned WHERE b='1'");
- verifyResults(ptn_data_1);
- run("SELECT a from " + dbName + "_dupe.ptned WHERE b='2'");
- verifyResults(ptn_data_2);
- run("SELECT a from " + dbName + "_dupe.ptned2 WHERE b='1'");
- verifyResults(ptn_data_1);
- run("SELECT a from " + dbName + "_dupe.ptned2 WHERE b='2'");
- verifyResults(ptn_data_2);
-
- run("CREATE TABLE " + dbName + ".unptned_copy" + " AS SELECT a FROM " + dbName + ".unptned");
- run("CREATE TABLE " + dbName + ".ptned_copy" + " LIKE " + dbName + ".ptned");
+ run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + dbName + ".unptned", driver);
+ run("SELECT * from " + dbName + ".unptned", driver);
+ verifyResults(unptn_data, driver);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b='1')", driver);
+ run("SELECT a from " + dbName + ".ptned WHERE b='1'", driver);
+ verifyResults(ptn_data_1, driver);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b='2')", driver);
+ run("SELECT a from " + dbName + ".ptned WHERE b='2'", driver);
+ verifyResults(ptn_data_2, driver);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned2 PARTITION(b='1')", driver);
+ run("SELECT a from " + dbName + ".ptned2 WHERE b='1'", driver);
+ verifyResults(ptn_data_1, driver);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned2 PARTITION(b='2')", driver);
+ run("SELECT a from " + dbName + ".ptned2 WHERE b='2'", driver);
+ verifyResults(ptn_data_2, driver);
+
+ advanceDumpDir();
+ run("REPL DUMP " + dbName, driver);
+ String replDumpLocn = getResult(0,0,driver);
+ String replDumpId = getResult(0,1,true,driver);
+ run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'", driverMirror);
+ printOutput(driverMirror);
+ run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'", driverMirror);
+
+ run("REPL STATUS " + dbName + "_dupe", driverMirror);
+ verifyResults(new String[] {replDumpId}, driverMirror);
+
+ run("SELECT * from " + dbName + "_dupe.unptned", driverMirror);
+ verifyResults(unptn_data, driverMirror);
+ run("SELECT a from " + dbName + "_dupe.ptned WHERE b='1'", driverMirror);
+ verifyResults(ptn_data_1, driverMirror);
+ run("SELECT a from " + dbName + "_dupe.ptned WHERE b='2'", driverMirror);
+ verifyResults(ptn_data_2, driverMirror);
+ run("SELECT a from " + dbName + "_dupe.ptned2 WHERE b='1'", driverMirror);
+ verifyResults(ptn_data_1, driverMirror);
+ run("SELECT a from " + dbName + "_dupe.ptned2 WHERE b='2'", driverMirror);
+ verifyResults(ptn_data_2, driverMirror);
+
+ run("CREATE TABLE " + dbName + ".unptned_copy" + " AS SELECT a FROM " + dbName + ".unptned", driver);
+ run("CREATE TABLE " + dbName + ".ptned_copy" + " LIKE " + dbName + ".ptned", driver);
run("INSERT INTO TABLE " + dbName + ".ptned_copy" + " PARTITION(b='1') SELECT a FROM " +
- dbName + ".ptned WHERE b='1'");
- run("SELECT a from " + dbName + ".unptned_copy");
- verifyResults(unptn_data);
- run("SELECT a from " + dbName + ".ptned_copy");
- verifyResults(ptn_data_1);
-
- run("DROP TABLE " + dbName + ".unptned");
- run("ALTER TABLE " + dbName + ".ptned DROP PARTITION (b='2')");
- run("DROP TABLE " + dbName + ".ptned2");
- run("SELECT a from " + dbName + ".ptned WHERE b=2");
- verifyResults(empty);
- run("SELECT a from " + dbName + ".ptned");
- verifyResults(ptn_data_1);
-
- advanceDumpDir();
- run("REPL DUMP " + dbName + " FROM " + replDumpId);
- String postDropReplDumpLocn = getResult(0,0);
- String postDropReplDumpId = getResult(0,1,true);
+ dbName + ".ptned WHERE b='1'", driver);
+ run("SELECT a from " + dbName + ".unptned_copy", driver);
+ verifyResults(unptn_data, driver);
+ run("SELECT a from " + dbName + ".ptned_copy", driver);
+ verifyResults(ptn_data_1, driver);
+
+ run("DROP TABLE " + dbName + ".unptned", driver);
+ run("ALTER TABLE " + dbName + ".ptned DROP PARTITION (b='2')", driver);
+ run("DROP TABLE " + dbName + ".ptned2", driver);
+ run("SELECT a from " + dbName + ".ptned WHERE b=2", driver);
+ verifyResults(empty, driver);
+ run("SELECT a from " + dbName + ".ptned", driver);
+ verifyResults(ptn_data_1, driver);
+
+ advanceDumpDir();
+ run("REPL DUMP " + dbName + " FROM " + replDumpId, driver);
+ String postDropReplDumpLocn = getResult(0,0,driver);
+ String postDropReplDumpId = getResult(0,1,true,driver);
LOG.info("Dumped to {} with id {}->{}", postDropReplDumpLocn, replDumpId, postDropReplDumpId);
// Drop table after dump
- run("DROP TABLE " + dbName + ".unptned_copy");
+ run("DROP TABLE " + dbName + ".unptned_copy", driver);
// Drop partition after dump
- run("ALTER TABLE " + dbName + ".ptned_copy DROP PARTITION(b='1')");
+ run("ALTER TABLE " + dbName + ".ptned_copy DROP PARTITION(b='1')", driver);
- run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + postDropReplDumpLocn + "'");
- printOutput();
- run("REPL LOAD " + dbName + "_dupe FROM '" + postDropReplDumpLocn + "'");
+ run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + postDropReplDumpLocn + "'", driverMirror);
+ printOutput(driverMirror);
+ run("REPL LOAD " + dbName + "_dupe FROM '" + postDropReplDumpLocn + "'", driverMirror);
Exception e = null;
try {
- Table tbl = metaStoreClient.getTable(dbName + "_dupe", "unptned");
+ Table tbl = metaStoreClientMirror.getTable(dbName + "_dupe", "unptned");
assertNull(tbl);
} catch (TException te) {
e = te;
@@ -922,28 +947,28 @@ public class TestReplicationScenarios {
assertNotNull(e);
assertEquals(NoSuchObjectException.class, e.getClass());
- run("SELECT a from " + dbName + "_dupe.ptned WHERE b=2");
- verifyResults(empty);
- run("SELECT a from " + dbName + "_dupe.ptned");
- verifyResults(ptn_data_1);
+ run("SELECT a from " + dbName + "_dupe.ptned WHERE b=2", driverMirror);
+ verifyResults(empty, driverMirror);
+ run("SELECT a from " + dbName + "_dupe.ptned", driverMirror);
+ verifyResults(ptn_data_1, driverMirror);
- verifyIfTableNotExist(dbName +"_dupe", "ptned2");
+ verifyIfTableNotExist(dbName +"_dupe", "ptned2", metaStoreClientMirror);
- run("SELECT a from " + dbName + "_dupe.unptned_copy");
- verifyResults(unptn_data);
- run("SELECT a from " + dbName + "_dupe.ptned_copy");
- verifyResults(ptn_data_1);
+ run("SELECT a from " + dbName + "_dupe.unptned_copy", driverMirror);
+ verifyResults(unptn_data, driverMirror);
+ run("SELECT a from " + dbName + "_dupe.ptned_copy", driverMirror);
+ verifyResults(ptn_data_1, driverMirror);
}
@Test
public void testAlters() throws IOException {
String testName = "alters";
- String dbName = createDB(testName);
- run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE");
- run("CREATE TABLE " + dbName + ".unptned2(a string) STORED AS TEXTFILE");
- run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b string) STORED AS TEXTFILE");
- run("CREATE TABLE " + dbName + ".ptned2(a string) partitioned by (b string) STORED AS TEXTFILE");
+ String dbName = createDB(testName, driver);
+ run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver);
+ run("CREATE TABLE " + dbName + ".unptned2(a string) STORED AS TEXTFILE", driver);
+ run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b string) STORED AS TEXTFILE", driver);
+ run("CREATE TABLE " + dbName + ".ptned2(a string) partitioned by (b string) STORED AS TEXTFILE", driver);
String[] unptn_data = new String[]{ "eleven" , "twelve" };
String[] ptn_data_1 = new String[]{ "thirteen", "fourteen", "fifteen"};
@@ -958,51 +983,51 @@ public class TestReplicationScenarios {
createTestDataFile(ptn_locn_1, ptn_data_1);
createTestDataFile(ptn_locn_2, ptn_data_2);
- run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + dbName + ".unptned");
- verifySetup("SELECT * from " + dbName + ".unptned", unptn_data);
- run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + dbName + ".unptned2");
- verifySetup("SELECT * from " + dbName + ".unptned2", unptn_data);
+ run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + dbName + ".unptned", driver);
+ verifySetup("SELECT * from " + dbName + ".unptned", unptn_data, driver);
+ run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + dbName + ".unptned2", driver);
+ verifySetup("SELECT * from " + dbName + ".unptned2", unptn_data, driver);
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b='1')");
- verifySetup("SELECT a from " + dbName + ".ptned WHERE b='1'", ptn_data_1);
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b='2')");
- verifySetup("SELECT a from " + dbName + ".ptned WHERE b='2'", ptn_data_2);
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned2 PARTITION(b='1')");
- verifySetup("SELECT a from " + dbName + ".ptned2 WHERE b='1'",ptn_data_1);
- run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned2 PARTITION(b='2')");
- verifySetup("SELECT a from " + dbName + ".ptned2 WHERE b='2'", ptn_data_2);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b='1')", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b='1'", ptn_data_1, driver);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b='2')", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b='2'", ptn_data_2, driver);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned2 PARTITION(b='1')", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned2 WHERE b='1'",ptn_data_1, driver);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned2 PARTITION(b='2')", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned2 WHERE b='2'", ptn_data_2, driver);
// base tables set up, let's replicate them over
advanceDumpDir();
- run("REPL DUMP " + dbName);
- String replDumpLocn = getResult(0,0);
- String replDumpId = getResult(0,1,true);
- run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'");
- printOutput();
- run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'");
+ run("REPL DUMP " + dbName, driver);
+ String replDumpLocn = getResult(0,0,driver);
+ String replDumpId = getResult(0,1,true,driver);
+ run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'", driverMirror);
+ printOutput(driverMirror);
+ run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'", driverMirror);
- run("REPL STATUS " + dbName + "_dupe");
- verifyResults(new String[] {replDumpId});
+ run("REPL STATUS " + dbName + "_dupe", driverMirror);
+ verifyResults(new String[] {replDumpId}, driverMirror);
- verifySetup("SELECT * from " + dbName + "_dupe.unptned", unptn_data);
- verifySetup("SELECT * from " + dbName + "_dupe.unptned2", unptn_data);
- verifySetup("SELECT a from " + dbName + "_dupe.ptned WHERE b='1'", ptn_data_1);
- verifySetup("SELECT a from " + dbName + "_dupe.ptned WHERE b='2'", ptn_data_2);
- verifySetup("SELECT a from " + dbName + "_dupe.ptned2 WHERE b='1'", ptn_data_1);
- verifySetup("SELECT a from " + dbName + "_dupe.ptned2 WHERE b='2'", ptn_data_2);
+ verifySetup("SELECT * from " + dbName + "_dupe.unptned", unptn_data, driverMirror);
+ verifySetup("SELECT * from " + dbName + "_dupe.unptned2", unptn_data, driverMirror);
+ verifySetup("SELECT a from " + dbName + "_dupe.ptned WHERE b='1'", ptn_data_1, driverMirror);
+ verifySetup("SELECT a from " + dbName + "_dupe.ptned WHERE b='2'", ptn_data_2, driverMirror);
+ verifySetup("SELECT a from " + dbName + "_dupe.ptned2 WHERE b='1'", ptn_data_1, driverMirror);
+ verifySetup("SELECT a from " + dbName + "_dupe.ptned2 WHERE b='2'", ptn_data_2, driverMirror);
// tables have been replicated over, and verified to be identical. Now, we do a couple of
// alters on the source
// Rename unpartitioned table
- run("ALTER TABLE " + dbName + ".unptned RENAME TO " + dbName + ".unptned_rn");
- verifySetup("SELECT * from " + dbName + ".unptned_rn", unptn_data);
+ run("ALTER TABLE " + dbName + ".unptned RENAME TO " + dbName + ".unptned_rn", driver);
+ verifySetup("SELECT * from " + dbName + ".unptned_rn", unptn_data, driver);
// Alter unpartitioned table set table property
String testKey = "blah";
String testVal = "foo";
- run("ALTER TABLE " + dbName + ".unptned2 SET TBLPROPERTIES ('" + testKey + "' = '" + testVal + "')");
+ run("ALTER TABLE " + dbName + ".unptned2 SET TBLPROPERTIES ('" + testKey + "' = '" + testVal + "')", driver);
if (VERIFY_SETUP_STEPS){
try {
Table unptn2 = metaStoreClient.getTable(dbName,"unptned2");
@@ -1014,12 +1039,12 @@ public class TestReplicationScenarios {
}
// alter partitioned table, rename partition
- run("ALTER TABLE " + dbName + ".ptned PARTITION (b='2') RENAME TO PARTITION (b='22')");
- verifySetup("SELECT a from " + dbName + ".ptned WHERE b=2", empty);
- verifySetup("SELECT a from " + dbName + ".ptned WHERE b=22", ptn_data_2);
+ run("ALTER TABLE " + dbName + ".ptned PARTITION (b='2') RENAME TO PARTITION (b='22')", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=2", empty, driver);
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=22", ptn_data_2, driver);
// alter partitioned table set table property
- run("ALTER TABLE " + dbName + ".ptned SET TBLPROPERTIES ('" + testKey + "' = '" + testVal + "')");
+ run("ALTER TABLE " + dbName + ".ptned SET TBLPROPERTIES ('" + testKey + "' = '" + testVal + "')", driver);
if (VERIFY_SETUP_STEPS){
try {
Table ptned = metaStoreClient.getTable(dbName,"ptned");
@@ -1043,46 +1068,46 @@ public class TestReplicationScenarios {
}
// rename partitioned table
- verifySetup("SELECT a from " + dbName + ".ptned2 WHERE b=2", ptn_data_2);
- run("ALTER TABLE " + dbName + ".ptned2 RENAME TO " + dbName + ".ptned2_rn");
- verifySetup("SELECT a from " + dbName + ".ptned2_rn WHERE b=2", ptn_data_2);
+ verifySetup("SELECT a from " + dbName + ".ptned2 WHERE b=2", ptn_data_2, driver);
+ run("ALTER TABLE " + dbName + ".ptned2 RENAME TO " + dbName + ".ptned2_rn", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned2_rn WHERE b=2", ptn_data_2, driver);
// All alters done, now we replicate them over.
advanceDumpDir();
- run("REPL DUMP " + dbName + " FROM " + replDumpId);
- String postAlterReplDumpLocn = getResult(0,0);
- String postAlterReplDumpId = getResult(0,1,true);
+ run("REPL DUMP " + dbName + " FROM " + replDumpId, driver);
+ String postAlterReplDumpLocn = getResult(0,0,driver);
+ String postAlterReplDumpId = getResult(0,1,true,driver);
LOG.info("Dumped to {} with id {}->{}", postAlterReplDumpLocn, replDumpId, postAlterReplDumpId);
- run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + postAlterReplDumpLocn + "'");
- printOutput();
- run("REPL LOAD " + dbName + "_dupe FROM '" + postAlterReplDumpLocn + "'");
+ run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + postAlterReplDumpLocn + "'", driverMirror);
+ printOutput(driverMirror);
+ run("REPL LOAD " + dbName + "_dupe FROM '" + postAlterReplDumpLocn + "'", driverMirror);
// Replication done, we now do the following verifications:
// verify that unpartitioned table rename succeeded.
- verifyIfTableNotExist(dbName + "_dupe", "unptned");
- verifyRun("SELECT * from " + dbName + "_dupe.unptned_rn", unptn_data);
+ verifyIfTableNotExist(dbName + "_dupe", "unptned", metaStoreClientMirror);
+ verifyRun("SELECT * from " + dbName + "_dupe.unptned_rn", unptn_data, driverMirror);
// verify that partition rename succeded.
try {
- Table unptn2 = metaStoreClient.getTable(dbName + "_dupe" , "unptned2");
+ Table unptn2 = metaStoreClientMirror.getTable(dbName + "_dupe" , "unptned2");
assertTrue(unptn2.getParameters().containsKey(testKey));
assertEquals(testVal,unptn2.getParameters().get(testKey));
} catch (TException te) {
assertNull(te);
}
- verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=2", empty);
- verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=22", ptn_data_2);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=2", empty, driverMirror);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=22", ptn_data_2, driverMirror);
// verify that ptned table rename succeded.
- verifyIfTableNotExist(dbName + "_dupe", "ptned2");
- verifyRun("SELECT a from " + dbName + "_dupe.ptned2_rn WHERE b=2", ptn_data_2);
+ verifyIfTableNotExist(dbName + "_dupe", "ptned2", metaStoreClientMirror);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned2_rn WHERE b=2", ptn_data_2, driverMirror);
// verify that ptned table property set worked
try {
- Table ptned = metaStoreClient.getTable(dbName + "_dupe" , "ptned");
+ Table ptned = metaStoreClientMirror.getTable(dbName + "_dupe" , "ptned");
assertTrue(ptned.getParameters().containsKey(testKey));
assertEquals(testVal, ptned.getParameters().get(testKey));
} catch (TException te) {
@@ -1093,7 +1118,7 @@ public class TestReplicationScenarios {
try {
List<String> ptnVals1 = new ArrayList<String>();
ptnVals1.add("1");
- Partition ptn1 = metaStoreClient.getPartition(dbName + "_dupe", "ptned", ptnVals1);
+ Partition ptn1 = metaStoreClientMirror.getPartition(dbName + "_dupe", "ptned", ptnVals1);
assertTrue(ptn1.getParameters().containsKey(testKey));
assertEquals(testVal,ptn1.getParameters().get(testKey));
} catch (TException te) {
@@ -1105,20 +1130,20 @@ public class TestReplicationScenarios {
@Test
public void testIncrementalLoad() throws IOException {
String testName = "incrementalLoad";
- String dbName = createDB(testName);
+ String dbName = createDB(testName, driver);
- run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE");
- run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE");
- run("CREATE TABLE " + dbName + ".unptned_empty(a string) STORED AS TEXTFILE");
+ run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver);
+ run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE", driver);
+ run("CREATE TABLE " + dbName + ".unptned_empty(a string) STORED AS TEXTFILE", driver);
run("CREATE TABLE " + dbName
- + ".ptned_empty(a string) partitioned by (b int) STORED AS TEXTFILE");
+ + ".ptned_empty(a string) partitioned by (b int) STORED AS TEXTFILE", driver);
advanceDumpDir();
- run("REPL DUMP " + dbName);
- String replDumpLocn = getResult(0, 0);
- String replDumpId = getResult(0, 1, true);
+ run("REPL DUMP " + dbName, driver);
+ String replDumpLocn = getResult(0, 0,driver);
+ String replDumpId = getResult(0, 1, true, driver);
LOG.info("Bootstrap-Dump: Dumped to {} with id {}", replDumpLocn, replDumpId);
- run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'");
+ run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'", driverMirror);
String[] unptn_data = new String[] { "eleven", "twelve" };
String[] ptn_data_1 = new String[] { "thirteen", "fourteen", "fifteen" };
@@ -1133,129 +1158,129 @@ public class TestReplicationScenarios {
createTestDataFile(ptn_locn_1, ptn_data_1);
createTestDataFile(ptn_locn_2, ptn_data_2);
- verifySetup("SELECT a from " + dbName + ".ptned_empty", empty);
- verifySetup("SELECT * from " + dbName + ".unptned_empty", empty);
+ verifySetup("SELECT a from " + dbName + ".ptned_empty", empty, driverMirror);
+ verifySetup("SELECT * from " + dbName + ".unptned_empty", empty, driverMirror);
- run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + dbName + ".unptned");
- verifySetup("SELECT * from " + dbName + ".unptned", unptn_data);
- run("CREATE TABLE " + dbName + ".unptned_late LIKE " + dbName + ".unptned");
- run("INSERT INTO TABLE " + dbName + ".unptned_late SELECT * FROM " + dbName + ".unptned");
- verifySetup("SELECT * from " + dbName + ".unptned_late", unptn_data);
+ run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + dbName + ".unptned", driver);
+ verifySetup("SELECT * from " + dbName + ".unptned", unptn_data, driver);
+ run("CREATE TABLE " + dbName + ".unptned_late LIKE " + dbName + ".unptned", driver);
+ run("INSERT INTO TABLE " + dbName + ".unptned_late SELECT * FROM " + dbName + ".unptned", driver);
+ verifySetup("SELECT * from " + dbName + ".unptned_late", unptn_data, driver);
advanceDumpDir();
- run("REPL DUMP " + dbName + " FROM " + replDumpId);
- String incrementalDumpLocn = getResult(0, 0);
- String incrementalDumpId = getResult(0, 1, true);
+ run("REPL DUMP " + dbName + " FROM " + replDumpId, driver);
+ String incrementalDumpLocn = getResult(0, 0, driver);
+ String incrementalDumpId = getResult(0, 1, true, driver);
LOG.info("Incremental-Dump: Dumped to {} with id {} from {}", incrementalDumpLocn, incrementalDumpId, replDumpId);
replDumpId = incrementalDumpId;
- run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'");
- printOutput();
- run("REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'");
- verifyRun("SELECT * from " + dbName + "_dupe.unptned_late", unptn_data);
+ run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'", driverMirror);
+ printOutput(driverMirror);
+ run("REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'", driverMirror);
+ verifyRun("SELECT * from " + dbName + "_dupe.unptned_late", unptn_data, driverMirror);
- run("ALTER TABLE " + dbName + ".ptned ADD PARTITION (b=1)");
+ run("ALTER TABLE " + dbName + ".ptned ADD PARTITION (b=1)", driver);
run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName
- + ".ptned PARTITION(b=1)");
- verifySetup("SELECT a from " + dbName + ".ptned WHERE b=1", ptn_data_1);
+ + ".ptned PARTITION(b=1)", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=1", ptn_data_1, driver);
run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName
- + ".ptned PARTITION(b=2)");
- verifySetup("SELECT a from " + dbName + ".ptned WHERE b=2", ptn_data_2);
+ + ".ptned PARTITION(b=2)", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=2", ptn_data_2, driver);
run("CREATE TABLE " + dbName
- + ".ptned_late(a string) PARTITIONED BY (b int) STORED AS TEXTFILE");
+ + ".ptned_late(a string) PARTITIONED BY (b int) STORED AS TEXTFILE", driver);
run("INSERT INTO TABLE " + dbName + ".ptned_late PARTITION(b=1) SELECT a FROM " + dbName
- + ".ptned WHERE b=1");
- verifySetup("SELECT a from " + dbName + ".ptned_late WHERE b=1", ptn_data_1);
+ + ".ptned WHERE b=1", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned_late WHERE b=1", ptn_data_1, driver);
run("INSERT INTO TABLE " + dbName + ".ptned_late PARTITION(b=2) SELECT a FROM " + dbName
- + ".ptned WHERE b=2");
- verifySetup("SELECT a from " + dbName + ".ptned_late WHERE b=2", ptn_data_2);
+ + ".ptned WHERE b=2", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned_late WHERE b=2", ptn_data_2, driver);
advanceDumpDir();
- run("REPL DUMP " + dbName + " FROM " + replDumpId);
- incrementalDumpLocn = getResult(0, 0);
- incrementalDumpId = getResult(0, 1, true);
+ run("REPL DUMP " + dbName + " FROM " + replDumpId, driver);
+ incrementalDumpLocn = getResult(0, 0, driver);
+ incrementalDumpId = getResult(0, 1, true, driver);
LOG.info("Incremental-Dump: Dumped to {} with id {} from {}", incrementalDumpLocn, incrementalDumpId, replDumpId);
replDumpId = incrementalDumpId;
- run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'");
- printOutput();
- run("REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'");
+ run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'", driverMirror);
+ printOutput(driverMirror);
+ run("REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'", driverMirror);
- verifyRun("SELECT a from " + dbName + "_dupe.ptned_late WHERE b=1", ptn_data_1);
- verifyRun("SELECT a from " + dbName + "_dupe.ptned_late WHERE b=2", ptn_data_2);
- verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=1", ptn_data_1);
- verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=2", ptn_data_2);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned_late WHERE b=1", ptn_data_1, driverMirror);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned_late WHERE b=2", ptn_data_2, driverMirror);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=1", ptn_data_1, driverMirror);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=2", ptn_data_2, driverMirror);
}
@Test
public void testIncrementalInserts() throws IOException {
String testName = "incrementalInserts";
- String dbName = createDB(testName);
- run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE");
+ String dbName = createDB(testName, driver);
+ run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver);
advanceDumpDir();
- run("REPL DUMP " + dbName);
- String replDumpLocn = getResult(0, 0);
- String replDumpId = getResult(0, 1, true);
+ run("REPL DUMP " + dbName, driver);
+ String replDumpLocn = getResult(0, 0, driver);
+ String replDumpId = getResult(0, 1, true, driver);
LOG.info("Bootstrap-Dump: Dumped to {} with id {}", replDumpLocn, replDumpId);
- run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'");
+ run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'", driverMirror);
String[] unptn_data = new String[] { "eleven", "twelve" };
- run("INSERT INTO TABLE " + dbName + ".unptned values('" + unptn_data[0] + "')");
- run("INSERT INTO TABLE " + dbName + ".unptned values('" + unptn_data[1] + "')");
- verifySetup("SELECT a from " + dbName + ".unptned ORDER BY a", unptn_data);
+ run("INSERT INTO TABLE " + dbName + ".unptned values('" + unptn_data[0] + "')", driver);
+ run("INSERT INTO TABLE " + dbName + ".unptned values('" + unptn_data[1] + "')", driver);
+ verifySetup("SELECT a from " + dbName + ".unptned ORDER BY a", unptn_data, driver);
- run("CREATE TABLE " + dbName + ".unptned_late LIKE " + dbName + ".unptned");
- run("INSERT INTO TABLE " + dbName + ".unptned_late SELECT * FROM " + dbName + ".unptned");
- verifySetup("SELECT * from " + dbName + ".unptned_late ORDER BY a", unptn_data);
+ run("CREATE TABLE " + dbName + ".unptned_late LIKE " + dbName + ".unptned", driver);
+ run("INSERT INTO TABLE " + dbName + ".unptned_late SELECT * FROM " + dbName + ".unptned", driver);
+ verifySetup("SELECT * from " + dbName + ".unptned_late ORDER BY a", unptn_data, driver);
advanceDumpDir();
- run("REPL DUMP " + dbName + " FROM " + replDumpId);
- String incrementalDumpLocn = getResult(0, 0);
- String incrementalDumpId = getResult(0, 1, true);
+ run("REPL DUMP " + dbName + " FROM " + replDumpId, driver);
+ String incrementalDumpLocn = getResult(0, 0, driver);
+ String incrementalDumpId = getResult(0, 1, true, driver);
LOG.info("Incremental-Dump: Dumped to {} with id {} from {}", incrementalDumpLocn, incrementalDumpId, replDumpId);
replDumpId = incrementalDumpId;
- run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'");
- printOutput();
- run("REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'");
- verifyRun("SELECT a from " + dbName + ".unptned ORDER BY a", unptn_data);
- verifyRun("SELECT a from " + dbName + ".unptned_late ORDER BY a", unptn_data);
- verifyRun("SELECT a from " + dbName + "_dupe.unptned ORDER BY a", unptn_data);
- verifyRun("SELECT a from " + dbName + "_dupe.unptned_late ORDER BY a", unptn_data);
+ run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'", driverMirror);
+ printOutput(driverMirror);
+ run("REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'", driverMirror);
+ verifyRun("SELECT a from " + dbName + ".unptned ORDER BY a", unptn_data, driver);
+ verifyRun("SELECT a from " + dbName + ".unptned_late ORDER BY a", unptn_data, driver);
+ verifyRun("SELECT a from " + dbName + "_dupe.unptned ORDER BY a", unptn_data, driverMirror);
+ verifyRun("SELECT a from " + dbName + "_dupe.unptned_late ORDER BY a", unptn_data, driverMirror);
String[] unptn_data_after_ins = new String[] { "eleven", "thirteen", "twelve" };
String[] data_after_ovwrite = new String[] { "hundred" };
- run("INSERT INTO TABLE " + dbName + ".unptned_late values('" + unptn_data_after_ins[1] + "')");
- verifySetup("SELECT a from " + dbName + ".unptned_late ORDER BY a", unptn_data_after_ins);
- run("INSERT OVERWRITE TABLE " + dbName + ".unptned values('" + data_after_ovwrite[0] + "')");
- verifySetup("SELECT a from " + dbName + ".unptned", data_after_ovwrite);
+ run("INSERT INTO TABLE " + dbName + ".unptned_late values('" + unptn_data_after_ins[1] + "')", driver);
+ verifySetup("SELECT a from " + dbName + ".unptned_late ORDER BY a", unptn_data_after_ins, driver);
+ run("INSERT OVERWRITE TABLE " + dbName + ".unptned values('" + data_after_ovwrite[0] + "')", driver);
+ verifySetup("SELECT a from " + dbName + ".unptned", data_after_ovwrite, driver);
advanceDumpDir();
- run("REPL DUMP " + dbName + " FROM " + replDumpId);
- incrementalDumpLocn = getResult(0, 0);
- incrementalDumpId = getResult(0, 1, true);
+ run("REPL DUMP " + dbName + " FROM " + replDumpId, driver);
+ incrementalDumpLocn = getResult(0, 0, driver);
+ incrementalDumpId = getResult(0, 1, true, driver);
LOG.info("Incremental-Dump: Dumped to {} with id {} from {}", incrementalDumpLocn, incrementalDumpId, replDumpId);
replDumpId = incrementalDumpId;
- run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'");
- printOutput();
- run("REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'");
+ run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'", driverMirror);
+ printOutput(driverMirror);
+ run("REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'", driverMirror);
- verifyRun("SELECT a from " + dbName + "_dupe.unptned_late ORDER BY a", unptn_data_after_ins);
+ verifyRun("SELECT a from " + dbName + "_dupe.unptned_late ORDER BY a", unptn_data_after_ins, driverMirror);
- verifyRun("SELECT a from " + dbName + "_dupe.unptned", data_after_ovwrite);
+ verifyRun("SELECT a from " + dbName + "_dupe.unptned", data_after_ovwrite, driverMirror);
}
@Test
public void testEventTypesForDynamicAddPartitionByInsert() throws IOException {
String name = testName.getMethodName();
- final String dbName = createDB(name);
+ final String dbName = createDB(name, driver);
String replDbName = dbName + "_dupe";
- run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE");
+ run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE", driver);
Tuple bootstrap = bootstrapLoadAndVerify(dbName, replDbName);
String[] ptn_data = new String[]{ "ten"};
- run("INSERT INTO TABLE " + dbName + ".ptned partition(b=1) values('" + ptn_data[0] + "')");
+ run("INSERT INTO TABLE " + dbName + ".ptned partition(b=1) values('" + ptn_data[0] + "')", driver);
// Inject a behaviour where it throws exception if an INSERT event is found
// As we dynamically add a partition through INSERT INTO cmd, it should just add ADD_PARTITION
@@ -1291,7 +1316,7 @@ public class TestReplicationScenarios {
eventTypeValidator.assertInjectionsPerformed(true,false);
InjectableBehaviourObjectStore.resetGetNextNotificationBehaviour(); // reset the behaviour
- verifyRun("SELECT a from " + replDbName + ".ptned where (b=1) ORDER BY a", ptn_data);
+ verifyRun("SELECT a from " + replDbName + ".ptned where (b=1) ORDER BY a", ptn_data, driverMirror);
}
@Test
@@ -1300,64 +1325,64 @@ public class TestReplicationScenarios {
LOG.info("Testing " + testName);
String dbName = testName + "_" + tid;
- run("CREATE DATABASE " + dbName);
- run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE");
+ run("CREATE DATABASE " + dbName, driver);
+ run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE", driver);
advanceDumpDir();
- run("REPL DUMP " + dbName);
- String replDumpLocn = getResult(0, 0);
- String replDumpId = getResult(0, 1, true);
+ run("REPL DUMP " + dbName, driver);
+ String replDumpLocn = getResult(0, 0, driver);
+ String replDumpId = getResult(0, 1, true, driver);
LOG.info("Bootstrap-Dump: Dumped to {} with id {}", replDumpLocn, replDumpId);
- run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'");
+ run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'", driverMirror);
String[] ptn_data_1 = new String[] { "fifteen", "fourteen", "thirteen" };
String[] ptn_data_2 = new String[] { "fifteen", "seventeen", "sixteen" };
- run("INSERT INTO TABLE " + dbName + ".ptned partition(b=1) values('" + ptn_data_1[0] + "')");
- run("INSERT INTO TABLE " + dbName + ".ptned partition(b=1) values('" + ptn_data_1[1] + "')");
- run("INSERT INTO TABLE " + dbName + ".ptned partition(b=1) values('" + ptn_data_1[2] + "')");
+ run("INSERT INTO TABLE " + dbName + ".ptned partition(b=1) values('" + ptn_data_1[0] + "')", driver);
+ run("INSERT INTO TABLE " + dbName + ".ptned partition(b=1) values('" + ptn_data_1[1] + "')", driver);
+ run("INSERT INTO TABLE " + dbName + ".ptned partition(b=1) values('" + ptn_data_1[2] + "')", driver);
- run("ALTER TABLE " + dbName + ".ptned ADD PARTITION (b=2)");
- run("INSERT INTO TABLE " + dbName + ".ptned partition(b=2) values('" + ptn_data_2[0] + "')");
- run("INSERT INTO TABLE " + dbName + ".ptned partition(b=2) values('" + ptn_data_2[1] + "')");
- run("INSERT INTO TABLE " + dbName + ".ptned partition(b=2) values('" + ptn_data_2[2] + "')");
- verifySetup("SELECT a from " + dbName + ".ptned where (b=1) ORDER BY a", ptn_data_1);
- verifySetup("SELECT a from " + dbName + ".ptned where (b=2) ORDER BY a", ptn_data_2);
+ run("ALTER TABLE " + dbName + ".ptned ADD PARTITION (b=2)", driver);
+ run("INSERT INTO TABLE " + dbName + ".ptned partition(b=2) values('" + ptn_data_2[0] + "')", driver);
+ run("INSERT INTO TABLE " + dbName + ".ptned partition(b=2) values('" + ptn_data_2[1] + "')", driver);
+ run("INSERT INTO TABLE " + dbName + ".ptned partition(b=2) values('" + ptn_data_2[2] + "')", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned where (b=1) ORDER BY a", ptn_data_1, driver);
+ verifySetup("SELECT a from " + dbName + ".ptned where (b=2) ORDER BY a", ptn_data_2, driver);
advanceDumpDir();
- run("REPL DUMP " + dbName + " FROM " + replDumpId);
- String incrementalDumpLocn = getResult(0, 0);
- String incrementalDumpId = getResult(0, 1, true);
+ run("REPL DUMP " + dbName + " FROM " + replDumpId, driver);
+ String incrementalDumpLocn = getResult(0, 0, driver);
+ String incrementalDumpId = getResult(0, 1, true, driver);
LOG.info("Incremental-Dump: Dumped to {} with id {} from {}", incrementalDumpLocn, incrementalDumpId, replD
<TRUNCATED>
[4/4] hive git commit: HIVE-15705: Event replication for constraints
(Daniel Dai, reviewed by Sankar Hariappan, Thejas Nair)
Posted by da...@apache.org.
HIVE-15705: Event replication for constraints (Daniel Dai, reviewed by Sankar Hariappan, Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/90a92b74
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/90a92b74
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/90a92b74
Branch: refs/heads/master
Commit: 90a92b7462f17bddcc55a02736a0af884db2d018
Parents: 16bfb9c
Author: Daniel Dai <da...@hortonworks.com>
Authored: Wed Aug 9 12:35:49 2017 -0700
Committer: Daniel Dai <da...@hortonworks.com>
Committed: Wed Aug 9 12:35:49 2017 -0700
----------------------------------------------------------------------
.../listener/DbNotificationListener.java | 94 +
.../listener/DummyRawStoreFailEvent.java | 15 +-
.../hive/ql/parse/TestReplicationScenarios.java | 2342 +++++++++---------
.../hadoop/hive/metastore/HiveMetaStore.java | 230 +-
.../hive/metastore/MetaStoreEventListener.java | 40 +
.../metastore/MetaStoreListenerNotifier.java | 28 +
.../hadoop/hive/metastore/ObjectStore.java | 66 +-
.../apache/hadoop/hive/metastore/RawStore.java | 10 +-
.../hive/metastore/cache/CachedStore.java | 21 +-
.../metastore/events/AddForeignKeyEvent.java | 37 +
.../events/AddNotNullConstraintEvent.java | 37 +
.../metastore/events/AddPrimaryKeyEvent.java | 38 +
.../events/AddUniqueConstraintEvent.java | 37 +
.../metastore/events/DropConstraintEvent.java | 47 +
.../messaging/AddForeignKeyMessage.java | 36 +
.../messaging/AddNotNullConstraintMessage.java | 36 +
.../messaging/AddPrimaryKeyMessage.java | 35 +
.../messaging/AddUniqueConstraintMessage.java | 36 +
.../messaging/DropConstraintMessage.java | 29 +
.../hive/metastore/messaging/EventMessage.java | 7 +-
.../messaging/MessageDeserializer.java | 36 +-
.../metastore/messaging/MessageFactory.java | 53 +-
.../json/JSONAddForeignKeyMessage.java | 101 +
.../json/JSONAddNotNullConstraintMessage.java | 79 +
.../json/JSONAddPrimaryKeyMessage.java | 101 +
.../json/JSONAddUniqueConstraintMessage.java | 98 +
.../json/JSONDropConstraintMessage.java | 90 +
.../messaging/json/JSONMessageDeserializer.java | 50 +
.../messaging/json/JSONMessageFactory.java | 56 +
.../DummyRawStoreControlledCommit.java | 17 +-
.../DummyRawStoreForJdoConnection.java | 15 +-
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 13 +-
.../hive/ql/parse/DDLSemanticAnalyzer.java | 4 +-
.../hadoop/hive/ql/parse/repl/DumpType.java | 35 +
.../repl/dump/events/AddForeignKeyHandler.java | 41 +
.../events/AddNotNullConstraintHandler.java | 41 +
.../repl/dump/events/AddPrimaryKeyHandler.java | 41 +
.../dump/events/AddUniqueConstraintHandler.java | 41 +
.../repl/dump/events/DropConstraintHandler.java | 41 +
.../repl/dump/events/EventHandlerFactory.java | 5 +
.../hive/ql/parse/repl/load/DumpMetaData.java | 14 +-
.../repl/load/message/AddForeignKeyHandler.java | 71 +
.../message/AddNotNullConstraintHandler.java | 72 +
.../repl/load/message/AddPrimaryKeyHandler.java | 69 +
.../message/AddUniqueConstraintHandler.java | 70 +
.../load/message/DropConstraintHandler.java | 51 +
.../hadoop/hive/ql/plan/AlterTableDesc.java | 19 +-
47 files changed, 3352 insertions(+), 1193 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
----------------------------------------------------------------------
diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
index 351546c..2a2cc7b 100644
--- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
+++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
@@ -41,9 +41,17 @@ import org.apache.hadoop.hive.metastore.api.Index;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
+import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
+import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
+import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.events.AddForeignKeyEvent;
import org.apache.hadoop.hive.metastore.events.AddIndexEvent;
+import org.apache.hadoop.hive.metastore.events.AddNotNullConstraintEvent;
import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.AddPrimaryKeyEvent;
+import org.apache.hadoop.hive.metastore.events.AddUniqueConstraintEvent;
import org.apache.hadoop.hive.metastore.events.AlterIndexEvent;
import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent;
import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
@@ -51,6 +59,7 @@ import org.apache.hadoop.hive.metastore.events.ConfigChangeEvent;
import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent;
import org.apache.hadoop.hive.metastore.events.CreateFunctionEvent;
import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
+import org.apache.hadoop.hive.metastore.events.DropConstraintEvent;
import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
import org.apache.hadoop.hive.metastore.events.DropFunctionEvent;
import org.apache.hadoop.hive.metastore.events.DropIndexEvent;
@@ -459,6 +468,91 @@ public class DbNotificationListener extends MetaStoreEventListener {
// then load data into it.
}
+ /***
+ * @param addPrimaryKeyEvent add primary key event
+ * @throws MetaException
+ */
+ @Override
+ public void onAddPrimaryKey(AddPrimaryKeyEvent addPrimaryKeyEvent) throws MetaException {
+ List<SQLPrimaryKey> cols = addPrimaryKeyEvent.getPrimaryKeyCols();
+ if (cols.size() > 0) {
+ NotificationEvent event =
+ new NotificationEvent(0, now(), EventType.ADD_PRIMARYKEY.toString(), msgFactory
+ .buildAddPrimaryKeyMessage(addPrimaryKeyEvent.getPrimaryKeyCols()).toString());
+ event.setDbName(cols.get(0).getTable_db());
+ event.setTableName(cols.get(0).getTable_name());
+ process(event, addPrimaryKeyEvent);
+ }
+ }
+
+ /***
+ * @param addForeignKeyEvent add foreign key event
+ * @throws MetaException
+ */
+ @Override
+ public void onAddForeignKey(AddForeignKeyEvent addForeignKeyEvent) throws MetaException {
+ List<SQLForeignKey> cols = addForeignKeyEvent.getForeignKeyCols();
+ if (cols.size() > 0) {
+ NotificationEvent event =
+ new NotificationEvent(0, now(), EventType.ADD_FOREIGNKEY.toString(), msgFactory
+ .buildAddForeignKeyMessage(addForeignKeyEvent.getForeignKeyCols()).toString());
+ event.setDbName(cols.get(0).getPktable_db());
+ event.setTableName(cols.get(0).getPktable_name());
+ process(event, addForeignKeyEvent);
+ }
+ }
+
+ /***
+ * @param addUniqueConstraintEvent add unique constraint event
+ * @throws MetaException
+ */
+ @Override
+ public void onAddUniqueConstraint(AddUniqueConstraintEvent addUniqueConstraintEvent) throws MetaException {
+ List<SQLUniqueConstraint> cols = addUniqueConstraintEvent.getUniqueConstraintCols();
+ if (cols.size() > 0) {
+ NotificationEvent event =
+ new NotificationEvent(0, now(), EventType.ADD_UNIQUECONSTRAINT.toString(), msgFactory
+ .buildAddUniqueConstraintMessage(addUniqueConstraintEvent.getUniqueConstraintCols()).toString());
+ event.setDbName(cols.get(0).getTable_db());
+ event.setTableName(cols.get(0).getTable_name());
+ process(event, addUniqueConstraintEvent);
+ }
+ }
+
+ /***
+ * @param addNotNullConstraintEvent add not null constraint event
+ * @throws MetaException
+ */
+ @Override
+ public void onAddNotNullConstraint(AddNotNullConstraintEvent addNotNullConstraintEvent) throws MetaException {
+ List<SQLNotNullConstraint> cols = addNotNullConstraintEvent.getNotNullConstraintCols();
+ if (cols.size() > 0) {
+ NotificationEvent event =
+ new NotificationEvent(0, now(), EventType.ADD_NOTNULLCONSTRAINT.toString(), msgFactory
+ .buildAddNotNullConstraintMessage(addNotNullConstraintEvent.getNotNullConstraintCols()).toString());
+ event.setDbName(cols.get(0).getTable_db());
+ event.setTableName(cols.get(0).getTable_name());
+ process(event, addNotNullConstraintEvent);
+ }
+ }
+
+ /***
+ * @param dropConstraintEvent drop constraint event
+ * @throws MetaException
+ */
+ @Override
+ public void onDropConstraint(DropConstraintEvent dropConstraintEvent) throws MetaException {
+ String dbName = dropConstraintEvent.getDbName();
+ String tableName = dropConstraintEvent.getTableName();
+ String constraintName = dropConstraintEvent.getConstraintName();
+ NotificationEvent event =
+ new NotificationEvent(0, now(), EventType.DROP_CONSTRAINT.toString(), msgFactory
+ .buildDropConstraintMessage(dbName, tableName, constraintName).toString());
+ event.setDbName(dbName);
+ event.setTableName(tableName);
+ process(event, dropConstraintEvent);
+ }
+
private int now() {
long millis = System.currentTimeMillis();
millis /= 1000;
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
----------------------------------------------------------------------
diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
index 35989f5..5d7cfad 100644
--- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
+++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
@@ -912,11 +912,12 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable {
}
@Override
- public void createTableWithConstraints(Table tbl,
+ public List<String> createTableWithConstraints(Table tbl,
List<SQLPrimaryKey> primaryKeys, List<SQLForeignKey> foreignKeys,
List<SQLUniqueConstraint> uniqueConstraints,
List<SQLNotNullConstraint> notNullConstraints)
throws InvalidObjectException, MetaException {
+ return null;
}
@Override
@@ -925,23 +926,27 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable {
}
@Override
- public void addPrimaryKeys(List<SQLPrimaryKey> pks)
+ public List<String> addPrimaryKeys(List<SQLPrimaryKey> pks)
throws InvalidObjectException, MetaException {
+ return null;
}
@Override
- public void addForeignKeys(List<SQLForeignKey> fks)
+ public List<String> addForeignKeys(List<SQLForeignKey> fks)
throws InvalidObjectException, MetaException {
+ return null;
}
@Override
- public void addUniqueConstraints(List<SQLUniqueConstraint> uks)
+ public List<String> addUniqueConstraints(List<SQLUniqueConstraint> uks)
throws InvalidObjectException, MetaException {
+ return null;
}
@Override
- public void addNotNullConstraints(List<SQLNotNullConstraint> nns)
+ public List<String> addNotNullConstraints(List<SQLNotNullConstraint> nns)
throws InvalidObjectException, MetaException {
+ return null;
}
@Override
[2/4] hive git commit: HIVE-15705: Event replication for constraints
(Daniel Dai, reviewed by Sankar Hariappan, Thejas Nair)
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index e2a7d7d..b43b4df 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -85,8 +85,12 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.io.HdfsUtils;
import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.hadoop.hive.metastore.events.AddForeignKeyEvent;
import org.apache.hadoop.hive.metastore.events.AddIndexEvent;
+import org.apache.hadoop.hive.metastore.events.AddNotNullConstraintEvent;
import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.AddPrimaryKeyEvent;
+import org.apache.hadoop.hive.metastore.events.AddUniqueConstraintEvent;
import org.apache.hadoop.hive.metastore.events.AlterIndexEvent;
import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent;
import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
@@ -94,6 +98,7 @@ import org.apache.hadoop.hive.metastore.events.ConfigChangeEvent;
import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent;
import org.apache.hadoop.hive.metastore.events.CreateFunctionEvent;
import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
+import org.apache.hadoop.hive.metastore.events.DropConstraintEvent;
import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
import org.apache.hadoop.hive.metastore.events.DropFunctionEvent;
import org.apache.hadoop.hive.metastore.events.DropIndexEvent;
@@ -1540,16 +1545,64 @@ public class HiveMetaStore extends ThriftHiveMetastore {
&& uniqueConstraints == null && notNullConstraints == null) {
ms.createTable(tbl);
} else {
- ms.createTableWithConstraints(tbl, primaryKeys, foreignKeys,
+ // Set constraint name if null before sending to listener
+ List<String> constraintNames = ms.createTableWithConstraints(tbl, primaryKeys, foreignKeys,
uniqueConstraints, notNullConstraints);
+ int primaryKeySize = 0;
+ if (primaryKeys != null) {
+ primaryKeySize = primaryKeys.size();
+ for (int i = 0; i < primaryKeys.size(); i++) {
+ if (primaryKeys.get(i).getPk_name() == null) {
+ primaryKeys.get(i).setPk_name(constraintNames.get(i));
+ }
+ }
+ }
+ int foreignKeySize = 0;
+ if (foreignKeys != null) {
+ foreignKeySize = foreignKeys.size();
+ for (int i = 0; i < foreignKeySize; i++) {
+ if (foreignKeys.get(i).getFk_name() == null) {
+ foreignKeys.get(i).setFk_name(constraintNames.get(primaryKeySize + i));
+ }
+ }
+ }
+ int uniqueConstraintSize = 0;
+ if (uniqueConstraints != null) {
+ uniqueConstraintSize = uniqueConstraints.size();
+ for (int i = 0; i < uniqueConstraintSize; i++) {
+ if (uniqueConstraints.get(i).getUk_name() == null) {
+ uniqueConstraints.get(i).setUk_name(constraintNames.get(primaryKeySize + foreignKeySize + i));
+ }
+ }
+ }
+ if (notNullConstraints != null) {
+ for (int i = 0; i < notNullConstraints.size(); i++) {
+ if (notNullConstraints.get(i).getNn_name() == null) {
+ notNullConstraints.get(i).setNn_name(constraintNames.get(primaryKeySize + foreignKeySize + uniqueConstraintSize + i));
+ }
+ }
+ }
}
if (!transactionalListeners.isEmpty()) {
- transactionalListenerResponses =
- MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
- EventType.CREATE_TABLE,
- new CreateTableEvent(tbl, true, this),
- envContext);
+ transactionalListenerResponses = MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
+ EventType.CREATE_TABLE, new CreateTableEvent(tbl, true, this), envContext);
+ if (primaryKeys != null && !primaryKeys.isEmpty()) {
+ MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventType.ADD_PRIMARYKEY,
+ new AddPrimaryKeyEvent(primaryKeys, true, this), envContext);
+ }
+ if (foreignKeys != null && !foreignKeys.isEmpty()) {
+ MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventType.ADD_FOREIGNKEY,
+ new AddForeignKeyEvent(foreignKeys, true, this), envContext);
+ }
+ if (uniqueConstraints != null && !uniqueConstraints.isEmpty()) {
+ MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventType.ADD_UNIQUECONSTRAINT,
+ new AddUniqueConstraintEvent(uniqueConstraints, true, this), envContext);
+ }
+ if (notNullConstraints != null && !notNullConstraints.isEmpty()) {
+ MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventType.ADD_NOTNULLCONSTRAINT,
+ new AddNotNullConstraintEvent(notNullConstraints, true, this), envContext);
+ }
}
success = ms.commitTransaction();
@@ -1562,11 +1615,24 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
if (!listeners.isEmpty()) {
- MetaStoreListenerNotifier.notifyEvent(listeners,
- EventType.CREATE_TABLE,
- new CreateTableEvent(tbl, success, this),
- envContext,
- transactionalListenerResponses, ms);
+ MetaStoreListenerNotifier.notifyEvent(listeners, EventType.CREATE_TABLE,
+ new CreateTableEvent(tbl, success, this), envContext, transactionalListenerResponses, ms);
+ if (primaryKeys != null && !primaryKeys.isEmpty()) {
+ MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ADD_PRIMARYKEY,
+ new AddPrimaryKeyEvent(primaryKeys, success, this), envContext);
+ }
+ if (foreignKeys != null && !foreignKeys.isEmpty()) {
+ MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ADD_FOREIGNKEY,
+ new AddForeignKeyEvent(foreignKeys, success, this), envContext);
+ }
+ if (uniqueConstraints != null && !uniqueConstraints.isEmpty()) {
+ MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ADD_UNIQUECONSTRAINT,
+ new AddUniqueConstraintEvent(uniqueConstraints, success, this), envContext);
+ }
+ if (notNullConstraints != null && !notNullConstraints.isEmpty()) {
+ MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ADD_NOTNULLCONSTRAINT,
+ new AddNotNullConstraintEvent(notNullConstraints, success, this), envContext);
+ }
}
}
}
@@ -1647,9 +1713,18 @@ public class HiveMetaStore extends ThriftHiveMetastore {
startFunction("drop_constraint", ": " + constraintName.toString());
boolean success = false;
Exception ex = null;
+ RawStore ms = getMS();
try {
- getMS().dropConstraint(dbName, tableName, constraintName);
- success = true;
+ ms.openTransaction();
+ ms.dropConstraint(dbName, tableName, constraintName);
+ if (transactionalListeners.size() > 0) {
+ DropConstraintEvent dropConstraintEvent = new DropConstraintEvent(dbName,
+ tableName, constraintName, true, this);
+ for (MetaStoreEventListener transactionalListener : transactionalListeners) {
+ transactionalListener.onDropConstraint(dropConstraintEvent);
+ }
+ }
+ success = ms.commitTransaction();
} catch (NoSuchObjectException e) {
ex = e;
throw new InvalidObjectException(e.getMessage());
@@ -1663,6 +1738,15 @@ public class HiveMetaStore extends ThriftHiveMetastore {
throw newMetaException(e);
}
} finally {
+ if (!success) {
+ ms.rollbackTransaction();
+ } else {
+ for (MetaStoreEventListener listener : listeners) {
+ DropConstraintEvent dropConstraintEvent = new DropConstraintEvent(dbName,
+ tableName, constraintName, true, this);
+ listener.onDropConstraint(dropConstraintEvent);
+ }
+ }
endFunction("drop_constraint", success, ex, constraintName);
}
}
@@ -1676,9 +1760,27 @@ public class HiveMetaStore extends ThriftHiveMetastore {
startFunction("add_primary_key", ": " + constraintName);
boolean success = false;
Exception ex = null;
+ RawStore ms = getMS();
try {
- getMS().addPrimaryKeys(primaryKeyCols);
- success = true;
+ ms.openTransaction();
+ List<String> constraintNames = ms.addPrimaryKeys(primaryKeyCols);
+ // Set primary key name if null before sending to listener
+ if (primaryKeyCols != null) {
+ for (int i = 0; i < primaryKeyCols.size(); i++) {
+ if (primaryKeyCols.get(i).getPk_name() == null) {
+ primaryKeyCols.get(i).setPk_name(constraintNames.get(i));
+ }
+ }
+ }
+ if (transactionalListeners.size() > 0) {
+ if (primaryKeyCols != null && primaryKeyCols.size() > 0) {
+ AddPrimaryKeyEvent addPrimaryKeyEvent = new AddPrimaryKeyEvent(primaryKeyCols, true, this);
+ for (MetaStoreEventListener transactionalListener : transactionalListeners) {
+ transactionalListener.onAddPrimaryKey(addPrimaryKeyEvent);
+ }
+ }
+ }
+ success = ms.commitTransaction();
} catch (Exception e) {
ex = e;
if (e instanceof MetaException) {
@@ -1689,6 +1791,14 @@ public class HiveMetaStore extends ThriftHiveMetastore {
throw newMetaException(e);
}
} finally {
+ if (!success) {
+ ms.rollbackTransaction();
+ } else if (primaryKeyCols != null && primaryKeyCols.size() > 0) {
+ for (MetaStoreEventListener listener : listeners) {
+ AddPrimaryKeyEvent addPrimaryKeyEvent = new AddPrimaryKeyEvent(primaryKeyCols, true, this);
+ listener.onAddPrimaryKey(addPrimaryKeyEvent);
+ }
+ }
endFunction("add_primary_key", success, ex, constraintName);
}
}
@@ -1702,9 +1812,27 @@ public class HiveMetaStore extends ThriftHiveMetastore {
startFunction("add_foreign_key", ": " + constraintName);
boolean success = false;
Exception ex = null;
+ RawStore ms = getMS();
try {
- getMS().addForeignKeys(foreignKeyCols);
- success = true;
+ ms.openTransaction();
+ List<String> constraintNames = ms.addForeignKeys(foreignKeyCols);
+ // Set foreign key name if null before sending to listener
+ if (foreignKeyCols != null) {
+ for (int i = 0; i < foreignKeyCols.size(); i++) {
+ if (foreignKeyCols.get(i).getFk_name() == null) {
+ foreignKeyCols.get(i).setFk_name(constraintNames.get(i));
+ }
+ }
+ }
+ if (transactionalListeners.size() > 0) {
+ if (foreignKeyCols != null && foreignKeyCols.size() > 0) {
+ AddForeignKeyEvent addForeignKeyEvent = new AddForeignKeyEvent(foreignKeyCols, true, this);
+ for (MetaStoreEventListener transactionalListener : transactionalListeners) {
+ transactionalListener.onAddForeignKey(addForeignKeyEvent);
+ }
+ }
+ }
+ success = ms.commitTransaction();
} catch (Exception e) {
ex = e;
if (e instanceof MetaException) {
@@ -1715,6 +1843,14 @@ public class HiveMetaStore extends ThriftHiveMetastore {
throw newMetaException(e);
}
} finally {
+ if (!success) {
+ ms.rollbackTransaction();
+ } else if (foreignKeyCols != null && foreignKeyCols.size() > 0) {
+ for (MetaStoreEventListener listener : listeners) {
+ AddForeignKeyEvent addForeignKeyEvent = new AddForeignKeyEvent(foreignKeyCols, true, this);
+ listener.onAddForeignKey(addForeignKeyEvent);
+ }
+ }
endFunction("add_foreign_key", success, ex, constraintName);
}
}
@@ -1728,9 +1864,27 @@ public class HiveMetaStore extends ThriftHiveMetastore {
startFunction("add_unique_constraint", ": " + constraintName);
boolean success = false;
Exception ex = null;
+ RawStore ms = getMS();
try {
- getMS().addUniqueConstraints(uniqueConstraintCols);
- success = true;
+ ms.openTransaction();
+ List<String> constraintNames = ms.addUniqueConstraints(uniqueConstraintCols);
+ // Set unique constraint name if null before sending to listener
+ if (uniqueConstraintCols != null) {
+ for (int i = 0; i < uniqueConstraintCols.size(); i++) {
+ if (uniqueConstraintCols.get(i).getUk_name() == null) {
+ uniqueConstraintCols.get(i).setUk_name(constraintNames.get(i));
+ }
+ }
+ }
+ if (transactionalListeners.size() > 0) {
+ if (uniqueConstraintCols != null && uniqueConstraintCols.size() > 0) {
+ AddUniqueConstraintEvent addUniqueConstraintEvent = new AddUniqueConstraintEvent(uniqueConstraintCols, true, this);
+ for (MetaStoreEventListener transactionalListener : transactionalListeners) {
+ transactionalListener.onAddUniqueConstraint(addUniqueConstraintEvent);
+ }
+ }
+ }
+ success = ms.commitTransaction();
} catch (Exception e) {
ex = e;
if (e instanceof MetaException) {
@@ -1741,6 +1895,14 @@ public class HiveMetaStore extends ThriftHiveMetastore {
throw newMetaException(e);
}
} finally {
+ if (!success) {
+ ms.rollbackTransaction();
+ } else if (uniqueConstraintCols != null && uniqueConstraintCols.size() > 0) {
+ for (MetaStoreEventListener listener : listeners) {
+ AddUniqueConstraintEvent addUniqueConstraintEvent = new AddUniqueConstraintEvent(uniqueConstraintCols, true, this);
+ listener.onAddUniqueConstraint(addUniqueConstraintEvent);
+ }
+ }
endFunction("add_unique_constraint", success, ex, constraintName);
}
}
@@ -1754,9 +1916,27 @@ public class HiveMetaStore extends ThriftHiveMetastore {
startFunction("add_not_null_constraint", ": " + constraintName);
boolean success = false;
Exception ex = null;
+ RawStore ms = getMS();
try {
- getMS().addNotNullConstraints(notNullConstraintCols);
- success = true;
+ ms.openTransaction();
+ List<String> constraintNames = ms.addNotNullConstraints(notNullConstraintCols);
+ // Set not null constraint name if null before sending to listener
+ if (notNullConstraintCols != null) {
+ for (int i = 0; i < notNullConstraintCols.size(); i++) {
+ if (notNullConstraintCols.get(i).getNn_name() == null) {
+ notNullConstraintCols.get(i).setNn_name(constraintNames.get(i));
+ }
+ }
+ }
+ if (transactionalListeners.size() > 0) {
+ if (notNullConstraintCols != null && notNullConstraintCols.size() > 0) {
+ AddNotNullConstraintEvent addNotNullConstraintEvent = new AddNotNullConstraintEvent(notNullConstraintCols, true, this);
+ for (MetaStoreEventListener transactionalListener : transactionalListeners) {
+ transactionalListener.onAddNotNullConstraint(addNotNullConstraintEvent);
+ }
+ }
+ }
+ success = ms.commitTransaction();
} catch (Exception e) {
ex = e;
if (e instanceof MetaException) {
@@ -1767,6 +1947,14 @@ public class HiveMetaStore extends ThriftHiveMetastore {
throw newMetaException(e);
}
} finally {
+ if (!success) {
+ ms.rollbackTransaction();
+ } else if (notNullConstraintCols != null && notNullConstraintCols.size() > 0) {
+ for (MetaStoreEventListener listener : listeners) {
+ AddNotNullConstraintEvent addNotNullConstraintEvent = new AddNotNullConstraintEvent(notNullConstraintCols, true, this);
+ listener.onAddNotNullConstraint(addNotNullConstraintEvent);
+ }
+ }
endFunction("add_not_null_constraint", success, ex, constraintName);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java
index c2594f7..5a72082 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java
@@ -23,7 +23,11 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hadoop.hive.common.classification.InterfaceStability;
import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.events.AddForeignKeyEvent;
import org.apache.hadoop.hive.metastore.events.AddIndexEvent;
+import org.apache.hadoop.hive.metastore.events.AddNotNullConstraintEvent;
+import org.apache.hadoop.hive.metastore.events.AddPrimaryKeyEvent;
+import org.apache.hadoop.hive.metastore.events.AddUniqueConstraintEvent;
import org.apache.hadoop.hive.metastore.events.AlterIndexEvent;
import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent;
@@ -32,6 +36,7 @@ import org.apache.hadoop.hive.metastore.events.ConfigChangeEvent;
import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent;
import org.apache.hadoop.hive.metastore.events.CreateFunctionEvent;
import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
+import org.apache.hadoop.hive.metastore.events.DropConstraintEvent;
import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
import org.apache.hadoop.hive.metastore.events.DropFunctionEvent;
import org.apache.hadoop.hive.metastore.events.DropIndexEvent;
@@ -172,6 +177,41 @@ public abstract class MetaStoreEventListener implements Configurable {
}
+ /**
+ * @param addPrimaryKeyEvent add primary key event
+ * @throws MetaException
+ */
+ public void onAddPrimaryKey(AddPrimaryKeyEvent addPrimaryKeyEvent) throws MetaException {
+ }
+
+ /**
+ * @param addForeignKeyEvent add foreign key event
+ * @throws MetaException
+ */
+ public void onAddForeignKey(AddForeignKeyEvent addForeignKeyEvent) throws MetaException {
+ }
+
+ /**
+ * @param addUniqueConstraintEvent add unique constraint event
+ * @throws MetaException
+ */
+ public void onAddUniqueConstraint(AddUniqueConstraintEvent addUniqueConstraintEvent) throws MetaException {
+ }
+
+ /**
+ * @param addNotNullConstraintEvent add not null constraint event
+ * @throws MetaException
+ */
+ public void onAddNotNullConstraint(AddNotNullConstraintEvent addNotNullConstraintEvent) throws MetaException {
+ }
+
+ /**
+ * @param dropConstraintEvent drop constraint event
+ * @throws MetaException
+ */
+ public void onDropConstraint(DropConstraintEvent dropConstraintEvent) throws MetaException {
+ }
+
@Override
public Configuration getConf() {
return this.conf;
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java
index 37327f8..75dc89c 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java
@@ -24,8 +24,12 @@ import com.google.common.collect.Maps;
import org.apache.hadoop.hive.common.classification.InterfaceAudience.Private;
import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.events.AddForeignKeyEvent;
import org.apache.hadoop.hive.metastore.events.AddIndexEvent;
+import org.apache.hadoop.hive.metastore.events.AddNotNullConstraintEvent;
import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.AddPrimaryKeyEvent;
+import org.apache.hadoop.hive.metastore.events.AddUniqueConstraintEvent;
import org.apache.hadoop.hive.metastore.events.AlterIndexEvent;
import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent;
import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
@@ -141,6 +145,30 @@ public class MetaStoreListenerNotifier {
listener.onAlterIndex((AlterIndexEvent)event);
}
})
+ .put(EventType.ADD_PRIMARYKEY, new EventNotifier() {
+ @Override
+ public void notify(MetaStoreEventListener listener, ListenerEvent event) throws MetaException {
+ listener.onAddPrimaryKey((AddPrimaryKeyEvent)event);
+ }
+ })
+ .put(EventType.ADD_FOREIGNKEY, new EventNotifier() {
+ @Override
+ public void notify(MetaStoreEventListener listener, ListenerEvent event) throws MetaException {
+ listener.onAddForeignKey((AddForeignKeyEvent)event);
+ }
+ })
+ .put(EventType.ADD_UNIQUECONSTRAINT, new EventNotifier() {
+ @Override
+ public void notify(MetaStoreEventListener listener, ListenerEvent event) throws MetaException {
+ listener.onAddUniqueConstraint((AddUniqueConstraintEvent)event);
+ }
+ })
+ .put(EventType.ADD_NOTNULLCONSTRAINT, new EventNotifier() {
+ @Override
+ public void notify(MetaStoreEventListener listener, ListenerEvent event) throws MetaException {
+ listener.onAddNotNullConstraint((AddNotNullConstraintEvent)event);
+ }
+ })
.build()
);
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 043e2df..30f9e57 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -188,6 +188,7 @@ import com.google.common.collect.Maps;
public class ObjectStore implements RawStore, Configurable {
private static Properties prop = null;
private static PersistenceManagerFactory pmf = null;
+ private static boolean forTwoMetastoreTesting = false;
private static Lock pmfPropLock = new ReentrantLock();
/**
@@ -290,8 +291,10 @@ public class ObjectStore implements RawStore, Configurable {
if (propsChanged) {
if (pmf != null){
clearOutPmfClassLoaderCache(pmf);
- // close the underlying connection pool to avoid leaks
- pmf.close();
+ if (!forTwoMetastoreTesting) {
+ // close the underlying connection pool to avoid leaks
+ pmf.close();
+ }
}
pmf = null;
prop = null;
@@ -979,7 +982,7 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
- public void createTableWithConstraints(Table tbl,
+ public List<String> createTableWithConstraints(Table tbl,
List<SQLPrimaryKey> primaryKeys, List<SQLForeignKey> foreignKeys,
List<SQLUniqueConstraint> uniqueConstraints,
List<SQLNotNullConstraint> notNullConstraints)
@@ -991,11 +994,12 @@ public class ObjectStore implements RawStore, Configurable {
// Add constraints.
// We need not do a deep retrieval of the Table Column Descriptor while persisting the
// constraints since this transaction involving create table is not yet committed.
- addPrimaryKeys(primaryKeys, false);
- addForeignKeys(foreignKeys, false);
- addUniqueConstraints(uniqueConstraints, false);
- addNotNullConstraints(notNullConstraints, false);
+ List<String> constraintNames = addPrimaryKeys(primaryKeys, false);
+ constraintNames.addAll(addForeignKeys(foreignKeys, false));
+ constraintNames.addAll(addUniqueConstraints(uniqueConstraints, false));
+ constraintNames.addAll(addNotNullConstraints(notNullConstraints, false));
success = commitTransaction();
+ return constraintNames;
} finally {
if (!success) {
rollbackTransaction();
@@ -3506,7 +3510,7 @@ public class ObjectStore implements RawStore, Configurable {
}
private String generateConstraintName(String... parameters) throws MetaException {
- int hashcode = ArrayUtils.toString(parameters).hashCode();
+ int hashcode = ArrayUtils.toString(parameters).hashCode() & 0xfffffff;
int counter = 0;
final int MAX_RETRIES = 10;
while (counter < MAX_RETRIES) {
@@ -3520,9 +3524,9 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
- public void addForeignKeys(
+ public List<String> addForeignKeys(
List<SQLForeignKey> fks) throws InvalidObjectException, MetaException {
- addForeignKeys(fks, true);
+ return addForeignKeys(fks, true);
}
@Override
@@ -3596,9 +3600,10 @@ public class ObjectStore implements RawStore, Configurable {
return null;
}
- private void addForeignKeys(
+ private List<String> addForeignKeys(
List<SQLForeignKey> fks, boolean retrieveCD) throws InvalidObjectException,
MetaException {
+ List<String> fkNames = new ArrayList<String>();
List<MConstraint> mpkfks = new ArrayList<MConstraint>();
String currentConstraintName = null;
@@ -3653,6 +3658,7 @@ public class ObjectStore implements RawStore, Configurable {
} else {
currentConstraintName = HiveStringUtils.normalizeIdentifier(fks.get(i).getFk_name());
}
+ fkNames.add(currentConstraintName);
Integer updateRule = fks.get(i).getUpdate_rule();
Integer deleteRule = fks.get(i).getDelete_rule();
int enableValidateRely = (fks.get(i).isEnable_cstr() ? 4 : 0) +
@@ -3674,16 +3680,18 @@ public class ObjectStore implements RawStore, Configurable {
mpkfks.add(mpkfk);
}
pm.makePersistentAll(mpkfks);
+ return fkNames;
}
@Override
- public void addPrimaryKeys(List<SQLPrimaryKey> pks) throws InvalidObjectException,
+ public List<String> addPrimaryKeys(List<SQLPrimaryKey> pks) throws InvalidObjectException,
MetaException {
- addPrimaryKeys(pks, true);
+ return addPrimaryKeys(pks, true);
}
- private void addPrimaryKeys(List<SQLPrimaryKey> pks, boolean retrieveCD) throws InvalidObjectException,
+ private List<String> addPrimaryKeys(List<SQLPrimaryKey> pks, boolean retrieveCD) throws InvalidObjectException,
MetaException {
+ List<String> pkNames = new ArrayList<String>();
List<MConstraint> mpks = new ArrayList<MConstraint>();
String constraintName = null;
@@ -3719,7 +3727,7 @@ public class ObjectStore implements RawStore, Configurable {
} else {
constraintName = HiveStringUtils.normalizeIdentifier(pks.get(i).getPk_name());
}
-
+ pkNames.add(constraintName);
int enableValidateRely = (pks.get(i).isEnable_cstr() ? 4 : 0) +
(pks.get(i).isValidate_cstr() ? 2 : 0) + (pks.get(i).isRely_cstr() ? 1 : 0);
MConstraint mpk = new MConstraint(
@@ -3738,16 +3746,18 @@ public class ObjectStore implements RawStore, Configurable {
mpks.add(mpk);
}
pm.makePersistentAll(mpks);
+ return pkNames;
}
@Override
- public void addUniqueConstraints(List<SQLUniqueConstraint> uks)
+ public List<String> addUniqueConstraints(List<SQLUniqueConstraint> uks)
throws InvalidObjectException, MetaException {
- addUniqueConstraints(uks, true);
+ return addUniqueConstraints(uks, true);
}
- private void addUniqueConstraints(List<SQLUniqueConstraint> uks, boolean retrieveCD)
+ private List<String> addUniqueConstraints(List<SQLUniqueConstraint> uks, boolean retrieveCD)
throws InvalidObjectException, MetaException {
+ List<String> ukNames = new ArrayList<String>();
List<MConstraint> cstrs = new ArrayList<MConstraint>();
String constraintName = null;
@@ -3777,6 +3787,7 @@ public class ObjectStore implements RawStore, Configurable {
} else {
constraintName = HiveStringUtils.normalizeIdentifier(uks.get(i).getUk_name());
}
+ ukNames.add(constraintName);
int enableValidateRely = (uks.get(i).isEnable_cstr() ? 4 : 0) +
(uks.get(i).isValidate_cstr() ? 2 : 0) + (uks.get(i).isRely_cstr() ? 1 : 0);
@@ -3796,16 +3807,18 @@ public class ObjectStore implements RawStore, Configurable {
cstrs.add(muk);
}
pm.makePersistentAll(cstrs);
+ return ukNames;
}
@Override
- public void addNotNullConstraints(List<SQLNotNullConstraint> nns)
+ public List<String> addNotNullConstraints(List<SQLNotNullConstraint> nns)
throws InvalidObjectException, MetaException {
- addNotNullConstraints(nns, true);
+ return addNotNullConstraints(nns, true);
}
- private void addNotNullConstraints(List<SQLNotNullConstraint> nns, boolean retrieveCD)
+ private List<String> addNotNullConstraints(List<SQLNotNullConstraint> nns, boolean retrieveCD)
throws InvalidObjectException, MetaException {
+ List<String> nnNames = new ArrayList<String>();
List<MConstraint> cstrs = new ArrayList<MConstraint>();
String constraintName = null;
@@ -3833,6 +3846,7 @@ public class ObjectStore implements RawStore, Configurable {
} else {
constraintName = HiveStringUtils.normalizeIdentifier(nns.get(i).getNn_name());
}
+ nnNames.add(constraintName);
int enableValidateRely = (nns.get(i).isEnable_cstr() ? 4 : 0) +
(nns.get(i).isValidate_cstr() ? 2 : 0) + (nns.get(i).isRely_cstr() ? 1 : 0);
@@ -3852,6 +3866,7 @@ public class ObjectStore implements RawStore, Configurable {
cstrs.add(muk);
}
pm.makePersistentAll(cstrs);
+ return nnNames;
}
@Override
@@ -8869,4 +8884,13 @@ public class ObjectStore implements RawStore, Configurable {
}
}
}
+
+ /**
+ * To make possible to run multiple metastore in unit test
+ * @param twoMetastoreTesting if we are using multiple metastore in unit test
+ */
+ @VisibleForTesting
+ public static void setTwoMetastoreTesting(boolean twoMetastoreTesting) {
+ forTwoMetastoreTesting = twoMetastoreTesting;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
index 34f12a6..a2ae4c5 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
@@ -710,19 +710,19 @@ public interface RawStore extends Configurable {
public abstract List<SQLNotNullConstraint> getNotNullConstraints(String db_name,
String tbl_name) throws MetaException;
- void createTableWithConstraints(Table tbl, List<SQLPrimaryKey> primaryKeys,
+ List<String> createTableWithConstraints(Table tbl, List<SQLPrimaryKey> primaryKeys,
List<SQLForeignKey> foreignKeys, List<SQLUniqueConstraint> uniqueConstraints,
List<SQLNotNullConstraint> notNullConstraints) throws InvalidObjectException, MetaException;
void dropConstraint(String dbName, String tableName, String constraintName) throws NoSuchObjectException;
- void addPrimaryKeys(List<SQLPrimaryKey> pks) throws InvalidObjectException, MetaException;
+ List<String> addPrimaryKeys(List<SQLPrimaryKey> pks) throws InvalidObjectException, MetaException;
- void addForeignKeys(List<SQLForeignKey> fks) throws InvalidObjectException, MetaException;
+ List<String> addForeignKeys(List<SQLForeignKey> fks) throws InvalidObjectException, MetaException;
- void addUniqueConstraints(List<SQLUniqueConstraint> uks) throws InvalidObjectException, MetaException;
+ List<String> addUniqueConstraints(List<SQLUniqueConstraint> uks) throws InvalidObjectException, MetaException;
- void addNotNullConstraints(List<SQLNotNullConstraint> nns) throws InvalidObjectException, MetaException;
+ List<String> addNotNullConstraints(List<SQLNotNullConstraint> nns) throws InvalidObjectException, MetaException;
/**
* Gets the unique id of the backing datastore for the metadata
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
index fb98ccf..ce98a6e 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
@@ -1895,16 +1895,17 @@ public class CachedStore implements RawStore, Configurable {
}
@Override
- public void createTableWithConstraints(Table tbl,
+ public List<String> createTableWithConstraints(Table tbl,
List<SQLPrimaryKey> primaryKeys, List<SQLForeignKey> foreignKeys,
List<SQLUniqueConstraint> uniqueConstraints,
List<SQLNotNullConstraint> notNullConstraints)
throws InvalidObjectException, MetaException {
// TODO constraintCache
- rawStore.createTableWithConstraints(tbl, primaryKeys, foreignKeys,
+ List<String> constraintNames = rawStore.createTableWithConstraints(tbl, primaryKeys, foreignKeys,
uniqueConstraints, notNullConstraints);
SharedCache.addTableToCache(HiveStringUtils.normalizeIdentifier(tbl.getDbName()),
HiveStringUtils.normalizeIdentifier(tbl.getTableName()), tbl);
+ return constraintNames;
}
@Override
@@ -1915,31 +1916,31 @@ public class CachedStore implements RawStore, Configurable {
}
@Override
- public void addPrimaryKeys(List<SQLPrimaryKey> pks)
+ public List<String> addPrimaryKeys(List<SQLPrimaryKey> pks)
throws InvalidObjectException, MetaException {
// TODO constraintCache
- rawStore.addPrimaryKeys(pks);
+ return rawStore.addPrimaryKeys(pks);
}
@Override
- public void addForeignKeys(List<SQLForeignKey> fks)
+ public List<String> addForeignKeys(List<SQLForeignKey> fks)
throws InvalidObjectException, MetaException {
// TODO constraintCache
- rawStore.addForeignKeys(fks);
+ return rawStore.addForeignKeys(fks);
}
@Override
- public void addUniqueConstraints(List<SQLUniqueConstraint> uks)
+ public List<String> addUniqueConstraints(List<SQLUniqueConstraint> uks)
throws InvalidObjectException, MetaException {
// TODO constraintCache
- rawStore.addUniqueConstraints(uks);
+ return rawStore.addUniqueConstraints(uks);
}
@Override
- public void addNotNullConstraints(List<SQLNotNullConstraint> nns)
+ public List<String> addNotNullConstraints(List<SQLNotNullConstraint> nns)
throws InvalidObjectException, MetaException {
// TODO constraintCache
- rawStore.addNotNullConstraints(nns);
+ return rawStore.addNotNullConstraints(nns);
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddForeignKeyEvent.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddForeignKeyEvent.java b/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddForeignKeyEvent.java
new file mode 100644
index 0000000..1dc9588
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddForeignKeyEvent.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.events;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler;
+import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
+
+public class AddForeignKeyEvent extends ListenerEvent {
+ private final List<SQLForeignKey> fks;
+
+ public AddForeignKeyEvent(List<SQLForeignKey> fks, boolean status, HMSHandler handler) {
+ super(status, handler);
+ this.fks = fks;
+ }
+
+ public List<SQLForeignKey> getForeignKeyCols() {
+ return fks;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddNotNullConstraintEvent.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddNotNullConstraintEvent.java b/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddNotNullConstraintEvent.java
new file mode 100644
index 0000000..c01083c
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddNotNullConstraintEvent.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.events;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler;
+import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
+
+public class AddNotNullConstraintEvent extends ListenerEvent {
+ private final List<SQLNotNullConstraint> nns;
+
+ public AddNotNullConstraintEvent(List<SQLNotNullConstraint> nns, boolean status, HMSHandler handler) {
+ super(status, handler);
+ this.nns = nns;
+ }
+
+ public List<SQLNotNullConstraint> getNotNullConstraintCols() {
+ return nns;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddPrimaryKeyEvent.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddPrimaryKeyEvent.java b/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddPrimaryKeyEvent.java
new file mode 100644
index 0000000..cb0f562
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddPrimaryKeyEvent.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.events;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler;
+import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
+
+public class AddPrimaryKeyEvent extends ListenerEvent {
+
+ private final List<SQLPrimaryKey> pks;
+
+ public AddPrimaryKeyEvent(List<SQLPrimaryKey> pks, boolean status, HMSHandler handler) {
+ super(status, handler);
+ this.pks = pks;
+ }
+
+ public List<SQLPrimaryKey> getPrimaryKeyCols() {
+ return pks;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddUniqueConstraintEvent.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddUniqueConstraintEvent.java b/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddUniqueConstraintEvent.java
new file mode 100644
index 0000000..c02a309
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddUniqueConstraintEvent.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.events;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler;
+import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
+
+public class AddUniqueConstraintEvent extends ListenerEvent {
+ private final List<SQLUniqueConstraint> uks;
+
+ public AddUniqueConstraintEvent(List<SQLUniqueConstraint> uks, boolean status, HMSHandler handler) {
+ super(status, handler);
+ this.uks = uks;
+ }
+
+ public List<SQLUniqueConstraint> getUniqueConstraintCols() {
+ return uks;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropConstraintEvent.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropConstraintEvent.java b/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropConstraintEvent.java
new file mode 100644
index 0000000..5396863
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropConstraintEvent.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.events;
+
+import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler;
+
+public class DropConstraintEvent extends ListenerEvent {
+
+ private final String dbName;
+ private final String tableName;
+ private final String constraintName;
+ public DropConstraintEvent(String dbName, String tableName, String constraintName,
+ boolean status, HMSHandler handler) {
+ super(status, handler);
+ this.dbName = dbName;
+ this.tableName = tableName;
+ this.constraintName = constraintName;
+ }
+
+ public String getDbName() {
+ return dbName;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public String getConstraintName() {
+ return constraintName;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddForeignKeyMessage.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddForeignKeyMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddForeignKeyMessage.java
new file mode 100644
index 0000000..2eb14a1
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddForeignKeyMessage.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.messaging;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
+
+public abstract class AddForeignKeyMessage extends EventMessage {
+ protected AddForeignKeyMessage() {
+ super(EventType.ADD_FOREIGNKEY);
+ }
+
+ /**
+ * Getter for list of foreign keys.
+ * @return List of SQLForeignKey
+ */
+ public abstract List<SQLForeignKey> getForeignKeys() throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddNotNullConstraintMessage.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddNotNullConstraintMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddNotNullConstraintMessage.java
new file mode 100644
index 0000000..28ee610
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddNotNullConstraintMessage.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.messaging;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
+
+public abstract class AddNotNullConstraintMessage extends EventMessage {
+ protected AddNotNullConstraintMessage() {
+ super(EventType.ADD_NOTNULLCONSTRAINT);
+ }
+
+ /**
+ * Getter for list of not null constraints.
+ * @return List of SQLNotNullConstraint
+ */
+ public abstract List<SQLNotNullConstraint> getNotNullConstraints() throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPrimaryKeyMessage.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPrimaryKeyMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPrimaryKeyMessage.java
new file mode 100644
index 0000000..0e899ad
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPrimaryKeyMessage.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hive.metastore.messaging;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
+
+public abstract class AddPrimaryKeyMessage extends EventMessage {
+ protected AddPrimaryKeyMessage() {
+ super(EventType.ADD_PRIMARYKEY);
+ }
+
+ /**
+ * Getter for list of primary keys.
+ * @return List of SQLPrimaryKey
+ */
+ public abstract List<SQLPrimaryKey> getPrimaryKeys() throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddUniqueConstraintMessage.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddUniqueConstraintMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddUniqueConstraintMessage.java
new file mode 100644
index 0000000..8072d84f
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddUniqueConstraintMessage.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.messaging;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
+
+public abstract class AddUniqueConstraintMessage extends EventMessage {
+ protected AddUniqueConstraintMessage() {
+ super(EventType.ADD_UNIQUECONSTRAINT);
+ }
+
+ /**
+ * Getter for list of unique constraints.
+ * @return List of SQLUniqueConstraint
+ */
+ public abstract List<SQLUniqueConstraint> getUniqueConstraints() throws Exception;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropConstraintMessage.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropConstraintMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropConstraintMessage.java
new file mode 100644
index 0000000..6e691e9
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropConstraintMessage.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hive.metastore.messaging;
+
+public abstract class DropConstraintMessage extends EventMessage {
+ protected DropConstraintMessage() {
+ super(EventType.DROP_CONSTRAINT);
+ }
+
+ public abstract String getTable();
+
+ public abstract String getConstraint();
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java
index 1ec0de0..3b37cb3 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java
@@ -43,7 +43,12 @@ public abstract class EventMessage {
DROP_FUNCTION(MessageFactory.DROP_FUNCTION_EVENT),
CREATE_INDEX(MessageFactory.CREATE_INDEX_EVENT),
DROP_INDEX(MessageFactory.DROP_INDEX_EVENT),
- ALTER_INDEX(MessageFactory.ALTER_INDEX_EVENT);
+ ALTER_INDEX(MessageFactory.ALTER_INDEX_EVENT),
+ ADD_PRIMARYKEY(MessageFactory.ADD_PRIMARYKEY_EVENT),
+ ADD_FOREIGNKEY(MessageFactory.ADD_FOREIGNKEY_EVENT),
+ ADD_UNIQUECONSTRAINT(MessageFactory.ADD_UNIQUECONSTRAINT_EVENT),
+ ADD_NOTNULLCONSTRAINT(MessageFactory.ADD_NOTNULLCONSTRAINT_EVENT),
+ DROP_CONSTRAINT(MessageFactory.DROP_CONSTRAINT_EVENT);
private String typeString;
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java
index 515c455..50a2728 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java
@@ -58,7 +58,16 @@ public abstract class MessageDeserializer {
return getAlterIndexMessage(messageBody);
case INSERT:
return getInsertMessage(messageBody);
-
+ case ADD_PRIMARYKEY:
+ return getAddPrimaryKeyMessage(messageBody);
+ case ADD_FOREIGNKEY:
+ return getAddForeignKeyMessage(messageBody);
+ case ADD_UNIQUECONSTRAINT:
+ return getAddUniqueConstraintMessage(messageBody);
+ case ADD_NOTNULLCONSTRAINT:
+ return getAddNotNullConstraintMessage(messageBody);
+ case DROP_CONSTRAINT:
+ return getDropConstraintMessage(messageBody);
default:
throw new IllegalArgumentException("Unsupported event-type: " + eventTypeString);
}
@@ -140,6 +149,31 @@ public abstract class MessageDeserializer {
*/
public abstract InsertMessage getInsertMessage(String messageBody);
+ /**
+ * Method to de-serialize AddPrimaryKeyMessage instance.
+ */
+ public abstract AddPrimaryKeyMessage getAddPrimaryKeyMessage(String messageBody);
+
+ /**
+ * Method to de-serialize AddForeignKeyMessage instance.
+ */
+ public abstract AddForeignKeyMessage getAddForeignKeyMessage(String messageBody);
+
+ /**
+ * Method to de-serialize AddUniqueConstraintMessage instance.
+ */
+ public abstract AddUniqueConstraintMessage getAddUniqueConstraintMessage(String messageBody);
+
+ /**
+ * Method to de-serialize AddNotNullConstraintMessage instance.
+ */
+ public abstract AddNotNullConstraintMessage getAddNotNullConstraintMessage(String messageBody);
+
+ /**
+ * Method to de-serialize DropConstraintMessage instance.
+ */
+ public abstract DropConstraintMessage getDropConstraintMessage(String messageBody);
+
// Protection against construction.
protected MessageDeserializer() {}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java
index 9437e8b..e639e48 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java
@@ -25,10 +25,15 @@ import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.Function;
import org.apache.hadoop.hive.metastore.api.Index;
import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
+import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
+import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
+import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.util.ReflectionUtils;
import java.util.Iterator;
+import java.util.List;
/**
* Abstract Factory for the construction of HCatalog message instances.
@@ -50,7 +55,11 @@ public abstract class MessageFactory {
public static final String CREATE_INDEX_EVENT = "CREATE_INDEX";
public static final String DROP_INDEX_EVENT = "DROP_INDEX";
public static final String ALTER_INDEX_EVENT = "ALTER_INDEX";
-
+ public static final String ADD_PRIMARYKEY_EVENT = "ADD_PRIMARYKEY";
+ public static final String ADD_FOREIGNKEY_EVENT = "ADD_FOREIGNKEY";
+ public static final String ADD_UNIQUECONSTRAINT_EVENT = "ADD_UNIQUECONSTRAINT";
+ public static final String ADD_NOTNULLCONSTRAINT_EVENT = "ADD_NOTNULLCONSTRAINT";
+ public static final String DROP_CONSTRAINT_EVENT = "DROP_CONSTRAINT";
private static MessageFactory instance = null;
@@ -237,4 +246,46 @@ public abstract class MessageFactory {
*/
public abstract InsertMessage buildInsertMessage(Table tableObj, Partition ptnObj,
boolean replace, Iterator<String> files);
+
+ /***
+ * Factory method for building add primary key message
+ *
+ * @param pks list of primary keys
+ * @return instance of AddPrimaryKeyMessage
+ */
+ public abstract AddPrimaryKeyMessage buildAddPrimaryKeyMessage(List<SQLPrimaryKey> pks);
+
+ /***
+ * Factory method for building add foreign key message
+ *
+ * @param fks list of foreign keys
+ * @return instance of AddForeignKeyMessage
+ */
+ public abstract AddForeignKeyMessage buildAddForeignKeyMessage(List<SQLForeignKey> fks);
+
+ /***
+ * Factory method for building add unique constraint message
+ *
+ * @param uks list of unique constraints
+ * @return instance of SQLUniqueConstraint
+ */
+ public abstract AddUniqueConstraintMessage buildAddUniqueConstraintMessage(List<SQLUniqueConstraint> uks);
+
+ /***
+ * Factory method for building add not null constraint message
+ *
+ * @param nns list of not null constraints
+ * @return instance of SQLNotNullConstraint
+ */
+ public abstract AddNotNullConstraintMessage buildAddNotNullConstraintMessage(List<SQLNotNullConstraint> nns);
+
+ /***
+ * Factory method for building drop constraint message
+ * @param dbName
+ * @param tableName
+ * @param constraintName
+ * @return
+ */
+ public abstract DropConstraintMessage buildDropConstraintMessage(String dbName, String tableName,
+ String constraintName);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddForeignKeyMessage.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddForeignKeyMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddForeignKeyMessage.java
new file mode 100644
index 0000000..9c04c15
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddForeignKeyMessage.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hive.metastore.messaging.json;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
+import org.apache.hadoop.hive.metastore.messaging.AddForeignKeyMessage;
+import org.apache.thrift.TException;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+/**
+ * JSON implementation of AddForeignKeyMessage
+ */
+public class JSONAddForeignKeyMessage extends AddForeignKeyMessage {
+
+ @JsonProperty
+ String server, servicePrincipal;
+
+ @JsonProperty
+ Long timestamp;
+
+ @JsonProperty
+ List<String> foreignKeyListJson;
+
+ /**
+ * Default constructor, needed for Jackson.
+ */
+ public JSONAddForeignKeyMessage() {
+ }
+
+ public JSONAddForeignKeyMessage(String server, String servicePrincipal, List<SQLForeignKey> fks,
+ Long timestamp) {
+ this.server = server;
+ this.servicePrincipal = servicePrincipal;
+ this.timestamp = timestamp;
+ this.foreignKeyListJson = new ArrayList<String>();
+ try {
+ for (SQLForeignKey pk : fks) {
+ foreignKeyListJson.add(JSONMessageFactory.createForeignKeyObjJson(pk));
+ }
+ } catch (TException e) {
+ throw new IllegalArgumentException("Could not serialize: ", e);
+ }
+ }
+
+ @Override
+ public String getServer() {
+ return server;
+ }
+
+ @Override
+ public String getServicePrincipal() {
+ return servicePrincipal;
+ }
+
+ @Override
+ public String getDB() {
+ return null;
+ }
+
+ @Override
+ public Long getTimestamp() {
+ return timestamp;
+ }
+
+ @Override
+ public List<SQLForeignKey> getForeignKeys() throws Exception {
+ List<SQLForeignKey> fks = new ArrayList<SQLForeignKey>();
+ for (String pkJson : foreignKeyListJson) {
+ fks.add((SQLForeignKey)JSONMessageFactory.getTObj(pkJson, SQLForeignKey.class));
+ }
+ return fks;
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return JSONMessageDeserializer.mapper.writeValueAsString(this);
+ } catch (Exception exception) {
+ throw new IllegalArgumentException("Could not serialize: ", exception);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddNotNullConstraintMessage.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddNotNullConstraintMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddNotNullConstraintMessage.java
new file mode 100644
index 0000000..acc94ce
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddNotNullConstraintMessage.java
@@ -0,0 +1,79 @@
+package org.apache.hadoop.hive.metastore.messaging.json;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
+import org.apache.hadoop.hive.metastore.messaging.AddNotNullConstraintMessage;
+import org.apache.thrift.TException;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class JSONAddNotNullConstraintMessage extends AddNotNullConstraintMessage {
+ @JsonProperty
+ String server, servicePrincipal;
+
+ @JsonProperty
+ Long timestamp;
+
+ @JsonProperty
+ List<String> notNullConstraintListJson;
+
+ /**
+ * Default constructor, needed for Jackson.
+ */
+ public JSONAddNotNullConstraintMessage() {
+ }
+
+ public JSONAddNotNullConstraintMessage(String server, String servicePrincipal, List<SQLNotNullConstraint> nns,
+ Long timestamp) {
+ this.server = server;
+ this.servicePrincipal = servicePrincipal;
+ this.timestamp = timestamp;
+ this.notNullConstraintListJson = new ArrayList<String>();
+ try {
+ for (SQLNotNullConstraint nn : nns) {
+ notNullConstraintListJson.add(JSONMessageFactory.createNotNullConstraintObjJson(nn));
+ }
+ } catch (TException e) {
+ throw new IllegalArgumentException("Could not serialize: ", e);
+ }
+ }
+
+ @Override
+ public String getServer() {
+ return server;
+ }
+
+ @Override
+ public String getServicePrincipal() {
+ return servicePrincipal;
+ }
+
+ @Override
+ public String getDB() {
+ return null;
+ }
+
+ @Override
+ public Long getTimestamp() {
+ return timestamp;
+ }
+
+ @Override
+ public List<SQLNotNullConstraint> getNotNullConstraints() throws Exception {
+ List<SQLNotNullConstraint> nns = new ArrayList<SQLNotNullConstraint>();
+ for (String nnJson : notNullConstraintListJson) {
+ nns.add((SQLNotNullConstraint)JSONMessageFactory.getTObj(nnJson, SQLNotNullConstraint.class));
+ }
+ return nns;
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return JSONMessageDeserializer.mapper.writeValueAsString(this);
+ } catch (Exception exception) {
+ throw new IllegalArgumentException("Could not serialize: ", exception);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPrimaryKeyMessage.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPrimaryKeyMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPrimaryKeyMessage.java
new file mode 100644
index 0000000..2551cbf
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPrimaryKeyMessage.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hive.metastore.messaging.json;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
+import org.apache.hadoop.hive.metastore.messaging.AddPrimaryKeyMessage;
+import org.apache.thrift.TException;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+/**
+ * JSON implementation of AddPrimaryKeyMessage
+ */
+public class JSONAddPrimaryKeyMessage extends AddPrimaryKeyMessage {
+
+ @JsonProperty
+ String server, servicePrincipal;
+
+ @JsonProperty
+ Long timestamp;
+
+ @JsonProperty
+ List<String> primaryKeyListJson;
+
+ /**
+ * Default constructor, needed for Jackson.
+ */
+ public JSONAddPrimaryKeyMessage() {
+ }
+
+ public JSONAddPrimaryKeyMessage(String server, String servicePrincipal, List<SQLPrimaryKey> pks,
+ Long timestamp) {
+ this.server = server;
+ this.servicePrincipal = servicePrincipal;
+ this.timestamp = timestamp;
+ this.primaryKeyListJson = new ArrayList<String>();
+ try {
+ for (SQLPrimaryKey pk : pks) {
+ primaryKeyListJson.add(JSONMessageFactory.createPrimaryKeyObjJson(pk));
+ }
+ } catch (TException e) {
+ throw new IllegalArgumentException("Could not serialize: ", e);
+ }
+ }
+
+ @Override
+ public String getServer() {
+ return server;
+ }
+
+ @Override
+ public String getServicePrincipal() {
+ return servicePrincipal;
+ }
+
+ @Override
+ public String getDB() {
+ return null;
+ }
+
+ @Override
+ public Long getTimestamp() {
+ return timestamp;
+ }
+
+ @Override
+ public List<SQLPrimaryKey> getPrimaryKeys() throws Exception {
+ List<SQLPrimaryKey> pks = new ArrayList<SQLPrimaryKey>();
+ for (String pkJson : primaryKeyListJson) {
+ pks.add((SQLPrimaryKey)JSONMessageFactory.getTObj(pkJson, SQLPrimaryKey.class));
+ }
+ return pks;
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return JSONMessageDeserializer.mapper.writeValueAsString(this);
+ } catch (Exception exception) {
+ throw new IllegalArgumentException("Could not serialize: ", exception);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddUniqueConstraintMessage.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddUniqueConstraintMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddUniqueConstraintMessage.java
new file mode 100644
index 0000000..37f6b81
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddUniqueConstraintMessage.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.messaging.json;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
+import org.apache.hadoop.hive.metastore.messaging.AddUniqueConstraintMessage;
+import org.apache.thrift.TException;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class JSONAddUniqueConstraintMessage extends AddUniqueConstraintMessage {
+ @JsonProperty
+ String server, servicePrincipal;
+
+ @JsonProperty
+ Long timestamp;
+
+ @JsonProperty
+ List<String> uniqueConstraintListJson;
+
+ /**
+ * Default constructor, needed for Jackson.
+ */
+ public JSONAddUniqueConstraintMessage() {
+ }
+
+ public JSONAddUniqueConstraintMessage(String server, String servicePrincipal, List<SQLUniqueConstraint> uks,
+ Long timestamp) {
+ this.server = server;
+ this.servicePrincipal = servicePrincipal;
+ this.timestamp = timestamp;
+ this.uniqueConstraintListJson = new ArrayList<String>();
+ try {
+ for (SQLUniqueConstraint uk : uks) {
+ uniqueConstraintListJson.add(JSONMessageFactory.createUniqueConstraintObjJson(uk));
+ }
+ } catch (TException e) {
+ throw new IllegalArgumentException("Could not serialize: ", e);
+ }
+ }
+
+ @Override
+ public String getServer() {
+ return server;
+ }
+
+ @Override
+ public String getServicePrincipal() {
+ return servicePrincipal;
+ }
+
+ @Override
+ public String getDB() {
+ return null;
+ }
+
+ @Override
+ public Long getTimestamp() {
+ return timestamp;
+ }
+
+ @Override
+ public List<SQLUniqueConstraint> getUniqueConstraints() throws Exception {
+ List<SQLUniqueConstraint> uks = new ArrayList<SQLUniqueConstraint>();
+ for (String pkJson : uniqueConstraintListJson) {
+ uks.add((SQLUniqueConstraint)JSONMessageFactory.getTObj(pkJson, SQLUniqueConstraint.class));
+ }
+ return uks;
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return JSONMessageDeserializer.mapper.writeValueAsString(this);
+ } catch (Exception exception) {
+ throw new IllegalArgumentException("Could not serialize: ", exception);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java
new file mode 100644
index 0000000..4d3422b
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.messaging.json;
+
+import org.apache.hadoop.hive.metastore.messaging.DropConstraintMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+/**
+ * JSON implementation of DropConstraintMessage
+ */
+public class JSONDropConstraintMessage extends DropConstraintMessage {
+
+ @JsonProperty
+ String server, servicePrincipal, dbName, tableName, constraintName;
+
+ @JsonProperty
+ Long timestamp;
+
+ /**
+ * Default constructor, needed for Jackson.
+ */
+ public JSONDropConstraintMessage() {
+ }
+
+ public JSONDropConstraintMessage(String server, String servicePrincipal, String dbName,
+ String tableName, String constraintName, Long timestamp) {
+ this.server = server;
+ this.servicePrincipal = servicePrincipal;
+ this.timestamp = timestamp;
+ this.dbName = dbName;
+ this.tableName = tableName;
+ this.constraintName = constraintName;
+ }
+
+ @Override
+ public String getServer() {
+ return server;
+ }
+
+ @Override
+ public String getServicePrincipal() {
+ return servicePrincipal;
+ }
+
+ @Override
+ public String getDB() {
+ return dbName;
+ }
+
+ @Override
+ public String getTable() {
+ return tableName;
+ }
+
+ @Override
+ public String getConstraint() {
+ return constraintName;
+ }
+
+ @Override
+ public Long getTimestamp() {
+ return timestamp;
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return JSONMessageDeserializer.mapper.writeValueAsString(this);
+ } catch (Exception exception) {
+ throw new IllegalArgumentException("Could not serialize: ", exception);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java
index 40ef5fb..7f588a0 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java
@@ -19,7 +19,11 @@
package org.apache.hadoop.hive.metastore.messaging.json;
+import org.apache.hadoop.hive.metastore.messaging.AddForeignKeyMessage;
+import org.apache.hadoop.hive.metastore.messaging.AddNotNullConstraintMessage;
import org.apache.hadoop.hive.metastore.messaging.AddPartitionMessage;
+import org.apache.hadoop.hive.metastore.messaging.AddPrimaryKeyMessage;
+import org.apache.hadoop.hive.metastore.messaging.AddUniqueConstraintMessage;
import org.apache.hadoop.hive.metastore.messaging.AlterIndexMessage;
import org.apache.hadoop.hive.metastore.messaging.AlterPartitionMessage;
import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage;
@@ -27,6 +31,7 @@ import org.apache.hadoop.hive.metastore.messaging.CreateDatabaseMessage;
import org.apache.hadoop.hive.metastore.messaging.CreateFunctionMessage;
import org.apache.hadoop.hive.metastore.messaging.CreateIndexMessage;
import org.apache.hadoop.hive.metastore.messaging.CreateTableMessage;
+import org.apache.hadoop.hive.metastore.messaging.DropConstraintMessage;
import org.apache.hadoop.hive.metastore.messaging.DropDatabaseMessage;
import org.apache.hadoop.hive.metastore.messaging.DropFunctionMessage;
import org.apache.hadoop.hive.metastore.messaging.DropIndexMessage;
@@ -190,4 +195,49 @@ public class JSONMessageDeserializer extends MessageDeserializer {
throw new IllegalArgumentException("Could not construct InsertMessage", e);
}
}
+
+ @Override
+ public AddPrimaryKeyMessage getAddPrimaryKeyMessage(String messageBody) {
+ try {
+ return mapper.readValue(messageBody, JSONAddPrimaryKeyMessage.class);
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Could not construct AddPrimaryKeyMessage", e);
+ }
+ }
+
+ @Override
+ public AddForeignKeyMessage getAddForeignKeyMessage(String messageBody) {
+ try {
+ return mapper.readValue(messageBody, JSONAddForeignKeyMessage.class);
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Could not construct AddForeignKeyMessage", e);
+ }
+ }
+
+ @Override
+ public AddUniqueConstraintMessage getAddUniqueConstraintMessage(String messageBody) {
+ try {
+ return mapper.readValue(messageBody, JSONAddUniqueConstraintMessage.class);
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Could not construct AddUniqueConstraintMessage", e);
+ }
+ }
+
+ @Override
+ public AddNotNullConstraintMessage getAddNotNullConstraintMessage(String messageBody) {
+ try {
+ return mapper.readValue(messageBody, JSONAddNotNullConstraintMessage.class);
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Could not construct AddNotNullConstraintMessage", e);
+ }
+ }
+
+ @Override
+ public DropConstraintMessage getDropConstraintMessage(String messageBody) {
+ try {
+ return mapper.readValue(messageBody, JSONDropConstraintMessage.class);
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Could not construct DropConstraintMessage", e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java
index b24d1e3..71e25ed 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java
@@ -33,8 +33,16 @@ import org.apache.hadoop.hive.metastore.api.Function;
import org.apache.hadoop.hive.metastore.api.Index;
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
+import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
+import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
+import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.messaging.AddForeignKeyMessage;
+import org.apache.hadoop.hive.metastore.messaging.AddNotNullConstraintMessage;
import org.apache.hadoop.hive.metastore.messaging.AddPartitionMessage;
+import org.apache.hadoop.hive.metastore.messaging.AddPrimaryKeyMessage;
+import org.apache.hadoop.hive.metastore.messaging.AddUniqueConstraintMessage;
import org.apache.hadoop.hive.metastore.messaging.AlterIndexMessage;
import org.apache.hadoop.hive.metastore.messaging.AlterPartitionMessage;
import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage;
@@ -42,6 +50,7 @@ import org.apache.hadoop.hive.metastore.messaging.CreateDatabaseMessage;
import org.apache.hadoop.hive.metastore.messaging.CreateFunctionMessage;
import org.apache.hadoop.hive.metastore.messaging.CreateIndexMessage;
import org.apache.hadoop.hive.metastore.messaging.CreateTableMessage;
+import org.apache.hadoop.hive.metastore.messaging.DropConstraintMessage;
import org.apache.hadoop.hive.metastore.messaging.DropDatabaseMessage;
import org.apache.hadoop.hive.metastore.messaging.DropFunctionMessage;
import org.apache.hadoop.hive.metastore.messaging.DropIndexMessage;
@@ -166,6 +175,33 @@ public class JSONMessageFactory extends MessageFactory {
return new JSONInsertMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, tableObj, partObj, replace, fileIter, now());
}
+ @Override
+ public AddPrimaryKeyMessage buildAddPrimaryKeyMessage(List<SQLPrimaryKey> pks) {
+ return new JSONAddPrimaryKeyMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, pks, now());
+ }
+
+ @Override
+ public AddForeignKeyMessage buildAddForeignKeyMessage(List<SQLForeignKey> fks) {
+ return new JSONAddForeignKeyMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, fks, now());
+ }
+
+ @Override
+ public AddUniqueConstraintMessage buildAddUniqueConstraintMessage(List<SQLUniqueConstraint> uks) {
+ return new JSONAddUniqueConstraintMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, uks, now());
+ }
+
+ @Override
+ public AddNotNullConstraintMessage buildAddNotNullConstraintMessage(List<SQLNotNullConstraint> nns) {
+ return new JSONAddNotNullConstraintMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, nns, now());
+ }
+
+ @Override
+ public DropConstraintMessage buildDropConstraintMessage(String dbName, String tableName,
+ String constraintName) {
+ return new JSONDropConstraintMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, dbName, tableName,
+ constraintName, now());
+ }
+
private long now() {
return System.currentTimeMillis() / 1000;
}
@@ -188,6 +224,26 @@ public class JSONMessageFactory extends MessageFactory {
}));
}
+ static String createPrimaryKeyObjJson(SQLPrimaryKey primaryKeyObj) throws TException {
+ TSerializer serializer = new TSerializer(new TJSONProtocol.Factory());
+ return serializer.toString(primaryKeyObj, "UTF-8");
+ }
+
+ static String createForeignKeyObjJson(SQLForeignKey foreignKeyObj) throws TException {
+ TSerializer serializer = new TSerializer(new TJSONProtocol.Factory());
+ return serializer.toString(foreignKeyObj, "UTF-8");
+ }
+
+ static String createUniqueConstraintObjJson(SQLUniqueConstraint uniqueConstraintObj) throws TException {
+ TSerializer serializer = new TSerializer(new TJSONProtocol.Factory());
+ return serializer.toString(uniqueConstraintObj, "UTF-8");
+ }
+
+ static String createNotNullConstraintObjJson(SQLNotNullConstraint notNullConstaintObj) throws TException {
+ TSerializer serializer = new TSerializer(new TJSONProtocol.Factory());
+ return serializer.toString(notNullConstaintObj, "UTF-8");
+ }
+
static String createTableObjJson(Table tableObj) throws TException {
TSerializer serializer = new TSerializer(new TJSONProtocol.Factory());
return serializer.toString(tableObj, "UTF-8");