You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by we...@apache.org on 2017/05/08 22:17:27 UTC
[10/50] [abbrv] hive git commit: HIVE-16267 : Enable bootstrap
function metadata to be loaded in repl load (Anishek Agarwal,
reviewed by Sushanth Sowmyan)
http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/PartitionSerializer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/PartitionSerializer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/PartitionSerializer.java
new file mode 100644
index 0000000..077d39b
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/PartitionSerializer.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.io;
+
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.thrift.TException;
+import org.apache.thrift.TSerializer;
+import org.apache.thrift.protocol.TJSONProtocol;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class PartitionSerializer implements JsonWriter.Serializer {
+ public static final String FIELD_NAME="partitions";
+ private Partition partition;
+
+ PartitionSerializer(Partition partition) {
+ this.partition = partition;
+ }
+
+ @Override
+ public void writeTo(JsonWriter writer, ReplicationSpec additionalPropertiesProvider)
+ throws SemanticException, IOException {
+ TSerializer serializer = new TSerializer(new TJSONProtocol.Factory());
+ try {
+ if (additionalPropertiesProvider.isInReplicationScope()) {
+ partition.putToParameters(
+ ReplicationSpec.KEY.CURR_STATE_ID.toString(),
+ additionalPropertiesProvider.getCurrentReplicationState());
+ if (isPartitionExternal()) {
+ // Replication destination will not be external
+ partition.putToParameters("EXTERNAL", "FALSE");
+ }
+ }
+ writer.jsonGenerator.writeString(serializer.toString(partition, UTF_8));
+ writer.jsonGenerator.flush();
+ } catch (TException e) {
+ throw new SemanticException(ErrorMsg.ERROR_SERIALIZE_METASTORE.getMsg(), e);
+ }
+ }
+
+ private boolean isPartitionExternal() {
+ Map<String, String> params = partition.getParameters();
+ return params.containsKey("EXTERNAL")
+ && params.get("EXTERNAL").equalsIgnoreCase("TRUE");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/ReplicationSpecSerializer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/ReplicationSpecSerializer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/ReplicationSpecSerializer.java
new file mode 100644
index 0000000..3a92e8a
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/ReplicationSpecSerializer.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.io;
+
+import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+
+import java.io.IOException;
+
+public class ReplicationSpecSerializer implements JsonWriter.Serializer {
+ @Override
+ public void writeTo(JsonWriter writer, ReplicationSpec additionalPropertiesProvider)
+ throws SemanticException, IOException {
+ for (ReplicationSpec.KEY key : ReplicationSpec.KEY.values()) {
+ String value = additionalPropertiesProvider.get(key);
+ if (value != null) {
+ writer.jsonGenerator.writeStringField(key.toString(), value);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java
new file mode 100644
index 0000000..948cb39
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.io;
+
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.thrift.TException;
+import org.apache.thrift.TSerializer;
+import org.apache.thrift.protocol.TJSONProtocol;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class TableSerializer implements JsonWriter.Serializer {
+ public static final String FIELD_NAME = "table";
+ private final org.apache.hadoop.hive.ql.metadata.Table tableHandle;
+ private final Iterable<Partition> partitions;
+
+ public TableSerializer(org.apache.hadoop.hive.ql.metadata.Table tableHandle,
+ Iterable<Partition> partitions) {
+ this.tableHandle = tableHandle;
+ this.partitions = partitions;
+ }
+
+ @Override
+ public void writeTo(JsonWriter writer, ReplicationSpec additionalPropertiesProvider)
+ throws SemanticException, IOException {
+ if (cannotReplicateTable(additionalPropertiesProvider)) {
+ return;
+ }
+
+ Table tTable = tableHandle.getTTable();
+ tTable = addPropertiesToTable(tTable, additionalPropertiesProvider);
+ try {
+ TSerializer serializer = new TSerializer(new TJSONProtocol.Factory());
+ writer.jsonGenerator
+ .writeStringField(FIELD_NAME, serializer.toString(tTable, UTF_8));
+ writer.jsonGenerator.writeFieldName(PartitionSerializer.FIELD_NAME);
+ writePartitions(writer, additionalPropertiesProvider);
+ } catch (TException e) {
+ throw new SemanticException(ErrorMsg.ERROR_SERIALIZE_METASTORE.getMsg(), e);
+ }
+ }
+
+ private boolean cannotReplicateTable(ReplicationSpec additionalPropertiesProvider) {
+ return tableHandle == null || additionalPropertiesProvider.isNoop();
+ }
+
+ private Table addPropertiesToTable(Table table, ReplicationSpec additionalPropertiesProvider)
+ throws SemanticException, IOException {
+ if (additionalPropertiesProvider.isInReplicationScope()) {
+ table.putToParameters(
+ ReplicationSpec.KEY.CURR_STATE_ID.toString(),
+ additionalPropertiesProvider.getCurrentReplicationState());
+ if (isExternalTable(table)) {
+ // Replication destination will not be external - override if set
+ table.putToParameters("EXTERNAL", "FALSE");
+ }
+ if (isExternalTableType(table)) {
+ // Replication dest will not be external - override if set
+ table.setTableType(TableType.MANAGED_TABLE.toString());
+ }
+ } else {
+ // ReplicationSpec.KEY scopeKey = ReplicationSpec.KEY.REPL_SCOPE;
+ // write(out, ",\""+ scopeKey.toString() +"\":\"" + replicationSpec.get(scopeKey) + "\"");
+ // TODO: if we want to be explicit about this dump not being a replication dump, we can
+ // uncomment this else section, but currently unnneeded. Will require a lot of golden file
+ // regen if we do so.
+ }
+ return table;
+ }
+
+ private boolean isExternalTableType(org.apache.hadoop.hive.metastore.api.Table table) {
+ return table.isSetTableType()
+ && table.getTableType().equalsIgnoreCase(TableType.EXTERNAL_TABLE.toString());
+ }
+
+ private boolean isExternalTable(org.apache.hadoop.hive.metastore.api.Table table) {
+ Map<String, String> params = table.getParameters();
+ return params.containsKey("EXTERNAL")
+ && params.get("EXTERNAL").equalsIgnoreCase("TRUE");
+ }
+
+ private void writePartitions(JsonWriter writer, ReplicationSpec additionalPropertiesProvider)
+ throws SemanticException, IOException {
+ writer.jsonGenerator.writeStartArray();
+ if (partitions != null) {
+ for (org.apache.hadoop.hive.ql.metadata.Partition partition : partitions) {
+ new PartitionSerializer(partition.getTPartition())
+ .writeTo(writer, additionalPropertiesProvider);
+ }
+ }
+ writer.jsonGenerator.writeEndArray();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/VersionCompatibleSerializer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/VersionCompatibleSerializer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/VersionCompatibleSerializer.java
new file mode 100644
index 0000000..8201173
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/VersionCompatibleSerializer.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.io;
+
+import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.hive.ql.parse.EximUtil.METADATA_FORMAT_FORWARD_COMPATIBLE_VERSION;
+
+/**
+ * This is not used as of now as the conditional which lead to its usage is always false
+ * hence we have removed the conditional and the usage of this class, but might be required in future.
+ */
+public class VersionCompatibleSerializer implements JsonWriter.Serializer {
+ @Override
+ public void writeTo(JsonWriter writer, ReplicationSpec additionalPropertiesProvider)
+ throws SemanticException, IOException {
+ writer.jsonGenerator.writeStringField("fcversion", METADATA_FORMAT_FORWARD_COMPATIBLE_VERSION);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AddPartitionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AddPartitionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AddPartitionHandler.java
index 9a4f8b9..1616ab9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AddPartitionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AddPartitionHandler.java
@@ -35,7 +35,7 @@ import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.Iterator;
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DUMPTYPE;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
public class AddPartitionHandler extends AbstractHandler {
protected AddPartitionHandler(NotificationEvent notificationEvent) {
@@ -108,7 +108,7 @@ public class AddPartitionHandler extends AbstractHandler {
}
@Override
- public DUMPTYPE dumpType() {
- return DUMPTYPE.EVENT_ADD_PARTITION;
+ public DumpType dumpType() {
+ return DumpType.EVENT_ADD_PARTITION;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterPartitionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterPartitionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterPartitionHandler.java
index 20d04dc..b6c3496 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterPartitionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterPartitionHandler.java
@@ -23,14 +23,14 @@ import org.apache.hadoop.hive.metastore.messaging.AlterPartitionMessage;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.EximUtil;
-import org.apache.hadoop.hive.ql.parse.SemanticException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DUMPTYPE;
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DumpMetaData;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+
+import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
public class AlterPartitionHandler extends AbstractHandler {
private final org.apache.hadoop.hive.metastore.api.Partition after;
@@ -51,24 +51,24 @@ public class AlterPartitionHandler extends AbstractHandler {
private enum Scenario {
ALTER {
@Override
- DUMPTYPE dumpType() {
- return DUMPTYPE.EVENT_ALTER_PARTITION;
+ DumpType dumpType() {
+ return DumpType.EVENT_ALTER_PARTITION;
}
},
RENAME {
@Override
- DUMPTYPE dumpType() {
- return DUMPTYPE.EVENT_RENAME_PARTITION;
+ DumpType dumpType() {
+ return DumpType.EVENT_RENAME_PARTITION;
}
},
TRUNCATE {
@Override
- DUMPTYPE dumpType() {
- return DUMPTYPE.EVENT_TRUNCATE_PARTITION;
+ DumpType dumpType() {
+ return DumpType.EVENT_TRUNCATE_PARTITION;
}
};
- abstract DUMPTYPE dumpType();
+ abstract DumpType dumpType();
}
private Scenario scenarioType(org.apache.hadoop.hive.metastore.api.Partition before,
@@ -90,14 +90,14 @@ public class AlterPartitionHandler extends AbstractHandler {
if (Scenario.ALTER == scenario) {
withinContext.replicationSpec.setIsMetadataOnly(true);
Table qlMdTable = new Table(tableObject);
- List<Partition> qlPtns = new ArrayList<>();
- qlPtns.add(new Partition(qlMdTable, after));
+ List<Partition> partitions = new ArrayList<>();
+ partitions.add(new Partition(qlMdTable, after));
Path metaDataPath = new Path(withinContext.eventRoot, EximUtil.METADATA_NAME);
EximUtil.createExportDump(
metaDataPath.getFileSystem(withinContext.hiveConf),
metaDataPath,
qlMdTable,
- qlPtns,
+ partitions,
withinContext.replicationSpec);
}
DumpMetaData dmd = withinContext.createDmd(this);
@@ -106,7 +106,7 @@ public class AlterPartitionHandler extends AbstractHandler {
}
@Override
- public DUMPTYPE dumpType() {
+ public DumpType dumpType() {
return scenario.dumpType();
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterTableHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterTableHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterTableHandler.java
index bfe0181..d553240 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterTableHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterTableHandler.java
@@ -22,13 +22,12 @@ import org.apache.hadoop.hive.metastore.api.NotificationEvent;
import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.EximUtil;
-import org.apache.hadoop.hive.ql.parse.SemanticException;
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DUMPTYPE;
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DumpMetaData;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+
+import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
public class AlterTableHandler extends AbstractHandler {
- private final org.apache.hadoop.hive.metastore.api.Table before;
private final org.apache.hadoop.hive.metastore.api.Table after;
private final boolean isTruncateOp;
private final Scenario scenario;
@@ -36,30 +35,30 @@ public class AlterTableHandler extends AbstractHandler {
private enum Scenario {
ALTER {
@Override
- DUMPTYPE dumpType() {
- return DUMPTYPE.EVENT_ALTER_TABLE;
+ DumpType dumpType() {
+ return DumpType.EVENT_ALTER_TABLE;
}
},
RENAME {
@Override
- DUMPTYPE dumpType() {
- return DUMPTYPE.EVENT_RENAME_TABLE;
+ DumpType dumpType() {
+ return DumpType.EVENT_RENAME_TABLE;
}
},
TRUNCATE {
@Override
- DUMPTYPE dumpType() {
- return DUMPTYPE.EVENT_TRUNCATE_TABLE;
+ DumpType dumpType() {
+ return DumpType.EVENT_TRUNCATE_TABLE;
}
};
- abstract DUMPTYPE dumpType();
+ abstract DumpType dumpType();
}
AlterTableHandler(NotificationEvent event) throws Exception {
super(event);
AlterTableMessage atm = deserializer.getAlterTableMessage(event.getMessage());
- before = atm.getTableObjBefore();
+ org.apache.hadoop.hive.metastore.api.Table before = atm.getTableObjBefore();
after = atm.getTableObjAfter();
isTruncateOp = atm.getIsTruncateOp();
scenario = scenarioType(before, after);
@@ -97,7 +96,7 @@ public class AlterTableHandler extends AbstractHandler {
}
@Override
- public DUMPTYPE dumpType() {
+ public DumpType dumpType() {
return scenario.dumpType();
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/CreateTableHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/CreateTableHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/CreateTableHandler.java
index 03f400d..88600fd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/CreateTableHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/CreateTableHandler.java
@@ -28,7 +28,7 @@ import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DUMPTYPE;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
public class CreateTableHandler extends AbstractHandler {
@@ -80,7 +80,7 @@ public class CreateTableHandler extends AbstractHandler {
}
@Override
- public DUMPTYPE dumpType() {
- return DUMPTYPE.EVENT_CREATE_TABLE;
+ public DumpType dumpType() {
+ return DumpType.EVENT_CREATE_TABLE;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DefaultHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DefaultHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DefaultHandler.java
index 61c5f37..78cd74f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DefaultHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DefaultHandler.java
@@ -19,8 +19,9 @@ package org.apache.hadoop.hive.ql.parse.repl.events;
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DUMPTYPE;
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DumpMetaData;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+
+import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
public class DefaultHandler extends AbstractHandler {
@@ -37,7 +38,7 @@ public class DefaultHandler extends AbstractHandler {
}
@Override
- public DUMPTYPE dumpType() {
- return DUMPTYPE.EVENT_UNKNOWN;
+ public DumpType dumpType() {
+ return DumpType.EVENT_UNKNOWN;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DropPartitionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DropPartitionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DropPartitionHandler.java
index 3ad794e..c4a0908 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DropPartitionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DropPartitionHandler.java
@@ -19,8 +19,9 @@ package org.apache.hadoop.hive.ql.parse.repl.events;
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DUMPTYPE;
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DumpMetaData;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+
+import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
public class DropPartitionHandler extends AbstractHandler {
@@ -37,7 +38,7 @@ public class DropPartitionHandler extends AbstractHandler {
}
@Override
- public DUMPTYPE dumpType() {
- return DUMPTYPE.EVENT_DROP_PARTITION;
+ public DumpType dumpType() {
+ return DumpType.EVENT_DROP_PARTITION;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DropTableHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DropTableHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DropTableHandler.java
index cae379b..e3addaf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DropTableHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DropTableHandler.java
@@ -19,8 +19,9 @@ package org.apache.hadoop.hive.ql.parse.repl.events;
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DUMPTYPE;
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DumpMetaData;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+
+import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
public class DropTableHandler extends AbstractHandler {
@@ -37,7 +38,7 @@ public class DropTableHandler extends AbstractHandler {
}
@Override
- public DUMPTYPE dumpType() {
- return DUMPTYPE.EVENT_DROP_TABLE;
+ public DumpType dumpType() {
+ return DumpType.EVENT_DROP_TABLE;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/EventHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/EventHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/EventHandler.java
index 199145a..29f3b42 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/EventHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/EventHandler.java
@@ -22,8 +22,8 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DumpMetaData;
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DUMPTYPE;
+import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
public interface EventHandler {
void handle(Context withinContext) throws Exception;
@@ -32,7 +32,7 @@ public interface EventHandler {
long toEventId();
- DUMPTYPE dumpType();
+ DumpType dumpType();
class Context {
final Path eventRoot, cmRoot;
http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/InsertHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/InsertHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/InsertHandler.java
index e9f2a6a..910b396 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/InsertHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/InsertHandler.java
@@ -32,8 +32,9 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DUMPTYPE;
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DumpMetaData;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+
+import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
public class InsertHandler extends AbstractHandler {
@@ -103,7 +104,7 @@ public class InsertHandler extends AbstractHandler {
}
@Override
- public DUMPTYPE dumpType() {
- return DUMPTYPE.EVENT_INSERT;
+ public DumpType dumpType() {
+ return DumpType.EVENT_INSERT;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java
new file mode 100644
index 0000000..12ad19b
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.ReplChangeManager;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.repl.dump.Utils;
+
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Arrays;
+
+public class DumpMetaData {
+ // wrapper class for reading and writing metadata about a dump
+ // responsible for _dumpmetadata files
+ private static final String DUMP_METADATA = "_dumpmetadata";
+
+ private DumpType dumpType;
+ private Long eventFrom = null;
+ private Long eventTo = null;
+ private String payload = null;
+ private boolean initialized = false;
+
+ private final Path dumpFile;
+ private final HiveConf hiveConf;
+ private Path cmRoot;
+
+ public DumpMetaData(Path dumpRoot, HiveConf hiveConf) {
+ this.hiveConf = hiveConf;
+ dumpFile = new Path(dumpRoot, DUMP_METADATA);
+ }
+
+ public DumpMetaData(Path dumpRoot, DumpType lvl, Long eventFrom, Long eventTo, Path cmRoot,
+ HiveConf hiveConf) {
+ this(dumpRoot, hiveConf);
+ setDump(lvl, eventFrom, eventTo, cmRoot);
+ }
+
+ public void setDump(DumpType lvl, Long eventFrom, Long eventTo, Path cmRoot) {
+ this.dumpType = lvl;
+ this.eventFrom = eventFrom;
+ this.eventTo = eventTo;
+ this.initialized = true;
+ this.cmRoot = cmRoot;
+ }
+
+ private void loadDumpFromFile() throws SemanticException {
+ try {
+ // read from dumpfile and instantiate self
+ FileSystem fs = dumpFile.getFileSystem(hiveConf);
+ BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(dumpFile)));
+ String line = null;
+ if ((line = br.readLine()) != null) {
+ String[] lineContents = line.split("\t", 5);
+ setDump(DumpType.valueOf(lineContents[0]), Long.valueOf(lineContents[1]),
+ Long.valueOf(lineContents[2]),
+ new Path(lineContents[3]));
+ setPayload(lineContents[4].equals(Utilities.nullStringOutput) ? null : lineContents[4]);
+ ReplChangeManager.setCmRoot(cmRoot);
+ } else {
+ throw new IOException(
+ "Unable to read valid values from dumpFile:" + dumpFile.toUri().toString());
+ }
+ } catch (IOException ioe) {
+ throw new SemanticException(ioe);
+ }
+ }
+
+ public DumpType getDumpType() throws SemanticException {
+ initializeIfNot();
+ return this.dumpType;
+ }
+
+ public String getPayload() throws SemanticException {
+ initializeIfNot();
+ return this.payload;
+ }
+
+ public void setPayload(String payload) {
+ this.payload = payload;
+ }
+
+ public Long getEventFrom() throws SemanticException {
+ initializeIfNot();
+ return eventFrom;
+ }
+
+ public Long getEventTo() throws SemanticException {
+ initializeIfNot();
+ return eventTo;
+ }
+
+ public Path getDumpFilePath() {
+ return dumpFile;
+ }
+
+ public boolean isIncrementalDump() throws SemanticException {
+ initializeIfNot();
+ return (this.dumpType == DumpType.INCREMENTAL);
+ }
+
+ private void initializeIfNot() throws SemanticException {
+ if (!initialized) {
+ loadDumpFromFile();
+ }
+ }
+
+
+ public void write() throws SemanticException {
+ Utils.writeOutput(
+ Arrays.asList(
+ dumpType.toString(),
+ eventFrom.toString(),
+ eventTo.toString(),
+ cmRoot.toString(),
+ payload),
+ dumpFile,
+ hiveConf
+ );
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/MetaData.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/MetaData.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/MetaData.java
new file mode 100644
index 0000000..fc02dfd
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/MetaData.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load;
+
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.Function;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
+
+/**
+ * Utility class to help return complex value from readMetaData function
+ */
+public class MetaData {
+ private final Database db;
+ private final Table table;
+ private final Iterable<Partition> partitions;
+ private final ReplicationSpec replicationSpec;
+ public final Function function;
+
+ public MetaData() {
+ this(null, null, null, new ReplicationSpec(), null);
+ }
+
+ MetaData(Database db, Table table, Iterable<Partition> partitions,
+ ReplicationSpec replicationSpec, Function function) {
+ this.db = db;
+ this.table = table;
+ this.partitions = partitions;
+ this.replicationSpec = replicationSpec;
+ this.function = function;
+ }
+
+ public Database getDatabase() {
+ return db;
+ }
+
+ public Table getTable() {
+ return table;
+ }
+
+ public Iterable<Partition> getPartitions() {
+ return partitions;
+ }
+
+ public ReplicationSpec getReplicationSpec() {
+ return replicationSpec;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/MetadataJson.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/MetadataJson.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/MetadataJson.java
new file mode 100644
index 0000000..b7a5680
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/MetadataJson.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load;
+
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.Function;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.parse.EximUtil;
+import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.repl.dump.io.DBSerializer;
+import org.apache.hadoop.hive.ql.parse.repl.dump.io.FunctionSerializer;
+import org.apache.hadoop.hive.ql.parse.repl.dump.io.PartitionSerializer;
+import org.apache.hadoop.hive.ql.parse.repl.dump.io.TableSerializer;
+import org.apache.thrift.TBase;
+import org.apache.thrift.TDeserializer;
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TJSONProtocol;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import javax.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.hadoop.hive.ql.parse.repl.dump.io.JsonWriter.Serializer.UTF_8;
+
+public class MetadataJson {
+ private final JSONObject json;
+ private final TDeserializer deserializer;
+ private final String tableDesc;
+
+ public MetadataJson(String message) throws JSONException, SemanticException {
+ deserializer = new TDeserializer(new TJSONProtocol.Factory());
+ json = new JSONObject(message);
+ checkCompatibility();
+ tableDesc = jsonEntry(TableSerializer.FIELD_NAME);
+ }
+
+ public MetaData getMetaData() throws TException, JSONException {
+ return new MetaData(
+ database(),
+ table(),
+ partitions(),
+ readReplicationSpec(),
+ function()
+ );
+ }
+
+ private Function function() throws TException {
+ return deserialize(new Function(), jsonEntry(FunctionSerializer.FIELD_NAME));
+ }
+
+ private Database database() throws TException {
+ return deserialize(new Database(), jsonEntry(DBSerializer.FIELD_NAME));
+ }
+
+ private Table table() throws TException {
+ return deserialize(new Table(), tableDesc);
+ }
+
+ private <T extends TBase> T deserialize(T intoObject, String json) throws TException {
+ if (json == null) {
+ return null;
+ }
+ deserializer.deserialize(intoObject, json, UTF_8);
+ return intoObject;
+ }
+
+ private List<Partition> partitions() throws JSONException, TException {
+ if (tableDesc == null) {
+ return null;
+ }
+ // TODO : jackson-streaming-iterable-redo this
+ JSONArray jsonPartitions = new JSONArray(json.getString(PartitionSerializer.FIELD_NAME));
+ List<Partition> partitionsList = new ArrayList<>(jsonPartitions.length());
+ for (int i = 0; i < jsonPartitions.length(); ++i) {
+ String partDesc = jsonPartitions.getString(i);
+ partitionsList.add(deserialize(new Partition(), partDesc));
+ }
+ return partitionsList;
+ }
+
+ private ReplicationSpec readReplicationSpec() {
+ com.google.common.base.Function<String, String> keyFetcher =
+ new com.google.common.base.Function<String, String>() {
+ @Override
+ public String apply(@Nullable String s) {
+ return jsonEntry(s);
+ }
+ };
+ return new ReplicationSpec(keyFetcher);
+ }
+
+ private void checkCompatibility() throws SemanticException, JSONException {
+ String version = json.getString("version");
+ String fcVersion = jsonEntry("fcversion");
+ EximUtil.doCheckCompatibility(
+ EximUtil.METADATA_FORMAT_VERSION,
+ version,
+ fcVersion);
+ }
+
+ private String jsonEntry(String forName) {
+ try {
+ return json.getString(forName);
+ } catch (JSONException ignored) {
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/dump/HiveWrapperTest.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/dump/HiveWrapperTest.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/dump/HiveWrapperTest.java
new file mode 100644
index 0000000..3028e76
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/dump/HiveWrapperTest.java
@@ -0,0 +1,27 @@
+package org.apache.hadoop.hive.ql.parse.repl.dump;
+
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InOrder;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class HiveWrapperTest {
+ @Mock
+ private HiveWrapper.Tuple.Function<ReplicationSpec> specFunction;
+ @Mock
+ private HiveWrapper.Tuple.Function<Table> tableFunction;
+
+ @Test
+ public void replicationIdIsRequestedBeforeObjectDefinition() throws HiveException {
+ new HiveWrapper.Tuple<>(specFunction, tableFunction);
+ InOrder inOrder = Mockito.inOrder(specFunction, tableFunction);
+ inOrder.verify(specFunction).fromMetaStore();
+ inOrder.verify(tableFunction).fromMetaStore();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/events/TestEventHandlerFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/events/TestEventHandlerFactory.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/events/TestEventHandlerFactory.java
index d44cb79..4b802c4 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/events/TestEventHandlerFactory.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/events/TestEventHandlerFactory.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.hive.ql.parse.repl.events;
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
-import org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
import org.junit.Test;
import static org.junit.Assert.assertTrue;
@@ -44,7 +44,7 @@ public class TestEventHandlerFactory {
}
@Override
- public ReplicationSemanticAnalyzer.DUMPTYPE dumpType() {
+ public DumpType dumpType() {
return null;
}
}