You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by an...@apache.org on 2020/05/06 08:28:44 UTC
[hive] branch master updated: HIVE-23309: Lazy Initialization of
Hadoop Shims (Aasha Medhi, reviewed by Anishek Agarwal)
This is an automated email from the ASF dual-hosted git repository.
anishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 48c0110 HIVE-23309: Lazy Initialization of Hadoop Shims (Aasha Medhi, reviewed by Anishek Agarwal)
48c0110 is described below
commit 48c01107cd18d80867369e5addfa1fc5b4e7f698
Author: Aasha Medhi <aa...@gmail.com>
AuthorDate: Wed May 6 13:58:29 2020 +0530
HIVE-23309: Lazy Initialization of Hadoop Shims (Aasha Medhi, reviewed by Anishek Agarwal)
---
.../hcatalog/listener/DbNotificationListener.java | 9 +++---
.../TestMetaStoreMultipleEncryptionZones.java | 32 +++++++++++-----------
.../ql/parse/repl/dump/io/FunctionSerializer.java | 3 +-
.../hadoop/hive/metastore/ReplChangeManager.java | 7 +++--
4 files changed, 27 insertions(+), 24 deletions(-)
diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
index 3a8cb39..ac8922b 100644
--- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
+++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
@@ -302,9 +302,9 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener
try {
FileStatus file = files[i];
i++;
- return ReplChangeManager.encodeFileUri(file.getPath().toString(),
+ return ReplChangeManager.getInstance(conf).encodeFileUri(file.getPath().toString(),
ReplChangeManager.checksumFor(file.getPath(), fs), null);
- } catch (IOException e) {
+ } catch (IOException | MetaException e) {
throw new RuntimeException(e);
}
}
@@ -521,9 +521,10 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener
public String next() {
String result;
try {
- result = ReplChangeManager.encodeFileUri(files.get(i), (chksums != null && !chksums.isEmpty()) ? chksums.get(i) : null,
+ result = ReplChangeManager.getInstance(conf).
+ encodeFileUri(files.get(i), (chksums != null && !chksums.isEmpty()) ? chksums.get(i) : null,
subDirs != null ? subDirs.get(i) : null);
- } catch (IOException e) {
+ } catch (IOException | MetaException e) {
// File operations failed
LOG.error("Encoding file URI failed with error " + e.getMessage());
throw new RuntimeException(e.getMessage());
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java
index 41a1ce9..8f1bc40 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java
@@ -1302,29 +1302,29 @@ public class TestMetaStoreMultipleEncryptionZones {
ReplChangeManager.getInstance(hiveConfCmClearer).recycle(dirTbl3, RecycleType.MOVE, true);
assertTrue(fsWarehouse.exists(ReplChangeManager.getCMPath(hiveConfCmClearer, part11.getName(), fileChksum11,
- ReplChangeManager.getCmRoot(part11).toString())));
+ ReplChangeManager.getInstance(conf).getCmRoot(part11).toString())));
assertTrue(fsWarehouse.exists(ReplChangeManager.getCMPath(hiveConfCmClearer, part12.getName(), fileChksum12,
- ReplChangeManager.getCmRoot(part12).toString())));
+ ReplChangeManager.getInstance(conf).getCmRoot(part12).toString())));
assertTrue(fsWarehouse.exists(ReplChangeManager.getCMPath(hiveConfCmClearer, part21.getName(), fileChksum21,
- ReplChangeManager.getCmRoot(part21).toString())));
+ ReplChangeManager.getInstance(conf).getCmRoot(part21).toString())));
assertTrue(fsWarehouse.exists(ReplChangeManager.getCMPath(hiveConfCmClearer, part22.getName(), fileChksum22,
- ReplChangeManager.getCmRoot(part22).toString())));
+ ReplChangeManager.getInstance(conf).getCmRoot(part22).toString())));
assertTrue(fsWarehouse.exists(ReplChangeManager.getCMPath(hiveConfCmClearer, part31.getName(), fileChksum31,
- ReplChangeManager.getCmRoot(part31).toString())));
+ ReplChangeManager.getInstance(conf).getCmRoot(part31).toString())));
assertTrue(fsWarehouse.exists(ReplChangeManager.getCMPath(hiveConfCmClearer, part32.getName(), fileChksum32,
- ReplChangeManager.getCmRoot(part32).toString())));
+ ReplChangeManager.getInstance(conf).getCmRoot(part32).toString())));
fsWarehouse.setTimes(ReplChangeManager.getCMPath(hiveConfCmClearer, part11.getName(), fileChksum11,
- ReplChangeManager.getCmRoot(part11).toString()),
+ ReplChangeManager.getInstance(conf).getCmRoot(part11).toString()),
now - 86400*1000*2, now - 86400*1000*2);
fsWarehouse.setTimes(ReplChangeManager.getCMPath(hiveConfCmClearer, part21.getName(), fileChksum21,
- ReplChangeManager.getCmRoot(part21).toString()),
+ ReplChangeManager.getInstance(conf).getCmRoot(part21).toString()),
now - 86400*1000*2, now - 86400*1000*2);
fsWarehouse.setTimes(ReplChangeManager.getCMPath(hiveConfCmClearer, part31.getName(), fileChksum31,
- ReplChangeManager.getCmRoot(part31).toString()),
+ ReplChangeManager.getInstance(conf).getCmRoot(part31).toString()),
now - 86400*1000*2, now - 86400*1000*2);
fsWarehouse.setTimes(ReplChangeManager.getCMPath(hiveConfCmClearer, part32.getName(), fileChksum32,
- ReplChangeManager.getCmRoot(part32).toString()),
+ ReplChangeManager.getInstance(conf).getCmRoot(part32).toString()),
now - 86400*1000*2, now - 86400*1000*2);
ReplChangeManager.scheduleCMClearer(hiveConfCmClearer);
@@ -1339,17 +1339,17 @@ public class TestMetaStoreMultipleEncryptionZones {
Assert.fail("timeout, cmroot has not been cleared");
}
if (!fsWarehouse.exists(ReplChangeManager.getCMPath(hiveConfCmClearer, part11.getName(), fileChksum11,
- ReplChangeManager.getCmRoot(part11).toString())) &&
+ ReplChangeManager.getInstance(conf).getCmRoot(part11).toString())) &&
fsWarehouse.exists(ReplChangeManager.getCMPath(hiveConfCmClearer, part12.getName(), fileChksum12,
- ReplChangeManager.getCmRoot(part12).toString())) &&
+ ReplChangeManager.getInstance(conf).getCmRoot(part12).toString())) &&
!fsWarehouse.exists(ReplChangeManager.getCMPath(hiveConfCmClearer, part21.getName(), fileChksum21,
- ReplChangeManager.getCmRoot(part21).toString())) &&
+ ReplChangeManager.getInstance(conf).getCmRoot(part21).toString())) &&
fsWarehouse.exists(ReplChangeManager.getCMPath(hiveConfCmClearer, part22.getName(), fileChksum22,
- ReplChangeManager.getCmRoot(part22).toString())) &&
+ ReplChangeManager.getInstance(conf).getCmRoot(part22).toString())) &&
!fsWarehouse.exists(ReplChangeManager.getCMPath(hiveConfCmClearer, part31.getName(), fileChksum31,
- ReplChangeManager.getCmRoot(part31).toString())) &&
+ ReplChangeManager.getInstance(conf).getCmRoot(part31).toString())) &&
!fsWarehouse.exists(ReplChangeManager.getCMPath(hiveConfCmClearer, part32.getName(), fileChksum32,
- ReplChangeManager.getCmRoot(part32).toString()))) {
+ ReplChangeManager.getInstance(conf).getCmRoot(part32).toString()))) {
cleared = true;
}
} while (!cleared);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FunctionSerializer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FunctionSerializer.java
index 420c9bf..733bab5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FunctionSerializer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FunctionSerializer.java
@@ -58,7 +58,8 @@ public class FunctionSerializer implements JsonWriter.Serializer {
FileSystem fileSystem = inputPath.getFileSystem(hiveConf);
Path qualifiedUri = PathBuilder.fullyQualifiedHDFSUri(inputPath, fileSystem);
String checkSum = ReplChangeManager.checksumFor(qualifiedUri, fileSystem);
- String newFileUri = ReplChangeManager.encodeFileUri(qualifiedUri.toString(), checkSum, null);
+ String newFileUri = ReplChangeManager.getInstance(hiveConf)
+ .encodeFileUri(qualifiedUri.toString(), checkSum, null);
resourceUris.add(new ResourceUri(uri.getResourceType(), newFileUri));
} else {
resourceUris.add(uri);
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java
index 8e1bb4e..690b1f3 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java
@@ -60,7 +60,7 @@ public class ReplChangeManager {
private static boolean inited = false;
private static boolean enabled = false;
private static Map<String, String> encryptionZoneToCmrootMapping = new HashMap<>();
- private static HadoopShims hadoopShims = ShimLoader.getHadoopShims();
+ private HadoopShims hadoopShims;
private static Configuration conf;
private String msUser;
private String msGroup;
@@ -153,6 +153,7 @@ public class ReplChangeManager {
if (MetastoreConf.getBoolVar(conf, ConfVars.REPLCMENABLED)) {
ReplChangeManager.enabled = true;
ReplChangeManager.conf = conf;
+ hadoopShims = ShimLoader.getHadoopShims();
cmRootDir = MetastoreConf.getVar(conf, ConfVars.REPLCMDIR);
encryptedCmRootDir = MetastoreConf.getVar(conf, ConfVars.REPLCMENCRYPTEDDIR);
fallbackNonEncryptedCmRootDir = MetastoreConf.getVar(conf, ConfVars.REPLCMFALLBACKNONENCRYPTEDDIR);
@@ -403,7 +404,7 @@ public class ReplChangeManager {
*/
// TODO: this needs to be enhanced once change management based filesystem is implemented
// Currently using fileuri#checksum#cmrooturi#subdirs as the format
- public static String encodeFileUri(String fileUriStr, String fileChecksum, String encodedSubDir)
+ public String encodeFileUri(String fileUriStr, String fileChecksum, String encodedSubDir)
throws IOException {
if (instance == null) {
throw new IllegalStateException("Uninitialized ReplChangeManager instance.");
@@ -551,7 +552,7 @@ public class ReplChangeManager {
}
@VisibleForTesting
- static Path getCmRoot(Path path) throws IOException {
+ Path getCmRoot(Path path) throws IOException {
Path cmroot = null;
//Default path if hive.repl.cm dir is encrypted
String cmrootDir = fallbackNonEncryptedCmRootDir;