You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by an...@apache.org on 2020/08/11 11:18:20 UTC
[hive] branch master updated: HIVE-24014: Need to delete
DumpDirectoryCleanerTask (Arko Sharma, reviewed by Aasha Medhi)
This is an automated email from the ASF dual-hosted git repository.
anishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new d4af384 HIVE-24014: Need to delete DumpDirectoryCleanerTask (Arko Sharma, reviewed by Aasha Medhi)
d4af384 is described below
commit d4af3840f89408edea886a28ee4ae7d79a6f16f8
Author: Anishek Agarwal <an...@gmail.com>
AuthorDate: Tue Aug 11 16:48:10 2020 +0530
HIVE-24014: Need to delete DumpDirectoryCleanerTask (Arko Sharma, reviewed by Aasha Medhi)
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 6 --
.../hive/metastore/repl/DumpDirCleanerTask.java | 74 ----------------------
.../hadoop/hive/metastore/conf/MetastoreConf.java | 1 -
3 files changed, 81 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 2e7247b..ab46865 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -536,12 +536,6 @@ public class HiveConf extends Configuration {
REPL_FILE_LIST_CACHE_SIZE("hive.repl.file.list.cache.size", 10000,
"This config indicates threshold for the maximum number of data copy locations to be kept in memory. \n"
+ "When the config 'hive.repl.data.copy.lazy' is set to true, this config is not considered."),
- REPL_DUMPDIR_CLEAN_FREQ("hive.repl.dumpdir.clean.freq", "0s",
- new TimeValidator(TimeUnit.SECONDS),
- "Frequency at which timer task runs to purge expired dump dirs."),
- REPL_DUMPDIR_TTL("hive.repl.dumpdir.ttl", "7d",
- new TimeValidator(TimeUnit.DAYS),
- "TTL of dump dirs before cleanup."),
REPL_DUMP_METADATA_ONLY("hive.repl.dump.metadata.only", false,
"Indicates whether replication dump only metadata information or data + metadata. \n"
+ "This config makes hive.repl.include.external.tables config ineffective."),
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/repl/DumpDirCleanerTask.java b/metastore/src/java/org/apache/hadoop/hive/metastore/repl/DumpDirCleanerTask.java
deleted file mode 100644
index daf67e4..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/repl/DumpDirCleanerTask.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.repl;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.MetastoreTaskThread;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.concurrent.TimeUnit;
-
-public class DumpDirCleanerTask implements MetastoreTaskThread {
- public static final Logger LOG = LoggerFactory.getLogger(DumpDirCleanerTask.class);
- private Configuration conf;
- private Path dumpRoot;
- private long ttl;
-
- @Override
- public void setConf(Configuration conf) {
- this.conf = conf;
- dumpRoot = new Path(HiveConf.getVar(conf, ConfVars.REPLDIR));
- ttl = HiveConf.getTimeVar(conf, ConfVars.REPL_DUMPDIR_TTL, TimeUnit.MILLISECONDS);
- }
-
- @Override
- public Configuration getConf() {
- return conf;
- }
-
- @Override
- public long runFrequency(TimeUnit unit) {
- return HiveConf.getTimeVar(conf, ConfVars.REPL_DUMPDIR_CLEAN_FREQ, unit);
- }
-
- @Override
- public void run() {
- LOG.debug("Trying to delete old dump dirs");
- try {
- FileSystem fs = FileSystem.get(dumpRoot.toUri(), conf);
- FileStatus[] statuses = fs.listStatus(dumpRoot);
- for (FileStatus status : statuses)
- {
- if (status.getModificationTime() < System.currentTimeMillis() - ttl)
- {
- fs.delete(status.getPath(), true);
- LOG.info("Deleted old dump dir: " + status.getPath());
- }
- }
- } catch (IOException e) {
- LOG.error("Error while trying to delete dump dir", e);
- }
- }
-}
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index cc4e68f..0583208 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -1091,7 +1091,6 @@ public class MetastoreConf {
+ " quoted table names.\nThe default value is true."),
TASK_THREADS_ALWAYS("metastore.task.threads.always", "metastore.task.threads.always",
EVENT_CLEANER_TASK_CLASS + "," + RUNTIME_STATS_CLEANER_TASK_CLASS + "," +
- "org.apache.hadoop.hive.metastore.repl.DumpDirCleanerTask" + "," +
"org.apache.hadoop.hive.metastore.HiveProtoEventsCleanerTask" + ","
+ "org.apache.hadoop.hive.metastore.ScheduledQueryExecutionsMaintTask" + ","
+ "org.apache.hadoop.hive.metastore.ReplicationMetricsMaintTask",