You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by jl...@apache.org on 2019/01/11 22:56:09 UTC
[incubator-pinot] 01/01: Add touch method in PinotFS;
Call touch when moving deleted segments.
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a commit to branch handle-delete-aged-segments
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit ea1412c887f16d404c8a7e23bac02548ee8669ba
Author: Jack Li(Analytics Engineering) <jl...@jlli-mn1.linkedin.biz>
AuthorDate: Fri Jan 11 14:03:40 2019 -0800
Add touch method in PinotFS; Call touch when moving deleted segments.
---
.../pinot/controller/helix/core/SegmentDeletionManager.java | 4 +++-
.../main/java/org/apache/pinot/filesystem/LocalPinotFS.java | 9 +++++++++
.../src/main/java/org/apache/pinot/filesystem/PinotFS.java | 8 +++++++-
.../java/org/apache/pinot/filesystem/HadoopPinotFS.java | 13 +++++++++++++
4 files changed, 32 insertions(+), 2 deletions(-)
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java
index f405e68..5554da6 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java
@@ -185,6 +185,8 @@ public class SegmentDeletionManager {
if (pinotFS.exists(fileToMoveURI)) {
// Overwrites the file if it already exists in the target directory.
pinotFS.move(fileToMoveURI, deletedSegmentDestURI, true);
+ // Updates last modified
+ pinotFS.touch(deletedSegmentDestURI);
LOGGER.info("Moved segment {} from {} to {}", segmentId, fileToMoveURI.toString(), deletedSegmentDestURI.toString());
} else {
if (!SegmentName.isHighLevelConsumerSegmentName(segmentId)) {
@@ -248,7 +250,7 @@ public class SegmentDeletionManager {
}
}
} catch (IOException e) {
- LOGGER.error("Had trouble deleting directories", deletedDirURI.toString());
+ LOGGER.error("Had trouble deleting directories: {}", deletedDirURI.toString(), e.toString());
}
} else {
LOGGER.info("dataDir is not configured, won't delete any expired segments from deleted directory.");
diff --git a/pinot-filesystem/src/main/java/org/apache/pinot/filesystem/LocalPinotFS.java b/pinot-filesystem/src/main/java/org/apache/pinot/filesystem/LocalPinotFS.java
index 987125f..e060d88 100644
--- a/pinot-filesystem/src/main/java/org/apache/pinot/filesystem/LocalPinotFS.java
+++ b/pinot-filesystem/src/main/java/org/apache/pinot/filesystem/LocalPinotFS.java
@@ -161,6 +161,15 @@ public class LocalPinotFS extends PinotFS {
return file.lastModified();
}
+ @Override
+ public boolean touch(URI uri) throws IOException {
+ File file = new File(decodeURI(uri.getRawPath()));
+ if (!exists(uri)) {
+ return file.createNewFile();
+ }
+ return file.setLastModified(System.currentTimeMillis());
+ }
+
private String encodeURI(String uri) {
String encodedStr;
try {
diff --git a/pinot-filesystem/src/main/java/org/apache/pinot/filesystem/PinotFS.java b/pinot-filesystem/src/main/java/org/apache/pinot/filesystem/PinotFS.java
index abcf494..4a5815a 100644
--- a/pinot-filesystem/src/main/java/org/apache/pinot/filesystem/PinotFS.java
+++ b/pinot-filesystem/src/main/java/org/apache/pinot/filesystem/PinotFS.java
@@ -137,7 +137,7 @@ public abstract class PinotFS implements Closeable {
/**
* Returns the age of the file
- * @param uri
+ * @param uri location of file or directory
* @return A long value representing the time the file was last modified, measured in milliseconds since epoch
* (00:00:00 GMT, January 1, 1970) or 0L if the file does not exist or if an I/O error occurs
* @throws Exception if uri is not valid or present
@@ -145,6 +145,12 @@ public abstract class PinotFS implements Closeable {
public abstract long lastModified(URI uri);
/**
+ * Creates a new empty file if it doesn't exist, or updates the last modified on existing file or directory.
+ * @param uri location of file or directory
+ */
+ public abstract boolean touch(URI uri) throws IOException;
+
+ /**
* For certain filesystems, we may need to close the filesystem and do relevant operations to prevent leaks.
* By default, this method does nothing.
* @throws IOException
diff --git a/pinot-hadoop-filesystem/src/main/java/org/apache/pinot/filesystem/HadoopPinotFS.java b/pinot-hadoop-filesystem/src/main/java/org/apache/pinot/filesystem/HadoopPinotFS.java
index c58a16a..fb64770 100644
--- a/pinot-hadoop-filesystem/src/main/java/org/apache/pinot/filesystem/HadoopPinotFS.java
+++ b/pinot-hadoop-filesystem/src/main/java/org/apache/pinot/filesystem/HadoopPinotFS.java
@@ -19,6 +19,7 @@
package org.apache.pinot.filesystem;
import com.google.common.base.Strings;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.pinot.common.utils.retry.RetryPolicies;
import org.apache.pinot.common.utils.retry.RetryPolicy;
import java.io.File;
@@ -191,6 +192,18 @@ public class HadoopPinotFS extends PinotFS {
}
}
+ @Override
+ public boolean touch(URI uri) throws IOException {
+ Path path = new Path(uri);
+ if (!exists(uri)) {
+ FSDataOutputStream fos = _hadoopFS.create(path);
+ fos.close();
+ } else {
+ _hadoopFS.setTimes(path, System.currentTimeMillis(), -1);
+ }
+ return true;
+ }
+
private void authenticate(org.apache.hadoop.conf.Configuration hadoopConf, org.apache.commons.configuration.Configuration configs) {
String principal = configs.getString(PRINCIPAL);
String keytab = configs.getString(KEYTAB);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org