You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by jl...@apache.org on 2019/01/11 22:04:01 UTC
[incubator-pinot] branch handle-delete-aged-segments created (now
8c3e7d6)
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a change to branch handle-delete-aged-segments
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
at 8c3e7d6 Add touch method in PinotFS
This branch includes the following new commits:
new 8c3e7d6 Add touch method in PinotFS
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: Add touch method in PinotFS
Posted by jl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a commit to branch handle-delete-aged-segments
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 8c3e7d6e19d0433ca057b4b26a31ebdb5d626406
Author: Jack Li(Analytics Engineering) <jl...@jlli-mn1.linkedin.biz>
AuthorDate: Fri Jan 11 14:03:40 2019 -0800
Add touch method in PinotFS
---
.../pinot/controller/helix/core/SegmentDeletionManager.java | 2 +-
.../main/java/org/apache/pinot/filesystem/LocalPinotFS.java | 9 +++++++++
.../src/main/java/org/apache/pinot/filesystem/PinotFS.java | 8 +++++++-
.../java/org/apache/pinot/filesystem/HadoopPinotFS.java | 13 +++++++++++++
4 files changed, 30 insertions(+), 2 deletions(-)
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java
index f405e68..cfef049 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java
@@ -248,7 +248,7 @@ public class SegmentDeletionManager {
}
}
} catch (IOException e) {
- LOGGER.error("Had trouble deleting directories", deletedDirURI.toString());
+ LOGGER.error("Had trouble deleting directories: {}", deletedDirURI.toString(), e.toString());
}
} else {
LOGGER.info("dataDir is not configured, won't delete any expired segments from deleted directory.");
diff --git a/pinot-filesystem/src/main/java/org/apache/pinot/filesystem/LocalPinotFS.java b/pinot-filesystem/src/main/java/org/apache/pinot/filesystem/LocalPinotFS.java
index 987125f..e060d88 100644
--- a/pinot-filesystem/src/main/java/org/apache/pinot/filesystem/LocalPinotFS.java
+++ b/pinot-filesystem/src/main/java/org/apache/pinot/filesystem/LocalPinotFS.java
@@ -161,6 +161,15 @@ public class LocalPinotFS extends PinotFS {
return file.lastModified();
}
+ @Override
+ public boolean touch(URI uri) throws IOException {
+ File file = new File(decodeURI(uri.getRawPath()));
+ if (!exists(uri)) {
+ return file.createNewFile();
+ }
+ return file.setLastModified(System.currentTimeMillis());
+ }
+
private String encodeURI(String uri) {
String encodedStr;
try {
diff --git a/pinot-filesystem/src/main/java/org/apache/pinot/filesystem/PinotFS.java b/pinot-filesystem/src/main/java/org/apache/pinot/filesystem/PinotFS.java
index abcf494..4a5815a 100644
--- a/pinot-filesystem/src/main/java/org/apache/pinot/filesystem/PinotFS.java
+++ b/pinot-filesystem/src/main/java/org/apache/pinot/filesystem/PinotFS.java
@@ -137,7 +137,7 @@ public abstract class PinotFS implements Closeable {
/**
* Returns the age of the file
- * @param uri
+ * @param uri location of file or directory
* @return A long value representing the time the file was last modified, measured in milliseconds since epoch
* (00:00:00 GMT, January 1, 1970) or 0L if the file does not exist or if an I/O error occurs
* @throws Exception if uri is not valid or present
@@ -145,6 +145,12 @@ public abstract class PinotFS implements Closeable {
public abstract long lastModified(URI uri);
/**
+ * Creates a new empty file if it doesn't exist, or updates the last modified on existing file or directory.
+ * @param uri location of file or directory
+ */
+ public abstract boolean touch(URI uri) throws IOException;
+
+ /**
* For certain filesystems, we may need to close the filesystem and do relevant operations to prevent leaks.
* By default, this method does nothing.
* @throws IOException
diff --git a/pinot-hadoop-filesystem/src/main/java/org/apache/pinot/filesystem/HadoopPinotFS.java b/pinot-hadoop-filesystem/src/main/java/org/apache/pinot/filesystem/HadoopPinotFS.java
index c58a16a..fb64770 100644
--- a/pinot-hadoop-filesystem/src/main/java/org/apache/pinot/filesystem/HadoopPinotFS.java
+++ b/pinot-hadoop-filesystem/src/main/java/org/apache/pinot/filesystem/HadoopPinotFS.java
@@ -19,6 +19,7 @@
package org.apache.pinot.filesystem;
import com.google.common.base.Strings;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.pinot.common.utils.retry.RetryPolicies;
import org.apache.pinot.common.utils.retry.RetryPolicy;
import java.io.File;
@@ -191,6 +192,18 @@ public class HadoopPinotFS extends PinotFS {
}
}
+ @Override
+ public boolean touch(URI uri) throws IOException {
+ Path path = new Path(uri);
+ if (!exists(uri)) {
+ FSDataOutputStream fos = _hadoopFS.create(path);
+ fos.close();
+ } else {
+ _hadoopFS.setTimes(path, System.currentTimeMillis(), -1);
+ }
+ return true;
+ }
+
private void authenticate(org.apache.hadoop.conf.Configuration hadoopConf, org.apache.commons.configuration.Configuration configs) {
String principal = configs.getString(PRINCIPAL);
String keytab = configs.getString(KEYTAB);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org