You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by je...@apache.org on 2018/12/03 21:47:57 UTC
[incubator-pinot] branch segmentcommitend updated (5a2d273 ->
8ef66a6)
This is an automated email from the ASF dual-hosted git repository.
jenniferdai pushed a change to branch segmentcommitend
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
discard 5a2d273 Changing segmentCommitEnd to support deep storage
new 8ef66a6 Changing segmentCommitEnd to support deep storage
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (5a2d273)
\
N -- N -- N refs/heads/segmentcommitend (8ef66a6)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: Changing segmentCommitEnd to support deep
storage
Posted by je...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jenniferdai pushed a commit to branch segmentcommitend
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 8ef66a66fc686cf371cb9b3fda4d3063b345c064
Author: Jennifer Dai <jd...@linkedin.com>
AuthorDate: Mon Dec 3 13:45:54 2018 -0800
Changing segmentCommitEnd to support deep storage
---
.../linkedin/pinot/controller/ControllerConf.java | 2 +-
.../realtime/PinotLLCRealtimeSegmentManager.java | 40 ++++++++++------------
2 files changed, 20 insertions(+), 22 deletions(-)
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/ControllerConf.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/ControllerConf.java
index e18f13a..f7214bd 100644
--- a/pinot-controller/src/main/java/com/linkedin/pinot/controller/ControllerConf.java
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/ControllerConf.java
@@ -149,7 +149,7 @@ public class ControllerConf extends PropertiesConfiguration {
}
public String getLocalTempDir() {
- return getString(LOCAL_TEMP_DIR, null);
+ return getString(LOCAL_TEMP_DIR, getDataDir());
}
public void setPinotFSFactoryClasses(Configuration pinotFSFactoryClasses) {
diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
index 9cedc35..8f13b82 100644
--- a/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
+++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
@@ -59,11 +59,12 @@ import com.linkedin.pinot.core.realtime.stream.StreamConfigProperties;
import com.linkedin.pinot.core.segment.creator.impl.V1Constants;
import com.linkedin.pinot.core.segment.index.ColumnMetadata;
import com.linkedin.pinot.core.segment.index.SegmentMetadataImpl;
+import com.linkedin.pinot.filesystem.PinotFS;
+import com.linkedin.pinot.filesystem.PinotFSFactory;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.net.URI;
-import java.net.URISyntaxException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -349,11 +350,11 @@ public class PinotLLCRealtimeSegmentManager {
public boolean commitSegmentFile(String tableName, CommittingSegmentDescriptor committingSegmentDescriptor) {
String segmentName = committingSegmentDescriptor.getSegmentName();
String segmentLocation = committingSegmentDescriptor.getSegmentLocation();
- File segmentFile = convertURIToSegmentLocation(segmentLocation);
-
- File baseDir = new File(_controllerConf.getDataDir());
- File tableDir = new File(baseDir, tableName);
- File fileToMoveTo = new File(tableDir, segmentName);
+ URI segmentFileURI = ControllerConf.getUriFromPath(segmentLocation);
+ URI baseDirURI = ControllerConf.getUriFromPath(_controllerConf.getDataDir());
+ URI tableDirURI = ControllerConf.getUriFromPath(StringUtil.join("/", _controllerConf.getDataDir(), tableName));
+ URI uriToMoveTo = ControllerConf.getUriFromPath(StringUtil.join("/", tableDirURI.toString(), segmentName));
+ PinotFS pinotFS = PinotFSFactory.create(baseDirURI.getScheme());
if (!isConnected() || !isLeader()) {
// We can potentially log a different value than what we saw ....
@@ -364,27 +365,24 @@ public class PinotLLCRealtimeSegmentManager {
}
try {
- com.linkedin.pinot.common.utils.FileUtils.moveFileWithOverwrite(segmentFile, fileToMoveTo);
+ pinotFS.move(segmentFileURI, uriToMoveTo, true);
} catch (Exception e) {
- LOGGER.error("Could not move {} to {}", segmentFile, segmentName, e);
+ LOGGER.error("Could not move {} to {}", segmentLocation, segmentName, e);
return false;
}
- for (File file : tableDir.listFiles()) {
- if (file.getName().startsWith(SegmentCompletionUtils.getSegmentNamePrefix(segmentName))) {
- LOGGER.warn("Deleting " + file);
- FileUtils.deleteQuietly(file);
- }
- }
- return true;
- }
- private static File convertURIToSegmentLocation(String segmentLocation) {
try {
- URI uri = new URI(segmentLocation);
- return new File(uri.getPath());
- } catch (URISyntaxException e) {
- throw new RuntimeException("Could not convert URI " + segmentLocation + " to segment location", e);
+ for (String uri : pinotFS.listFiles(tableDirURI, true)) {
+ if (uri.startsWith(SegmentCompletionUtils.getSegmentNamePrefix(segmentName))) {
+ LOGGER.warn("Deleting " + uri);
+ pinotFS.delete(new URI(uri), true);
+ }
+ }
+ } catch (Exception e) {
+ LOGGER.warn("Could not delete tmp segment files");
}
+
+ return true;
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org