You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2021/01/17 21:22:13 UTC
[incubator-pinot] branch master updated: Fix gcs listFiles (#6426)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 950295a Fix gcs listFiles (#6426)
950295a is described below
commit 950295ab40844e628c5802649e99fc4feec4af54
Author: Elon Azoulay <el...@gmail.com>
AuthorDate: Sun Jan 17 13:21:49 2021 -0800
Fix gcs listFiles (#6426)
---
.../apache/pinot/plugin/filesystem/GcsPinotFS.java | 19 ++++++++++++-------
1 file changed, 12 insertions(+), 7 deletions(-)
diff --git a/pinot-plugins/pinot-file-system/pinot-gcs/src/main/java/org/apache/pinot/plugin/filesystem/GcsPinotFS.java b/pinot-plugins/pinot-file-system/pinot-gcs/src/main/java/org/apache/pinot/plugin/filesystem/GcsPinotFS.java
index b36c1f4..a585b42 100644
--- a/pinot-plugins/pinot-file-system/pinot-gcs/src/main/java/org/apache/pinot/plugin/filesystem/GcsPinotFS.java
+++ b/pinot-plugins/pinot-file-system/pinot-gcs/src/main/java/org/apache/pinot/plugin/filesystem/GcsPinotFS.java
@@ -58,6 +58,7 @@ public class GcsPinotFS extends PinotFS {
public static final String GCP_KEY = "gcpKey";
private static final Logger LOGGER = LoggerFactory.getLogger(GcsPinotFS.class);
+ private static final String SCHEME = "gs";
private static final String DELIMITER = "/";
private static final int BUFFER_SIZE = 128 * 1024;
private Storage storage;
@@ -130,7 +131,7 @@ public class GcsPinotFS extends PinotFS {
private URI getBase(URI uri) throws IOException {
try {
- return new URI(uri.getScheme(), uri.getHost(), null, null);
+ return new URI(SCHEME, uri.getHost(), DELIMITER, null);
} catch (URISyntaxException e) {
throw new IOException(e);
}
@@ -204,11 +205,11 @@ public class GcsPinotFS extends PinotFS {
LOGGER.info("Deleting uri {} force {}", segmentUri, forceDelete);
try {
if (!exists(segmentUri)) {
- return false;
+ return forceDelete;
}
if (isDirectory(segmentUri)) {
- if (!forceDelete) {
- checkState(isEmptyDirectory(segmentUri), "ForceDelete flag is not set and directory '%s' is not empty", segmentUri);
+ if (!forceDelete && !isEmptyDirectory(segmentUri)) {
+ return false;
}
String prefix = normalizeToDirectoryPrefix(segmentUri);
Page<Blob> page;
@@ -292,7 +293,7 @@ public class GcsPinotFS extends PinotFS {
public long length(URI fileUri) throws IOException {
try {
checkState(!isPathTerminatedByDelimiter(fileUri), "URI is a directory");
- Blob blob = getBucket(fileUri).get(fileUri.getPath());
+ Blob blob = getBlob(fileUri);
checkState(existsBlob(blob), "File '%s' does not exist", fileUri);
return blob.getSize();
} catch (Throwable t) {
@@ -313,8 +314,12 @@ public class GcsPinotFS extends PinotFS {
}
page.iterateAll()
.forEach(blob -> {
- if (!blob.getName().equals(fileUri.getPath())) {
- builder.add(blob.getName());
+ if (!blob.getName().equals(prefix)) {
+ try {
+ builder.add(new URI(SCHEME, fileUri.getHost(), DELIMITER + blob.getName(), null).toString());
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
}
});
return builder.build().toArray(new String[0]);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org