You are viewing a plain text version of this content. The canonical link for it is here.
Posted to github@arrow.apache.org by GitBox <gi...@apache.org> on 2021/10/11 14:17:50 UTC

[GitHub] [arrow] coryan commented on a change in pull request #11375: ARROW-14236: [C++] add GCS testbench for testing

coryan commented on a change in pull request #11375:
URL: https://github.com/apache/arrow/pull/11375#discussion_r726160001



##########
File path: cpp/src/arrow/filesystem/gcsfs_test.cc
##########
@@ -30,10 +34,58 @@ namespace arrow {
 namespace fs {
 namespace {
 
+namespace bp = boost::process;
+namespace gc = google::cloud;
+namespace gcs = google::cloud::storage;
+
 using ::testing::IsEmpty;
 using ::testing::Not;
 using ::testing::NotNull;
 
+auto const* kPreexistingBucket = "test-bucket-name";
+
+class GcsIntegrationTest : public ::testing::Test {
+ public:
+  ~GcsIntegrationTest() override {
+    if (server_process_.valid()) {
+      // Brutal shutdown
+      server_process_.terminate();
+      server_process_.wait();
+    }
+  }
+
+ protected:
+  void SetUp() override {
+    port_ = std::to_string(GetListenPort());
+    auto exe_path = bp::search_path("python3");
+    ASSERT_THAT(exe_path, Not(IsEmpty()));
+
+    server_process_ = bp::child(boost::this_process::environment(), exe_path, "-m",
+                                "testbench", "--port", port_);
+
+    // Create a bucket in the testbench so additional
+    auto client = gcs::Client(
+        google::cloud::Options{}
+            .set<gcs::RestEndpointOption>("http://127.0.0.1:" + port_)
+            .set<gc::UnifiedCredentialsOption>(gc::MakeInsecureCredentials()));
+    auto metadata = client.CreateBucketForProject(
+        kPreexistingBucket, "ignored-by-testbench", gcs::BucketMetadata{});
+    EXPECT_TRUE(metadata.ok()) << "Failed to create bucket <" << kPreexistingBucket

Review comment:
       Done.

##########
File path: cpp/src/arrow/filesystem/gcsfs.cc
##########
@@ -35,19 +68,85 @@ google::cloud::Options AsGoogleCloudOptions(const GcsOptions& o) {
   if (!o.endpoint_override.empty()) {
     std::string scheme = o.scheme;
     if (scheme.empty()) scheme = "https";
+    if (scheme == "https") {
+      options.set<google::cloud::UnifiedCredentialsOption>(
+          google::cloud::MakeGoogleDefaultCredentials());
+    } else {
+      options.set<google::cloud::UnifiedCredentialsOption>(
+          google::cloud::MakeInsecureCredentials());
+    }
     options.set<gcs::RestEndpointOption>(scheme + "://" + o.endpoint_override);
   }
   return options;
 }
 
+Status ToArrowStatus(google::cloud::Status const& s) {

Review comment:
       > there should probably be more detailed unit tests for this function?
   
   Done.  I tried to make it as little of a "change detection test" as possible, but it is hard for these things.
   
   > Do you think it will be important to retrieve the original status? Arrow status has an opaque shared pointer that can be used for this.
   
   I doubt it.  The client library automatically retries (subject to policies set by the application), by the time it returns the most common action is to log the error.  We can always add this later in any case.
   

##########
File path: cpp/src/arrow/filesystem/gcsfs.cc
##########
@@ -35,19 +68,85 @@ google::cloud::Options AsGoogleCloudOptions(const GcsOptions& o) {
   if (!o.endpoint_override.empty()) {
     std::string scheme = o.scheme;
     if (scheme.empty()) scheme = "https";
+    if (scheme == "https") {
+      options.set<google::cloud::UnifiedCredentialsOption>(
+          google::cloud::MakeGoogleDefaultCredentials());
+    } else {
+      options.set<google::cloud::UnifiedCredentialsOption>(
+          google::cloud::MakeInsecureCredentials());
+    }
     options.set<gcs::RestEndpointOption>(scheme + "://" + o.endpoint_override);
   }
   return options;
 }
 
+Status ToArrowStatus(google::cloud::Status const& s) {
+  std::ostringstream os;
+  os << "google::cloud::Status(" << s << ")";
+  switch (s.code()) {
+    case google::cloud::StatusCode::kOk:
+      break;
+    case google::cloud::StatusCode::kCancelled:
+      return Status::Cancelled(os.str());
+    case google::cloud::StatusCode::kUnknown:
+      return Status::UnknownError(os.str());
+    case google::cloud::StatusCode::kInvalidArgument:
+      return Status::Invalid(os.str());
+    case google::cloud::StatusCode::kDeadlineExceeded:
+    case google::cloud::StatusCode::kNotFound:
+      // TODO: it is unclear if a better mapping would be possible.
+      return Status::UnknownError(os.str());
+    case google::cloud::StatusCode::kAlreadyExists:
+      return Status::AlreadyExists(os.str());
+    case google::cloud::StatusCode::kPermissionDenied:
+    case google::cloud::StatusCode::kUnauthenticated:
+      return Status::UnknownError(os.str());
+    case google::cloud::StatusCode::kResourceExhausted:
+      return Status::CapacityError(os.str());
+    case google::cloud::StatusCode::kFailedPrecondition:
+    case google::cloud::StatusCode::kAborted:
+      return Status::UnknownError(os.str());
+    case google::cloud::StatusCode::kOutOfRange:
+      return Status::Invalid(os.str());
+    case google::cloud::StatusCode::kUnimplemented:
+      return Status::NotImplemented(os.str());
+    case google::cloud::StatusCode::kInternal:
+    case google::cloud::StatusCode::kUnavailable:
+    case google::cloud::StatusCode::kDataLoss:
+      return Status::UnknownError(os.str());
+  }
+  return Status::OK();
+}
+
 class GcsFileSystem::Impl {
  public:
   explicit Impl(GcsOptions o)
       : options_(std::move(o)), client_(AsGoogleCloudOptions(options_)) {}
 
   GcsOptions const& options() const { return options_; }
 
+  Result<FileInfo> GetFileInfo(GcsPath const& path) {
+    if (!path.object.empty()) {
+      auto meta = client_.GetObjectMetadata(path.bucket, path.object);
+      return GetFileInfoImpl(path, std::move(meta).status(), FileType::File);
+    }
+    auto meta = client_.GetBucketMetadata(path.bucket);
+    return GetFileInfoImpl(path, std::move(meta).status(), FileType::Directory);
+  }
+
  private:
+  Result<FileInfo> GetFileInfoImpl(GcsPath const& path, google::cloud::Status status,

Review comment:
       Done.

##########
File path: cpp/src/arrow/filesystem/gcsfs.cc
##########
@@ -35,19 +68,85 @@ google::cloud::Options AsGoogleCloudOptions(const GcsOptions& o) {
   if (!o.endpoint_override.empty()) {
     std::string scheme = o.scheme;
     if (scheme.empty()) scheme = "https";
+    if (scheme == "https") {
+      options.set<google::cloud::UnifiedCredentialsOption>(
+          google::cloud::MakeGoogleDefaultCredentials());
+    } else {
+      options.set<google::cloud::UnifiedCredentialsOption>(
+          google::cloud::MakeInsecureCredentials());
+    }
     options.set<gcs::RestEndpointOption>(scheme + "://" + o.endpoint_override);
   }
   return options;
 }
 
+Status ToArrowStatus(google::cloud::Status const& s) {
+  std::ostringstream os;
+  os << "google::cloud::Status(" << s << ")";
+  switch (s.code()) {
+    case google::cloud::StatusCode::kOk:
+      break;
+    case google::cloud::StatusCode::kCancelled:
+      return Status::Cancelled(os.str());
+    case google::cloud::StatusCode::kUnknown:
+      return Status::UnknownError(os.str());
+    case google::cloud::StatusCode::kInvalidArgument:
+      return Status::Invalid(os.str());
+    case google::cloud::StatusCode::kDeadlineExceeded:
+    case google::cloud::StatusCode::kNotFound:
+      // TODO: it is unclear if a better mapping would be possible.
+      return Status::UnknownError(os.str());
+    case google::cloud::StatusCode::kAlreadyExists:
+      return Status::AlreadyExists(os.str());
+    case google::cloud::StatusCode::kPermissionDenied:
+    case google::cloud::StatusCode::kUnauthenticated:
+      return Status::UnknownError(os.str());
+    case google::cloud::StatusCode::kResourceExhausted:
+      return Status::CapacityError(os.str());
+    case google::cloud::StatusCode::kFailedPrecondition:
+    case google::cloud::StatusCode::kAborted:
+      return Status::UnknownError(os.str());
+    case google::cloud::StatusCode::kOutOfRange:
+      return Status::Invalid(os.str());
+    case google::cloud::StatusCode::kUnimplemented:
+      return Status::NotImplemented(os.str());
+    case google::cloud::StatusCode::kInternal:
+    case google::cloud::StatusCode::kUnavailable:
+    case google::cloud::StatusCode::kDataLoss:
+      return Status::UnknownError(os.str());
+  }
+  return Status::OK();
+}
+
 class GcsFileSystem::Impl {
  public:
   explicit Impl(GcsOptions o)
       : options_(std::move(o)), client_(AsGoogleCloudOptions(options_)) {}
 
   GcsOptions const& options() const { return options_; }
 
+  Result<FileInfo> GetFileInfo(GcsPath const& path) {

Review comment:
       Fixed here and elsewhere.




-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org