You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2025/05/27 22:20:32 UTC
(hadoop) branch apache-github-HADOOP-19343 created (now 31e24808e6b)
This is an automated email from the ASF dual-hosted git repository.
cnauroth pushed a change to branch apache-github-HADOOP-19343
in repository https://gitbox.apache.org/repos/asf/hadoop.git
at 31e24808e6b HADOOP-19343: GCS / Add implementation for create() API
This branch includes the following new commits:
new 31e24808e6b HADOOP-19343: GCS / Add implementation for create() API
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 01/01: HADOOP-19343: GCS / Add implementation for create() API
Posted by cn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
cnauroth pushed a commit to branch apache-github-HADOOP-19343
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 31e24808e6bd7033804bf63ab2d710315c1c2a7b
Author: Arunkumar Chacko <ar...@google.com>
AuthorDate: Tue May 27 22:17:54 2025 +0000
HADOOP-19343: GCS / Add implementation for create() API
Closes #7656
Signed-off-by: Chris Nauroth <cn...@apache.org>
---
hadoop-project/pom.xml | 7 +-
.../hadoop-gcp/dev-support/findbugs-exclude.xml | 29 ++
hadoop-tools/hadoop-gcp/pom.xml | 503 ++++++++++++++++++++
.../src/config/checkstyle-suppressions.xml | 26 ++
.../java/org/apache/hadoop/fs/gs/Constants.java | 31 ++
.../org/apache/hadoop/fs/gs/CreateOptions.java | 121 +++++
.../apache/hadoop/fs/gs/ErrorTypeExtractor.java | 60 +++
.../java/org/apache/hadoop/fs/gs/FileInfo.java | 203 +++++++++
.../apache/hadoop/fs/gs/GoogleCloudStorage.java | 261 +++++++++++
.../gs/GoogleCloudStorageClientWriteChannel.java | 116 +++++
.../hadoop/fs/gs/GoogleCloudStorageFileSystem.java | 89 ++++
.../hadoop/fs/gs/GoogleCloudStorageItemInfo.java | 423 +++++++++++++++++
.../hadoop/fs/gs/GoogleHadoopFileSystem.java | 505 +++++++++++++++++++++
.../fs/gs/GoogleHadoopFileSystemConfiguration.java | 75 +++
.../hadoop/fs/gs/GoogleHadoopOutputStream.java | 124 +++++
.../hadoop/fs/gs/HadoopConfigurationProperty.java | 90 ++++
.../org/apache/hadoop/fs/gs/StorageResourceId.java | 328 +++++++++++++
.../java/org/apache/hadoop/fs/gs/StringPaths.java | 169 +++++++
.../java/org/apache/hadoop/fs/gs/UriPaths.java | 113 +++++
.../hadoop/fs/gs/VerificationAttributes.java | 68 +++
.../java/org/apache/hadoop/fs/gs/package-info.java | 23 +
.../apache/hadoop/fs/gs/TestStorageResourceId.java | 285 ++++++++++++
.../org/apache/hadoop/fs/gs/TestStringPaths.java | 164 +++++++
.../java/org/apache/hadoop/fs/gs/TestUriPaths.java | 150 ++++++
.../java/org/apache/hadoop/fs/gs/package-info.java | 22 +
hadoop-tools/pom.xml | 1 +
26 files changed, 3985 insertions(+), 1 deletion(-)
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 248e17f02c5..b0c4cc832da 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -108,7 +108,7 @@
<findbugs.version>3.0.5</findbugs.version>
<dnsjava.version>3.6.1</dnsjava.version>
- <guava.version>27.0-jre</guava.version>
+ <guava.version>33.1.0-jre</guava.version>
<guice.version>5.1.0</guice.version>
<bouncycastle.version>1.78.1</bouncycastle.version>
@@ -2142,6 +2142,11 @@
<artifactId>failsafe</artifactId>
<version>2.4.4</version>
</dependency>
+ <dependency>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>google-cloud-storage</artifactId>
+ <version>2.44.1</version>
+ </dependency>
</dependencies>
</dependencyManagement>
diff --git a/hadoop-tools/hadoop-gcp/dev-support/findbugs-exclude.xml b/hadoop-tools/hadoop-gcp/dev-support/findbugs-exclude.xml
new file mode 100644
index 00000000000..80be329bd6d
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/dev-support/findbugs-exclude.xml
@@ -0,0 +1,29 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<FindBugsFilter>
+ <!-- Redundant null check makes code clearer, future-proof here. -->
+ <Match>
+ <Class name="org.apache.hadoop.fs.gs.GoogleCloudStorage" />
+ <Method name="createItemInfoForBlob" />
+ <Bug pattern="RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.fs.gs.GoogleCloudStorage" />
+ <Method name="createItemInfoForBlob" />
+ <Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" />
+ </Match>
+</FindBugsFilter>
diff --git a/hadoop-tools/hadoop-gcp/pom.xml b/hadoop-tools/hadoop-gcp/pom.xml
new file mode 100644
index 00000000000..d5744f1f97c
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/pom.xml
@@ -0,0 +1,503 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-project</artifactId>
+ <version>3.5.0-SNAPSHOT</version>
+ <relativePath>../../hadoop-project</relativePath>
+ </parent>
+ <artifactId>hadoop-gcp</artifactId>
+ <version>3.5.0-SNAPSHOT</version>
+ <name>Apache Hadoop Google Cloud Platform support</name>
+ <description>
+ This module contains code to support integration with Google Cloud Platform.
+ It also declares the dependencies needed to work with Google Cloud Storage.
+ </description>
+ <packaging>jar</packaging>
+
+ <properties>
+ <file.encoding>UTF-8</file.encoding>
+ <downloadSources>true</downloadSources>
+ <hadoop.tmp.dir>${project.build.directory}/test</hadoop.tmp.dir>
+ </properties>
+
+ <profiles>
+ <profile>
+ <id>tests-off</id>
+ <activation>
+ <file>
+ <missing>src/test/resources/auth-keys.xml</missing>
+ </file>
+ </activation>
+ <properties>
+ <skipITs>true</skipITs>
+ </properties>
+ </profile>
+ <profile>
+ <id>tests-on</id>
+ <activation>
+ <file>
+ <exists>src/test/resources/auth-keys.xml</exists>
+ </file>
+ </activation>
+ <properties>
+ <skipITs>false</skipITs>
+ </properties>
+ </profile>
+ <profile>
+ <id>parallel-tests</id>
+ <activation>
+ <property>
+ <name>parallel-tests</name>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-maven-plugins</artifactId>
+ <executions>
+ <execution>
+ <id>parallel-tests-createdir</id>
+ <goals>
+ <goal>parallel-tests-createdir</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <forkCount>${testsThreadCount}</forkCount>
+ <reuseForks>false</reuseForks>
+ <trimStackTrace>false</trimStackTrace>
+ <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
+ <systemPropertyVariables>
+ <testsThreadCount>${testsThreadCount}</testsThreadCount>
+ <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
+ <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
+ <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
+ <test.unique.fork.id>job-${job.id}-fork-000${surefire.forkNumber}</test.unique.fork.id>
+ </systemPropertyVariables>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>default-integration-test</id>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ <configuration>
+ <forkCount>${testsThreadCount}</forkCount>
+ <reuseForks>false</reuseForks>
+ <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
+ <forkedProcessTimeoutInSeconds>${fs.gs.scale.test.timeout}</forkedProcessTimeoutInSeconds>
+ <trimStackTrace>false</trimStackTrace>
+ <systemPropertyVariables>
+ <!-- Tell tests that they are being executed in parallel -->
+ <test.parallel.execution>true</test.parallel.execution>
+ <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
+ <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
+ <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
+
+ <!-- Due to a Maven quirk, setting this to just -->
+ <!-- surefire.forkNumber won't do the parameter -->
+ <!-- substitution. Putting a prefix in front of it like -->
+ <!-- "fork-" makes it work. -->
+ <test.unique.fork.id>job-${job.id}-fork-000${surefire.forkNumber}</test.unique.fork.id>
+ <test.default.timeout>${test.integration.timeout}</test.default.timeout>
+ </systemPropertyVariables>
+ <includes>
+ <include>**/ITest*.java</include>
+ </includes>
+ </configuration>
+ </execution>
+ <execution>
+ <id>sequential-integration-tests</id>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ <configuration>
+ <forkedProcessTimeoutInSeconds>${fs.gs.scale.test.timeout}</forkedProcessTimeoutInSeconds>
+ <trimStackTrace>false</trimStackTrace>
+ <systemPropertyVariables>
+ <!-- Tell tests that they are being executed sequentially -->
+ <test.parallel.execution>false</test.parallel.execution>
+ <test.unique.fork.id>job-${job.id}</test.unique.fork.id>
+ </systemPropertyVariables>
+ <!-- Do a sequential run for tests that cannot handle -->
+ <!-- parallel execution. -->
+ <includes>
+ <include>**/ITest*.java</include>
+ </includes>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
+ <id>sequential-tests</id>
+ <activation>
+ <property>
+ <name>!parallel-tests</name>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ <configuration>
+ <trimStackTrace>false</trimStackTrace>
+ <systemPropertyVariables>
+ <test.unique.fork.id>job-${job.id}</test.unique.fork.id>
+ </systemPropertyVariables>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer"/>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ </transformers>
+ <filters>
+
+ <filter>
+ <artifact>com.google.auth:*</artifact>
+ <includes>
+ <include>**</include>
+ </includes>
+ </filter>
+ <filter>
+ <artifact>io.grpc:*</artifact>
+ <includes>
+ <include>**</include>
+ </includes>
+ </filter>
+ <filter>
+ <artifact>io.opencensus:*</artifact>
+ <includes>
+ <include>**</include>
+ </includes>
+ </filter>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>*.json</exclude>
+ <exclude>google/**</exclude>
+ <exclude>grpc/**</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <artifactSet>
+ <includes>
+ <include>com.google.api</include>
+ <include>com.google.api-client</include>
+ <include>com.google.api.grpc</include>
+ <include>com.google.apis</include>
+ <include>com.google.auth</include>
+ <include>com.google.cloud</include>
+ <include>com.google.cloud.bigdataoss</include>
+ <include>com.google.cloud.grpc</include>
+ <include>com.google.cloud.http</include>
+ <include>com.google.flogger</include>
+ <include>com.google.code.gson</include>
+ <include>com.google.guava</include>
+ <include>com.google.http-client</include>
+ <include>com.google.oauth-client</include>
+ <include>com.google.protobuf</include>
+ <include>com.google.re2j</include>
+ <include>com.google.storage.v2</include>
+ <include>com.lmax</include>
+ <include>io.grpc</include>
+ <include>io.opencensus</include>
+ <include>io.perfmark</include>
+ <include>org.apache.httpcomponents</include>
+ <include>org.threeten:threetenbp</include>
+ </includes>
+ </artifactSet>
+ <minimizeJar>true</minimizeJar>
+ <relocations>
+ <relocation>
+ <pattern>com</pattern>
+ <shadedPattern>com.google.cloud.hadoop.repackaged.ossgcs.com</shadedPattern>
+ <includes>
+ <include>com.google.api.**</include>
+ <include>com.google.api.gax.**</include>
+ <include>com.google.auth.**</include>
+ <include>com.google.cloud.*</include>
+ <include>com.google.cloud.audit.**</include>
+ <include>com.google.cloud.grpc.**</include>
+ <include>com.google.cloud.hadoop.gcsio.**</include>
+ <include>com.google.cloud.hadoop.util.**</include>
+ <include>com.google.cloud.http.**</include>
+ <include>com.google.cloud.monitoring.**</include>
+ <include>com.google.cloud.spi.**</include>
+ <include>com.google.cloud.storage.**</include>
+ <include>com.google.common.**</include>
+ <include>com.google.geo.**</include>
+ <include>com.google.gson.**</include>
+ <include>com.google.google.storage.**</include>
+ <include>com.google.iam.**</include>
+ <include>com.google.logging.**</include>
+ <include>com.google.longrunning.**</include>
+ <include>com.google.monitoring.**</include>
+ <include>com.google.protobuf.**</include>
+ <include>com.google.re2j.**</include>
+ <include>com.google.rpc.**</include>
+ <include>com.google.storage.**</include>
+ <include>com.google.thirdparty.**</include>
+ <include>com.google.type.**</include>
+ <include>com.lmax.disruptor.**</include>
+ </includes>
+ <excludes>
+ <exclude>com.google.cloud.hadoop.util.AccessTokenProvider</exclude>
+ <exclude>com.google.cloud.hadoop.util.AccessTokenProvider$AccessToken</exclude>
+ <exclude>com.google.cloud.hadoop.util.AccessTokenProvider$AccessTokenType</exclude>
+ <exclude>com.google.cloud.hadoop.util.AccessBoundary</exclude>
+ <exclude>com.google.cloud.hadoop.util.AccessBoundary$Action</exclude>
+ <exclude>com.google.cloud.hadoop.util.AutoValue_AccessBoundary</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>org</pattern>
+ <shadedPattern>com.google.cloud.hadoop.repackaged.ossgcs.org</shadedPattern>
+ <includes>
+ <include>org.apache.http.**</include>
+ <include>org.threeten.**</include>
+ </includes>
+ </relocation>
+
+ <relocation>
+ <pattern>io.grpc.netty.shaded</pattern>
+ <shadedPattern>
+ com.google.cloud.hadoop.repackaged.ossgcs.io.grpc.netty.shaded
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>io</pattern>
+ <shadedPattern>com.google.cloud.hadoop.repackaged.ossgcs.io</shadedPattern>
+ <includes>
+ <include>io.grpc.**</include>
+ <include>io.opencensus.**</include>
+ <include>io.perfmark.**</include>
+ </includes>
+ </relocation>
+ <relocation>
+ <pattern>META-INF/native/io_grpc_netty_shaded_</pattern>
+ <shadedPattern>
+ META-INF/native/com_google_cloud_hadoop_repackaged_gcs_io_grpc_netty_shaded_
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>META-INF/native/libio_grpc_netty_shaded_</pattern>
+ <shadedPattern>
+ META-INF/native/libcom_google_cloud_hadoop_repackaged_gcs_io_grpc_netty_shaded_
+ </shadedPattern>
+ </relocation>
+ </relocations>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>com.github.spotbugs</groupId>
+ <artifactId>spotbugs-maven-plugin</artifactId>
+ <configuration>
+ <xmlOutput>true</xmlOutput>
+ <excludeFilterFile>${basedir}/dev-support/findbugs-exclude.xml
+ </excludeFilterFile>
+ <effort>Max</effort>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <configuration>
+ <suppressionsLocation>src/config/checkstyle-suppressions.xml</suppressionsLocation>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <forkedProcessTimeoutInSeconds>3600</forkedProcessTimeoutInSeconds>
+ <systemPropertyVariables>
+ <test.default.timeout>${test.integration.timeout}</test.default.timeout>
+ </systemPropertyVariables>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>banned-illegal-imports</id>
+ <phase>process-sources</phase>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <restrictImports>
+ <includeTestCode>false</includeTestCode>
+ <reason>Restrict mapreduce imports to committer code</reason>
+ <exclusions>
+
+ </exclusions>
+ <bannedImports>
+ <bannedImport>org.apache.hadoop.mapreduce.**</bannedImport>
+ <bannedImport>org.apache.hadoop.mapred.**</bannedImport>
+ </bannedImports>
+ </restrictImports>
+ <restrictImports>
+ <includeTestCode>false</includeTestCode>
+ <reason>Restrict encryption client imports to encryption client factory</reason>
+ <exclusions>
+
+ </exclusions>
+ <bannedImports>
+
+ </bannedImports>
+ </restrictImports>
+ </rules>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+<dependencyManagement>
+ <dependencies>
+ <dependency>
+ <!--
+ We're using a specific Protobuf version to ensure compatibility with the
+ Google Cloud Storage (GCS) client. The GCS client often relies on
+ particular Long-Term Support (LTS) versions of Protobuf. When we upgrade
+ the GCS client, we'll likely need to update Protobuf too. To prevent
+ dependency conflicts, Protobuf will be shaded within the GCS connector's
+ fat JAR.
+ -->
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>3.25.5</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.enterprise</groupId>
+ <artifactId>cdi-api</artifactId>
+ </exclusion>
+ <!-- Exclude protobuf-java 2.5.0 -->
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.assertj</groupId>
+ <artifactId>assertj-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.platform</groupId>
+ <artifactId>junit-platform-launcher</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.vintage</groupId>
+ <artifactId>junit-vintage-engine</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>google-cloud-storage</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-engine</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
+
diff --git a/hadoop-tools/hadoop-gcp/src/config/checkstyle-suppressions.xml b/hadoop-tools/hadoop-gcp/src/config/checkstyle-suppressions.xml
new file mode 100644
index 00000000000..8c765bc97f9
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/config/checkstyle-suppressions.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE suppressions PUBLIC
+ "-//Checkstyle//DTD SuppressionFilter Configuration 1.0//EN"
+ "https://checkstyle.org/dtds/suppressions_1_0.dtd">
+
+
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<suppressions>
+ <suppress checks="ParameterNumber"
+ files="org[\\/]apache[\\/]hadoop[\\/]fs[\\/]gs[\\/]GoogleCloudStorageItemInfo.java"/>
+ <suppress checks="ParameterNumber"
+ files="org[\\/]apache[\\/]hadoop[\\/]fs[\\/]gs[\\/]TestStorageResourceId.java"/>
+</suppressions>
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/Constants.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/Constants.java
new file mode 100644
index 00000000000..34434b2859a
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/Constants.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+final class Constants {
+ private Constants() {}
+
+ // URI scheme for GCS.
+ static final String SCHEME = "gs";
+ static final String PATH_DELIMITER = "/";
+
+ static final String GCS_CONFIG_PREFIX = "fs.gs";
+
+ static final String BASE_KEY_PREFIX = "google.cloud";
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/CreateOptions.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/CreateOptions.java
new file mode 100644
index 00000000000..c9b44a1a481
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/CreateOptions.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument;
+
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
+
+import java.util.Map;
+import javax.annotation.Nullable;
+
+/**
+ * Options that can be specified when creating a file in the {@link GoogleCloudStorageFileSystem}.
+ */
+final class CreateOptions {
+ private final ImmutableMap<String, byte[]> attributes;
+ private final String contentType;
+ private final long overwriteGenerationId;
+ private final WriteMode mode;
+
+ private CreateOptions(CreateOperationOptionsBuilder builder) {
+ this.attributes = ImmutableMap.copyOf(builder.attributes);
+ this.contentType = builder.contentType;
+ this.overwriteGenerationId = builder.overwriteGenerationId;
+ this.mode = builder.writeMode;
+ }
+
+ boolean isOverwriteExisting() {
+ return this.mode == WriteMode.OVERWRITE;
+ }
+
+ enum WriteMode {
+ /**
+ * Creates a new file for write and fails if file already exists.
+ */
+ CREATE_NEW,
+ /**
+ * Creates a new file for write or overwrites an existing file if it already exists.
+ */
+ OVERWRITE
+ }
+
+ static CreateOperationOptionsBuilder builder() {
+ return new CreateOperationOptionsBuilder();
+ }
+
+ /**
+ * Extended attributes to set when creating a file.
+ */
+ ImmutableMap<String, byte[]> getAttributes() {
+ return attributes;
+ }
+
+ /**
+ * Content-type to set when creating a file.
+ */
+ @Nullable
+ String getContentType() {
+ return contentType;
+ }
+
+ /**
+ * Whether to overwrite an existing file with the same name.
+ */
+ WriteMode getWriteMode() {
+ return mode;
+ }
+
+ /**
+ * Generation of existing object to overwrite. Ignored if set to {@link
+ * StorageResourceId#UNKNOWN_GENERATION_ID}, but otherwise this is used instead of {@code
+ * overwriteExisting}, where 0 indicates no existing object, and otherwise an existing object will
+ * only be overwritten by the newly created file if its generation matches this provided
+ * generationId.
+ */
+ long getOverwriteGenerationId() {
+ return overwriteGenerationId;
+ }
+
+ static class CreateOperationOptionsBuilder {
+ private Map<String, byte[]> attributes = ImmutableMap.of();
+ private String contentType = "application/octet-stream";
+ private long overwriteGenerationId = StorageResourceId.UNKNOWN_GENERATION_ID;
+ private WriteMode writeMode = WriteMode.CREATE_NEW;
+
+ CreateOperationOptionsBuilder setWriteMode(WriteMode mode) {
+ this.writeMode = mode;
+ return this;
+ }
+
+ CreateOptions build() {
+ CreateOptions options = new CreateOptions(this);
+
+ checkArgument(!options.getAttributes().containsKey("Content-Type"),
+ "The Content-Type attribute must be set via the contentType option");
+ if (options.getWriteMode() != WriteMode.OVERWRITE) {
+ checkArgument(options.getOverwriteGenerationId() == StorageResourceId.UNKNOWN_GENERATION_ID,
+ "overwriteGenerationId is set to %s but it can be set only in OVERWRITE mode",
+ options.getOverwriteGenerationId());
+ }
+
+ return options;
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/ErrorTypeExtractor.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/ErrorTypeExtractor.java
new file mode 100644
index 00000000000..a4497734524
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/ErrorTypeExtractor.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import io.grpc.Status;
+
+/**
+ * Implementation for {@link ErrorTypeExtractor} for exception specifically thrown from gRPC path.
+ */
+final class ErrorTypeExtractor {
+
+ enum ErrorType {
+ NOT_FOUND, OUT_OF_RANGE, ALREADY_EXISTS, FAILED_PRECONDITION, INTERNAL, RESOURCE_EXHAUSTED,
+ UNAVAILABLE, UNKNOWN
+ }
+
+ // public static final ErrorTypeExtractor INSTANCE = new ErrorTypeExtractor();
+
+ private static final String BUCKET_ALREADY_EXISTS_MESSAGE =
+ "FAILED_PRECONDITION: Your previous request to create the named bucket succeeded and you "
+ + "already own it.";
+
+ private ErrorTypeExtractor() {
+ }
+
+ static ErrorType getErrorType(Exception error) {
+ switch (Status.fromThrowable(error).getCode()) {
+ case NOT_FOUND:
+ return ErrorType.NOT_FOUND;
+ case OUT_OF_RANGE:
+ return ErrorType.OUT_OF_RANGE;
+ case ALREADY_EXISTS:
+ return ErrorType.ALREADY_EXISTS;
+ case FAILED_PRECONDITION:
+ return ErrorType.FAILED_PRECONDITION;
+ case RESOURCE_EXHAUSTED:
+ return ErrorType.RESOURCE_EXHAUSTED;
+ case INTERNAL:
+ return ErrorType.INTERNAL;
+ case UNAVAILABLE:
+ return ErrorType.UNAVAILABLE;
+ default:
+ return ErrorType.UNKNOWN;
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/FileInfo.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/FileInfo.java
new file mode 100644
index 00000000000..df8d63f5eec
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/FileInfo.java
@@ -0,0 +1,203 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import java.net.URI;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Contains information about a file or a directory.
+ *
+ * <p>Note: This class wraps GoogleCloudStorageItemInfo, adds file system specific information and
+ * hides bucket/object specific information.
+ */
+final class FileInfo {
+
+ // Info about the root path.
+ static final FileInfo ROOT_INFO =
+ new FileInfo(GoogleCloudStorageFileSystem.GCSROOT, GoogleCloudStorageItemInfo.ROOT_INFO);
+
+ // Path of this file or directory.
+ private final URI path;
+
+ // Information about the underlying GCS item.
+ private final GoogleCloudStorageItemInfo itemInfo;
+
+ /**
+ * Constructs an instance of FileInfo.
+ *
+ * @param itemInfo Information about the underlying item.
+ */
+ private FileInfo(URI path, GoogleCloudStorageItemInfo itemInfo) {
+ this.itemInfo = itemInfo;
+
+ // Construct the path once.
+ this.path = path;
+ }
+
+ /**
+ * Gets the path of this file or directory.
+ */
+ URI getPath() {
+ return path;
+ }
+
+ /**
+ * Indicates whether this item is a directory.
+ */
+ boolean isDirectory() {
+ return itemInfo.isDirectory();
+ }
+
+ /**
+ * Indicates whether this item is an inferred directory.
+ */
+ boolean isInferredDirectory() {
+ return itemInfo.isInferredDirectory();
+ }
+
+ /**
+ * Indicates whether this instance has information about the unique, shared root of the underlying
+ * storage system.
+ */
+ boolean isGlobalRoot() {
+ return itemInfo.isGlobalRoot();
+ }
+
+ /**
+ * Gets creation time of this item.
+ *
+ * <p>Time is expressed as milliseconds since January 1, 1970 UTC.
+ */
+ long getCreationTime() {
+ return itemInfo.getCreationTime();
+ }
+
+ /**
+ * Gets the size of this file or directory.
+ *
+ * <p>For files, size is in number of bytes. For directories size is 0. For items that do not
+ * exist, size is -1.
+ */
+ long getSize() {
+ return itemInfo.getSize();
+ }
+
+ /**
+ * Gets the modification time of this file if one is set, otherwise the value of {@link
+ * #getCreationTime()} is returned.
+ *
+ * <p>Time is expressed as milliseconds since January 1, 1970 UTC.
+ */
+ long getModificationTime() {
+ return itemInfo.getModificationTime();
+ }
+
+ /**
+ * Retrieve file attributes for this file.
+ *
+ * @return A map of file attributes
+ */
+ Map<String, byte[]> getAttributes() {
+ return itemInfo.getMetadata();
+ }
+
+ /**
+ * Indicates whether this file or directory exists.
+ */
+ boolean exists() {
+ return itemInfo.exists();
+ }
+
+ /**
+ * Returns CRC32C checksum of the file or {@code null}.
+ */
+ byte[] getCrc32cChecksum() {
+ VerificationAttributes verificationAttributes = itemInfo.getVerificationAttributes();
+ return verificationAttributes == null ? null : verificationAttributes.getCrc32c();
+ }
+
+ /**
+ * Returns MD5 checksum of the file or {@code null}.
+ */
+ byte[] getMd5Checksum() {
+ VerificationAttributes verificationAttributes = itemInfo.getVerificationAttributes();
+ return verificationAttributes == null ? null : verificationAttributes.getMd5hash();
+ }
+
+ /**
+ * Gets information about the underlying item.
+ */
+ GoogleCloudStorageItemInfo getItemInfo() {
+ return itemInfo;
+ }
+
+ /**
+ * Gets string representation of this instance.
+ */
+ @Override
+ public String toString() {
+ return getPath() + (exists() ?
+ ": created on: " + Instant.ofEpochMilli(getCreationTime()) :
+ ": exists: no");
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof FileInfo)) {
+ return false;
+ }
+ FileInfo fileInfo = (FileInfo) o;
+ return Objects.equals(path, fileInfo.path) && Objects.equals(itemInfo, fileInfo.itemInfo);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(path, itemInfo);
+ }
+
+ /**
+ * Handy factory method for constructing a FileInfo from a GoogleCloudStorageItemInfo while
+ * potentially returning a singleton instead of really constructing an object for cases like ROOT.
+ */
+ static FileInfo fromItemInfo(GoogleCloudStorageItemInfo itemInfo) {
+ if (itemInfo.isRoot()) {
+ return ROOT_INFO;
+ }
+ URI path = UriPaths.fromResourceId(itemInfo.getResourceId(), /* allowEmptyObjectName= */ true);
+ return new FileInfo(path, itemInfo);
+ }
+
+ /**
+ * Handy factory method for constructing a list of FileInfo from a list of
+ * GoogleCloudStorageItemInfo.
+ */
+ static List<FileInfo> fromItemInfos(List<GoogleCloudStorageItemInfo> itemInfos) {
+ List<FileInfo> fileInfos = new ArrayList<>(itemInfos.size());
+ for (GoogleCloudStorageItemInfo itemInfo : itemInfos) {
+ fileInfos.add(fromItemInfo(itemInfo));
+ }
+ return fileInfos;
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorage.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorage.java
new file mode 100644
index 00000000000..9c15962b7ef
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorage.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.*;
+import static org.apache.hadoop.thirdparty.com.google.common.base.Strings.isNullOrEmpty;
+
+import com.google.cloud.storage.*;
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
+import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
+import org.apache.hadoop.thirdparty.com.google.common.io.BaseEncoding;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.nio.channels.WritableByteChannel;
+import java.nio.file.FileAlreadyExistsException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A wrapper around <a href="https://github.com/googleapis/java-storage">Google cloud storage
+ * client</a>.
+ */
+class GoogleCloudStorage {
+ public static final Logger LOG = LoggerFactory.getLogger(GoogleHadoopFileSystem.class);
+ static final List<Storage.BlobField> BLOB_FIELDS =
+ ImmutableList.of(Storage.BlobField.BUCKET, Storage.BlobField.CONTENT_ENCODING,
+ Storage.BlobField.CONTENT_TYPE, Storage.BlobField.CRC32C, Storage.BlobField.GENERATION,
+ Storage.BlobField.METADATA, Storage.BlobField.MD5HASH, Storage.BlobField.METAGENERATION,
+ Storage.BlobField.NAME, Storage.BlobField.SIZE, Storage.BlobField.TIME_CREATED,
+ Storage.BlobField.UPDATED);
+ private final Storage storage;
+ private final GoogleHadoopFileSystemConfiguration configuration;
+
+ /**
+ * Having an instance of gscImpl to redirect calls to Json client while new client implementation
+ * is in WIP.
+ */
+ GoogleCloudStorage(GoogleHadoopFileSystemConfiguration configuration) throws IOException {
+ // TODO: Set projectId
+ // TODO: Set credentials
+ this.storage = StorageOptions.newBuilder().build().getService();
+ this.configuration = configuration;
+ }
+
+ public WritableByteChannel create(final StorageResourceId resourceId, final CreateOptions options)
+ throws IOException {
+ LOG.trace("create({})", resourceId);
+
+ checkArgument(resourceId.isStorageObject(), "Expected full StorageObject id, got %s",
+ resourceId);
+ // Update resourceId if generationId is missing
+ StorageResourceId resourceIdWithGeneration = resourceId;
+ if (!resourceId.hasGenerationId()) {
+ resourceIdWithGeneration =
+ new StorageResourceId(resourceId.getBucketName(), resourceId.getObjectName(),
+ getWriteGeneration(resourceId, options.isOverwriteExisting()));
+ }
+
+ return new GoogleCloudStorageClientWriteChannel(storage, resourceIdWithGeneration, options);
+ }
+
+ /**
+ * Gets the object generation for a write operation
+ *
+ * <p>making getItemInfo call even if overwrite is disabled to fail fast in case file is existing.
+ *
+ * @param resourceId object for which generation info is requested
+ * @param overwrite whether existing object should be overwritten
+ * @return the generation of the object
+ * @throws IOException if the object already exists and cannot be overwritten
+ */
+ private long getWriteGeneration(StorageResourceId resourceId, boolean overwrite)
+ throws IOException {
+ LOG.trace("getWriteGeneration({}, {})", resourceId, overwrite);
+ GoogleCloudStorageItemInfo info = getItemInfo(resourceId);
+ if (!info.exists()) {
+ return 0L;
+ }
+ if (info.exists() && overwrite) {
+ long generation = info.getContentGeneration();
+ checkState(generation != 0, "Generation should not be 0 for an existing item");
+ return generation;
+ }
+
+ throw new FileAlreadyExistsException(String.format("Object %s already exists.", resourceId));
+ }
+
+ public void close() {
+ try {
+ storage.close();
+ } catch (Exception e) {
+ LOG.warn("Error occurred while closing the storage client", e);
+ }
+ }
+
+ public GoogleCloudStorageItemInfo getItemInfo(StorageResourceId resourceId) throws IOException {
+ LOG.trace("getItemInfo({})", resourceId);
+
+ // Handle ROOT case first.
+ if (resourceId.isRoot()) {
+ return GoogleCloudStorageItemInfo.ROOT_INFO;
+ }
+ GoogleCloudStorageItemInfo itemInfo = null;
+
+ if (resourceId.isBucket()) {
+ Bucket bucket = getBucket(resourceId.getBucketName());
+ if (bucket != null) {
+ itemInfo = createItemInfoForBucket(resourceId, bucket);
+ } else {
+ LOG.debug("getBucket({}): not found", resourceId.getBucketName());
+ }
+ } else {
+ Blob blob = getBlob(resourceId);
+ if (blob != null) {
+ itemInfo = createItemInfoForBlob(resourceId, blob);
+ } else {
+ LOG.debug("getObject({}): not found", resourceId);
+ }
+ }
+
+ if (itemInfo == null) {
+ itemInfo = GoogleCloudStorageItemInfo.createNotFound(resourceId);
+ }
+ LOG.debug("getItemInfo: {}", itemInfo);
+ return itemInfo;
+ }
+
+ /**
+ * Gets the bucket with the given name.
+ *
+ * @param bucketName name of the bucket to get
+ * @return the bucket with the given name or null if bucket not found
+ * @throws IOException if the bucket exists but cannot be accessed
+ */
+ @Nullable
+ private Bucket getBucket(String bucketName) throws IOException {
+ LOG.debug("getBucket({})", bucketName);
+ checkArgument(!isNullOrEmpty(bucketName), "bucketName must not be null or empty");
+ try {
+ return storage.get(bucketName);
+ } catch (StorageException e) {
+ if (ErrorTypeExtractor.getErrorType(e) == ErrorTypeExtractor.ErrorType.NOT_FOUND) {
+ return null;
+ }
+ throw new IOException("Error accessing Bucket " + bucketName, e);
+ }
+ }
+
+ private static GoogleCloudStorageItemInfo createItemInfoForBlob(StorageResourceId resourceId,
+ Blob blob) {
+ checkArgument(resourceId != null, "resourceId must not be null");
+ checkArgument(blob != null, "object must not be null");
+ checkArgument(resourceId.isStorageObject(),
+ "resourceId must be a StorageObject. resourceId: %s", resourceId);
+ checkArgument(resourceId.getBucketName().equals(blob.getBucket()),
+ "resourceId.getBucketName() must equal object.getBucket(): '%s' vs '%s'",
+ resourceId.getBucketName(), blob.getBucket());
+ checkArgument(resourceId.getObjectName().equals(blob.getName()),
+ "resourceId.getObjectName() must equal object.getName(): '%s' vs '%s'",
+ resourceId.getObjectName(), blob.getName());
+
+ Map<String, byte[]> decodedMetadata =
+ blob.getMetadata() == null ? null : decodeMetadata(blob.getMetadata());
+
+ byte[] md5Hash = null;
+ byte[] crc32c = null;
+
+ if (!isNullOrEmpty(blob.getCrc32c())) {
+ crc32c = BaseEncoding.base64().decode(blob.getCrc32c());
+ }
+
+ if (!isNullOrEmpty(blob.getMd5())) {
+ md5Hash = BaseEncoding.base64().decode(blob.getMd5());
+ }
+
+ return GoogleCloudStorageItemInfo.createObject(resourceId,
+ blob.getCreateTimeOffsetDateTime() == null ?
+ 0 :
+ blob.getCreateTimeOffsetDateTime().toInstant().toEpochMilli(),
+ blob.getUpdateTimeOffsetDateTime() == null ?
+ 0 :
+ blob.getUpdateTimeOffsetDateTime().toInstant().toEpochMilli(),
+ blob.getSize() == null ? 0 : blob.getSize(), blob.getContentType(),
+ blob.getContentEncoding(), decodedMetadata,
+ blob.getGeneration() == null ? 0 : blob.getGeneration(),
+ blob.getMetageneration() == null ? 0 : blob.getMetageneration(),
+ new VerificationAttributes(md5Hash, crc32c));
+ }
+
+ static Map<String, byte[]> decodeMetadata(Map<String, String> metadata) {
+ return Maps.transformValues(metadata, GoogleCloudStorage::decodeMetadataValues);
+ }
+
+ @Nullable
+ private static byte[] decodeMetadataValues(String value) {
+ try {
+ return BaseEncoding.base64().decode(value);
+ } catch (IllegalArgumentException iae) {
+ LOG.error("Failed to parse base64 encoded attribute value {}", value, iae);
+ return null;
+ }
+ }
+
+ /**
+ * Gets the object with the given resourceId.
+ *
+ * @param resourceId identifies a StorageObject
+ * @return the object with the given name or null if object not found
+ * @throws IOException if the object exists but cannot be accessed
+ */
+ @Nullable
+ Blob getBlob(StorageResourceId resourceId) throws IOException {
+ checkArgument(resourceId.isStorageObject(), "Expected full StorageObject id, got %s",
+ resourceId);
+ String bucketName = resourceId.getBucketName();
+ String objectName = resourceId.getObjectName();
+ Blob blob;
+ try {
+ blob = storage.get(BlobId.of(bucketName, objectName),
+ Storage.BlobGetOption.fields(BLOB_FIELDS.toArray(new Storage.BlobField[0])));
+ } catch (StorageException e) {
+ throw new IOException("Error accessing " + resourceId, e);
+ }
+ return blob;
+ }
+
+ private static GoogleCloudStorageItemInfo createItemInfoForBucket(StorageResourceId resourceId,
+ Bucket bucket) {
+ checkArgument(resourceId != null, "resourceId must not be null");
+ checkArgument(bucket != null, "bucket must not be null");
+ checkArgument(resourceId.isBucket(), "resourceId must be a Bucket. resourceId: %s", resourceId);
+ checkArgument(resourceId.getBucketName().equals(bucket.getName()),
+ "resourceId.getBucketName() must equal bucket.getName(): '%s' vs '%s'",
+ resourceId.getBucketName(), bucket.getName());
+
+ return GoogleCloudStorageItemInfo.createBucket(resourceId,
+ bucket.asBucketInfo().getCreateTimeOffsetDateTime().toInstant().toEpochMilli(),
+ bucket.asBucketInfo().getUpdateTimeOffsetDateTime().toInstant().toEpochMilli(),
+ bucket.getLocation(),
+ bucket.getStorageClass() == null ? null : bucket.getStorageClass().name());
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorageClientWriteChannel.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorageClientWriteChannel.java
new file mode 100644
index 00000000000..7956b6f0a82
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorageClientWriteChannel.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.BlobInfo;
+import com.google.cloud.storage.BlobWriteSession;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.Storage.BlobWriteOption;
+import com.google.cloud.storage.StorageException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.WritableByteChannel;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Implements WritableByteChannel to provide write access to GCS via java-storage client.
+ */
+class GoogleCloudStorageClientWriteChannel implements WritableByteChannel {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(GoogleCloudStorageClientWriteChannel.class);
+
+ private final StorageResourceId resourceId;
+ private WritableByteChannel writableByteChannel;
+
+ GoogleCloudStorageClientWriteChannel(final Storage storage,
+ final StorageResourceId resourceId, final CreateOptions createOptions) throws IOException {
+ this.resourceId = resourceId;
+ BlobWriteSession blobWriteSession = getBlobWriteSession(storage, resourceId, createOptions);
+ try {
+ this.writableByteChannel = blobWriteSession.open();
+ } catch (StorageException e) {
+ throw new IOException(e);
+ }
+ }
+
+ private static BlobInfo getBlobInfo(final StorageResourceId resourceId,
+ final CreateOptions createOptions) {
+ BlobInfo blobInfo = BlobInfo.newBuilder(
+ BlobId.of(resourceId.getBucketName(), resourceId.getObjectName(),
+ resourceId.getGenerationId())).setContentType(createOptions.getContentType())
+ // .setMetadata(encodeMetadata(createOptions.getMetadata())) // TODO:
+ .build();
+ return blobInfo;
+ }
+
+ private static BlobWriteSession getBlobWriteSession(final Storage storage,
+ final StorageResourceId resourceId, final CreateOptions createOptions) {
+ return storage.blobWriteSession(getBlobInfo(resourceId, createOptions),
+ generateWriteOptions(createOptions));
+ }
+
+ private static BlobWriteOption[] generateWriteOptions(final CreateOptions createOptions) {
+ List<BlobWriteOption> blobWriteOptions = new ArrayList<>();
+
+ blobWriteOptions.add(BlobWriteOption.disableGzipContent());
+ blobWriteOptions.add(BlobWriteOption.generationMatch());
+
+ //TODO: Enable KMS and checksum
+ return blobWriteOptions.toArray(new BlobWriteOption[blobWriteOptions.size()]);
+ }
+
+ @Override
+ public boolean isOpen() {
+ return writableByteChannel != null && writableByteChannel.isOpen();
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ if (!isOpen()) {
+ return;
+ }
+
+ writableByteChannel.close();
+ } catch (Exception e) {
+ throw new IOException(
+ String.format("Upload failed for '%s'. reason=%s", resourceId, e.getMessage()), e);
+ } finally {
+ writableByteChannel = null;
+ }
+ }
+
+ private int writeInternal(final ByteBuffer byteBuffer) throws IOException {
+ int bytesWritten = writableByteChannel.write(byteBuffer);
+ LOG.trace("{} bytes were written out of provided buffer of capacity {}", bytesWritten,
+ byteBuffer.limit());
+ return bytesWritten;
+ }
+
+ @Override
+ public int write(final ByteBuffer src) throws IOException {
+ return writeInternal(src);
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorageFileSystem.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorageFileSystem.java
new file mode 100644
index 00000000000..e411f22eb39
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorageFileSystem.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.*;
+import static org.apache.hadoop.fs.gs.Constants.SCHEME;
+
+import com.google.auth.Credentials;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.channels.WritableByteChannel;
+
+/**
+ * Provides FS semantics over GCS based on Objects API.
+ */
+class GoogleCloudStorageFileSystem {
+ private static final Logger LOG = LoggerFactory.getLogger(StorageResourceId.class);
+
+ // URI of the root path.
+ static final URI GCSROOT = URI.create(SCHEME + ":/");
+
+ // GCS access instance.
+ private GoogleCloudStorage gcs;
+
+ private static GoogleCloudStorage createCloudStorage(
+ final GoogleHadoopFileSystemConfiguration configuration, final Credentials credentials)
+ throws IOException {
+ checkNotNull(configuration, "configuration must not be null");
+
+ return new GoogleCloudStorage(configuration);
+ }
+
+ GoogleCloudStorageFileSystem(final GoogleHadoopFileSystemConfiguration configuration,
+ final Credentials credentials) throws IOException {
+ gcs = createCloudStorage(configuration, credentials);
+ }
+
+ WritableByteChannel create(final URI path, final CreateOptions createOptions)
+ throws IOException {
+ LOG.trace("create(path: {}, createOptions: {})", path, createOptions);
+ checkNotNull(path, "path could not be null");
+ StorageResourceId resourceId =
+ StorageResourceId.fromUriPath(path, /* allowEmptyObjectName=*/ true);
+
+ if (resourceId.isDirectory()) {
+ throw new IOException(
+ String.format("Cannot create a file whose name looks like a directory: '%s'",
+ resourceId));
+ }
+
+ if (createOptions.getOverwriteGenerationId() != StorageResourceId.UNKNOWN_GENERATION_ID) {
+ resourceId = new StorageResourceId(resourceId.getBucketName(), resourceId.getObjectName(),
+ createOptions.getOverwriteGenerationId());
+ }
+
+ return gcs.create(resourceId, createOptions);
+ }
+
+ void close() {
+ if (gcs == null) {
+ return;
+ }
+ LOG.trace("close()");
+ try {
+ gcs.close();
+ } finally {
+ gcs = null;
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorageItemInfo.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorageItemInfo.java
new file mode 100644
index 00000000000..887e68b05f9
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorageItemInfo.java
@@ -0,0 +1,423 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument;
+import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
+
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Contains information about an item in Google Cloud Storage.
+ */
+final class GoogleCloudStorageItemInfo {
+ // Info about the root of GCS namespace.
+ public static final GoogleCloudStorageItemInfo ROOT_INFO =
+ new GoogleCloudStorageItemInfo(StorageResourceId.ROOT,
+ /* creationTime= */ 0,
+ /* modificationTime= */ 0,
+ /* size= */ 0,
+ /* location= */ null,
+ /* storageClass= */ null,
+ /* contentType= */ null,
+ /* contentEncoding= */ null,
+ /* metadata= */ null,
+ /* contentGeneration= */ 0,
+ /* metaGeneration= */ 0,
+ /* verificationAttributes= */ null);
+
+ /**
+ * Factory method for creating a GoogleCloudStorageItemInfo for a bucket.
+ *
+ * @param resourceId Resource ID that identifies a bucket
+ * @param creationTime Time when a bucket was created (milliseconds since January 1, 1970
+ * UTC).
+ * @param modificationTime Time when a bucket was last modified (milliseconds since January 1,
+ * 1970 UTC).
+ * @param location Location of a bucket.
+ * @param storageClass Storage class of a bucket.
+ */
+ static GoogleCloudStorageItemInfo createBucket(StorageResourceId resourceId,
+ long creationTime, long modificationTime, String location, String storageClass) {
+ checkNotNull(resourceId, "resourceId must not be null");
+ checkArgument(resourceId.isBucket(), "expected bucket but got '%s'", resourceId);
+ return new GoogleCloudStorageItemInfo(resourceId, creationTime, modificationTime,
+ /* size= */ 0, location, storageClass,
+ /* contentType= */ null,
+ /* contentEncoding= */ null,
+ /* metadata= */ null,
+ /* contentGeneration= */ 0,
+ /* metaGeneration= */ 0,
+ /* verificationAttributes= */ null);
+ }
+
+ /**
+ * Factory method for creating a GoogleCloudStorageItemInfo for an object.
+ *
+ * @param resourceId identifies either root, a Bucket, or a StorageObject
+ * @param creationTime Time when object was created (milliseconds since January 1, 1970
+ * UTC).
+ * @param size Size of the given object (number of bytes) or -1 if the object
+ * does not exist.
+ * @param metadata User-supplied object metadata for this object.
+ */
+ static GoogleCloudStorageItemInfo createObject(StorageResourceId resourceId,
+ long creationTime, long modificationTime, long size, String contentType,
+ String contentEncoding, Map<String, byte[]> metadata, long contentGeneration,
+ long metaGeneration, VerificationAttributes verificationAttributes) {
+ checkNotNull(resourceId, "resourceId must not be null");
+ checkArgument(
+ !resourceId.isRoot(),
+ "expected object or directory but got '%s'", resourceId);
+ checkArgument(
+ !resourceId.isBucket(),
+ "expected object or directory but got '%s'", resourceId);
+ return new GoogleCloudStorageItemInfo(resourceId, creationTime, modificationTime, size,
+ /* location= */ null,
+ /* storageClass= */ null, contentType, contentEncoding, metadata, contentGeneration,
+ metaGeneration, verificationAttributes);
+ }
+
+ /**
+ * Factory method for creating a "found" GoogleCloudStorageItemInfo for an inferred directory.
+ *
+ * @param resourceId Resource ID that identifies an inferred directory
+ */
+ static GoogleCloudStorageItemInfo createInferredDirectory(StorageResourceId resourceId) {
+ return new GoogleCloudStorageItemInfo(resourceId,
+ /* creationTime= */ 0,
+ /* modificationTime= */ 0,
+ /* size= */ 0,
+ /* location= */ null,
+ /* storageClass= */ null,
+ /* contentType= */ null,
+ /* contentEncoding= */ null,
+ /* metadata= */ null,
+ /* contentGeneration= */ 0,
+ /* metaGeneration= */ 0,
+ /* verificationAttributes= */ null);
+ }
+
+ /**
+ * Factory method for creating a "not found" GoogleCloudStorageItemInfo for a bucket or an object.
+ *
+ * @param resourceId Resource ID that identifies an inferred directory
+ */
+ static GoogleCloudStorageItemInfo createNotFound(StorageResourceId resourceId) {
+ return new GoogleCloudStorageItemInfo(resourceId,
+ /* creationTime= */ 0,
+ /* modificationTime= */ 0,
+ /* size= */ -1,
+ /* location= */ null,
+ /* storageClass= */ null,
+ /* contentType= */ null,
+ /* contentEncoding= */ null,
+ /* metadata= */ null,
+ /* contentGeneration= */ 0,
+ /* metaGeneration= */ 0,
+ /* verificationAttributes= */ null);
+ }
+
+ // The Bucket and maybe StorageObject names of the GCS "item" referenced by this object. Not
+ // null.
+ private final StorageResourceId resourceId;
+
+ // Creation time of this item.
+ // Time is expressed as milliseconds since January 1, 1970 UTC.
+ private final long creationTime;
+
+ // Modification time of this item.
+ // Time is expressed as milliseconds since January 1, 1970 UTC.
+ private final long modificationTime;
+
+ // Size of an object (number of bytes).
+ // Size is -1 for items that do not exist.
+ private final long size;
+
+ // Location of this item.
+ private final String location;
+
+ // Storage class of this item.
+ private final String storageClass;
+
+ // Content-Type of this item
+ private final String contentType;
+
+ private final String contentEncoding;
+
+ // User-supplied metadata.
+ private final Map<String, byte[]> metadata;
+
+ private final long contentGeneration;
+
+ private final long metaGeneration;
+
+ private final VerificationAttributes verificationAttributes;
+
+ private GoogleCloudStorageItemInfo(StorageResourceId resourceId, long creationTime,
+ long modificationTime, long size, String location, String storageClass, String contentType,
+ String contentEncoding, Map<String, byte[]> metadata, long contentGeneration,
+ long metaGeneration, VerificationAttributes verificationAttributes) {
+ this.resourceId = checkNotNull(resourceId, "resourceId must not be null");
+ this.creationTime = creationTime;
+ this.modificationTime = modificationTime;
+ this.size = size;
+ this.location = location;
+ this.storageClass = storageClass;
+ this.contentType = contentType;
+ this.contentEncoding = contentEncoding;
+ this.metadata = (metadata == null) ? ImmutableMap.of() : metadata;
+ this.contentGeneration = contentGeneration;
+ this.metaGeneration = metaGeneration;
+ this.verificationAttributes = verificationAttributes;
+ }
+
+ /**
+ * Gets bucket name of this item.
+ */
+ String getBucketName() {
+ return resourceId.getBucketName();
+ }
+
+ /**
+ * Gets object name of this item.
+ */
+ String getObjectName() {
+ return resourceId.getObjectName();
+ }
+
+ /**
+ * Gets the resourceId that holds the (possibly null) bucketName and objectName of this object.
+ */
+ StorageResourceId getResourceId() {
+ return resourceId;
+ }
+
+ /**
+ * Gets creation time of this item.
+ *
+ * <p>Time is expressed as milliseconds since January 1, 1970 UTC.
+ */
+ long getCreationTime() {
+ return creationTime;
+ }
+
+ /**
+ * Gets modification time of this item.
+ *
+ * <p>Time is expressed as milliseconds since January 1, 1970 UTC.
+ */
+ long getModificationTime() {
+ return modificationTime;
+ }
+
+ /**
+ * Gets size of this item (number of bytes). Returns -1 if the object does not exist.
+ */
+ long getSize() {
+ return size;
+ }
+
+ /**
+ * Gets location of this item.
+ *
+ * <p>Note: Location is only supported for buckets. The value is always null for objects.
+ */
+ String getLocation() {
+ return location;
+ }
+
+ /**
+ * Gets storage class of this item.
+ *
+ * <p>Note: Storage-class is only supported for buckets. The value is always null for objects.
+ */
+ String getStorageClass() {
+ return storageClass;
+ }
+
+ /**
+ * Gets the content-type of this item, or null if unknown or inapplicable.
+ *
+ * <p>Note: content-type is only supported for objects, and will always be null for buckets.
+ */
+ String getContentType() {
+ return contentType;
+ }
+
+ /**
+ * Gets the content-encoding of this item, or null if unknown or inapplicable.
+ *
+ * <p>Note: content-encoding is only supported for objects, and will always be null for buckets.
+ */
+ String getContentEncoding() {
+ return contentEncoding;
+ }
+
+ /**
+ * Gets user-supplied metadata for this item.
+ *
+ * <p>Note: metadata is only supported for objects. This value is always an empty map for buckets.
+ */
+ Map<String, byte[]> getMetadata() {
+ return metadata;
+ }
+
+ /**
+ * Indicates whether this item is a bucket. Root is not considered to be a bucket.
+ */
+ boolean isBucket() {
+ return resourceId.isBucket();
+ }
+
+ /**
+ * Indicates whether this item refers to the GCS root (gs://).
+ */
+ boolean isRoot() {
+ return resourceId.isRoot();
+ }
+
+ /**
+ * Indicates whether this instance has information about the unique, shared root of the underlying
+ * storage system.
+ */
+ boolean isGlobalRoot() {
+ return isRoot() && exists();
+ }
+
+ /**
+ * Indicates whether {@code itemInfo} is a directory.
+ */
+ boolean isDirectory() {
+ return isGlobalRoot() || isBucket() || resourceId.isDirectory();
+ }
+
+ /**
+ * Indicates whether {@code itemInfo} is an inferred directory.
+ */
+ boolean isInferredDirectory() {
+ return creationTime == 0 && modificationTime == 0 && size == 0 && contentGeneration == 0
+ && metaGeneration == 0;
+ }
+
+ /**
+ * Get the content generation of the object.
+ */
+ long getContentGeneration() {
+ return contentGeneration;
+ }
+
+ /**
+ * Get the meta generation of the object.
+ */
+ long getMetaGeneration() {
+ return metaGeneration;
+ }
+
+ /**
+ * Get object validation attributes.
+ */
+ VerificationAttributes getVerificationAttributes() {
+ return verificationAttributes;
+ }
+
+ /**
+ * Indicates whether this item exists.
+ */
+ boolean exists() {
+ return size >= 0;
+ }
+
+ /**
+ * Helper for checking logical equality of metadata maps, checking equality of keySet() between
+ * this.metadata and otherMetadata, and then using Arrays.equals to compare contents of
+ * corresponding byte arrays.
+ */
+ @VisibleForTesting
+ public boolean metadataEquals(Map<String, byte[]> otherMetadata) {
+ if (metadata == otherMetadata) {
+ // Fast-path for common cases where the same actual default metadata instance may be
+ // used in
+ // multiple different item infos.
+ return true;
+ }
+ // No need to check if other `metadata` is not null,
+ // because previous `if` checks if both of them are null.
+ if (metadata == null || otherMetadata == null) {
+ return false;
+ }
+ if (!metadata.keySet().equals(otherMetadata.keySet())) {
+ return false;
+ }
+
+ // Compare each byte[] with Arrays.equals.
+ for (Map.Entry<String, byte[]> metadataEntry : metadata.entrySet()) {
+ if (!Arrays.equals(metadataEntry.getValue(), otherMetadata.get(metadataEntry.getKey()))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Gets string representation of this instance.
+ */
+ @Override
+ public String toString() {
+ return exists() ?
+ String.format("%s: created on: %s", resourceId, Instant.ofEpochMilli(creationTime)) :
+ String.format("%s: exists: no", resourceId);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof GoogleCloudStorageItemInfo) {
+ GoogleCloudStorageItemInfo other = (GoogleCloudStorageItemInfo) obj;
+ return resourceId.equals(other.resourceId) && creationTime == other.creationTime
+ && modificationTime == other.modificationTime && size == other.size && Objects.equals(
+ location, other.location) && Objects.equals(storageClass, other.storageClass)
+ && Objects.equals(verificationAttributes, other.verificationAttributes)
+ && metaGeneration == other.metaGeneration && contentGeneration == other.contentGeneration
+ && metadataEquals(other.getMetadata());
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + resourceId.hashCode();
+ result = prime * result + (int) creationTime;
+ result = prime * result + (int) modificationTime;
+ result = prime * result + (int) size;
+ result = prime * result + Objects.hashCode(location);
+ result = prime * result + Objects.hashCode(storageClass);
+ result = prime * result + Objects.hashCode(verificationAttributes);
+ result = prime * result + (int) metaGeneration;
+ result = prime * result + (int) contentGeneration;
+ result = prime * result + metadata.entrySet().stream()
+ .mapToInt(e -> Objects.hash(e.getKey()) + Arrays.hashCode(e.getValue())).sum();
+ return result;
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleHadoopFileSystem.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleHadoopFileSystem.java
new file mode 100644
index 00000000000..1c2fc19d2b5
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleHadoopFileSystem.java
@@ -0,0 +1,505 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import static org.apache.hadoop.fs.gs.Constants.GCS_CONFIG_PREFIX;
+import static org.apache.hadoop.fs.gs.GoogleHadoopFileSystemConfiguration.GCS_WORKING_DIRECTORY;
+
+import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument;
+import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkState;
+import static org.apache.hadoop.thirdparty.com.google.common.base.Strings.isNullOrEmpty;
+
+import com.google.auth.oauth2.GoogleCredentials;
+import org.apache.hadoop.thirdparty.com.google.common.base.Ascii;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.util.EnumSet;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.ProviderUtils;
+import org.apache.hadoop.util.Progressable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * GoogleHadoopFileSystem is rooted in a single bucket at initialization time; in this case, Hadoop
+ * paths no longer correspond directly to general GCS paths, and all Hadoop operations going through
+ * this FileSystem will never touch any GCS bucket other than the bucket on which this FileSystem is
+ * rooted.
+ *
+ * <p>This implementation sacrifices a small amount of cross-bucket interoperability in favor of
+ * more straightforward FileSystem semantics and compatibility with existing Hadoop applications. In
+ * particular, it is not subject to bucket-naming constraints, and files are allowed to be placed in
+ * root.
+ */
+public class GoogleHadoopFileSystem extends FileSystem {
+
+ public static final Logger LOG = LoggerFactory.getLogger(GoogleHadoopFileSystem.class);
+
+ /**
+ * URI scheme for GoogleHadoopFileSystem.
+ */
+ private static final String SCHEME = Constants.SCHEME;
+
+ /**
+ * Default value of replication factor.
+ */
+ static final short REPLICATION_FACTOR_DEFAULT = 3;
+
+ // TODO: Take this from config
+ private static final int PERMISSIONS_TO_REPORT = 700;
+
+ /**
+ * The URI the File System is passed in initialize.
+ */
+ private URI initUri;
+
+ /**
+ * Default block size. Note that this is the size that is reported to Hadoop FS clients. It does
+ * not modify the actual block size of an underlying GCS object, because GCS JSON API does not
+ * allow modifying or querying the value. Modifying this value allows one to control how many
+ * mappers are used to process a given file.
+ */
+ private long defaultBlockSize = GoogleHadoopFileSystemConfiguration.BLOCK_SIZE.getDefault();
+
+ // The bucket the file system is rooted in used for default values of:
+ // -- working directory
+ // -- user home directories (only for Hadoop purposes).
+ private Path fsRoot;
+
+ /**
+ * Current working directory; overridden in initialize() if {@link
+ * GoogleHadoopFileSystemConfiguration#GCS_WORKING_DIRECTORY} is set.
+ */
+ private Path workingDirectory;
+ private GoogleCloudStorageFileSystem gcsFs;
+ private boolean isClosed;
+ private FsPermission reportedPermissions;
+
+ public GoogleHadoopFileSystemConfiguration getFileSystemConfiguration() {
+ return fileSystemConfiguration;
+ }
+
+ private GoogleHadoopFileSystemConfiguration fileSystemConfiguration;
+
+ @Override
+ public void initialize(final URI path, Configuration config) throws IOException {
+ LOG.trace("initialize(path: {}, config: {})", path, config);
+
+ checkArgument(path != null, "path must not be null");
+ checkArgument(config != null, "config must not be null");
+ checkArgument(path.getScheme() != null, "scheme of path must not be null");
+ checkArgument(path.getScheme().equals(getScheme()), "URI scheme not supported: {}", path);
+
+ config =
+ ProviderUtils.excludeIncompatibleCredentialProviders(config, GoogleHadoopFileSystem.class);
+ super.initialize(path, config);
+
+ initUri = path;
+
+ // Set this configuration as the default config for this instance; configure()
+ // will perform some file-system-specific adjustments, but the original should
+ // be sufficient (and is required) for the delegation token binding initialization.
+ setConf(config);
+
+ this.reportedPermissions = new FsPermission(PERMISSIONS_TO_REPORT);
+
+ initializeFsRoot();
+
+ this.fileSystemConfiguration = new GoogleHadoopFileSystemConfiguration(config);
+ initializeWorkingDirectory(fileSystemConfiguration);
+ initializeGcsFs(fileSystemConfiguration);
+ }
+
+ private void initializeFsRoot() {
+ String rootBucket = initUri.getAuthority();
+ checkArgument(rootBucket != null, "No bucket specified in GCS URI: {}", initUri);
+ // Validate root bucket name
+ URI rootUri = UriPaths.fromStringPathComponents(rootBucket, /* objectName= */
+ null, /* allowEmptyObjectName= */ true);
+ fsRoot = new Path(rootUri);
+ LOG.trace("Configured FS root: '{}'", fsRoot);
+ }
+
+ private void initializeWorkingDirectory(final GoogleHadoopFileSystemConfiguration config) {
+ String configWorkingDirectory = config.getWorkingDirectory();
+ if (isNullOrEmpty(configWorkingDirectory)) {
+ LOG.warn("No working directory configured, using default: '{}'", workingDirectory);
+ }
+ // Use the public method to ensure proper behavior of normalizing and resolving the new
+ // working directory relative to the initial filesystem-root directory.
+ setWorkingDirectory(
+ isNullOrEmpty(configWorkingDirectory) ? fsRoot : new Path(configWorkingDirectory));
+ LOG.trace("Configured working directory: {} = {}", GCS_WORKING_DIRECTORY.getKey(),
+ getWorkingDirectory());
+ }
+
+ private synchronized void initializeGcsFs(final GoogleHadoopFileSystemConfiguration config)
+ throws IOException {
+ this.gcsFs = createGcsFs(config);
+ }
+
+ private GoogleCloudStorageFileSystem createGcsFs(final GoogleHadoopFileSystemConfiguration config)
+ throws IOException {
+ GoogleCredentials credentials = getCredentials(config);
+ return new GoogleCloudStorageFileSystem(config, credentials);
+ }
+
+ private GoogleCredentials getCredentials(GoogleHadoopFileSystemConfiguration config)
+ throws IOException {
+ return getCredentials(config, GCS_CONFIG_PREFIX);
+ }
+
+ public static GoogleCredentials getCredentials(GoogleHadoopFileSystemConfiguration config,
+ String... keyPrefixesVararg) throws IOException {
+ return GoogleCredentials.getApplicationDefault(); // TODO: Add other Auth mechanisms
+ }
+
+ @Override
+ protected void checkPath(final Path path) {
+ LOG.trace("checkPath(path: {})", path);
+ // Validate scheme
+ URI uri = path.toUri();
+
+ String scheme = uri.getScheme();
+ if (scheme != null && !scheme.equalsIgnoreCase(getScheme())) {
+ throw new IllegalArgumentException(
+ String.format("Wrong scheme: {}, in path: {}, expected scheme: {}", scheme, path,
+ getScheme()));
+ }
+
+ String bucket = uri.getAuthority();
+ String rootBucket = fsRoot.toUri().getAuthority();
+
+ // Bucket-less URIs will be qualified later
+ if (bucket == null || bucket.equals(rootBucket)) {
+ return;
+ }
+
+ throw new IllegalArgumentException(
+ String.format("Wrong bucket: {}, in path: {}, expected bucket: {}", bucket, path,
+ rootBucket));
+ }
+
+ /**
+ * Validates that GCS path belongs to this file system. The bucket must match the root bucket
+ * provided at initialization time.
+ */
+ Path getHadoopPath(final URI gcsPath) {
+ LOG.trace("getHadoopPath(gcsPath: {})", gcsPath);
+
+ // Handle root. Delegate to getGcsPath on "gs:/" to resolve the appropriate gs://<bucket> URI.
+ if (gcsPath.equals(getGcsPath(fsRoot))) {
+ return fsRoot;
+ }
+
+ StorageResourceId resourceId = StorageResourceId.fromUriPath(gcsPath, true);
+
+ checkArgument(!resourceId.isRoot(), "Missing authority in gcsPath '{}'", gcsPath);
+ String rootBucket = fsRoot.toUri().getAuthority();
+ checkArgument(resourceId.getBucketName().equals(rootBucket),
+ "Authority of URI '{}' doesn't match root bucket '{}'", resourceId.getBucketName(),
+ rootBucket);
+
+ Path hadoopPath = new Path(fsRoot,
+ new Path(/* schema= */ null, /* authority= */ null, resourceId.getObjectName()));
+ LOG.trace("getHadoopPath(gcsPath: {}): {}", gcsPath, hadoopPath);
+ return hadoopPath;
+ }
+
+ /**
+ * Translates a "gs:/" style hadoopPath (or relative path which is not fully-qualified) into the
+ * appropriate GCS path which is compatible with the underlying GcsFs.
+ */
+ URI getGcsPath(final Path hadoopPath) {
+ LOG.trace("getGcsPath(hadoopPath: {})", hadoopPath);
+
+ // Convert to fully qualified absolute path; the Path object will call back to get our current
+ // workingDirectory as part of fully resolving the path.
+ Path resolvedPath = makeQualified(hadoopPath);
+
+ String objectName = resolvedPath.toUri().getPath();
+ if (objectName != null && resolvedPath.isAbsolute()) {
+ // Strip off leading '/' because GoogleCloudStorageFileSystem.getPath appends it explicitly
+ // between bucket and objectName.
+ objectName = objectName.substring(1);
+ }
+
+ // Construct GCS path URI
+ String rootBucket = fsRoot.toUri().getAuthority();
+ URI gcsPath =
+ UriPaths.fromStringPathComponents(rootBucket, objectName, /* allowEmptyObjectName= */ true);
+ LOG.trace("getGcsPath(hadoopPath: {}): {}", hadoopPath, gcsPath);
+ return gcsPath;
+ }
+
+ @Override
+ public String getScheme() {
+ return SCHEME;
+ }
+
+ @Override
+ public FSDataInputStream open(final Path path, final int bufferSize) throws IOException {
+ LOG.trace("open({})", path);
+ throw new UnsupportedOperationException(path.toString());
+ }
+
+ @Override
+ public FSDataOutputStream create(Path hadoopPath, FsPermission permission, boolean overwrite,
+ int bufferSize, short replication, long blockSize, Progressable progress) throws IOException {
+ checkArgument(hadoopPath != null, "hadoopPath must not be null");
+ checkArgument(replication > 0, "replication must be a positive integer: %s", replication);
+ checkArgument(blockSize > 0, "blockSize must be a positive integer: %s", blockSize);
+
+ System.out.println(String.format("create(%s)", hadoopPath));
+ checkOpen();
+
+ LOG.trace("create(hadoopPath: {}, overwrite: {}, bufferSize: {} [ignored])", hadoopPath,
+ overwrite, bufferSize);
+
+ CreateOptions.WriteMode writeMode =
+ overwrite ? CreateOptions.WriteMode.OVERWRITE : CreateOptions.WriteMode.CREATE_NEW;
+ FSDataOutputStream response = new FSDataOutputStream(
+ new GoogleHadoopOutputStream(this, getGcsPath(hadoopPath),
+ CreateOptions.builder().setWriteMode(writeMode).build(), statistics), statistics);
+
+ return response;
+ }
+
+ @Override
+ public FSDataOutputStream createNonRecursive(Path hadoopPath, FsPermission permission,
+ EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize,
+ Progressable progress) throws IOException {
+ throw new UnsupportedOperationException(hadoopPath.toString());
+ }
+
+ @Override
+ public FSDataOutputStream append(final Path path, final int i, final Progressable progressable)
+ throws IOException {
+ throw new UnsupportedOperationException(path.toString());
+ }
+
+ @Override
+ public boolean rename(final Path path, final Path path1) throws IOException {
+ LOG.trace("rename({}, {})", path, path1);
+ throw new UnsupportedOperationException(path.toString());
+ }
+
+ @Override
+ public boolean delete(final Path path, final boolean recursive) throws IOException {
+ LOG.trace("delete({}, {})", path, recursive);
+ throw new UnsupportedOperationException(path.toString());
+ }
+
+ @Override
+ public FileStatus[] listStatus(final Path path) throws FileNotFoundException, IOException {
+ checkArgument(path != null, "hadoopPath must not be null");
+
+ checkOpen();
+
+ LOG.trace("listStatus(hadoopPath: {})", path);
+ throw new UnsupportedOperationException(path.toString());
+ }
+
+ /**
+ * Overridden to make root its own parent. This is POSIX compliant, but more importantly guards
+ * against poor directory accounting in the PathData class of Hadoop 2's FsShell.
+ */
+ @Override
+ public Path makeQualified(final Path path) {
+ Path qualifiedPath = super.makeQualified(path);
+
+ URI uri = qualifiedPath.toUri();
+
+ checkState("".equals(uri.getPath()) || qualifiedPath.isAbsolute(),
+ "Path '{}' must be fully qualified.", qualifiedPath);
+
+ Path result;
+ String upath = uri.getPath();
+
+ // Strip initial '..'s to make root is its own parent.
+ int i = 0;
+ while (upath.startsWith("/../", i)) {
+ // Leave a preceding slash, so path is still absolute.
+ i += 3;
+ }
+ if (i == upath.length() || upath.substring(i).equals("/..")) {
+ // Allow a Path of gs://someBucket to map to gs://someBucket/
+ result = new Path(uri.getScheme(), uri.getAuthority(), "/");
+ } else if (i == 0) {
+ result = qualifiedPath;
+ } else {
+ result = new Path(uri.getScheme(), uri.getAuthority(), upath.substring(i));
+ }
+
+ LOG.trace("makeQualified(path: {}): {}", path, result);
+ return result;
+ }
+
+ /**
+ * Returns a URI of the root of this FileSystem.
+ */
+ @Override
+ public URI getUri() {
+ return fsRoot.toUri();
+ }
+
+ /**
+ * The default port is listed as -1 as an indication that ports are not used.
+ */
+ @Override
+ protected int getDefaultPort() {
+ int result = -1;
+ LOG.trace("getDefaultPort(): %d", result);
+ return result;
+ }
+
+ @Override
+ public boolean hasPathCapability(final Path path, final String capability) {
+ checkNotNull(path, "path must not be null");
+ checkArgument(!isNullOrEmpty(capability), "capability must not be null or empty string for {}",
+ path);
+ switch (Ascii.toLowerCase(capability)) {
+ case CommonPathCapabilities.FS_APPEND:
+ case CommonPathCapabilities.FS_CONCAT:
+ return false;
+ default:
+ return false;
+ }
+ }
+
+ /**
+ * Gets the current working directory.
+ *
+ * @return The current working directory.
+ */
+ @Override
+ public Path getWorkingDirectory() {
+ LOG.trace("getWorkingDirectory(): {}", workingDirectory);
+ return workingDirectory;
+ }
+
+ @Override
+ public boolean mkdirs(final Path path, final FsPermission fsPermission) throws IOException {
+ LOG.trace("mkdirs({})", path);
+ throw new UnsupportedOperationException(path.toString());
+ }
+
+// /**
+// * Gets the default replication factor.
+// */
+// @Override
+// public short getDefaultReplication() {
+// return REPLICATION_FACTOR_DEFAULT;
+// }
+
+ @Override
+ public FileStatus getFileStatus(final Path path) throws IOException {
+ checkArgument(path != null, "path must not be null");
+
+ checkOpen();
+
+ URI gcsPath = getGcsPath(path);
+
+ LOG.trace("getFileStatus(): {}", gcsPath);
+
+ throw new UnsupportedOperationException(path.toString());
+ }
+
+ /**
+ * Returns home directory of the current user.
+ *
+ * <p>Note: This directory is only used for Hadoop purposes. It is not the same as a user's OS
+ * home directory.
+ */
+ @Override
+ public Path getHomeDirectory() {
+ Path result = new Path(fsRoot, "user/" + System.getProperty("user.name"));
+ LOG.trace("getHomeDirectory(): {}", result);
+ return result;
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * <p>Returns the service if delegation tokens are configured, otherwise, null.
+ */
+ @Override
+ public String getCanonicalServiceName() {
+ // TODO: Add delegation token support
+ return null;
+ }
+
+ /**
+ * Gets GCS FS instance.
+ */
+ GoogleCloudStorageFileSystem getGcsFs() {
+ return gcsFs;
+ }
+
+ /**
+ * Assert that the FileSystem has been initialized and not close()d.
+ */
+ private void checkOpen() throws IOException {
+ if (isClosed) {
+ throw new IOException("GoogleHadoopFileSystem has been closed or not initialized.");
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ LOG.trace("close()");
+ if (isClosed) {
+ return;
+ }
+
+ super.close();
+
+ getGcsFs().close();
+
+ this.isClosed = true;
+ }
+
+ @Override
+ public long getUsed() throws IOException {
+ long result = super.getUsed();
+ LOG.trace("getUsed(): {}", result);
+ return result;
+ }
+
+// @Override
+// public long getDefaultBlockSize() {
+// LOG.trace("getDefaultBlockSize(): {}", defaultBlockSize);
+// return defaultBlockSize;
+// }
+
+ @Override
+ public void setWorkingDirectory(final Path hadoopPath) {
+ checkArgument(hadoopPath != null, "hadoopPath must not be null");
+ URI gcsPath = UriPaths.toDirectory(getGcsPath(hadoopPath));
+ workingDirectory = getHadoopPath(gcsPath);
+ LOG.trace("setWorkingDirectory(hadoopPath: {}): {}", hadoopPath, workingDirectory);
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleHadoopFileSystemConfiguration.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleHadoopFileSystemConfiguration.java
new file mode 100644
index 00000000000..16d940b16f4
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleHadoopFileSystemConfiguration.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import static java.lang.Math.toIntExact;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * This class provides a configuration for the {@link GoogleHadoopFileSystem} implementations.
+ */
+class GoogleHadoopFileSystemConfiguration {
+ /**
+ * Configuration key for default block size of a file.
+ *
+ * <p>Note that this is the size that is reported to Hadoop FS clients. It does not modify the
+ * actual block size of an underlying GCS object, because GCS JSON API does not allow modifying or
+ * querying the value. Modifying this value allows one to control how many mappers are used to
+ * process a given file.
+ */
+ public static final HadoopConfigurationProperty<Long> BLOCK_SIZE =
+ new HadoopConfigurationProperty<>("fs.gs.block.size", 64 * 1024 * 1024L);
+
+ /**
+ * Configuration key for GCS project ID. Default value: none
+ */
+ public static final HadoopConfigurationProperty<String> GCS_PROJECT_ID =
+ new HadoopConfigurationProperty<>("fs.gs.project.id");
+
+ /**
+ * Configuration key for initial working directory of a GHFS instance. Default value: '/'
+ */
+ public static final HadoopConfigurationProperty<String> GCS_WORKING_DIRECTORY =
+ new HadoopConfigurationProperty<>("fs.gs.working.dir", "/");
+
+ /**
+ * Configuration key for setting write buffer size.
+ */
+ public static final HadoopConfigurationProperty<Long> GCS_OUTPUT_STREAM_BUFFER_SIZE =
+ new HadoopConfigurationProperty<>("fs.gs.outputstream.buffer.size", 8L * 1024 * 1024);
+
+ private final String workingDirectory;
+
+ public int getOutStreamBufferSize() {
+ return outStreamBufferSize;
+ }
+
+ private final int outStreamBufferSize;
+
+ GoogleHadoopFileSystemConfiguration(Configuration config) {
+ this.workingDirectory = GCS_WORKING_DIRECTORY.get(config, config::get);
+ this.outStreamBufferSize =
+ toIntExact(GCS_OUTPUT_STREAM_BUFFER_SIZE.get(config, config::getLongBytes));
+ }
+
+ public String getWorkingDirectory() {
+ return this.workingDirectory;
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleHadoopOutputStream.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleHadoopOutputStream.java
new file mode 100644
index 00000000000..747d9f001c5
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleHadoopOutputStream.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.channels.Channels;
+import java.nio.channels.ClosedChannelException;
+import java.nio.channels.WritableByteChannel;
+import javax.annotation.Nonnull;
+
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileSystem;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class GoogleHadoopOutputStream extends OutputStream {
+ public static final Logger LOG = LoggerFactory.getLogger(StorageResourceId.class);
+
+ private final GoogleHadoopFileSystem ghfs;
+
+ // Path of the file to write to.
+ private final URI dstGcsPath;
+
+ private OutputStream outputStream;
+
+ // Statistics tracker provided by the parent GoogleHadoopFileSystem for recording
+ // numbers of bytes written.
+ private final FileSystem.Statistics statistics;
+
+ /**
+ * Constructs an instance of GoogleHadoopOutputStream object.
+ *
+ * @param ghfs Instance of {@link GoogleHadoopFileSystem}.
+ * @param dstGcsPath Path of the file to write to.
+ * @param statistics File system statistics object.
+ * @param createFileOptions options for file creation
+ * @throws IOException if an IO error occurs.
+ */
+ GoogleHadoopOutputStream(GoogleHadoopFileSystem ghfs, URI dstGcsPath,
+ CreateOptions createFileOptions, FileSystem.Statistics statistics) throws IOException {
+ LOG.trace("GoogleHadoopOutputStream(gcsPath: {}, createFileOptions: {})", dstGcsPath,
+ createFileOptions);
+ this.ghfs = ghfs;
+ this.dstGcsPath = dstGcsPath;
+ this.statistics = statistics;
+
+ this.outputStream = createOutputStream(ghfs.getGcsFs(), dstGcsPath, createFileOptions,
+ ghfs.getFileSystemConfiguration());
+ }
+
+ private static OutputStream createOutputStream(GoogleCloudStorageFileSystem gcsfs, URI gcsPath,
+ CreateOptions options, GoogleHadoopFileSystemConfiguration fileSystemConfiguration)
+ throws IOException {
+ WritableByteChannel channel;
+ try {
+ channel = gcsfs.create(gcsPath, options);
+ } catch (java.nio.file.FileAlreadyExistsException e) {
+
+ throw (FileAlreadyExistsException) new FileAlreadyExistsException(
+ String.format("'%s' already exists", gcsPath)).initCause(e);
+ }
+ OutputStream outputStream = Channels.newOutputStream(channel);
+ int bufferSize = fileSystemConfiguration.getOutStreamBufferSize();
+ return bufferSize > 0 ? new BufferedOutputStream(outputStream, bufferSize) : outputStream;
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ throwIfNotOpen();
+ outputStream.write(b);
+ statistics.incrementBytesWritten(1);
+ statistics.incrementWriteOps(1);
+ }
+
+ @Override
+ public void write(@Nonnull byte[] b, int offset, int len) throws IOException {
+ throwIfNotOpen();
+ outputStream.write(b, offset, len);
+ statistics.incrementBytesWritten(len);
+ statistics.incrementWriteOps(1);
+ }
+
+ @Override
+ public void close() throws IOException {
+ LOG.trace("close(): final destination: {}", dstGcsPath);
+
+ if (outputStream == null) {
+ LOG.trace("close(): Ignoring; stream already closed.");
+ return;
+ }
+
+ try {
+ outputStream.close();
+ } finally {
+ outputStream = null;
+ }
+ }
+
+ private void throwIfNotOpen() throws IOException {
+ if (outputStream == null) {
+ throw new ClosedChannelException();
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/HadoopConfigurationProperty.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/HadoopConfigurationProperty.java
new file mode 100644
index 00000000000..9360290a09c
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/HadoopConfigurationProperty.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
+
+import java.util.List;
+import java.util.function.BiFunction;
+
+import org.apache.hadoop.conf.Configuration;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Hadoop configuration property.
+ */
+class HadoopConfigurationProperty<T> {
+ private static final Logger LOG = LoggerFactory.getLogger(HadoopConfigurationProperty.class);
+
+ private final String key;
+ private final List<String> deprecatedKeys;
+ private final T defaultValue;
+
+ private List<String> keyPrefixes = ImmutableList.of("");
+
+ HadoopConfigurationProperty(String key) {
+ this(key, null);
+ }
+
+ HadoopConfigurationProperty(String key, T defaultValue, String... deprecatedKeys) {
+ this.key = key;
+ this.deprecatedKeys =
+ deprecatedKeys == null ? ImmutableList.of() : ImmutableList.copyOf(deprecatedKeys);
+ this.defaultValue = defaultValue;
+ }
+
+ String getKey() {
+ return key;
+ }
+
+ T getDefault() {
+ return defaultValue;
+ }
+
+ T get(Configuration config, BiFunction<String, T, T> getterFn) {
+ String lookupKey = getLookupKey(config, key, (c, k) -> c.get(k) != null);
+ return logProperty(lookupKey, getterFn.apply(lookupKey, defaultValue));
+ }
+
+ private String getLookupKey(Configuration config, String lookupKey,
+ BiFunction<Configuration, String, Boolean> checkFn) {
+ for (String prefix : keyPrefixes) {
+ String prefixedKey = prefix + lookupKey;
+ if (checkFn.apply(config, prefixedKey)) {
+ return prefixedKey;
+ }
+ for (String deprecatedKey : deprecatedKeys) {
+ String prefixedDeprecatedKey = prefix + deprecatedKey;
+ if (checkFn.apply(config, prefixedDeprecatedKey)) {
+ LOG.warn("Using deprecated key '{}', use '{}' key instead.", prefixedDeprecatedKey,
+ prefixedKey);
+ return prefixedDeprecatedKey;
+ }
+ }
+ }
+ return keyPrefixes.get(0) + lookupKey;
+ }
+
+ private static <S> S logProperty(String key, S value) {
+ LOG.trace("{} = {}", key, value);
+ return value;
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/StorageResourceId.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/StorageResourceId.java
new file mode 100644
index 00000000000..5935564feed
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/StorageResourceId.java
@@ -0,0 +1,328 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument;
+import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.hadoop.thirdparty.com.google.common.base.Strings.isNullOrEmpty;
+import static org.apache.hadoop.fs.gs.Constants.SCHEME;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URI;
+import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Data struct representing either a GCS StorageObject, a GCS Bucket or the GCS root (gs://). If
+ * both bucketName and objectName are null, the StorageResourceId refers to GCS root (gs://). If
+ * bucketName is non-null, and objectName is null, then this refers to a GCS Bucket. Otherwise, if
+ * bucketName and objectName are both non-null, this refers to a GCS StorageObject.
+ */
+class StorageResourceId {
+
+ private static final Logger LOG = LoggerFactory.getLogger(StorageResourceId.class);
+
+ // The generationId used to denote "unknown"; if given to a method expecting generationId
+ // constraints, the method may perform extra low-level GETs to determine an existing
+ // generationId
+ // if idempotency constraints require doing so.
+ static final long UNKNOWN_GENERATION_ID = -1L;
+
+ // Pattern that parses out bucket and object names.
+ // Given 'gs://foo-bucket/foo/bar/baz', matcher.group(x) will return:
+ // 0 = gs://foo-bucket/foo/bar/baz
+ // 1 = foo-bucket/foo/bar/baz
+ // 2 = foo-bucket
+ // 3 = /foo/bar/baz
+ // 4 = foo/bar/baz
+ // Groups 2 and 4 can be used to create an instance.
+ private static final Pattern GCS_PATH_PATTERN = Pattern.compile("gs://(([^/]+)(/(.+)?)?)?");
+
+ // The singleton instance identifying the GCS root (gs://). Both getObjectName() and
+ // getBucketName() will return null.
+ static final StorageResourceId ROOT = new StorageResourceId();
+
+ // Bucket name of this storage resource to be used with the Google Cloud Storage API.
+ private final String bucketName;
+
+ // Object name of this storage resource to be used with the Google Cloud Storage API.
+ private final String objectName;
+
+ // Human-readable String to be returned by toString(); kept as 'final' member for efficiency.
+ private final String stringPath;
+
+ // The generationId to be used with precondition checks when using this StorageResourceId
+ // as an identifier for mutation requests.
+ private final long generationId;
+
+ /**
+ * Constructor for a StorageResourceId that refers to the GCS root (gs://). Private because all
+ * external users should just use the singleton StorageResourceId.ROOT.
+ */
+ private StorageResourceId() {
+ this.bucketName = null;
+ this.objectName = null;
+ this.stringPath = StringPaths.fromComponents(bucketName, objectName);
+ this.generationId = UNKNOWN_GENERATION_ID;
+ }
+
+ /**
+ * Constructor for a StorageResourceId representing a Bucket; {@code getObjectName()} will return
+ * null for a StorageResourceId that represents a Bucket.
+ *
+ * @param bucketName The bucket name of the resource. Must be non-empty and non-null.
+ */
+ StorageResourceId(String bucketName) {
+ checkArgument(!isNullOrEmpty(bucketName), "bucketName must not be null or empty");
+
+ this.bucketName = bucketName;
+ this.objectName = null;
+ this.stringPath = StringPaths.fromComponents(bucketName, objectName);
+ this.generationId = UNKNOWN_GENERATION_ID;
+ }
+
+ /**
+ * Constructor for a StorageResourceId representing a full StorageObject, including bucketName and
+ * objectName.
+ *
+ * @param bucketName The bucket name of the resource. Must be non-empty and non-null.
+ * @param objectName The object name of the resource. Must be non-empty and non-null.
+ */
+ StorageResourceId(String bucketName, String objectName) {
+ checkArgument(!isNullOrEmpty(bucketName), "bucketName must not be null or empty");
+ checkArgument(!isNullOrEmpty(objectName), "objectName must not be null or empty");
+
+ this.bucketName = bucketName;
+ this.objectName = objectName;
+ this.stringPath = StringPaths.fromComponents(bucketName, objectName);
+ this.generationId = UNKNOWN_GENERATION_ID;
+ }
+
+ /**
+ * Constructor for a StorageResourceId representing a full StorageObject, including bucketName and
+ * objectName.
+ *
+ * @param bucketName The bucket name of the resource. Must be non-empty and non-null.
+ * @param objectName The object name of the resource. Must be non-empty and non-null.
+ * @param generationId The generationId to be used with precondition checks when using this
+ * StorageResourceId as an identifier for mutation requests.
+ */
+ StorageResourceId(String bucketName, String objectName, long generationId) {
+ checkArgument(!isNullOrEmpty(bucketName), "bucketName must not be null or empty");
+ checkArgument(!isNullOrEmpty(objectName), "objectName must not be null or empty");
+
+ this.bucketName = bucketName;
+ this.objectName = objectName;
+ this.stringPath = StringPaths.fromComponents(bucketName, objectName);
+ this.generationId = generationId;
+ }
+
+ /**
+ * Constructor for a StorageResourceId representing a full StorageObject, including bucketName and
+ * objectName.
+ *
+ * @param bucketName The bucket name of the resource. Must be non-empty and non-null.
+ * @param generationId The generationId to be used with precondition checks when using this
+ * StorageResourceId as an identifier for mutation requests.
+ */
+ StorageResourceId(String bucketName, long generationId) {
+ checkArgument(!isNullOrEmpty(bucketName), "bucketName must not be null or empty");
+ this.bucketName = bucketName;
+ this.objectName = null;
+ this.stringPath = StringPaths.fromComponents(bucketName, objectName);
+ this.generationId = generationId;
+ }
+
+ /**
+ * Returns true if this StorageResourceId represents a GCS StorageObject; if true, both {@code
+ * getBucketName} and {@code getObjectName} will be non-empty and non-null.
+ */
+ boolean isStorageObject() {
+ return bucketName != null && objectName != null;
+ }
+
+ /**
+ * Returns true if this StorageResourceId represents a GCS Bucket; if true, then {@code
+ * getObjectName} will return null.
+ */
+ boolean isBucket() {
+ return bucketName != null && objectName == null;
+ }
+
+ /**
+ * Returns true if this StorageResourceId represents the GCS root (gs://); if true, then both
+ * {@code getBucketName} and {@code getObjectName} will be null.
+ */
+ boolean isRoot() {
+ return bucketName == null && objectName == null;
+ }
+
+ /**
+ * Indicates if this StorageResourceId corresponds to a 'directory'; similar to {@link
+ * FileInfo#isDirectory} except deals entirely with pathnames instead of also checking for
+ * exists() to be true on a corresponding GoogleCloudStorageItemInfo.
+ */
+ boolean isDirectory() {
+ return isRoot() || isBucket() || StringPaths.isDirectoryPath(objectName);
+ }
+
+ /**
+ * Gets the bucket name component of this resource identifier.
+ */
+ String getBucketName() {
+ return bucketName;
+ }
+
+ /**
+ * Gets the object name component of this resource identifier.
+ */
+ String getObjectName() {
+ return objectName;
+ }
+
+ /**
+ * The generationId to be used with precondition checks when using this StorageResourceId as an
+ * identifier for mutation requests. The generationId is *not* used when determining equals() or
+ * hashCode().
+ */
+ long getGenerationId() {
+ return generationId;
+ }
+
+ /**
+ * Returns true if generationId is not UNKNOWN_GENERATION_ID.
+ */
+ boolean hasGenerationId() {
+ return generationId != UNKNOWN_GENERATION_ID;
+ }
+
+ /**
+ * Returns a string of the form {@code gs://<bucketName>/<objectName>}.
+ */
+ @Override
+ public String toString() {
+ return stringPath;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof StorageResourceId) {
+ StorageResourceId other = (StorageResourceId) obj;
+ return Objects.equals(bucketName, other.bucketName) && Objects.equals(objectName,
+ other.objectName);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return stringPath.hashCode();
+ }
+
+ /**
+ * Converts StorageResourceId instance to look like a directory path. If the path already looks
+ * like a directory path then this call is a no-op.
+ *
+ * @return A resourceId with a directory path corresponding to the given resourceId.
+ */
+ StorageResourceId toDirectoryId() {
+ if (isStorageObject() && !StringPaths.isDirectoryPath(getObjectName())) {
+ return new StorageResourceId(getBucketName(), StringPaths.toDirectoryPath(getObjectName()));
+ }
+ return this;
+ }
+
+ /**
+ * Parses {@link StorageResourceId} from specified string.
+ */
+ static StorageResourceId fromStringPath(String path) {
+ return fromStringPath(path, UNKNOWN_GENERATION_ID);
+ }
+
+ /**
+ * Parses {@link StorageResourceId} from specified string and generationId.
+ */
+ static StorageResourceId fromStringPath(String path, long generationId) {
+ checkArgument(path != null, "path must not be null");
+
+ Matcher matcher = GCS_PATH_PATTERN.matcher(path);
+ checkArgument(matcher.matches(), "'%s' is not a valid GCS object name.", path);
+
+ String bucketName = matcher.group(2);
+ String relativePath = matcher.group(4);
+ if (bucketName == null) {
+ checkArgument(generationId == UNKNOWN_GENERATION_ID,
+ "Cannot specify generationId '%s' for root object '%s'", generationId, path);
+ return ROOT;
+ } else if (relativePath != null) {
+ return new StorageResourceId(bucketName, relativePath, generationId);
+ }
+ checkArgument(generationId == UNKNOWN_GENERATION_ID,
+ "Cannot specify generationId '%s' for bucket '%s'", generationId, path);
+ return new StorageResourceId(bucketName);
+ }
+
+ /**
+ * Validates the given URI and if valid, returns the associated StorageResourceId.
+ *
+ * @param path The GCS URI to validate.
+ * @param allowEmptyObjectName If true, a missing object name is not considered invalid.
+ * @return a StorageResourceId that may be the GCS root, a Bucket, or a StorageObject.
+ */
+ static StorageResourceId fromUriPath(URI path, boolean allowEmptyObjectName) {
+ return fromUriPath(path, allowEmptyObjectName, UNKNOWN_GENERATION_ID);
+ }
+
+ /**
+ * Validates the given URI and if valid, returns the associated StorageResourceId.
+ *
+ * @param path The GCS URI to validate.
+ * @param allowEmptyObjectName If true, a missing object name is not considered invalid.
+ * @param generationId The generationId to be used with precondition checks when
+ * using this
+ * @return a StorageResourceId that may be the GCS root, a Bucket, or a StorageObject.
+ */
+ static StorageResourceId fromUriPath(URI path, boolean allowEmptyObjectName,
+ long generationId) {
+ LOG.trace("fromUriPath('{}', {})", path, allowEmptyObjectName);
+ checkNotNull(path);
+
+ if (!SCHEME.equals(path.getScheme())) {
+ throw new IllegalArgumentException(
+ String.format("GCS path supports only '%s' scheme, instead got '%s' from '%s'.", SCHEME,
+ path.getScheme(), path));
+ }
+
+ if (path.equals(GoogleCloudStorageFileSystem.GCSROOT)) {
+ return ROOT;
+ }
+
+ String bucketName = StringPaths.validateBucketName(path.getAuthority());
+ // Note that we're using getPath here instead of rawPath, etc. This is because it is assumed
+ // that the path was properly encoded in getPath (or another similar method):
+ String objectName = StringPaths.validateObjectName(path.getPath(), allowEmptyObjectName);
+
+ return isNullOrEmpty(objectName) ?
+ new StorageResourceId(bucketName, generationId) :
+ new StorageResourceId(bucketName, objectName, generationId);
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/StringPaths.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/StringPaths.java
new file mode 100644
index 00000000000..80682c3ed2a
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/StringPaths.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import static org.apache.hadoop.thirdparty.com.google.common.base.Strings.isNullOrEmpty;
+import static org.apache.hadoop.fs.gs.Constants.PATH_DELIMITER;
+
+import org.apache.hadoop.thirdparty.com.google.common.base.CharMatcher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility methods for String GCS paths.
+ */
+final class StringPaths {
+
+ public static final Logger LOG = LoggerFactory.getLogger(StorageResourceId.class);
+
+ private StringPaths() {
+ }
+
+ // 14x faster (20ns vs 280ns) than "^[a-z0-9_.-]+$" regex
+ private static final CharMatcher BUCKET_NAME_CHAR_MATCHER = CharMatcher.ascii()
+ .and(CharMatcher.inRange('0', '9').or(CharMatcher.inRange('a', 'z'))
+ .or(CharMatcher.anyOf("_.-")))
+ .precomputed();
+
+ /**
+ * Validate the given bucket name to make sure that it can be used as a part of a file system
+ * path.
+ *
+ * <p>Note: this is not designed to duplicate the exact checks that GCS would perform on the
+ * server side. We make some checks that are relevant to using GCS as a file system.
+ *
+ * @param bucketName Bucket name to check.
+ */
+ static String validateBucketName(String bucketName) {
+ // If the name ends with '/', remove it.
+ bucketName = toFilePath(bucketName);
+
+ if (isNullOrEmpty(bucketName)) {
+ throw new IllegalArgumentException("GCS bucket name cannot be empty.");
+ }
+
+ if (!BUCKET_NAME_CHAR_MATCHER.matchesAllOf(bucketName)) {
+ throw new IllegalArgumentException(String.format(
+ "Invalid GCS bucket name '%s': bucket name must contain only 'a-z0-9_.-' characters.",
+ bucketName));
+ }
+
+ return bucketName;
+ }
+
+ /**
+ * Validate the given object name to make sure that it can be used as a part of a file system
+ * path.
+ *
+ * <p>Note: this is not designed to duplicate the exact checks that GCS would perform on the
+ * server side. We make some checks that are relevant to using GCS as a file system.
+ *
+ * @param objectName Object name to check.
+ * @param allowEmptyObjectName If true, a missing object name is not considered invalid.
+ */
+ static String validateObjectName(String objectName, boolean allowEmptyObjectName) {
+ LOG.trace("validateObjectName('{}', {})", objectName, allowEmptyObjectName);
+
+ if (isNullOrEmpty(objectName) || objectName.equals(PATH_DELIMITER)) {
+ if (allowEmptyObjectName) {
+ objectName = "";
+ } else {
+ throw new IllegalArgumentException(String.format(
+ "GCS path must include non-empty object name [objectName='%s',"
+ + " allowEmptyObjectName=%s]", objectName, allowEmptyObjectName));
+ }
+ }
+
+ // We want objectName to look like a traditional file system path,
+ // therefore, disallow objectName with consecutive '/' chars.
+ for (int i = 0; i < (objectName.length() - 1); i++) {
+ if (objectName.charAt(i) == '/' && objectName.charAt(i + 1) == '/') {
+ throw new IllegalArgumentException(
+ String.format("GCS path must not have consecutive '/' characters: '%s'", objectName));
+ }
+ }
+
+ // Remove leading '/' if it exists.
+ if (objectName.startsWith(PATH_DELIMITER)) {
+ objectName = objectName.substring(1);
+ }
+
+ LOG.trace("validateObjectName -> '{}'", objectName);
+ return objectName;
+ }
+
+ /**
+ * Helper for standardizing the way various human-readable messages in logs/exceptions that refer
+ * to a bucket/object pair.
+ */
+ public static String fromComponents(String bucketName, String objectName) {
+ if (bucketName == null && objectName != null) {
+ throw new IllegalArgumentException(
+ String.format("Invalid bucketName/objectName pair: gs://%s/%s", "<null>", objectName));
+ }
+ // TODO(user): Unify this method with other methods that convert bucketName/objectName
+ // to a URI; maybe use the single slash for compatibility.
+ StringBuilder result = new StringBuilder("gs://");
+ if (bucketName != null) {
+ result.append(bucketName);
+ }
+ if (objectName != null) {
+ result.append('/').append(objectName);
+ }
+ return result.toString();
+ }
+
+ /**
+ * Indicates whether the given object name looks like a directory path.
+ *
+ * @param path Name of the object to inspect.
+ * @return Whether the given object name looks like a directory path.
+ */
+ public static boolean isDirectoryPath(String path) {
+ return !isNullOrEmpty(path) && path.endsWith(PATH_DELIMITER);
+ }
+
+ /**
+ * Converts the given object name to look like a file path. If the object name already looks like
+ * a file path then this call is a no-op.
+ *
+ * <p>If the object name is null or empty, it is returned as-is.
+ *
+ * @param path Name of the object to inspect.
+ * @return File path for the given path.
+ */
+ public static String toFilePath(String path) {
+ return !isNullOrEmpty(path) && isDirectoryPath(path) ?
+ path.substring(0, path.length() - 1) :
+ path;
+ }
+
+ /**
+ * Converts the given object name to look like a directory path. If the object name already looks
+ * like a directory path then this call is a no-op.
+ *
+ * <p>If the object name is null or empty, it is returned as-is.
+ *
+ * @param path Name of the object to inspect.
+ * @return Directory path for the given path.
+ */
+ static String toDirectoryPath(String path) {
+ return isNullOrEmpty(path) || isDirectoryPath(path) ? path : path + PATH_DELIMITER;
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/UriPaths.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/UriPaths.java
new file mode 100644
index 00000000000..30e13cb33cb
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/UriPaths.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.hadoop.fs.gs.Constants.PATH_DELIMITER;
+import static org.apache.hadoop.fs.gs.Constants.SCHEME;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+
+/**
+ * Utility methods for URI GCS paths.
+ */
+final class UriPaths {
+
+ private UriPaths() {
+ }
+
+ /**
+ * Converts the given path to look like a directory path. If the path already looks like a
+ * directory path then this call is a no-op.
+ *
+ * @param path Path to convert.
+ * @return Directory path for the given path.
+ */
+ public static URI toDirectory(URI path) {
+ StorageResourceId resourceId = StorageResourceId.fromUriPath(path, true);
+
+ if (resourceId.isStorageObject() && !resourceId.isDirectory()) {
+ resourceId = resourceId.toDirectoryId();
+ path = fromResourceId(resourceId, /* allowEmptyObjectName= */ false);
+ }
+ return path;
+ }
+
+ /**
+ * Gets the parent directory of the given path.
+ *
+ * @param path Path to convert.
+ * @return Path of parent directory of the given item or null for root path.
+ */
+ public static URI getParentPath(URI path) {
+ checkNotNull(path);
+
+ // Root path has no parent.
+ if (path.equals(GoogleCloudStorageFileSystem.GCSROOT)) {
+ return null;
+ }
+
+ StorageResourceId resourceId = StorageResourceId.fromUriPath(path, true);
+
+ if (resourceId.isBucket()) {
+ return GoogleCloudStorageFileSystem.GCSROOT;
+ }
+
+ String objectName = resourceId.getObjectName();
+ int index = StringPaths.isDirectoryPath(objectName) ?
+ objectName.lastIndexOf(PATH_DELIMITER, objectName.length() - 2) :
+ objectName.lastIndexOf(PATH_DELIMITER);
+ return index < 0 ?
+ fromStringPathComponents(resourceId.getBucketName(), /* objectName= */
+ null, /* allowEmptyObjectName= */ true) :
+ fromStringPathComponents(resourceId.getBucketName(), objectName.substring(0, index + 1),
+ /* allowEmptyObjectName= */ false);
+ }
+
+ /**
+ * Constructs and returns full path for the given bucket and object names.
+ */
+ public static URI fromResourceId(StorageResourceId resourceId, boolean allowEmptyObjectName) {
+ return fromStringPathComponents(resourceId.getBucketName(), resourceId.getObjectName(),
+ allowEmptyObjectName);
+ }
+
+ /**
+ * Constructs and returns full path for the given bucket and object names.
+ */
+ public static URI fromStringPathComponents(String bucketName, String objectName,
+ boolean allowEmptyObjectName) {
+ if (allowEmptyObjectName && bucketName == null && objectName == null) {
+ return GoogleCloudStorageFileSystem.GCSROOT;
+ }
+
+ String authority = StringPaths.validateBucketName(bucketName);
+ String path = PATH_DELIMITER + StringPaths.validateObjectName(objectName, allowEmptyObjectName);
+
+ try {
+ return new URI(SCHEME, authority, path,
+ /* query= */ null,
+ /* fragment= */ null);
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException(
+ String.format("Invalid bucket name (%s) or object name (%s)", bucketName, objectName), e);
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/VerificationAttributes.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/VerificationAttributes.java
new file mode 100644
index 00000000000..4155482fc7d
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/VerificationAttributes.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2016 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import java.util.Arrays;
+import javax.annotation.Nullable;
+
+/**
+ * GCS provided validation attributes for a single object.
+ */
+class VerificationAttributes {
+ private final byte[] md5hash;
+ private final byte[] crc32c;
+
+ VerificationAttributes(@Nullable byte[] md5hash, @Nullable byte[] crc32c) {
+ this.md5hash = md5hash;
+ this.crc32c = crc32c;
+ }
+
+ /**
+ * MD5 hash of an object, if available.
+ */
+ @Nullable
+ byte[] getMd5hash() {
+ return md5hash;
+ }
+
+ /**
+ * CRC32c checksum of an object, if available.
+ */
+ @Nullable
+ byte[] getCrc32c() {
+ return crc32c;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof VerificationAttributes)) {
+ return false;
+ }
+ VerificationAttributes that = (VerificationAttributes) o;
+ return Arrays.equals(md5hash, that.md5hash) && Arrays.equals(crc32c, that.crc32c);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = Arrays.hashCode(md5hash);
+ result = 31 * result + Arrays.hashCode(crc32c);
+ return result;
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/package-info.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/package-info.java
new file mode 100644
index 00000000000..eedfb7822ac
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Google Cloud Storage Filesystem.
+ */
+
+package org.apache.hadoop.fs.gs;
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/TestStorageResourceId.java b/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/TestStorageResourceId.java
new file mode 100644
index 00000000000..e0a39b2d7e4
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/TestStorageResourceId.java
@@ -0,0 +1,285 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import java.net.URI;
+
+import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestStorageResourceId {
+ @Test
+ public void testConstructorInvalid() {
+ assertThrows(IllegalArgumentException.class, () -> {
+ new StorageResourceId(null);
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ new StorageResourceId("");
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ new StorageResourceId(null, null);
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ new StorageResourceId("foo", null);
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ new StorageResourceId("", null);
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ new StorageResourceId(null, null, 0L);
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ new StorageResourceId("foo", null, 0L);
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ new StorageResourceId("", null, 0L);
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ new StorageResourceId(null, 0L);
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ new StorageResourceId("", 0L);
+ });
+ }
+
+ @Test
+ public void testFromStringPathInvalid() {
+ assertThrows(IllegalArgumentException.class, () -> {
+ StorageResourceId.fromStringPath(null);
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ StorageResourceId.fromStringPath("");
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ StorageResourceId.fromStringPath("foo");
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ StorageResourceId.fromStringPath("/foo/bar");
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ StorageResourceId.fromStringPath("gs:///foo/bar");
+ });
+ }
+
+ @Test
+ public void testConstructor() {
+ String bucketName = "testbucketname";
+ String objectName = "a/b/c.txt";
+
+ verify(new StorageResourceId(bucketName), bucketName,
+ StorageResourceId.UNKNOWN_GENERATION_ID, null, false,
+ true, true, false, false);
+
+ verify(new StorageResourceId(bucketName, objectName), bucketName,
+ StorageResourceId.UNKNOWN_GENERATION_ID, objectName, false,
+ false, false, true, false);
+
+ long genId = System.currentTimeMillis();
+ verify(new StorageResourceId(bucketName, objectName, genId), bucketName,
+ genId, objectName, true,
+ false, false, true, false);
+
+ verify(new StorageResourceId(bucketName, genId), bucketName,
+ genId, null, true,
+ true, true, false, false);
+ }
+
+ @Test
+ public void testEqualsBucket() {
+ StorageResourceId bucket1 = new StorageResourceId("test-bucket");
+ StorageResourceId bucket2 = new StorageResourceId("test-bucket");
+ assertTrue(bucket1.equals(bucket2));
+ assertEquals(bucket1.hashCode(), bucket2.hashCode());
+ }
+
+ @Test
+ public void testEqualsObject() {
+ StorageResourceId obj1 = new StorageResourceId("test-bucket", "test-object");
+ StorageResourceId obj2 = new StorageResourceId("test-bucket", "test-object");
+ assertTrue(obj1.equals(obj2));
+ assertEquals(obj1.hashCode(), obj2.hashCode());
+ }
+
+ @Test
+ public void testEqualsDifferentBucket() {
+ StorageResourceId bucket1 = new StorageResourceId("test-bucket");
+ StorageResourceId bucket2 = new StorageResourceId("other-bucket");
+ assertFalse(bucket1.equals(bucket2));
+ }
+
+ @Test
+ public void testEqualsDifferentObject() {
+ StorageResourceId obj1 = new StorageResourceId("test-bucket", "test-object");
+ StorageResourceId obj2 = new StorageResourceId("test-bucket", "other-object");
+ assertFalse(obj1.equals(obj2));
+ }
+
+ @Test
+ public void testToDirectoryIdFromFile() {
+ StorageResourceId fileId = new StorageResourceId("my-bucket", "path/to/file.txt");
+ StorageResourceId dirId = fileId.toDirectoryId();
+
+ assertNotSame(fileId, dirId); // Should return a new instance
+ assertTrue(dirId.isDirectory());
+ assertEquals("my-bucket", dirId.getBucketName());
+ assertEquals("path/to/file.txt/", dirId.getObjectName());
+ assertEquals(fileId.getGenerationId(), dirId.getGenerationId());
+ }
+
+ @Test
+ public void testToDirectoryIdFromDirectoryObject() {
+ StorageResourceId dirIdOriginal = new StorageResourceId("my-bucket", "path/to/dir/");
+ StorageResourceId dirIdConverted = dirIdOriginal.toDirectoryId();
+
+ assertSame(dirIdOriginal, dirIdConverted); // Should return the same instance
+ assertTrue(dirIdConverted.isDirectory());
+ assertEquals("path/to/dir/", dirIdConverted.getObjectName());
+ }
+
+ @Test
+ public void testToDirectoryIdFromBucket() {
+ StorageResourceId bucketId = new StorageResourceId("my-bucket");
+ StorageResourceId convertedId = bucketId.toDirectoryId();
+ assertSame(bucketId, convertedId);
+ assertTrue(convertedId.isBucket());
+ }
+
+ @Test
+ public void testFromStringPathRoot() {
+ StorageResourceId id = StorageResourceId.fromStringPath("gs://");
+ assertTrue(id.isRoot());
+ }
+
+ @Test
+ public void testFromStringPathBucket() {
+ StorageResourceId id = StorageResourceId.fromStringPath("gs://my-bucket");
+ assertTrue(id.isBucket());
+ assertEquals("my-bucket", id.getBucketName());
+ assertNull(id.getObjectName());
+ assertEquals(StorageResourceId.UNKNOWN_GENERATION_ID, id.getGenerationId());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "gs://my-bucket/object",
+ "gs://my-bucket/folder/file.txt",
+ "gs://my-bucket/folder/"
+ })
+ public void testFromStringPathObject(String path) {
+ String expectedBucket = path.split("/")[2];
+ String expectedObject =
+ path.substring(path.indexOf(expectedBucket) + expectedBucket.length() + 1);
+
+ StorageResourceId id = StorageResourceId.fromStringPath(path);
+ assertTrue(id.isStorageObject());
+ assertEquals(expectedBucket, id.getBucketName());
+ assertEquals(expectedObject, id.getObjectName());
+ assertEquals(StorageResourceId.UNKNOWN_GENERATION_ID, id.getGenerationId());
+ }
+
+ @Test
+ public void testFromStringPathObjectWithGenerationId() {
+ long genId = 12345L;
+ StorageResourceId id = StorageResourceId.fromStringPath("gs://my-bucket/object.txt", genId);
+ assertTrue(id.isStorageObject());
+ assertEquals("my-bucket", id.getBucketName());
+ assertEquals("object.txt", id.getObjectName());
+ assertEquals(genId, id.getGenerationId());
+ assertTrue(id.hasGenerationId());
+ }
+
+ @Test
+ public void testFromUriPathBucket() throws Exception {
+ URI uri = new URI("gs://my-bucket");
+ StorageResourceId id = StorageResourceId.fromUriPath(uri, true);
+ assertTrue(id.isBucket());
+ assertEquals("my-bucket", id.getBucketName());
+ assertNull(id.getObjectName());
+ }
+
+ @Test
+ public void testFromUriPathObject() throws Exception {
+ URI uri = new URI("gs://my-bucket/path/to/file.txt");
+ StorageResourceId id = StorageResourceId.fromUriPath(uri, false);
+ assertTrue(id.isStorageObject());
+ assertEquals("my-bucket", id.getBucketName());
+ assertEquals("path/to/file.txt", id.getObjectName());
+ }
+
+ @Test
+ public void testFromUriPathObjectWithGenerationId() throws Exception {
+ URI uri = new URI("gs://my-bucket/object.txt");
+ long genId = 54321L;
+ StorageResourceId id = StorageResourceId.fromUriPath(uri, false, genId);
+ assertTrue(id.isStorageObject());
+ assertEquals("my-bucket", id.getBucketName());
+ assertEquals("object.txt", id.getObjectName());
+ assertEquals(genId, id.getGenerationId());
+ assertTrue(id.hasGenerationId());
+ }
+
+ @Test
+ public void testFromUriPathBucketWithGenerationId() throws Exception {
+ assertThrows(IllegalArgumentException.class, () -> {
+ URI uri = new URI("gs://my-bucket");
+ long genId = 54321L;
+ StorageResourceId.fromUriPath(uri, false, genId);
+ });
+ }
+
+ private static void verify(
+ StorageResourceId id,
+ String bucketName,
+ long generationId,
+ String objectName,
+ boolean hasGenerationId,
+ boolean isBucket,
+ boolean isDirectory,
+ boolean isStorageObject,
+ boolean isRoot) {
+ assertEquals(bucketName, id.getBucketName());
+ assertEquals(generationId, id.getGenerationId());
+ assertEquals(objectName, id.getObjectName());
+ assertEquals(hasGenerationId, id.hasGenerationId());
+ assertEquals(isBucket, id.isBucket());
+ assertEquals(isDirectory, id.isDirectory());
+ assertEquals(isStorageObject, id.isStorageObject());
+ assertEquals(isRoot, id.isRoot());
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/TestStringPaths.java b/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/TestStringPaths.java
new file mode 100644
index 00000000000..16234e0ce1d
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/TestStringPaths.java
@@ -0,0 +1,164 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import org.junit.Test;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+public class TestStringPaths {
+ @Test
+ public void testValidateBucketNameValid() {
+ assertEquals("my-bucket", StringPaths.validateBucketName("my-bucket"));
+ assertEquals("my.bucket", StringPaths.validateBucketName("my.bucket"));
+ assertEquals("my_bucket", StringPaths.validateBucketName("my_bucket"));
+ assertEquals("bucket123", StringPaths.validateBucketName("bucket123"));
+ assertEquals("a", StringPaths.validateBucketName("a"));
+ assertEquals("long-bucket-name-with-numbers-123",
+ StringPaths.validateBucketName("long-bucket-name-with-numbers-123"));
+ }
+
+ @Test
+ public void testValidateBucketNameEndsWithSlash() {
+ assertEquals("my-bucket", StringPaths.validateBucketName("my-bucket/"));
+ assertEquals("another-bucket", StringPaths.validateBucketName("another-bucket/"));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testValidateBucketNameEmpty() {
+ StringPaths.validateBucketName("");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testValidateBucketNameNull() {
+ StringPaths.validateBucketName(null);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testValidateBucketNameInvalidChars() {
+ StringPaths.validateBucketName("my bucket"); // Space
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testValidateBucketNameInvalidChars2() {
+ StringPaths.validateBucketName("my@bucket"); // @ symbol
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testValidateBucketNameUpperCase() {
+ StringPaths.validateBucketName("MyBucket"); // Uppercase
+ }
+
+ @Test
+ public void testValidateObjectNameValid() {
+ assertEquals("path/to/object",
+ StringPaths.validateObjectName("path/to/object", false));
+ assertEquals("object", StringPaths.validateObjectName("object", false));
+ assertEquals("dir/",
+ StringPaths.validateObjectName("dir/", false)); // Still valid after validation
+ assertEquals("", StringPaths.validateObjectName("/", true)); // Slash becomes empty if allowed
+ assertEquals("", StringPaths.validateObjectName("", true));
+ }
+
+ @Test
+ public void testValidateObjectNameLeadingSlash() {
+ assertEquals("path/to/object", StringPaths.validateObjectName("/path/to/object", false));
+ assertEquals("object", StringPaths.validateObjectName("/object", false));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testValidateObjectNameEmptyNotAllowed() {
+ StringPaths.validateObjectName("", false);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testValidateObjectNameNullNotAllowed() {
+ StringPaths.validateObjectName(null, false);
+ }
+
+ @Test
+ public void testValidateObjectNameEmptyAllowed() {
+ assertEquals("", StringPaths.validateObjectName("", true));
+ assertEquals("", StringPaths.validateObjectName(null, true));
+ assertEquals("", StringPaths.validateObjectName("/", true)); // Single slash becomes empty
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testValidateObjectNameConsecutiveSlashes() {
+ StringPaths.validateObjectName("path//to/object", false);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testValidateObjectNameConsecutiveSlashesAtStart() {
+ StringPaths.validateObjectName("//path/to/object", false);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testValidateObjectNameConsecutiveSlashesAtEnd() {
+ StringPaths.validateObjectName("path/to/object//", false);
+ }
+
+ @Test
+ public void testFromComponentsValid() {
+ assertEquals("gs://my-bucket/path/to/object",
+ StringPaths.fromComponents("my-bucket", "path/to/object"));
+ assertEquals("gs://my-bucket/dir/", StringPaths.fromComponents("my-bucket", "dir/"));
+ assertEquals("gs://my-bucket/", StringPaths.fromComponents("my-bucket", ""));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testFromComponentsNullBucketNonNullObject() {
+ StringPaths.fromComponents(null, "path/to/object");
+ }
+
+ @Test
+ public void testFromComponentsNullBucketAndObject() {
+ assertEquals("gs://", StringPaths.fromComponents(null, null));
+ }
+
+ @Test
+ public void testIsDirectoryPath() {
+ assertTrue(StringPaths.isDirectoryPath("dir/"));
+ assertTrue(StringPaths.isDirectoryPath("path/to/dir/"));
+ assertFalse(StringPaths.isDirectoryPath("file.txt"));
+ assertFalse(StringPaths.isDirectoryPath("path/to/file.txt"));
+ assertFalse(StringPaths.isDirectoryPath(""));
+ assertFalse(StringPaths.isDirectoryPath(null));
+ }
+
+ @Test
+ public void testToFilePath() {
+ assertEquals("path/to/file", StringPaths.toFilePath("path/to/file/"));
+ assertEquals("file.txt", StringPaths.toFilePath("file.txt"));
+ assertEquals("dir", StringPaths.toFilePath("dir/"));
+ assertEquals("", StringPaths.toFilePath(""));
+ assertNull(StringPaths.toFilePath(null));
+ }
+
+ // --- Tests for toDirectoryPath ---
+
+ @Test
+ public void testToDirectoryPath() {
+ assertEquals("path/to/dir/", StringPaths.toDirectoryPath("path/to/dir"));
+ assertEquals("dir/", StringPaths.toDirectoryPath("dir/"));
+ assertEquals("file/", StringPaths.toDirectoryPath("file"));
+ assertEquals("", StringPaths.toDirectoryPath(""));
+ assertNull(StringPaths.toDirectoryPath(null));
+ }
+}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/TestUriPaths.java b/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/TestUriPaths.java
new file mode 100644
index 00000000000..fe93a28dc43
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/TestUriPaths.java
@@ -0,0 +1,150 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import java.net.URI;
+
+import org.junit.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class TestUriPaths {
+ @Test
+ public void testToDirectoryFile() throws Exception {
+ URI fileUri = new URI("gs://my-bucket/path/to/file.txt");
+ URI expectedDirUri = new URI("gs://my-bucket/path/to/file.txt/");
+ // Temporarily override the behavior for testing purposes
+ // This is not a clean mocking strategy for static methods, but demonstrates the test intent.
+ // In a real environment, you'd use PowerMock or refactor.
+ URI result = UriPaths.toDirectory(fileUri);
+ assertEquals(expectedDirUri, result);
+ }
+
+ @Test
+ public void testToDirectoryAlreadyDirectory() throws Exception {
+ URI dirUri = new URI("gs://my-bucket/path/to/dir/");
+ URI result = UriPaths.toDirectory(dirUri);
+ assertEquals(dirUri, result);
+ }
+
+ @Test
+ public void testToDirectoryRootBucket() throws Exception {
+ URI bucketUri = new URI("gs://my-bucket");
+ URI result = UriPaths.toDirectory(bucketUri);
+ assertEquals(bucketUri, result); // Buckets are implicitly directories
+ }
+
+ @Test
+ public void testGetParentPathFile() throws Exception {
+ URI uri = new URI("gs://my-bucket/path/to/file.txt");
+ URI expectedParent = new URI("gs://my-bucket/path/to/");
+ assertEquals(expectedParent, UriPaths.getParentPath(uri));
+ }
+
+ @Test
+ public void testGetParentPathDirectory() throws Exception {
+ URI uri = new URI("gs://my-bucket/path/to/dir/");
+ URI expectedParent = new URI("gs://my-bucket/path/to/");
+ assertEquals(expectedParent, UriPaths.getParentPath(uri));
+ }
+
+ @Test
+ public void testGetParentPathObjectAtBucketRoot() throws Exception {
+ URI uri = new URI("gs://my-bucket/file.txt");
+ URI expectedParent = new URI("gs://my-bucket/");
+ assertEquals(expectedParent, UriPaths.getParentPath(uri));
+ }
+
+ @Test
+ public void testGetParentPathDirectoryAtBucketRoot() throws Exception {
+ URI uri = new URI("gs://my-bucket/dir/");
+ URI expectedParent = new URI("gs://my-bucket/");
+ assertEquals(expectedParent, UriPaths.getParentPath(uri));
+ }
+
+ @Test
+ public void testGetParentPathBucket() throws Exception {
+ URI uri = new URI("gs://my-bucket");
+ assertEquals(GoogleCloudStorageFileSystem.GCSROOT, UriPaths.getParentPath(uri));
+ }
+
+ @Test
+ public void testFromResourceIdObject() throws Exception {
+ StorageResourceId resourceId = new StorageResourceId("my-bucket", "path/to/object");
+ URI expectedUri = new URI("gs://my-bucket/path/to/object");
+ assertEquals(expectedUri, UriPaths.fromResourceId(resourceId, false));
+ }
+
+ @Test
+ public void testFromResourceIdDirectory() throws Exception {
+ StorageResourceId resourceId = new StorageResourceId("my-bucket", "path/to/dir/");
+ URI expectedUri = new URI("gs://my-bucket/path/to/dir/");
+ assertEquals(expectedUri, UriPaths.fromResourceId(resourceId, false));
+ }
+
+ @Test
+ public void testFromResourceIdBucket() throws Exception {
+ StorageResourceId resourceId = new StorageResourceId("my-bucket");
+ URI expectedUri = new URI("gs://my-bucket/");
+ assertEquals(expectedUri, UriPaths.fromResourceId(resourceId, true));
+ }
+
+ @Test
+ public void testFromResourceIdEmptyObjectAllowed() throws Exception {
+ StorageResourceId resourceId = new StorageResourceId("my-bucket");
+ URI expectedUri = new URI("gs://my-bucket/");
+ assertEquals(expectedUri, UriPaths.fromResourceId(resourceId, true));
+ }
+
+ @Test
+ public void testFromResourceIdNullObjectAllowed() throws Exception {
+ StorageResourceId resourceId = new StorageResourceId("my-bucket");
+ URI expectedUri = new URI("gs://my-bucket/");
+ assertEquals(expectedUri, UriPaths.fromResourceId(resourceId, true));
+ }
+
+ @Test
+ public void testFromStringPathComponentsValid() throws Exception {
+ assertEquals(new URI("gs://my-bucket/path/to/object"),
+ UriPaths.fromStringPathComponents("my-bucket", "path/to/object", false));
+ assertEquals(new URI("gs://my-bucket/path/to/dir/"),
+ UriPaths.fromStringPathComponents("my-bucket", "path/to/dir/", false));
+ assertEquals(new URI("gs://my-bucket/"),
+ UriPaths.fromStringPathComponents("my-bucket", null, true));
+ assertEquals(new URI("gs://my-bucket/"),
+ UriPaths.fromStringPathComponents("my-bucket", "", true));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testFromStringPathComponentsNullBucketNameNotAllowed() {
+ UriPaths.fromStringPathComponents(null, "object", false);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testFromStringPathComponentsEmptyObjectNameNotAllowed() {
+ UriPaths.fromStringPathComponents("my-bucket", "", false);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testFromStringPathComponentsConsecutiveSlashes() {
+ UriPaths.fromStringPathComponents("my-bucket", "path//to/object", false);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testFromStringPathComponentsInvalidBucketName() {
+ UriPaths.fromStringPathComponents("MyBucket", "object", false); // Uppercase
+ }
+}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/package-info.java b/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/package-info.java
new file mode 100644
index 00000000000..fe289cc6d3d
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Google Cloud Storage Filesystem tests.
+ */
+package org.apache.hadoop.fs.gs;
\ No newline at end of file
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index 8c1256a177c..70df9b02d0f 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -53,6 +53,7 @@
<module>hadoop-fs2img</module>
<module>hadoop-benchmark</module>
<module>hadoop-compat-bench</module>
+ <module>hadoop-gcp</module>
</modules>
<build>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org