You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2025/07/08 23:21:22 UTC
(hadoop) 01/07: HADOOP-19343: GCS / Add implementation for create() API
This is an automated email from the ASF dual-hosted git repository.
cnauroth pushed a commit to branch HADOOP-19343
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit b79cf93fca0943560ac7b69ee4ef642318a420f0
Author: Arunkumar Chacko <ar...@google.com>
AuthorDate: Tue May 27 22:17:54 2025 +0000
HADOOP-19343: GCS / Add implementation for create() API
Closes #7656
Signed-off-by: Chris Nauroth <cn...@apache.org>
---
hadoop-project/pom.xml | 7 +-
.../hadoop-gcp/dev-support/findbugs-exclude.xml | 29 ++
hadoop-tools/hadoop-gcp/pom.xml | 503 ++++++++++++++++++++
.../src/config/checkstyle-suppressions.xml | 26 ++
.../java/org/apache/hadoop/fs/gs/Constants.java | 31 ++
.../org/apache/hadoop/fs/gs/CreateOptions.java | 121 +++++
.../apache/hadoop/fs/gs/ErrorTypeExtractor.java | 60 +++
.../java/org/apache/hadoop/fs/gs/FileInfo.java | 203 +++++++++
.../apache/hadoop/fs/gs/GoogleCloudStorage.java | 261 +++++++++++
.../gs/GoogleCloudStorageClientWriteChannel.java | 116 +++++
.../hadoop/fs/gs/GoogleCloudStorageFileSystem.java | 89 ++++
.../hadoop/fs/gs/GoogleCloudStorageItemInfo.java | 423 +++++++++++++++++
.../hadoop/fs/gs/GoogleHadoopFileSystem.java | 505 +++++++++++++++++++++
.../fs/gs/GoogleHadoopFileSystemConfiguration.java | 75 +++
.../hadoop/fs/gs/GoogleHadoopOutputStream.java | 124 +++++
.../hadoop/fs/gs/HadoopConfigurationProperty.java | 90 ++++
.../org/apache/hadoop/fs/gs/StorageResourceId.java | 328 +++++++++++++
.../java/org/apache/hadoop/fs/gs/StringPaths.java | 169 +++++++
.../java/org/apache/hadoop/fs/gs/UriPaths.java | 113 +++++
.../hadoop/fs/gs/VerificationAttributes.java | 68 +++
.../java/org/apache/hadoop/fs/gs/package-info.java | 23 +
.../apache/hadoop/fs/gs/TestStorageResourceId.java | 285 ++++++++++++
.../org/apache/hadoop/fs/gs/TestStringPaths.java | 164 +++++++
.../java/org/apache/hadoop/fs/gs/TestUriPaths.java | 150 ++++++
.../java/org/apache/hadoop/fs/gs/package-info.java | 22 +
hadoop-tools/pom.xml | 1 +
26 files changed, 3985 insertions(+), 1 deletion(-)
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index a66c5535939..bbc44c04a52 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -108,7 +108,7 @@
<findbugs.version>3.0.5</findbugs.version>
<dnsjava.version>3.6.1</dnsjava.version>
- <guava.version>27.0-jre</guava.version>
+ <guava.version>33.1.0-jre</guava.version>
<guice.version>5.1.0</guice.version>
<bouncycastle.version>1.78.1</bouncycastle.version>
@@ -2153,6 +2153,11 @@
<artifactId>failsafe</artifactId>
<version>2.4.4</version>
</dependency>
+ <dependency>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>google-cloud-storage</artifactId>
+ <version>2.44.1</version>
+ </dependency>
</dependencies>
</dependencyManagement>
diff --git a/hadoop-tools/hadoop-gcp/dev-support/findbugs-exclude.xml b/hadoop-tools/hadoop-gcp/dev-support/findbugs-exclude.xml
new file mode 100644
index 00000000000..80be329bd6d
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/dev-support/findbugs-exclude.xml
@@ -0,0 +1,29 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<FindBugsFilter>
+ <!-- Redundant null check makes code clearer, future-proof here. -->
+ <Match>
+ <Class name="org.apache.hadoop.fs.gs.GoogleCloudStorage" />
+ <Method name="createItemInfoForBlob" />
+ <Bug pattern="RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.fs.gs.GoogleCloudStorage" />
+ <Method name="createItemInfoForBlob" />
+ <Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" />
+ </Match>
+</FindBugsFilter>
diff --git a/hadoop-tools/hadoop-gcp/pom.xml b/hadoop-tools/hadoop-gcp/pom.xml
new file mode 100644
index 00000000000..d5744f1f97c
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/pom.xml
@@ -0,0 +1,503 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-project</artifactId>
+ <version>3.5.0-SNAPSHOT</version>
+ <relativePath>../../hadoop-project</relativePath>
+ </parent>
+ <artifactId>hadoop-gcp</artifactId>
+ <version>3.5.0-SNAPSHOT</version>
+ <name>Apache Hadoop Google Cloud Platform support</name>
+ <description>
+ This module contains code to support integration with Google Cloud Platform.
+ It also declares the dependencies needed to work with Google Cloud Storage.
+ </description>
+ <packaging>jar</packaging>
+
+ <properties>
+ <file.encoding>UTF-8</file.encoding>
+ <downloadSources>true</downloadSources>
+ <hadoop.tmp.dir>${project.build.directory}/test</hadoop.tmp.dir>
+ </properties>
+
+ <profiles>
+ <profile>
+ <id>tests-off</id>
+ <activation>
+ <file>
+ <missing>src/test/resources/auth-keys.xml</missing>
+ </file>
+ </activation>
+ <properties>
+ <skipITs>true</skipITs>
+ </properties>
+ </profile>
+ <profile>
+ <id>tests-on</id>
+ <activation>
+ <file>
+ <exists>src/test/resources/auth-keys.xml</exists>
+ </file>
+ </activation>
+ <properties>
+ <skipITs>false</skipITs>
+ </properties>
+ </profile>
+ <profile>
+ <id>parallel-tests</id>
+ <activation>
+ <property>
+ <name>parallel-tests</name>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-maven-plugins</artifactId>
+ <executions>
+ <execution>
+ <id>parallel-tests-createdir</id>
+ <goals>
+ <goal>parallel-tests-createdir</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <forkCount>${testsThreadCount}</forkCount>
+ <reuseForks>false</reuseForks>
+ <trimStackTrace>false</trimStackTrace>
+ <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
+ <systemPropertyVariables>
+ <testsThreadCount>${testsThreadCount}</testsThreadCount>
+ <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
+ <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
+ <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
+ <test.unique.fork.id>job-${job.id}-fork-000${surefire.forkNumber}</test.unique.fork.id>
+ </systemPropertyVariables>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>default-integration-test</id>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ <configuration>
+ <forkCount>${testsThreadCount}</forkCount>
+ <reuseForks>false</reuseForks>
+ <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
+ <forkedProcessTimeoutInSeconds>${fs.gs.scale.test.timeout}</forkedProcessTimeoutInSeconds>
+ <trimStackTrace>false</trimStackTrace>
+ <systemPropertyVariables>
+ <!-- Tell tests that they are being executed in parallel -->
+ <test.parallel.execution>true</test.parallel.execution>
+ <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
+ <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
+ <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
+
+ <!-- Due to a Maven quirk, setting this to just -->
+ <!-- surefire.forkNumber won't do the parameter -->
+ <!-- substitution. Putting a prefix in front of it like -->
+ <!-- "fork-" makes it work. -->
+ <test.unique.fork.id>job-${job.id}-fork-000${surefire.forkNumber}</test.unique.fork.id>
+ <test.default.timeout>${test.integration.timeout}</test.default.timeout>
+ </systemPropertyVariables>
+ <includes>
+ <include>**/ITest*.java</include>
+ </includes>
+ </configuration>
+ </execution>
+ <execution>
+ <id>sequential-integration-tests</id>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ <configuration>
+ <forkedProcessTimeoutInSeconds>${fs.gs.scale.test.timeout}</forkedProcessTimeoutInSeconds>
+ <trimStackTrace>false</trimStackTrace>
+ <systemPropertyVariables>
+ <!-- Tell tests that they are being executed sequentially -->
+ <test.parallel.execution>false</test.parallel.execution>
+ <test.unique.fork.id>job-${job.id}</test.unique.fork.id>
+ </systemPropertyVariables>
+ <!-- Do a sequential run for tests that cannot handle -->
+ <!-- parallel execution. -->
+ <includes>
+ <include>**/ITest*.java</include>
+ </includes>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
+ <id>sequential-tests</id>
+ <activation>
+ <property>
+ <name>!parallel-tests</name>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ <configuration>
+ <trimStackTrace>false</trimStackTrace>
+ <systemPropertyVariables>
+ <test.unique.fork.id>job-${job.id}</test.unique.fork.id>
+ </systemPropertyVariables>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer"/>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ </transformers>
+ <filters>
+
+ <filter>
+ <artifact>com.google.auth:*</artifact>
+ <includes>
+ <include>**</include>
+ </includes>
+ </filter>
+ <filter>
+ <artifact>io.grpc:*</artifact>
+ <includes>
+ <include>**</include>
+ </includes>
+ </filter>
+ <filter>
+ <artifact>io.opencensus:*</artifact>
+ <includes>
+ <include>**</include>
+ </includes>
+ </filter>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>*.json</exclude>
+ <exclude>google/**</exclude>
+ <exclude>grpc/**</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <artifactSet>
+ <includes>
+ <include>com.google.api</include>
+ <include>com.google.api-client</include>
+ <include>com.google.api.grpc</include>
+ <include>com.google.apis</include>
+ <include>com.google.auth</include>
+ <include>com.google.cloud</include>
+ <include>com.google.cloud.bigdataoss</include>
+ <include>com.google.cloud.grpc</include>
+ <include>com.google.cloud.http</include>
+ <include>com.google.flogger</include>
+ <include>com.google.code.gson</include>
+ <include>com.google.guava</include>
+ <include>com.google.http-client</include>
+ <include>com.google.oauth-client</include>
+ <include>com.google.protobuf</include>
+ <include>com.google.re2j</include>
+ <include>com.google.storage.v2</include>
+ <include>com.lmax</include>
+ <include>io.grpc</include>
+ <include>io.opencensus</include>
+ <include>io.perfmark</include>
+ <include>org.apache.httpcomponents</include>
+ <include>org.threeten:threetenbp</include>
+ </includes>
+ </artifactSet>
+ <minimizeJar>true</minimizeJar>
+ <relocations>
+ <relocation>
+ <pattern>com</pattern>
+ <shadedPattern>com.google.cloud.hadoop.repackaged.ossgcs.com</shadedPattern>
+ <includes>
+ <include>com.google.api.**</include>
+ <include>com.google.api.gax.**</include>
+ <include>com.google.auth.**</include>
+ <include>com.google.cloud.*</include>
+ <include>com.google.cloud.audit.**</include>
+ <include>com.google.cloud.grpc.**</include>
+ <include>com.google.cloud.hadoop.gcsio.**</include>
+ <include>com.google.cloud.hadoop.util.**</include>
+ <include>com.google.cloud.http.**</include>
+ <include>com.google.cloud.monitoring.**</include>
+ <include>com.google.cloud.spi.**</include>
+ <include>com.google.cloud.storage.**</include>
+ <include>com.google.common.**</include>
+ <include>com.google.geo.**</include>
+ <include>com.google.gson.**</include>
+ <include>com.google.google.storage.**</include>
+ <include>com.google.iam.**</include>
+ <include>com.google.logging.**</include>
+ <include>com.google.longrunning.**</include>
+ <include>com.google.monitoring.**</include>
+ <include>com.google.protobuf.**</include>
+ <include>com.google.re2j.**</include>
+ <include>com.google.rpc.**</include>
+ <include>com.google.storage.**</include>
+ <include>com.google.thirdparty.**</include>
+ <include>com.google.type.**</include>
+ <include>com.lmax.disruptor.**</include>
+ </includes>
+ <excludes>
+ <exclude>com.google.cloud.hadoop.util.AccessTokenProvider</exclude>
+ <exclude>com.google.cloud.hadoop.util.AccessTokenProvider$AccessToken</exclude>
+ <exclude>com.google.cloud.hadoop.util.AccessTokenProvider$AccessTokenType</exclude>
+ <exclude>com.google.cloud.hadoop.util.AccessBoundary</exclude>
+ <exclude>com.google.cloud.hadoop.util.AccessBoundary$Action</exclude>
+ <exclude>com.google.cloud.hadoop.util.AutoValue_AccessBoundary</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>org</pattern>
+ <shadedPattern>com.google.cloud.hadoop.repackaged.ossgcs.org</shadedPattern>
+ <includes>
+ <include>org.apache.http.**</include>
+ <include>org.threeten.**</include>
+ </includes>
+ </relocation>
+
+ <relocation>
+ <pattern>io.grpc.netty.shaded</pattern>
+ <shadedPattern>
+ com.google.cloud.hadoop.repackaged.ossgcs.io.grpc.netty.shaded
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>io</pattern>
+ <shadedPattern>com.google.cloud.hadoop.repackaged.ossgcs.io</shadedPattern>
+ <includes>
+ <include>io.grpc.**</include>
+ <include>io.opencensus.**</include>
+ <include>io.perfmark.**</include>
+ </includes>
+ </relocation>
+ <relocation>
+ <pattern>META-INF/native/io_grpc_netty_shaded_</pattern>
+ <shadedPattern>
+ META-INF/native/com_google_cloud_hadoop_repackaged_gcs_io_grpc_netty_shaded_
+ </shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>META-INF/native/libio_grpc_netty_shaded_</pattern>
+ <shadedPattern>
+ META-INF/native/libcom_google_cloud_hadoop_repackaged_gcs_io_grpc_netty_shaded_
+ </shadedPattern>
+ </relocation>
+ </relocations>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>com.github.spotbugs</groupId>
+ <artifactId>spotbugs-maven-plugin</artifactId>
+ <configuration>
+ <xmlOutput>true</xmlOutput>
+ <excludeFilterFile>${basedir}/dev-support/findbugs-exclude.xml
+ </excludeFilterFile>
+ <effort>Max</effort>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <configuration>
+ <suppressionsLocation>src/config/checkstyle-suppressions.xml</suppressionsLocation>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <forkedProcessTimeoutInSeconds>3600</forkedProcessTimeoutInSeconds>
+ <systemPropertyVariables>
+ <test.default.timeout>${test.integration.timeout}</test.default.timeout>
+ </systemPropertyVariables>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>banned-illegal-imports</id>
+ <phase>process-sources</phase>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <restrictImports>
+ <includeTestCode>false</includeTestCode>
+ <reason>Restrict mapreduce imports to committer code</reason>
+ <exclusions>
+
+ </exclusions>
+ <bannedImports>
+ <bannedImport>org.apache.hadoop.mapreduce.**</bannedImport>
+ <bannedImport>org.apache.hadoop.mapred.**</bannedImport>
+ </bannedImports>
+ </restrictImports>
+ <restrictImports>
+ <includeTestCode>false</includeTestCode>
+ <reason>Restrict encryption client imports to encryption client factory</reason>
+ <exclusions>
+
+ </exclusions>
+ <bannedImports>
+
+ </bannedImports>
+ </restrictImports>
+ </rules>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+<dependencyManagement>
+ <dependencies>
+ <dependency>
+ <!--
+ We're using a specific Protobuf version to ensure compatibility with the
+ Google Cloud Storage (GCS) client. The GCS client often relies on
+ particular Long-Term Support (LTS) versions of Protobuf. When we upgrade
+ the GCS client, we'll likely need to update Protobuf too. To prevent
+ dependency conflicts, Protobuf will be shaded within the GCS connector's
+ fat JAR.
+ -->
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>3.25.5</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.enterprise</groupId>
+ <artifactId>cdi-api</artifactId>
+ </exclusion>
+ <!-- Exclude protobuf-java 2.5.0 -->
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.assertj</groupId>
+ <artifactId>assertj-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.platform</groupId>
+ <artifactId>junit-platform-launcher</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.vintage</groupId>
+ <artifactId>junit-vintage-engine</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>google-cloud-storage</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-engine</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
+
diff --git a/hadoop-tools/hadoop-gcp/src/config/checkstyle-suppressions.xml b/hadoop-tools/hadoop-gcp/src/config/checkstyle-suppressions.xml
new file mode 100644
index 00000000000..8c765bc97f9
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/config/checkstyle-suppressions.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE suppressions PUBLIC
+ "-//Checkstyle//DTD SuppressionFilter Configuration 1.0//EN"
+ "https://checkstyle.org/dtds/suppressions_1_0.dtd">
+
+
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<suppressions>
+ <suppress checks="ParameterNumber"
+ files="org[\\/]apache[\\/]hadoop[\\/]fs[\\/]gs[\\/]GoogleCloudStorageItemInfo.java"/>
+ <suppress checks="ParameterNumber"
+ files="org[\\/]apache[\\/]hadoop[\\/]fs[\\/]gs[\\/]TestStorageResourceId.java"/>
+</suppressions>
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/Constants.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/Constants.java
new file mode 100644
index 00000000000..34434b2859a
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/Constants.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+final class Constants {
+ private Constants() {}
+
+ // URI scheme for GCS.
+ static final String SCHEME = "gs";
+ static final String PATH_DELIMITER = "/";
+
+ static final String GCS_CONFIG_PREFIX = "fs.gs";
+
+ static final String BASE_KEY_PREFIX = "google.cloud";
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/CreateOptions.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/CreateOptions.java
new file mode 100644
index 00000000000..c9b44a1a481
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/CreateOptions.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument;
+
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
+
+import java.util.Map;
+import javax.annotation.Nullable;
+
+/**
+ * Options that can be specified when creating a file in the {@link GoogleCloudStorageFileSystem}.
+ */
+final class CreateOptions {
+ private final ImmutableMap<String, byte[]> attributes;
+ private final String contentType;
+ private final long overwriteGenerationId;
+ private final WriteMode mode;
+
+ private CreateOptions(CreateOperationOptionsBuilder builder) {
+ this.attributes = ImmutableMap.copyOf(builder.attributes);
+ this.contentType = builder.contentType;
+ this.overwriteGenerationId = builder.overwriteGenerationId;
+ this.mode = builder.writeMode;
+ }
+
+ boolean isOverwriteExisting() {
+ return this.mode == WriteMode.OVERWRITE;
+ }
+
+ enum WriteMode {
+ /**
+ * Creates a new file for write and fails if file already exists.
+ */
+ CREATE_NEW,
+ /**
+ * Creates a new file for write or overwrites an existing file if it already exists.
+ */
+ OVERWRITE
+ }
+
+ static CreateOperationOptionsBuilder builder() {
+ return new CreateOperationOptionsBuilder();
+ }
+
+ /**
+ * Extended attributes to set when creating a file.
+ */
+ ImmutableMap<String, byte[]> getAttributes() {
+ return attributes;
+ }
+
+ /**
+ * Content-type to set when creating a file.
+ */
+ @Nullable
+ String getContentType() {
+ return contentType;
+ }
+
+ /**
+ * Whether to overwrite an existing file with the same name.
+ */
+ WriteMode getWriteMode() {
+ return mode;
+ }
+
+ /**
+ * Generation of existing object to overwrite. Ignored if set to {@link
+ * StorageResourceId#UNKNOWN_GENERATION_ID}, but otherwise this is used instead of {@code
+ * overwriteExisting}, where 0 indicates no existing object, and otherwise an existing object will
+ * only be overwritten by the newly created file if its generation matches this provided
+ * generationId.
+ */
+ long getOverwriteGenerationId() {
+ return overwriteGenerationId;
+ }
+
+ static class CreateOperationOptionsBuilder {
+ private Map<String, byte[]> attributes = ImmutableMap.of();
+ private String contentType = "application/octet-stream";
+ private long overwriteGenerationId = StorageResourceId.UNKNOWN_GENERATION_ID;
+ private WriteMode writeMode = WriteMode.CREATE_NEW;
+
+ CreateOperationOptionsBuilder setWriteMode(WriteMode mode) {
+ this.writeMode = mode;
+ return this;
+ }
+
+ CreateOptions build() {
+ CreateOptions options = new CreateOptions(this);
+
+ checkArgument(!options.getAttributes().containsKey("Content-Type"),
+ "The Content-Type attribute must be set via the contentType option");
+ if (options.getWriteMode() != WriteMode.OVERWRITE) {
+ checkArgument(options.getOverwriteGenerationId() == StorageResourceId.UNKNOWN_GENERATION_ID,
+ "overwriteGenerationId is set to %s but it can be set only in OVERWRITE mode",
+ options.getOverwriteGenerationId());
+ }
+
+ return options;
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/ErrorTypeExtractor.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/ErrorTypeExtractor.java
new file mode 100644
index 00000000000..a4497734524
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/ErrorTypeExtractor.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import io.grpc.Status;
+
+/**
+ * Implementation for {@link ErrorTypeExtractor} for exception specifically thrown from gRPC path.
+ */
+final class ErrorTypeExtractor {
+
+ enum ErrorType {
+ NOT_FOUND, OUT_OF_RANGE, ALREADY_EXISTS, FAILED_PRECONDITION, INTERNAL, RESOURCE_EXHAUSTED,
+ UNAVAILABLE, UNKNOWN
+ }
+
+ // public static final ErrorTypeExtractor INSTANCE = new ErrorTypeExtractor();
+
+ private static final String BUCKET_ALREADY_EXISTS_MESSAGE =
+ "FAILED_PRECONDITION: Your previous request to create the named bucket succeeded and you "
+ + "already own it.";
+
+ private ErrorTypeExtractor() {
+ }
+
+ static ErrorType getErrorType(Exception error) {
+ switch (Status.fromThrowable(error).getCode()) {
+ case NOT_FOUND:
+ return ErrorType.NOT_FOUND;
+ case OUT_OF_RANGE:
+ return ErrorType.OUT_OF_RANGE;
+ case ALREADY_EXISTS:
+ return ErrorType.ALREADY_EXISTS;
+ case FAILED_PRECONDITION:
+ return ErrorType.FAILED_PRECONDITION;
+ case RESOURCE_EXHAUSTED:
+ return ErrorType.RESOURCE_EXHAUSTED;
+ case INTERNAL:
+ return ErrorType.INTERNAL;
+ case UNAVAILABLE:
+ return ErrorType.UNAVAILABLE;
+ default:
+ return ErrorType.UNKNOWN;
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/FileInfo.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/FileInfo.java
new file mode 100644
index 00000000000..df8d63f5eec
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/FileInfo.java
@@ -0,0 +1,203 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import java.net.URI;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Contains information about a file or a directory.
+ *
+ * <p>Note: This class wraps GoogleCloudStorageItemInfo, adds file system specific information and
+ * hides bucket/object specific information.
+ */
+final class FileInfo {
+
+ // Info about the root path.
+ static final FileInfo ROOT_INFO =
+ new FileInfo(GoogleCloudStorageFileSystem.GCSROOT, GoogleCloudStorageItemInfo.ROOT_INFO);
+
+ // Path of this file or directory.
+ private final URI path;
+
+ // Information about the underlying GCS item.
+ private final GoogleCloudStorageItemInfo itemInfo;
+
+ /**
+ * Constructs an instance of FileInfo.
+ *
+ * @param itemInfo Information about the underlying item.
+ */
+ private FileInfo(URI path, GoogleCloudStorageItemInfo itemInfo) {
+ this.itemInfo = itemInfo;
+
+ // Construct the path once.
+ this.path = path;
+ }
+
+ /**
+ * Gets the path of this file or directory.
+ */
+ URI getPath() {
+ return path;
+ }
+
+ /**
+ * Indicates whether this item is a directory.
+ */
+ boolean isDirectory() {
+ return itemInfo.isDirectory();
+ }
+
+ /**
+ * Indicates whether this item is an inferred directory.
+ */
+ boolean isInferredDirectory() {
+ return itemInfo.isInferredDirectory();
+ }
+
+ /**
+ * Indicates whether this instance has information about the unique, shared root of the underlying
+ * storage system.
+ */
+ boolean isGlobalRoot() {
+ return itemInfo.isGlobalRoot();
+ }
+
+ /**
+ * Gets creation time of this item.
+ *
+ * <p>Time is expressed as milliseconds since January 1, 1970 UTC.
+ */
+ long getCreationTime() {
+ return itemInfo.getCreationTime();
+ }
+
+ /**
+ * Gets the size of this file or directory.
+ *
+ * <p>For files, size is in number of bytes. For directories size is 0. For items that do not
+ * exist, size is -1.
+ */
+ long getSize() {
+ return itemInfo.getSize();
+ }
+
+ /**
+ * Gets the modification time of this file if one is set, otherwise the value of {@link
+ * #getCreationTime()} is returned.
+ *
+ * <p>Time is expressed as milliseconds since January 1, 1970 UTC.
+ */
+ long getModificationTime() {
+ return itemInfo.getModificationTime();
+ }
+
+ /**
+ * Retrieve file attributes for this file.
+ *
+ * @return A map of file attributes
+ */
+ Map<String, byte[]> getAttributes() {
+ return itemInfo.getMetadata();
+ }
+
+ /**
+ * Indicates whether this file or directory exists.
+ */
+ boolean exists() {
+ return itemInfo.exists();
+ }
+
+ /**
+ * Returns CRC32C checksum of the file or {@code null}.
+ */
+ byte[] getCrc32cChecksum() {
+ VerificationAttributes verificationAttributes = itemInfo.getVerificationAttributes();
+ return verificationAttributes == null ? null : verificationAttributes.getCrc32c();
+ }
+
+ /**
+ * Returns MD5 checksum of the file or {@code null}.
+ */
+ byte[] getMd5Checksum() {
+ VerificationAttributes verificationAttributes = itemInfo.getVerificationAttributes();
+ return verificationAttributes == null ? null : verificationAttributes.getMd5hash();
+ }
+
+ /**
+ * Gets information about the underlying item.
+ */
+ GoogleCloudStorageItemInfo getItemInfo() {
+ return itemInfo;
+ }
+
+ /**
+ * Gets string representation of this instance.
+ */
+ @Override
+ public String toString() {
+ return getPath() + (exists() ?
+ ": created on: " + Instant.ofEpochMilli(getCreationTime()) :
+ ": exists: no");
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof FileInfo)) {
+ return false;
+ }
+ FileInfo fileInfo = (FileInfo) o;
+ return Objects.equals(path, fileInfo.path) && Objects.equals(itemInfo, fileInfo.itemInfo);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(path, itemInfo);
+ }
+
+ /**
+ * Handy factory method for constructing a FileInfo from a GoogleCloudStorageItemInfo while
+ * potentially returning a singleton instead of really constructing an object for cases like ROOT.
+ */
+ static FileInfo fromItemInfo(GoogleCloudStorageItemInfo itemInfo) {
+ if (itemInfo.isRoot()) {
+ return ROOT_INFO;
+ }
+ URI path = UriPaths.fromResourceId(itemInfo.getResourceId(), /* allowEmptyObjectName= */ true);
+ return new FileInfo(path, itemInfo);
+ }
+
+ /**
+ * Handy factory method for constructing a list of FileInfo from a list of
+ * GoogleCloudStorageItemInfo.
+ */
+ static List<FileInfo> fromItemInfos(List<GoogleCloudStorageItemInfo> itemInfos) {
+ List<FileInfo> fileInfos = new ArrayList<>(itemInfos.size());
+ for (GoogleCloudStorageItemInfo itemInfo : itemInfos) {
+ fileInfos.add(fromItemInfo(itemInfo));
+ }
+ return fileInfos;
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorage.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorage.java
new file mode 100644
index 00000000000..9c15962b7ef
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorage.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.*;
+import static org.apache.hadoop.thirdparty.com.google.common.base.Strings.isNullOrEmpty;
+
+import com.google.cloud.storage.*;
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
+import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
+import org.apache.hadoop.thirdparty.com.google.common.io.BaseEncoding;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.nio.channels.WritableByteChannel;
+import java.nio.file.FileAlreadyExistsException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A wrapper around <a href="https://github.com/googleapis/java-storage">Google cloud storage
+ * client</a>.
+ */
+class GoogleCloudStorage {
+ public static final Logger LOG = LoggerFactory.getLogger(GoogleHadoopFileSystem.class);
+ static final List<Storage.BlobField> BLOB_FIELDS =
+ ImmutableList.of(Storage.BlobField.BUCKET, Storage.BlobField.CONTENT_ENCODING,
+ Storage.BlobField.CONTENT_TYPE, Storage.BlobField.CRC32C, Storage.BlobField.GENERATION,
+ Storage.BlobField.METADATA, Storage.BlobField.MD5HASH, Storage.BlobField.METAGENERATION,
+ Storage.BlobField.NAME, Storage.BlobField.SIZE, Storage.BlobField.TIME_CREATED,
+ Storage.BlobField.UPDATED);
+ private final Storage storage;
+ private final GoogleHadoopFileSystemConfiguration configuration;
+
+ /**
+ * Having an instance of gscImpl to redirect calls to Json client while new client implementation
+ * is in WIP.
+ */
+ GoogleCloudStorage(GoogleHadoopFileSystemConfiguration configuration) throws IOException {
+ // TODO: Set projectId
+ // TODO: Set credentials
+ this.storage = StorageOptions.newBuilder().build().getService();
+ this.configuration = configuration;
+ }
+
+ public WritableByteChannel create(final StorageResourceId resourceId, final CreateOptions options)
+ throws IOException {
+ LOG.trace("create({})", resourceId);
+
+ checkArgument(resourceId.isStorageObject(), "Expected full StorageObject id, got %s",
+ resourceId);
+ // Update resourceId if generationId is missing
+ StorageResourceId resourceIdWithGeneration = resourceId;
+ if (!resourceId.hasGenerationId()) {
+ resourceIdWithGeneration =
+ new StorageResourceId(resourceId.getBucketName(), resourceId.getObjectName(),
+ getWriteGeneration(resourceId, options.isOverwriteExisting()));
+ }
+
+ return new GoogleCloudStorageClientWriteChannel(storage, resourceIdWithGeneration, options);
+ }
+
+ /**
+ * Gets the object generation for a write operation
+ *
+ * <p>making getItemInfo call even if overwrite is disabled to fail fast in case file is existing.
+ *
+ * @param resourceId object for which generation info is requested
+ * @param overwrite whether existing object should be overwritten
+ * @return the generation of the object
+ * @throws IOException if the object already exists and cannot be overwritten
+ */
+ private long getWriteGeneration(StorageResourceId resourceId, boolean overwrite)
+ throws IOException {
+ LOG.trace("getWriteGeneration({}, {})", resourceId, overwrite);
+ GoogleCloudStorageItemInfo info = getItemInfo(resourceId);
+ if (!info.exists()) {
+ return 0L;
+ }
+ if (info.exists() && overwrite) {
+ long generation = info.getContentGeneration();
+ checkState(generation != 0, "Generation should not be 0 for an existing item");
+ return generation;
+ }
+
+ throw new FileAlreadyExistsException(String.format("Object %s already exists.", resourceId));
+ }
+
+ public void close() {
+ try {
+ storage.close();
+ } catch (Exception e) {
+ LOG.warn("Error occurred while closing the storage client", e);
+ }
+ }
+
+ public GoogleCloudStorageItemInfo getItemInfo(StorageResourceId resourceId) throws IOException {
+ LOG.trace("getItemInfo({})", resourceId);
+
+ // Handle ROOT case first.
+ if (resourceId.isRoot()) {
+ return GoogleCloudStorageItemInfo.ROOT_INFO;
+ }
+ GoogleCloudStorageItemInfo itemInfo = null;
+
+ if (resourceId.isBucket()) {
+ Bucket bucket = getBucket(resourceId.getBucketName());
+ if (bucket != null) {
+ itemInfo = createItemInfoForBucket(resourceId, bucket);
+ } else {
+ LOG.debug("getBucket({}): not found", resourceId.getBucketName());
+ }
+ } else {
+ Blob blob = getBlob(resourceId);
+ if (blob != null) {
+ itemInfo = createItemInfoForBlob(resourceId, blob);
+ } else {
+ LOG.debug("getObject({}): not found", resourceId);
+ }
+ }
+
+ if (itemInfo == null) {
+ itemInfo = GoogleCloudStorageItemInfo.createNotFound(resourceId);
+ }
+ LOG.debug("getItemInfo: {}", itemInfo);
+ return itemInfo;
+ }
+
+ /**
+ * Gets the bucket with the given name.
+ *
+ * @param bucketName name of the bucket to get
+ * @return the bucket with the given name or null if bucket not found
+ * @throws IOException if the bucket exists but cannot be accessed
+ */
+ @Nullable
+ private Bucket getBucket(String bucketName) throws IOException {
+ LOG.debug("getBucket({})", bucketName);
+ checkArgument(!isNullOrEmpty(bucketName), "bucketName must not be null or empty");
+ try {
+ return storage.get(bucketName);
+ } catch (StorageException e) {
+ if (ErrorTypeExtractor.getErrorType(e) == ErrorTypeExtractor.ErrorType.NOT_FOUND) {
+ return null;
+ }
+ throw new IOException("Error accessing Bucket " + bucketName, e);
+ }
+ }
+
+ private static GoogleCloudStorageItemInfo createItemInfoForBlob(StorageResourceId resourceId,
+ Blob blob) {
+ checkArgument(resourceId != null, "resourceId must not be null");
+ checkArgument(blob != null, "object must not be null");
+ checkArgument(resourceId.isStorageObject(),
+ "resourceId must be a StorageObject. resourceId: %s", resourceId);
+ checkArgument(resourceId.getBucketName().equals(blob.getBucket()),
+ "resourceId.getBucketName() must equal object.getBucket(): '%s' vs '%s'",
+ resourceId.getBucketName(), blob.getBucket());
+ checkArgument(resourceId.getObjectName().equals(blob.getName()),
+ "resourceId.getObjectName() must equal object.getName(): '%s' vs '%s'",
+ resourceId.getObjectName(), blob.getName());
+
+ Map<String, byte[]> decodedMetadata =
+ blob.getMetadata() == null ? null : decodeMetadata(blob.getMetadata());
+
+ byte[] md5Hash = null;
+ byte[] crc32c = null;
+
+ if (!isNullOrEmpty(blob.getCrc32c())) {
+ crc32c = BaseEncoding.base64().decode(blob.getCrc32c());
+ }
+
+ if (!isNullOrEmpty(blob.getMd5())) {
+ md5Hash = BaseEncoding.base64().decode(blob.getMd5());
+ }
+
+ return GoogleCloudStorageItemInfo.createObject(resourceId,
+ blob.getCreateTimeOffsetDateTime() == null ?
+ 0 :
+ blob.getCreateTimeOffsetDateTime().toInstant().toEpochMilli(),
+ blob.getUpdateTimeOffsetDateTime() == null ?
+ 0 :
+ blob.getUpdateTimeOffsetDateTime().toInstant().toEpochMilli(),
+ blob.getSize() == null ? 0 : blob.getSize(), blob.getContentType(),
+ blob.getContentEncoding(), decodedMetadata,
+ blob.getGeneration() == null ? 0 : blob.getGeneration(),
+ blob.getMetageneration() == null ? 0 : blob.getMetageneration(),
+ new VerificationAttributes(md5Hash, crc32c));
+ }
+
+ static Map<String, byte[]> decodeMetadata(Map<String, String> metadata) {
+ return Maps.transformValues(metadata, GoogleCloudStorage::decodeMetadataValues);
+ }
+
+ @Nullable
+ private static byte[] decodeMetadataValues(String value) {
+ try {
+ return BaseEncoding.base64().decode(value);
+ } catch (IllegalArgumentException iae) {
+ LOG.error("Failed to parse base64 encoded attribute value {}", value, iae);
+ return null;
+ }
+ }
+
+ /**
+ * Gets the object with the given resourceId.
+ *
+ * @param resourceId identifies a StorageObject
+ * @return the object with the given name or null if object not found
+ * @throws IOException if the object exists but cannot be accessed
+ */
+ @Nullable
+ Blob getBlob(StorageResourceId resourceId) throws IOException {
+ checkArgument(resourceId.isStorageObject(), "Expected full StorageObject id, got %s",
+ resourceId);
+ String bucketName = resourceId.getBucketName();
+ String objectName = resourceId.getObjectName();
+ Blob blob;
+ try {
+ blob = storage.get(BlobId.of(bucketName, objectName),
+ Storage.BlobGetOption.fields(BLOB_FIELDS.toArray(new Storage.BlobField[0])));
+ } catch (StorageException e) {
+ throw new IOException("Error accessing " + resourceId, e);
+ }
+ return blob;
+ }
+
+ private static GoogleCloudStorageItemInfo createItemInfoForBucket(StorageResourceId resourceId,
+ Bucket bucket) {
+ checkArgument(resourceId != null, "resourceId must not be null");
+ checkArgument(bucket != null, "bucket must not be null");
+ checkArgument(resourceId.isBucket(), "resourceId must be a Bucket. resourceId: %s", resourceId);
+ checkArgument(resourceId.getBucketName().equals(bucket.getName()),
+ "resourceId.getBucketName() must equal bucket.getName(): '%s' vs '%s'",
+ resourceId.getBucketName(), bucket.getName());
+
+ return GoogleCloudStorageItemInfo.createBucket(resourceId,
+ bucket.asBucketInfo().getCreateTimeOffsetDateTime().toInstant().toEpochMilli(),
+ bucket.asBucketInfo().getUpdateTimeOffsetDateTime().toInstant().toEpochMilli(),
+ bucket.getLocation(),
+ bucket.getStorageClass() == null ? null : bucket.getStorageClass().name());
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorageClientWriteChannel.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorageClientWriteChannel.java
new file mode 100644
index 00000000000..7956b6f0a82
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorageClientWriteChannel.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.BlobInfo;
+import com.google.cloud.storage.BlobWriteSession;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.Storage.BlobWriteOption;
+import com.google.cloud.storage.StorageException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.WritableByteChannel;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Implements WritableByteChannel to provide write access to GCS via java-storage client.
+ */
+class GoogleCloudStorageClientWriteChannel implements WritableByteChannel {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(GoogleCloudStorageClientWriteChannel.class);
+
+ private final StorageResourceId resourceId;
+ private WritableByteChannel writableByteChannel;
+
+ GoogleCloudStorageClientWriteChannel(final Storage storage,
+ final StorageResourceId resourceId, final CreateOptions createOptions) throws IOException {
+ this.resourceId = resourceId;
+ BlobWriteSession blobWriteSession = getBlobWriteSession(storage, resourceId, createOptions);
+ try {
+ this.writableByteChannel = blobWriteSession.open();
+ } catch (StorageException e) {
+ throw new IOException(e);
+ }
+ }
+
+ private static BlobInfo getBlobInfo(final StorageResourceId resourceId,
+ final CreateOptions createOptions) {
+ BlobInfo blobInfo = BlobInfo.newBuilder(
+ BlobId.of(resourceId.getBucketName(), resourceId.getObjectName(),
+ resourceId.getGenerationId())).setContentType(createOptions.getContentType())
+ // .setMetadata(encodeMetadata(createOptions.getMetadata())) // TODO:
+ .build();
+ return blobInfo;
+ }
+
+ private static BlobWriteSession getBlobWriteSession(final Storage storage,
+ final StorageResourceId resourceId, final CreateOptions createOptions) {
+ return storage.blobWriteSession(getBlobInfo(resourceId, createOptions),
+ generateWriteOptions(createOptions));
+ }
+
+ private static BlobWriteOption[] generateWriteOptions(final CreateOptions createOptions) {
+ List<BlobWriteOption> blobWriteOptions = new ArrayList<>();
+
+ blobWriteOptions.add(BlobWriteOption.disableGzipContent());
+ blobWriteOptions.add(BlobWriteOption.generationMatch());
+
+ //TODO: Enable KMS and checksum
+ return blobWriteOptions.toArray(new BlobWriteOption[blobWriteOptions.size()]);
+ }
+
+ @Override
+ public boolean isOpen() {
+ return writableByteChannel != null && writableByteChannel.isOpen();
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ if (!isOpen()) {
+ return;
+ }
+
+ writableByteChannel.close();
+ } catch (Exception e) {
+ throw new IOException(
+ String.format("Upload failed for '%s'. reason=%s", resourceId, e.getMessage()), e);
+ } finally {
+ writableByteChannel = null;
+ }
+ }
+
+ private int writeInternal(final ByteBuffer byteBuffer) throws IOException {
+ int bytesWritten = writableByteChannel.write(byteBuffer);
+ LOG.trace("{} bytes were written out of provided buffer of capacity {}", bytesWritten,
+ byteBuffer.limit());
+ return bytesWritten;
+ }
+
+ @Override
+ public int write(final ByteBuffer src) throws IOException {
+ return writeInternal(src);
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorageFileSystem.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorageFileSystem.java
new file mode 100644
index 00000000000..e411f22eb39
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorageFileSystem.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.*;
+import static org.apache.hadoop.fs.gs.Constants.SCHEME;
+
+import com.google.auth.Credentials;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.channels.WritableByteChannel;
+
+/**
+ * Provides FS semantics over GCS based on Objects API.
+ */
+class GoogleCloudStorageFileSystem {
+ private static final Logger LOG = LoggerFactory.getLogger(StorageResourceId.class);
+
+ // URI of the root path.
+ static final URI GCSROOT = URI.create(SCHEME + ":/");
+
+ // GCS access instance.
+ private GoogleCloudStorage gcs;
+
+ private static GoogleCloudStorage createCloudStorage(
+ final GoogleHadoopFileSystemConfiguration configuration, final Credentials credentials)
+ throws IOException {
+ checkNotNull(configuration, "configuration must not be null");
+
+ return new GoogleCloudStorage(configuration);
+ }
+
+ GoogleCloudStorageFileSystem(final GoogleHadoopFileSystemConfiguration configuration,
+ final Credentials credentials) throws IOException {
+ gcs = createCloudStorage(configuration, credentials);
+ }
+
+ WritableByteChannel create(final URI path, final CreateOptions createOptions)
+ throws IOException {
+ LOG.trace("create(path: {}, createOptions: {})", path, createOptions);
+ checkNotNull(path, "path could not be null");
+ StorageResourceId resourceId =
+ StorageResourceId.fromUriPath(path, /* allowEmptyObjectName=*/ true);
+
+ if (resourceId.isDirectory()) {
+ throw new IOException(
+ String.format("Cannot create a file whose name looks like a directory: '%s'",
+ resourceId));
+ }
+
+ if (createOptions.getOverwriteGenerationId() != StorageResourceId.UNKNOWN_GENERATION_ID) {
+ resourceId = new StorageResourceId(resourceId.getBucketName(), resourceId.getObjectName(),
+ createOptions.getOverwriteGenerationId());
+ }
+
+ return gcs.create(resourceId, createOptions);
+ }
+
+ void close() {
+ if (gcs == null) {
+ return;
+ }
+ LOG.trace("close()");
+ try {
+ gcs.close();
+ } finally {
+ gcs = null;
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorageItemInfo.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorageItemInfo.java
new file mode 100644
index 00000000000..887e68b05f9
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleCloudStorageItemInfo.java
@@ -0,0 +1,423 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument;
+import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
+
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Contains information about an item in Google Cloud Storage.
+ */
+final class GoogleCloudStorageItemInfo {
+ // Info about the root of GCS namespace.
+ public static final GoogleCloudStorageItemInfo ROOT_INFO =
+ new GoogleCloudStorageItemInfo(StorageResourceId.ROOT,
+ /* creationTime= */ 0,
+ /* modificationTime= */ 0,
+ /* size= */ 0,
+ /* location= */ null,
+ /* storageClass= */ null,
+ /* contentType= */ null,
+ /* contentEncoding= */ null,
+ /* metadata= */ null,
+ /* contentGeneration= */ 0,
+ /* metaGeneration= */ 0,
+ /* verificationAttributes= */ null);
+
+ /**
+ * Factory method for creating a GoogleCloudStorageItemInfo for a bucket.
+ *
+ * @param resourceId Resource ID that identifies a bucket
+ * @param creationTime Time when a bucket was created (milliseconds since January 1, 1970
+ * UTC).
+ * @param modificationTime Time when a bucket was last modified (milliseconds since January 1,
+ * 1970 UTC).
+ * @param location Location of a bucket.
+ * @param storageClass Storage class of a bucket.
+ */
+ static GoogleCloudStorageItemInfo createBucket(StorageResourceId resourceId,
+ long creationTime, long modificationTime, String location, String storageClass) {
+ checkNotNull(resourceId, "resourceId must not be null");
+ checkArgument(resourceId.isBucket(), "expected bucket but got '%s'", resourceId);
+ return new GoogleCloudStorageItemInfo(resourceId, creationTime, modificationTime,
+ /* size= */ 0, location, storageClass,
+ /* contentType= */ null,
+ /* contentEncoding= */ null,
+ /* metadata= */ null,
+ /* contentGeneration= */ 0,
+ /* metaGeneration= */ 0,
+ /* verificationAttributes= */ null);
+ }
+
+ /**
+ * Factory method for creating a GoogleCloudStorageItemInfo for an object.
+ *
+ * @param resourceId identifies either root, a Bucket, or a StorageObject
+ * @param creationTime Time when object was created (milliseconds since January 1, 1970
+ * UTC).
+ * @param size Size of the given object (number of bytes) or -1 if the object
+ * does not exist.
+ * @param metadata User-supplied object metadata for this object.
+ */
+ static GoogleCloudStorageItemInfo createObject(StorageResourceId resourceId,
+ long creationTime, long modificationTime, long size, String contentType,
+ String contentEncoding, Map<String, byte[]> metadata, long contentGeneration,
+ long metaGeneration, VerificationAttributes verificationAttributes) {
+ checkNotNull(resourceId, "resourceId must not be null");
+ checkArgument(
+ !resourceId.isRoot(),
+ "expected object or directory but got '%s'", resourceId);
+ checkArgument(
+ !resourceId.isBucket(),
+ "expected object or directory but got '%s'", resourceId);
+ return new GoogleCloudStorageItemInfo(resourceId, creationTime, modificationTime, size,
+ /* location= */ null,
+ /* storageClass= */ null, contentType, contentEncoding, metadata, contentGeneration,
+ metaGeneration, verificationAttributes);
+ }
+
+ /**
+ * Factory method for creating a "found" GoogleCloudStorageItemInfo for an inferred directory.
+ *
+ * @param resourceId Resource ID that identifies an inferred directory
+ */
+ static GoogleCloudStorageItemInfo createInferredDirectory(StorageResourceId resourceId) {
+ return new GoogleCloudStorageItemInfo(resourceId,
+ /* creationTime= */ 0,
+ /* modificationTime= */ 0,
+ /* size= */ 0,
+ /* location= */ null,
+ /* storageClass= */ null,
+ /* contentType= */ null,
+ /* contentEncoding= */ null,
+ /* metadata= */ null,
+ /* contentGeneration= */ 0,
+ /* metaGeneration= */ 0,
+ /* verificationAttributes= */ null);
+ }
+
+ /**
+ * Factory method for creating a "not found" GoogleCloudStorageItemInfo for a bucket or an object.
+ *
+ * @param resourceId Resource ID that identifies an inferred directory
+ */
+ static GoogleCloudStorageItemInfo createNotFound(StorageResourceId resourceId) {
+ return new GoogleCloudStorageItemInfo(resourceId,
+ /* creationTime= */ 0,
+ /* modificationTime= */ 0,
+ /* size= */ -1,
+ /* location= */ null,
+ /* storageClass= */ null,
+ /* contentType= */ null,
+ /* contentEncoding= */ null,
+ /* metadata= */ null,
+ /* contentGeneration= */ 0,
+ /* metaGeneration= */ 0,
+ /* verificationAttributes= */ null);
+ }
+
+ // The Bucket and maybe StorageObject names of the GCS "item" referenced by this object. Not
+ // null.
+ private final StorageResourceId resourceId;
+
+ // Creation time of this item.
+ // Time is expressed as milliseconds since January 1, 1970 UTC.
+ private final long creationTime;
+
+ // Modification time of this item.
+ // Time is expressed as milliseconds since January 1, 1970 UTC.
+ private final long modificationTime;
+
+ // Size of an object (number of bytes).
+ // Size is -1 for items that do not exist.
+ private final long size;
+
+ // Location of this item.
+ private final String location;
+
+ // Storage class of this item.
+ private final String storageClass;
+
+ // Content-Type of this item
+ private final String contentType;
+
+ private final String contentEncoding;
+
+ // User-supplied metadata.
+ private final Map<String, byte[]> metadata;
+
+ private final long contentGeneration;
+
+ private final long metaGeneration;
+
+ private final VerificationAttributes verificationAttributes;
+
+ private GoogleCloudStorageItemInfo(StorageResourceId resourceId, long creationTime,
+ long modificationTime, long size, String location, String storageClass, String contentType,
+ String contentEncoding, Map<String, byte[]> metadata, long contentGeneration,
+ long metaGeneration, VerificationAttributes verificationAttributes) {
+ this.resourceId = checkNotNull(resourceId, "resourceId must not be null");
+ this.creationTime = creationTime;
+ this.modificationTime = modificationTime;
+ this.size = size;
+ this.location = location;
+ this.storageClass = storageClass;
+ this.contentType = contentType;
+ this.contentEncoding = contentEncoding;
+ this.metadata = (metadata == null) ? ImmutableMap.of() : metadata;
+ this.contentGeneration = contentGeneration;
+ this.metaGeneration = metaGeneration;
+ this.verificationAttributes = verificationAttributes;
+ }
+
+ /**
+ * Gets bucket name of this item.
+ */
+ String getBucketName() {
+ return resourceId.getBucketName();
+ }
+
+ /**
+ * Gets object name of this item.
+ */
+ String getObjectName() {
+ return resourceId.getObjectName();
+ }
+
+ /**
+ * Gets the resourceId that holds the (possibly null) bucketName and objectName of this object.
+ */
+ StorageResourceId getResourceId() {
+ return resourceId;
+ }
+
+ /**
+ * Gets creation time of this item.
+ *
+ * <p>Time is expressed as milliseconds since January 1, 1970 UTC.
+ */
+ long getCreationTime() {
+ return creationTime;
+ }
+
+ /**
+ * Gets modification time of this item.
+ *
+ * <p>Time is expressed as milliseconds since January 1, 1970 UTC.
+ */
+ long getModificationTime() {
+ return modificationTime;
+ }
+
+ /**
+ * Gets size of this item (number of bytes). Returns -1 if the object does not exist.
+ */
+ long getSize() {
+ return size;
+ }
+
+ /**
+ * Gets location of this item.
+ *
+ * <p>Note: Location is only supported for buckets. The value is always null for objects.
+ */
+ String getLocation() {
+ return location;
+ }
+
+ /**
+ * Gets storage class of this item.
+ *
+ * <p>Note: Storage-class is only supported for buckets. The value is always null for objects.
+ */
+ String getStorageClass() {
+ return storageClass;
+ }
+
+ /**
+ * Gets the content-type of this item, or null if unknown or inapplicable.
+ *
+ * <p>Note: content-type is only supported for objects, and will always be null for buckets.
+ */
+ String getContentType() {
+ return contentType;
+ }
+
+ /**
+ * Gets the content-encoding of this item, or null if unknown or inapplicable.
+ *
+ * <p>Note: content-encoding is only supported for objects, and will always be null for buckets.
+ */
+ String getContentEncoding() {
+ return contentEncoding;
+ }
+
+ /**
+ * Gets user-supplied metadata for this item.
+ *
+ * <p>Note: metadata is only supported for objects. This value is always an empty map for buckets.
+ */
+ Map<String, byte[]> getMetadata() {
+ return metadata;
+ }
+
+ /**
+ * Indicates whether this item is a bucket. Root is not considered to be a bucket.
+ */
+ boolean isBucket() {
+ return resourceId.isBucket();
+ }
+
+ /**
+ * Indicates whether this item refers to the GCS root (gs://).
+ */
+ boolean isRoot() {
+ return resourceId.isRoot();
+ }
+
+ /**
+ * Indicates whether this instance has information about the unique, shared root of the underlying
+ * storage system.
+ */
+ boolean isGlobalRoot() {
+ return isRoot() && exists();
+ }
+
+ /**
+ * Indicates whether {@code itemInfo} is a directory.
+ */
+ boolean isDirectory() {
+ return isGlobalRoot() || isBucket() || resourceId.isDirectory();
+ }
+
+ /**
+ * Indicates whether {@code itemInfo} is an inferred directory.
+ */
+ boolean isInferredDirectory() {
+ return creationTime == 0 && modificationTime == 0 && size == 0 && contentGeneration == 0
+ && metaGeneration == 0;
+ }
+
+ /**
+ * Get the content generation of the object.
+ */
+ long getContentGeneration() {
+ return contentGeneration;
+ }
+
+ /**
+ * Get the meta generation of the object.
+ */
+ long getMetaGeneration() {
+ return metaGeneration;
+ }
+
+ /**
+ * Get object validation attributes.
+ */
+ VerificationAttributes getVerificationAttributes() {
+ return verificationAttributes;
+ }
+
+ /**
+ * Indicates whether this item exists.
+ */
+ boolean exists() {
+ return size >= 0;
+ }
+
+ /**
+ * Helper for checking logical equality of metadata maps, checking equality of keySet() between
+ * this.metadata and otherMetadata, and then using Arrays.equals to compare contents of
+ * corresponding byte arrays.
+ */
+ @VisibleForTesting
+ public boolean metadataEquals(Map<String, byte[]> otherMetadata) {
+ if (metadata == otherMetadata) {
+ // Fast-path for common cases where the same actual default metadata instance may be
+ // used in
+ // multiple different item infos.
+ return true;
+ }
+ // No need to check if other `metadata` is not null,
+ // because previous `if` checks if both of them are null.
+ if (metadata == null || otherMetadata == null) {
+ return false;
+ }
+ if (!metadata.keySet().equals(otherMetadata.keySet())) {
+ return false;
+ }
+
+ // Compare each byte[] with Arrays.equals.
+ for (Map.Entry<String, byte[]> metadataEntry : metadata.entrySet()) {
+ if (!Arrays.equals(metadataEntry.getValue(), otherMetadata.get(metadataEntry.getKey()))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Gets string representation of this instance.
+ */
+ @Override
+ public String toString() {
+ return exists() ?
+ String.format("%s: created on: %s", resourceId, Instant.ofEpochMilli(creationTime)) :
+ String.format("%s: exists: no", resourceId);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof GoogleCloudStorageItemInfo) {
+ GoogleCloudStorageItemInfo other = (GoogleCloudStorageItemInfo) obj;
+ return resourceId.equals(other.resourceId) && creationTime == other.creationTime
+ && modificationTime == other.modificationTime && size == other.size && Objects.equals(
+ location, other.location) && Objects.equals(storageClass, other.storageClass)
+ && Objects.equals(verificationAttributes, other.verificationAttributes)
+ && metaGeneration == other.metaGeneration && contentGeneration == other.contentGeneration
+ && metadataEquals(other.getMetadata());
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + resourceId.hashCode();
+ result = prime * result + (int) creationTime;
+ result = prime * result + (int) modificationTime;
+ result = prime * result + (int) size;
+ result = prime * result + Objects.hashCode(location);
+ result = prime * result + Objects.hashCode(storageClass);
+ result = prime * result + Objects.hashCode(verificationAttributes);
+ result = prime * result + (int) metaGeneration;
+ result = prime * result + (int) contentGeneration;
+ result = prime * result + metadata.entrySet().stream()
+ .mapToInt(e -> Objects.hash(e.getKey()) + Arrays.hashCode(e.getValue())).sum();
+ return result;
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleHadoopFileSystem.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleHadoopFileSystem.java
new file mode 100644
index 00000000000..1c2fc19d2b5
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleHadoopFileSystem.java
@@ -0,0 +1,505 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import static org.apache.hadoop.fs.gs.Constants.GCS_CONFIG_PREFIX;
+import static org.apache.hadoop.fs.gs.GoogleHadoopFileSystemConfiguration.GCS_WORKING_DIRECTORY;
+
+import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument;
+import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkState;
+import static org.apache.hadoop.thirdparty.com.google.common.base.Strings.isNullOrEmpty;
+
+import com.google.auth.oauth2.GoogleCredentials;
+import org.apache.hadoop.thirdparty.com.google.common.base.Ascii;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.util.EnumSet;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.ProviderUtils;
+import org.apache.hadoop.util.Progressable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * GoogleHadoopFileSystem is rooted in a single bucket at initialization time; in this case, Hadoop
+ * paths no longer correspond directly to general GCS paths, and all Hadoop operations going through
+ * this FileSystem will never touch any GCS bucket other than the bucket on which this FileSystem is
+ * rooted.
+ *
+ * <p>This implementation sacrifices a small amount of cross-bucket interoperability in favor of
+ * more straightforward FileSystem semantics and compatibility with existing Hadoop applications. In
+ * particular, it is not subject to bucket-naming constraints, and files are allowed to be placed in
+ * root.
+ */
+public class GoogleHadoopFileSystem extends FileSystem {
+
+ public static final Logger LOG = LoggerFactory.getLogger(GoogleHadoopFileSystem.class);
+
+ /**
+ * URI scheme for GoogleHadoopFileSystem.
+ */
+ private static final String SCHEME = Constants.SCHEME;
+
+ /**
+ * Default value of replication factor.
+ */
+ static final short REPLICATION_FACTOR_DEFAULT = 3;
+
+ // TODO: Take this from config
+ private static final int PERMISSIONS_TO_REPORT = 700;
+
+ /**
+ * The URI the File System is passed in initialize.
+ */
+ private URI initUri;
+
+ /**
+ * Default block size. Note that this is the size that is reported to Hadoop FS clients. It does
+ * not modify the actual block size of an underlying GCS object, because GCS JSON API does not
+ * allow modifying or querying the value. Modifying this value allows one to control how many
+ * mappers are used to process a given file.
+ */
+ private long defaultBlockSize = GoogleHadoopFileSystemConfiguration.BLOCK_SIZE.getDefault();
+
+ // The bucket the file system is rooted in used for default values of:
+ // -- working directory
+ // -- user home directories (only for Hadoop purposes).
+ private Path fsRoot;
+
+ /**
+ * Current working directory; overridden in initialize() if {@link
+ * GoogleHadoopFileSystemConfiguration#GCS_WORKING_DIRECTORY} is set.
+ */
+ private Path workingDirectory;
+ private GoogleCloudStorageFileSystem gcsFs;
+ private boolean isClosed;
+ private FsPermission reportedPermissions;
+
+ public GoogleHadoopFileSystemConfiguration getFileSystemConfiguration() {
+ return fileSystemConfiguration;
+ }
+
+ private GoogleHadoopFileSystemConfiguration fileSystemConfiguration;
+
+ @Override
+ public void initialize(final URI path, Configuration config) throws IOException {
+ LOG.trace("initialize(path: {}, config: {})", path, config);
+
+ checkArgument(path != null, "path must not be null");
+ checkArgument(config != null, "config must not be null");
+ checkArgument(path.getScheme() != null, "scheme of path must not be null");
+ checkArgument(path.getScheme().equals(getScheme()), "URI scheme not supported: {}", path);
+
+ config =
+ ProviderUtils.excludeIncompatibleCredentialProviders(config, GoogleHadoopFileSystem.class);
+ super.initialize(path, config);
+
+ initUri = path;
+
+ // Set this configuration as the default config for this instance; configure()
+ // will perform some file-system-specific adjustments, but the original should
+ // be sufficient (and is required) for the delegation token binding initialization.
+ setConf(config);
+
+ this.reportedPermissions = new FsPermission(PERMISSIONS_TO_REPORT);
+
+ initializeFsRoot();
+
+ this.fileSystemConfiguration = new GoogleHadoopFileSystemConfiguration(config);
+ initializeWorkingDirectory(fileSystemConfiguration);
+ initializeGcsFs(fileSystemConfiguration);
+ }
+
+ private void initializeFsRoot() {
+ String rootBucket = initUri.getAuthority();
+ checkArgument(rootBucket != null, "No bucket specified in GCS URI: {}", initUri);
+ // Validate root bucket name
+ URI rootUri = UriPaths.fromStringPathComponents(rootBucket, /* objectName= */
+ null, /* allowEmptyObjectName= */ true);
+ fsRoot = new Path(rootUri);
+ LOG.trace("Configured FS root: '{}'", fsRoot);
+ }
+
+ private void initializeWorkingDirectory(final GoogleHadoopFileSystemConfiguration config) {
+ String configWorkingDirectory = config.getWorkingDirectory();
+ if (isNullOrEmpty(configWorkingDirectory)) {
+ LOG.warn("No working directory configured, using default: '{}'", workingDirectory);
+ }
+ // Use the public method to ensure proper behavior of normalizing and resolving the new
+ // working directory relative to the initial filesystem-root directory.
+ setWorkingDirectory(
+ isNullOrEmpty(configWorkingDirectory) ? fsRoot : new Path(configWorkingDirectory));
+ LOG.trace("Configured working directory: {} = {}", GCS_WORKING_DIRECTORY.getKey(),
+ getWorkingDirectory());
+ }
+
+ private synchronized void initializeGcsFs(final GoogleHadoopFileSystemConfiguration config)
+ throws IOException {
+ this.gcsFs = createGcsFs(config);
+ }
+
+ private GoogleCloudStorageFileSystem createGcsFs(final GoogleHadoopFileSystemConfiguration config)
+ throws IOException {
+ GoogleCredentials credentials = getCredentials(config);
+ return new GoogleCloudStorageFileSystem(config, credentials);
+ }
+
+ private GoogleCredentials getCredentials(GoogleHadoopFileSystemConfiguration config)
+ throws IOException {
+ return getCredentials(config, GCS_CONFIG_PREFIX);
+ }
+
+ public static GoogleCredentials getCredentials(GoogleHadoopFileSystemConfiguration config,
+ String... keyPrefixesVararg) throws IOException {
+ return GoogleCredentials.getApplicationDefault(); // TODO: Add other Auth mechanisms
+ }
+
+ @Override
+ protected void checkPath(final Path path) {
+ LOG.trace("checkPath(path: {})", path);
+ // Validate scheme
+ URI uri = path.toUri();
+
+ String scheme = uri.getScheme();
+ if (scheme != null && !scheme.equalsIgnoreCase(getScheme())) {
+ throw new IllegalArgumentException(
+ String.format("Wrong scheme: {}, in path: {}, expected scheme: {}", scheme, path,
+ getScheme()));
+ }
+
+ String bucket = uri.getAuthority();
+ String rootBucket = fsRoot.toUri().getAuthority();
+
+ // Bucket-less URIs will be qualified later
+ if (bucket == null || bucket.equals(rootBucket)) {
+ return;
+ }
+
+ throw new IllegalArgumentException(
+ String.format("Wrong bucket: {}, in path: {}, expected bucket: {}", bucket, path,
+ rootBucket));
+ }
+
+ /**
+ * Validates that GCS path belongs to this file system. The bucket must match the root bucket
+ * provided at initialization time.
+ */
+ Path getHadoopPath(final URI gcsPath) {
+ LOG.trace("getHadoopPath(gcsPath: {})", gcsPath);
+
+ // Handle root. Delegate to getGcsPath on "gs:/" to resolve the appropriate gs://<bucket> URI.
+ if (gcsPath.equals(getGcsPath(fsRoot))) {
+ return fsRoot;
+ }
+
+ StorageResourceId resourceId = StorageResourceId.fromUriPath(gcsPath, true);
+
+ checkArgument(!resourceId.isRoot(), "Missing authority in gcsPath '{}'", gcsPath);
+ String rootBucket = fsRoot.toUri().getAuthority();
+ checkArgument(resourceId.getBucketName().equals(rootBucket),
+ "Authority of URI '{}' doesn't match root bucket '{}'", resourceId.getBucketName(),
+ rootBucket);
+
+ Path hadoopPath = new Path(fsRoot,
+ new Path(/* schema= */ null, /* authority= */ null, resourceId.getObjectName()));
+ LOG.trace("getHadoopPath(gcsPath: {}): {}", gcsPath, hadoopPath);
+ return hadoopPath;
+ }
+
+ /**
+ * Translates a "gs:/" style hadoopPath (or relative path which is not fully-qualified) into the
+ * appropriate GCS path which is compatible with the underlying GcsFs.
+ */
+ URI getGcsPath(final Path hadoopPath) {
+ LOG.trace("getGcsPath(hadoopPath: {})", hadoopPath);
+
+ // Convert to fully qualified absolute path; the Path object will call back to get our current
+ // workingDirectory as part of fully resolving the path.
+ Path resolvedPath = makeQualified(hadoopPath);
+
+ String objectName = resolvedPath.toUri().getPath();
+ if (objectName != null && resolvedPath.isAbsolute()) {
+ // Strip off leading '/' because GoogleCloudStorageFileSystem.getPath appends it explicitly
+ // between bucket and objectName.
+ objectName = objectName.substring(1);
+ }
+
+ // Construct GCS path URI
+ String rootBucket = fsRoot.toUri().getAuthority();
+ URI gcsPath =
+ UriPaths.fromStringPathComponents(rootBucket, objectName, /* allowEmptyObjectName= */ true);
+ LOG.trace("getGcsPath(hadoopPath: {}): {}", hadoopPath, gcsPath);
+ return gcsPath;
+ }
+
+ @Override
+ public String getScheme() {
+ return SCHEME;
+ }
+
+ @Override
+ public FSDataInputStream open(final Path path, final int bufferSize) throws IOException {
+ LOG.trace("open({})", path);
+ throw new UnsupportedOperationException(path.toString());
+ }
+
+ @Override
+ public FSDataOutputStream create(Path hadoopPath, FsPermission permission, boolean overwrite,
+ int bufferSize, short replication, long blockSize, Progressable progress) throws IOException {
+ checkArgument(hadoopPath != null, "hadoopPath must not be null");
+ checkArgument(replication > 0, "replication must be a positive integer: %s", replication);
+ checkArgument(blockSize > 0, "blockSize must be a positive integer: %s", blockSize);
+
+ System.out.println(String.format("create(%s)", hadoopPath));
+ checkOpen();
+
+ LOG.trace("create(hadoopPath: {}, overwrite: {}, bufferSize: {} [ignored])", hadoopPath,
+ overwrite, bufferSize);
+
+ CreateOptions.WriteMode writeMode =
+ overwrite ? CreateOptions.WriteMode.OVERWRITE : CreateOptions.WriteMode.CREATE_NEW;
+ FSDataOutputStream response = new FSDataOutputStream(
+ new GoogleHadoopOutputStream(this, getGcsPath(hadoopPath),
+ CreateOptions.builder().setWriteMode(writeMode).build(), statistics), statistics);
+
+ return response;
+ }
+
+ @Override
+ public FSDataOutputStream createNonRecursive(Path hadoopPath, FsPermission permission,
+ EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize,
+ Progressable progress) throws IOException {
+ throw new UnsupportedOperationException(hadoopPath.toString());
+ }
+
+ @Override
+ public FSDataOutputStream append(final Path path, final int i, final Progressable progressable)
+ throws IOException {
+ throw new UnsupportedOperationException(path.toString());
+ }
+
+ @Override
+ public boolean rename(final Path path, final Path path1) throws IOException {
+ LOG.trace("rename({}, {})", path, path1);
+ throw new UnsupportedOperationException(path.toString());
+ }
+
+ @Override
+ public boolean delete(final Path path, final boolean recursive) throws IOException {
+ LOG.trace("delete({}, {})", path, recursive);
+ throw new UnsupportedOperationException(path.toString());
+ }
+
+ @Override
+ public FileStatus[] listStatus(final Path path) throws FileNotFoundException, IOException {
+ checkArgument(path != null, "hadoopPath must not be null");
+
+ checkOpen();
+
+ LOG.trace("listStatus(hadoopPath: {})", path);
+ throw new UnsupportedOperationException(path.toString());
+ }
+
+ /**
+ * Overridden to make root its own parent. This is POSIX compliant, but more importantly guards
+ * against poor directory accounting in the PathData class of Hadoop 2's FsShell.
+ */
+ @Override
+ public Path makeQualified(final Path path) {
+ Path qualifiedPath = super.makeQualified(path);
+
+ URI uri = qualifiedPath.toUri();
+
+ checkState("".equals(uri.getPath()) || qualifiedPath.isAbsolute(),
+ "Path '{}' must be fully qualified.", qualifiedPath);
+
+ Path result;
+ String upath = uri.getPath();
+
+ // Strip initial '..'s to make root is its own parent.
+ int i = 0;
+ while (upath.startsWith("/../", i)) {
+ // Leave a preceding slash, so path is still absolute.
+ i += 3;
+ }
+ if (i == upath.length() || upath.substring(i).equals("/..")) {
+ // Allow a Path of gs://someBucket to map to gs://someBucket/
+ result = new Path(uri.getScheme(), uri.getAuthority(), "/");
+ } else if (i == 0) {
+ result = qualifiedPath;
+ } else {
+ result = new Path(uri.getScheme(), uri.getAuthority(), upath.substring(i));
+ }
+
+ LOG.trace("makeQualified(path: {}): {}", path, result);
+ return result;
+ }
+
+ /**
+ * Returns a URI of the root of this FileSystem.
+ */
+ @Override
+ public URI getUri() {
+ return fsRoot.toUri();
+ }
+
+ /**
+ * The default port is listed as -1 as an indication that ports are not used.
+ */
+ @Override
+ protected int getDefaultPort() {
+ int result = -1;
+ LOG.trace("getDefaultPort(): %d", result);
+ return result;
+ }
+
+ @Override
+ public boolean hasPathCapability(final Path path, final String capability) {
+ checkNotNull(path, "path must not be null");
+ checkArgument(!isNullOrEmpty(capability), "capability must not be null or empty string for {}",
+ path);
+ switch (Ascii.toLowerCase(capability)) {
+ case CommonPathCapabilities.FS_APPEND:
+ case CommonPathCapabilities.FS_CONCAT:
+ return false;
+ default:
+ return false;
+ }
+ }
+
+ /**
+ * Gets the current working directory.
+ *
+ * @return The current working directory.
+ */
+ @Override
+ public Path getWorkingDirectory() {
+ LOG.trace("getWorkingDirectory(): {}", workingDirectory);
+ return workingDirectory;
+ }
+
+ @Override
+ public boolean mkdirs(final Path path, final FsPermission fsPermission) throws IOException {
+ LOG.trace("mkdirs({})", path);
+ throw new UnsupportedOperationException(path.toString());
+ }
+
+// /**
+// * Gets the default replication factor.
+// */
+// @Override
+// public short getDefaultReplication() {
+// return REPLICATION_FACTOR_DEFAULT;
+// }
+
+ @Override
+ public FileStatus getFileStatus(final Path path) throws IOException {
+ checkArgument(path != null, "path must not be null");
+
+ checkOpen();
+
+ URI gcsPath = getGcsPath(path);
+
+ LOG.trace("getFileStatus(): {}", gcsPath);
+
+ throw new UnsupportedOperationException(path.toString());
+ }
+
+ /**
+ * Returns home directory of the current user.
+ *
+ * <p>Note: This directory is only used for Hadoop purposes. It is not the same as a user's OS
+ * home directory.
+ */
+ @Override
+ public Path getHomeDirectory() {
+ Path result = new Path(fsRoot, "user/" + System.getProperty("user.name"));
+ LOG.trace("getHomeDirectory(): {}", result);
+ return result;
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * <p>Returns the service if delegation tokens are configured, otherwise, null.
+ */
+ @Override
+ public String getCanonicalServiceName() {
+ // TODO: Add delegation token support
+ return null;
+ }
+
+ /**
+ * Gets GCS FS instance.
+ */
+ GoogleCloudStorageFileSystem getGcsFs() {
+ return gcsFs;
+ }
+
+ /**
+ * Assert that the FileSystem has been initialized and not close()d.
+ */
+ private void checkOpen() throws IOException {
+ if (isClosed) {
+ throw new IOException("GoogleHadoopFileSystem has been closed or not initialized.");
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ LOG.trace("close()");
+ if (isClosed) {
+ return;
+ }
+
+ super.close();
+
+ getGcsFs().close();
+
+ this.isClosed = true;
+ }
+
+ @Override
+ public long getUsed() throws IOException {
+ long result = super.getUsed();
+ LOG.trace("getUsed(): {}", result);
+ return result;
+ }
+
+// @Override
+// public long getDefaultBlockSize() {
+// LOG.trace("getDefaultBlockSize(): {}", defaultBlockSize);
+// return defaultBlockSize;
+// }
+
+ @Override
+ public void setWorkingDirectory(final Path hadoopPath) {
+ checkArgument(hadoopPath != null, "hadoopPath must not be null");
+ URI gcsPath = UriPaths.toDirectory(getGcsPath(hadoopPath));
+ workingDirectory = getHadoopPath(gcsPath);
+ LOG.trace("setWorkingDirectory(hadoopPath: {}): {}", hadoopPath, workingDirectory);
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleHadoopFileSystemConfiguration.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleHadoopFileSystemConfiguration.java
new file mode 100644
index 00000000000..16d940b16f4
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleHadoopFileSystemConfiguration.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import static java.lang.Math.toIntExact;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * This class provides a configuration for the {@link GoogleHadoopFileSystem} implementations.
+ */
+class GoogleHadoopFileSystemConfiguration {
+ /**
+ * Configuration key for default block size of a file.
+ *
+ * <p>Note that this is the size that is reported to Hadoop FS clients. It does not modify the
+ * actual block size of an underlying GCS object, because GCS JSON API does not allow modifying or
+ * querying the value. Modifying this value allows one to control how many mappers are used to
+ * process a given file.
+ */
+ public static final HadoopConfigurationProperty<Long> BLOCK_SIZE =
+ new HadoopConfigurationProperty<>("fs.gs.block.size", 64 * 1024 * 1024L);
+
+ /**
+ * Configuration key for GCS project ID. Default value: none
+ */
+ public static final HadoopConfigurationProperty<String> GCS_PROJECT_ID =
+ new HadoopConfigurationProperty<>("fs.gs.project.id");
+
+ /**
+ * Configuration key for initial working directory of a GHFS instance. Default value: '/'
+ */
+ public static final HadoopConfigurationProperty<String> GCS_WORKING_DIRECTORY =
+ new HadoopConfigurationProperty<>("fs.gs.working.dir", "/");
+
+ /**
+ * Configuration key for setting write buffer size.
+ */
+ public static final HadoopConfigurationProperty<Long> GCS_OUTPUT_STREAM_BUFFER_SIZE =
+ new HadoopConfigurationProperty<>("fs.gs.outputstream.buffer.size", 8L * 1024 * 1024);
+
+ private final String workingDirectory;
+
+ public int getOutStreamBufferSize() {
+ return outStreamBufferSize;
+ }
+
+ private final int outStreamBufferSize;
+
+ GoogleHadoopFileSystemConfiguration(Configuration config) {
+ this.workingDirectory = GCS_WORKING_DIRECTORY.get(config, config::get);
+ this.outStreamBufferSize =
+ toIntExact(GCS_OUTPUT_STREAM_BUFFER_SIZE.get(config, config::getLongBytes));
+ }
+
+ public String getWorkingDirectory() {
+ return this.workingDirectory;
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleHadoopOutputStream.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleHadoopOutputStream.java
new file mode 100644
index 00000000000..747d9f001c5
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/GoogleHadoopOutputStream.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.channels.Channels;
+import java.nio.channels.ClosedChannelException;
+import java.nio.channels.WritableByteChannel;
+import javax.annotation.Nonnull;
+
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileSystem;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class GoogleHadoopOutputStream extends OutputStream {
+ public static final Logger LOG = LoggerFactory.getLogger(StorageResourceId.class);
+
+ private final GoogleHadoopFileSystem ghfs;
+
+ // Path of the file to write to.
+ private final URI dstGcsPath;
+
+ private OutputStream outputStream;
+
+ // Statistics tracker provided by the parent GoogleHadoopFileSystem for recording
+ // numbers of bytes written.
+ private final FileSystem.Statistics statistics;
+
+ /**
+ * Constructs an instance of GoogleHadoopOutputStream object.
+ *
+ * @param ghfs Instance of {@link GoogleHadoopFileSystem}.
+ * @param dstGcsPath Path of the file to write to.
+ * @param statistics File system statistics object.
+ * @param createFileOptions options for file creation
+ * @throws IOException if an IO error occurs.
+ */
+ GoogleHadoopOutputStream(GoogleHadoopFileSystem ghfs, URI dstGcsPath,
+ CreateOptions createFileOptions, FileSystem.Statistics statistics) throws IOException {
+ LOG.trace("GoogleHadoopOutputStream(gcsPath: {}, createFileOptions: {})", dstGcsPath,
+ createFileOptions);
+ this.ghfs = ghfs;
+ this.dstGcsPath = dstGcsPath;
+ this.statistics = statistics;
+
+ this.outputStream = createOutputStream(ghfs.getGcsFs(), dstGcsPath, createFileOptions,
+ ghfs.getFileSystemConfiguration());
+ }
+
+ private static OutputStream createOutputStream(GoogleCloudStorageFileSystem gcsfs, URI gcsPath,
+ CreateOptions options, GoogleHadoopFileSystemConfiguration fileSystemConfiguration)
+ throws IOException {
+ WritableByteChannel channel;
+ try {
+ channel = gcsfs.create(gcsPath, options);
+ } catch (java.nio.file.FileAlreadyExistsException e) {
+
+ throw (FileAlreadyExistsException) new FileAlreadyExistsException(
+ String.format("'%s' already exists", gcsPath)).initCause(e);
+ }
+ OutputStream outputStream = Channels.newOutputStream(channel);
+ int bufferSize = fileSystemConfiguration.getOutStreamBufferSize();
+ return bufferSize > 0 ? new BufferedOutputStream(outputStream, bufferSize) : outputStream;
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ throwIfNotOpen();
+ outputStream.write(b);
+ statistics.incrementBytesWritten(1);
+ statistics.incrementWriteOps(1);
+ }
+
+ @Override
+ public void write(@Nonnull byte[] b, int offset, int len) throws IOException {
+ throwIfNotOpen();
+ outputStream.write(b, offset, len);
+ statistics.incrementBytesWritten(len);
+ statistics.incrementWriteOps(1);
+ }
+
+ @Override
+ public void close() throws IOException {
+ LOG.trace("close(): final destination: {}", dstGcsPath);
+
+ if (outputStream == null) {
+ LOG.trace("close(): Ignoring; stream already closed.");
+ return;
+ }
+
+ try {
+ outputStream.close();
+ } finally {
+ outputStream = null;
+ }
+ }
+
+ private void throwIfNotOpen() throws IOException {
+ if (outputStream == null) {
+ throw new ClosedChannelException();
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/HadoopConfigurationProperty.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/HadoopConfigurationProperty.java
new file mode 100644
index 00000000000..9360290a09c
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/HadoopConfigurationProperty.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
+
+import java.util.List;
+import java.util.function.BiFunction;
+
+import org.apache.hadoop.conf.Configuration;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Hadoop configuration property.
+ */
+class HadoopConfigurationProperty<T> {
+ private static final Logger LOG = LoggerFactory.getLogger(HadoopConfigurationProperty.class);
+
+ private final String key;
+ private final List<String> deprecatedKeys;
+ private final T defaultValue;
+
+ private List<String> keyPrefixes = ImmutableList.of("");
+
+ HadoopConfigurationProperty(String key) {
+ this(key, null);
+ }
+
+ HadoopConfigurationProperty(String key, T defaultValue, String... deprecatedKeys) {
+ this.key = key;
+ this.deprecatedKeys =
+ deprecatedKeys == null ? ImmutableList.of() : ImmutableList.copyOf(deprecatedKeys);
+ this.defaultValue = defaultValue;
+ }
+
+ String getKey() {
+ return key;
+ }
+
+ T getDefault() {
+ return defaultValue;
+ }
+
+ T get(Configuration config, BiFunction<String, T, T> getterFn) {
+ String lookupKey = getLookupKey(config, key, (c, k) -> c.get(k) != null);
+ return logProperty(lookupKey, getterFn.apply(lookupKey, defaultValue));
+ }
+
+ private String getLookupKey(Configuration config, String lookupKey,
+ BiFunction<Configuration, String, Boolean> checkFn) {
+ for (String prefix : keyPrefixes) {
+ String prefixedKey = prefix + lookupKey;
+ if (checkFn.apply(config, prefixedKey)) {
+ return prefixedKey;
+ }
+ for (String deprecatedKey : deprecatedKeys) {
+ String prefixedDeprecatedKey = prefix + deprecatedKey;
+ if (checkFn.apply(config, prefixedDeprecatedKey)) {
+ LOG.warn("Using deprecated key '{}', use '{}' key instead.", prefixedDeprecatedKey,
+ prefixedKey);
+ return prefixedDeprecatedKey;
+ }
+ }
+ }
+ return keyPrefixes.get(0) + lookupKey;
+ }
+
+ private static <S> S logProperty(String key, S value) {
+ LOG.trace("{} = {}", key, value);
+ return value;
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/StorageResourceId.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/StorageResourceId.java
new file mode 100644
index 00000000000..5935564feed
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/StorageResourceId.java
@@ -0,0 +1,328 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument;
+import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.hadoop.thirdparty.com.google.common.base.Strings.isNullOrEmpty;
+import static org.apache.hadoop.fs.gs.Constants.SCHEME;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URI;
+import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Data struct representing either a GCS StorageObject, a GCS Bucket or the GCS root (gs://). If
+ * both bucketName and objectName are null, the StorageResourceId refers to GCS root (gs://). If
+ * bucketName is non-null, and objectName is null, then this refers to a GCS Bucket. Otherwise, if
+ * bucketName and objectName are both non-null, this refers to a GCS StorageObject.
+ */
+class StorageResourceId {
+
+ private static final Logger LOG = LoggerFactory.getLogger(StorageResourceId.class);
+
+ // The generationId used to denote "unknown"; if given to a method expecting generationId
+ // constraints, the method may perform extra low-level GETs to determine an existing
+ // generationId
+ // if idempotency constraints require doing so.
+ static final long UNKNOWN_GENERATION_ID = -1L;
+
+ // Pattern that parses out bucket and object names.
+ // Given 'gs://foo-bucket/foo/bar/baz', matcher.group(x) will return:
+ // 0 = gs://foo-bucket/foo/bar/baz
+ // 1 = foo-bucket/foo/bar/baz
+ // 2 = foo-bucket
+ // 3 = /foo/bar/baz
+ // 4 = foo/bar/baz
+ // Groups 2 and 4 can be used to create an instance.
+ private static final Pattern GCS_PATH_PATTERN = Pattern.compile("gs://(([^/]+)(/(.+)?)?)?");
+
+ // The singleton instance identifying the GCS root (gs://). Both getObjectName() and
+ // getBucketName() will return null.
+ static final StorageResourceId ROOT = new StorageResourceId();
+
+ // Bucket name of this storage resource to be used with the Google Cloud Storage API.
+ private final String bucketName;
+
+ // Object name of this storage resource to be used with the Google Cloud Storage API.
+ private final String objectName;
+
+ // Human-readable String to be returned by toString(); kept as 'final' member for efficiency.
+ private final String stringPath;
+
+ // The generationId to be used with precondition checks when using this StorageResourceId
+ // as an identifier for mutation requests.
+ private final long generationId;
+
+ /**
+ * Constructor for a StorageResourceId that refers to the GCS root (gs://). Private because all
+ * external users should just use the singleton StorageResourceId.ROOT.
+ */
+ private StorageResourceId() {
+ this.bucketName = null;
+ this.objectName = null;
+ this.stringPath = StringPaths.fromComponents(bucketName, objectName);
+ this.generationId = UNKNOWN_GENERATION_ID;
+ }
+
+ /**
+ * Constructor for a StorageResourceId representing a Bucket; {@code getObjectName()} will return
+ * null for a StorageResourceId that represents a Bucket.
+ *
+ * @param bucketName The bucket name of the resource. Must be non-empty and non-null.
+ */
+ StorageResourceId(String bucketName) {
+ checkArgument(!isNullOrEmpty(bucketName), "bucketName must not be null or empty");
+
+ this.bucketName = bucketName;
+ this.objectName = null;
+ this.stringPath = StringPaths.fromComponents(bucketName, objectName);
+ this.generationId = UNKNOWN_GENERATION_ID;
+ }
+
+ /**
+ * Constructor for a StorageResourceId representing a full StorageObject, including bucketName and
+ * objectName.
+ *
+ * @param bucketName The bucket name of the resource. Must be non-empty and non-null.
+ * @param objectName The object name of the resource. Must be non-empty and non-null.
+ */
+ StorageResourceId(String bucketName, String objectName) {
+ checkArgument(!isNullOrEmpty(bucketName), "bucketName must not be null or empty");
+ checkArgument(!isNullOrEmpty(objectName), "objectName must not be null or empty");
+
+ this.bucketName = bucketName;
+ this.objectName = objectName;
+ this.stringPath = StringPaths.fromComponents(bucketName, objectName);
+ this.generationId = UNKNOWN_GENERATION_ID;
+ }
+
+ /**
+ * Constructor for a StorageResourceId representing a full StorageObject, including bucketName and
+ * objectName.
+ *
+ * @param bucketName The bucket name of the resource. Must be non-empty and non-null.
+ * @param objectName The object name of the resource. Must be non-empty and non-null.
+ * @param generationId The generationId to be used with precondition checks when using this
+ * StorageResourceId as an identifier for mutation requests.
+ */
+ StorageResourceId(String bucketName, String objectName, long generationId) {
+ checkArgument(!isNullOrEmpty(bucketName), "bucketName must not be null or empty");
+ checkArgument(!isNullOrEmpty(objectName), "objectName must not be null or empty");
+
+ this.bucketName = bucketName;
+ this.objectName = objectName;
+ this.stringPath = StringPaths.fromComponents(bucketName, objectName);
+ this.generationId = generationId;
+ }
+
+ /**
+ * Constructor for a StorageResourceId representing a full StorageObject, including bucketName and
+ * objectName.
+ *
+ * @param bucketName The bucket name of the resource. Must be non-empty and non-null.
+ * @param generationId The generationId to be used with precondition checks when using this
+ * StorageResourceId as an identifier for mutation requests.
+ */
+ StorageResourceId(String bucketName, long generationId) {
+ checkArgument(!isNullOrEmpty(bucketName), "bucketName must not be null or empty");
+ this.bucketName = bucketName;
+ this.objectName = null;
+ this.stringPath = StringPaths.fromComponents(bucketName, objectName);
+ this.generationId = generationId;
+ }
+
+ /**
+ * Returns true if this StorageResourceId represents a GCS StorageObject; if true, both {@code
+ * getBucketName} and {@code getObjectName} will be non-empty and non-null.
+ */
+ boolean isStorageObject() {
+ return bucketName != null && objectName != null;
+ }
+
+ /**
+ * Returns true if this StorageResourceId represents a GCS Bucket; if true, then {@code
+ * getObjectName} will return null.
+ */
+ boolean isBucket() {
+ return bucketName != null && objectName == null;
+ }
+
+ /**
+ * Returns true if this StorageResourceId represents the GCS root (gs://); if true, then both
+ * {@code getBucketName} and {@code getObjectName} will be null.
+ */
+ boolean isRoot() {
+ return bucketName == null && objectName == null;
+ }
+
+ /**
+ * Indicates if this StorageResourceId corresponds to a 'directory'; similar to {@link
+ * FileInfo#isDirectory} except deals entirely with pathnames instead of also checking for
+ * exists() to be true on a corresponding GoogleCloudStorageItemInfo.
+ */
+ boolean isDirectory() {
+ return isRoot() || isBucket() || StringPaths.isDirectoryPath(objectName);
+ }
+
+ /**
+ * Gets the bucket name component of this resource identifier.
+ */
+ String getBucketName() {
+ return bucketName;
+ }
+
+ /**
+ * Gets the object name component of this resource identifier.
+ */
+ String getObjectName() {
+ return objectName;
+ }
+
+ /**
+ * The generationId to be used with precondition checks when using this StorageResourceId as an
+ * identifier for mutation requests. The generationId is *not* used when determining equals() or
+ * hashCode().
+ */
+ long getGenerationId() {
+ return generationId;
+ }
+
+ /**
+ * Returns true if generationId is not UNKNOWN_GENERATION_ID.
+ */
+ boolean hasGenerationId() {
+ return generationId != UNKNOWN_GENERATION_ID;
+ }
+
+ /**
+ * Returns a string of the form {@code gs://<bucketName>/<objectName>}.
+ */
+ @Override
+ public String toString() {
+ return stringPath;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof StorageResourceId) {
+ StorageResourceId other = (StorageResourceId) obj;
+ return Objects.equals(bucketName, other.bucketName) && Objects.equals(objectName,
+ other.objectName);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return stringPath.hashCode();
+ }
+
+ /**
+ * Converts StorageResourceId instance to look like a directory path. If the path already looks
+ * like a directory path then this call is a no-op.
+ *
+ * @return A resourceId with a directory path corresponding to the given resourceId.
+ */
+ StorageResourceId toDirectoryId() {
+ if (isStorageObject() && !StringPaths.isDirectoryPath(getObjectName())) {
+ return new StorageResourceId(getBucketName(), StringPaths.toDirectoryPath(getObjectName()));
+ }
+ return this;
+ }
+
+ /**
+ * Parses {@link StorageResourceId} from specified string.
+ */
+ static StorageResourceId fromStringPath(String path) {
+ return fromStringPath(path, UNKNOWN_GENERATION_ID);
+ }
+
+ /**
+ * Parses {@link StorageResourceId} from specified string and generationId.
+ */
+ static StorageResourceId fromStringPath(String path, long generationId) {
+ checkArgument(path != null, "path must not be null");
+
+ Matcher matcher = GCS_PATH_PATTERN.matcher(path);
+ checkArgument(matcher.matches(), "'%s' is not a valid GCS object name.", path);
+
+ String bucketName = matcher.group(2);
+ String relativePath = matcher.group(4);
+ if (bucketName == null) {
+ checkArgument(generationId == UNKNOWN_GENERATION_ID,
+ "Cannot specify generationId '%s' for root object '%s'", generationId, path);
+ return ROOT;
+ } else if (relativePath != null) {
+ return new StorageResourceId(bucketName, relativePath, generationId);
+ }
+ checkArgument(generationId == UNKNOWN_GENERATION_ID,
+ "Cannot specify generationId '%s' for bucket '%s'", generationId, path);
+ return new StorageResourceId(bucketName);
+ }
+
+ /**
+ * Validates the given URI and if valid, returns the associated StorageResourceId.
+ *
+ * @param path The GCS URI to validate.
+ * @param allowEmptyObjectName If true, a missing object name is not considered invalid.
+ * @return a StorageResourceId that may be the GCS root, a Bucket, or a StorageObject.
+ */
+ static StorageResourceId fromUriPath(URI path, boolean allowEmptyObjectName) {
+ return fromUriPath(path, allowEmptyObjectName, UNKNOWN_GENERATION_ID);
+ }
+
+ /**
+ * Validates the given URI and if valid, returns the associated StorageResourceId.
+ *
+ * @param path The GCS URI to validate.
+ * @param allowEmptyObjectName If true, a missing object name is not considered invalid.
+ * @param generationId The generationId to be used with precondition checks when
+ * using this
+ * @return a StorageResourceId that may be the GCS root, a Bucket, or a StorageObject.
+ */
+ static StorageResourceId fromUriPath(URI path, boolean allowEmptyObjectName,
+ long generationId) {
+ LOG.trace("fromUriPath('{}', {})", path, allowEmptyObjectName);
+ checkNotNull(path);
+
+ if (!SCHEME.equals(path.getScheme())) {
+ throw new IllegalArgumentException(
+ String.format("GCS path supports only '%s' scheme, instead got '%s' from '%s'.", SCHEME,
+ path.getScheme(), path));
+ }
+
+ if (path.equals(GoogleCloudStorageFileSystem.GCSROOT)) {
+ return ROOT;
+ }
+
+ String bucketName = StringPaths.validateBucketName(path.getAuthority());
+ // Note that we're using getPath here instead of rawPath, etc. This is because it is assumed
+ // that the path was properly encoded in getPath (or another similar method):
+ String objectName = StringPaths.validateObjectName(path.getPath(), allowEmptyObjectName);
+
+ return isNullOrEmpty(objectName) ?
+ new StorageResourceId(bucketName, generationId) :
+ new StorageResourceId(bucketName, objectName, generationId);
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/StringPaths.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/StringPaths.java
new file mode 100644
index 00000000000..80682c3ed2a
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/StringPaths.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import static org.apache.hadoop.thirdparty.com.google.common.base.Strings.isNullOrEmpty;
+import static org.apache.hadoop.fs.gs.Constants.PATH_DELIMITER;
+
+import org.apache.hadoop.thirdparty.com.google.common.base.CharMatcher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility methods for String GCS paths.
+ */
+final class StringPaths {
+
+ public static final Logger LOG = LoggerFactory.getLogger(StorageResourceId.class);
+
+ private StringPaths() {
+ }
+
+ // 14x faster (20ns vs 280ns) than "^[a-z0-9_.-]+$" regex
+ private static final CharMatcher BUCKET_NAME_CHAR_MATCHER = CharMatcher.ascii()
+ .and(CharMatcher.inRange('0', '9').or(CharMatcher.inRange('a', 'z'))
+ .or(CharMatcher.anyOf("_.-")))
+ .precomputed();
+
+ /**
+ * Validate the given bucket name to make sure that it can be used as a part of a file system
+ * path.
+ *
+ * <p>Note: this is not designed to duplicate the exact checks that GCS would perform on the
+ * server side. We make some checks that are relevant to using GCS as a file system.
+ *
+ * @param bucketName Bucket name to check.
+ */
+ static String validateBucketName(String bucketName) {
+ // If the name ends with '/', remove it.
+ bucketName = toFilePath(bucketName);
+
+ if (isNullOrEmpty(bucketName)) {
+ throw new IllegalArgumentException("GCS bucket name cannot be empty.");
+ }
+
+ if (!BUCKET_NAME_CHAR_MATCHER.matchesAllOf(bucketName)) {
+ throw new IllegalArgumentException(String.format(
+ "Invalid GCS bucket name '%s': bucket name must contain only 'a-z0-9_.-' characters.",
+ bucketName));
+ }
+
+ return bucketName;
+ }
+
+ /**
+ * Validate the given object name to make sure that it can be used as a part of a file system
+ * path.
+ *
+ * <p>Note: this is not designed to duplicate the exact checks that GCS would perform on the
+ * server side. We make some checks that are relevant to using GCS as a file system.
+ *
+ * @param objectName Object name to check.
+ * @param allowEmptyObjectName If true, a missing object name is not considered invalid.
+ */
+ static String validateObjectName(String objectName, boolean allowEmptyObjectName) {
+ LOG.trace("validateObjectName('{}', {})", objectName, allowEmptyObjectName);
+
+ if (isNullOrEmpty(objectName) || objectName.equals(PATH_DELIMITER)) {
+ if (allowEmptyObjectName) {
+ objectName = "";
+ } else {
+ throw new IllegalArgumentException(String.format(
+ "GCS path must include non-empty object name [objectName='%s',"
+ + " allowEmptyObjectName=%s]", objectName, allowEmptyObjectName));
+ }
+ }
+
+ // We want objectName to look like a traditional file system path,
+ // therefore, disallow objectName with consecutive '/' chars.
+ for (int i = 0; i < (objectName.length() - 1); i++) {
+ if (objectName.charAt(i) == '/' && objectName.charAt(i + 1) == '/') {
+ throw new IllegalArgumentException(
+ String.format("GCS path must not have consecutive '/' characters: '%s'", objectName));
+ }
+ }
+
+ // Remove leading '/' if it exists.
+ if (objectName.startsWith(PATH_DELIMITER)) {
+ objectName = objectName.substring(1);
+ }
+
+ LOG.trace("validateObjectName -> '{}'", objectName);
+ return objectName;
+ }
+
+ /**
+ * Helper for standardizing the way various human-readable messages in logs/exceptions that refer
+ * to a bucket/object pair.
+ */
+ public static String fromComponents(String bucketName, String objectName) {
+ if (bucketName == null && objectName != null) {
+ throw new IllegalArgumentException(
+ String.format("Invalid bucketName/objectName pair: gs://%s/%s", "<null>", objectName));
+ }
+ // TODO(user): Unify this method with other methods that convert bucketName/objectName
+ // to a URI; maybe use the single slash for compatibility.
+ StringBuilder result = new StringBuilder("gs://");
+ if (bucketName != null) {
+ result.append(bucketName);
+ }
+ if (objectName != null) {
+ result.append('/').append(objectName);
+ }
+ return result.toString();
+ }
+
+ /**
+ * Indicates whether the given object name looks like a directory path.
+ *
+ * @param path Name of the object to inspect.
+ * @return Whether the given object name looks like a directory path.
+ */
+ public static boolean isDirectoryPath(String path) {
+ return !isNullOrEmpty(path) && path.endsWith(PATH_DELIMITER);
+ }
+
+ /**
+ * Converts the given object name to look like a file path. If the object name already looks like
+ * a file path then this call is a no-op.
+ *
+ * <p>If the object name is null or empty, it is returned as-is.
+ *
+ * @param path Name of the object to inspect.
+ * @return File path for the given path.
+ */
+ public static String toFilePath(String path) {
+ return !isNullOrEmpty(path) && isDirectoryPath(path) ?
+ path.substring(0, path.length() - 1) :
+ path;
+ }
+
+ /**
+ * Converts the given object name to look like a directory path. If the object name already looks
+ * like a directory path then this call is a no-op.
+ *
+ * <p>If the object name is null or empty, it is returned as-is.
+ *
+ * @param path Name of the object to inspect.
+ * @return Directory path for the given path.
+ */
+ static String toDirectoryPath(String path) {
+ return isNullOrEmpty(path) || isDirectoryPath(path) ? path : path + PATH_DELIMITER;
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/UriPaths.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/UriPaths.java
new file mode 100644
index 00000000000..30e13cb33cb
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/UriPaths.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.hadoop.fs.gs.Constants.PATH_DELIMITER;
+import static org.apache.hadoop.fs.gs.Constants.SCHEME;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+
+/**
+ * Utility methods for URI GCS paths.
+ */
+final class UriPaths {
+
+ private UriPaths() {
+ }
+
+ /**
+ * Converts the given path to look like a directory path. If the path already looks like a
+ * directory path then this call is a no-op.
+ *
+ * @param path Path to convert.
+ * @return Directory path for the given path.
+ */
+ public static URI toDirectory(URI path) {
+ StorageResourceId resourceId = StorageResourceId.fromUriPath(path, true);
+
+ if (resourceId.isStorageObject() && !resourceId.isDirectory()) {
+ resourceId = resourceId.toDirectoryId();
+ path = fromResourceId(resourceId, /* allowEmptyObjectName= */ false);
+ }
+ return path;
+ }
+
+ /**
+ * Gets the parent directory of the given path.
+ *
+ * @param path Path to convert.
+ * @return Path of parent directory of the given item or null for root path.
+ */
+ public static URI getParentPath(URI path) {
+ checkNotNull(path);
+
+ // Root path has no parent.
+ if (path.equals(GoogleCloudStorageFileSystem.GCSROOT)) {
+ return null;
+ }
+
+ StorageResourceId resourceId = StorageResourceId.fromUriPath(path, true);
+
+ if (resourceId.isBucket()) {
+ return GoogleCloudStorageFileSystem.GCSROOT;
+ }
+
+ String objectName = resourceId.getObjectName();
+ int index = StringPaths.isDirectoryPath(objectName) ?
+ objectName.lastIndexOf(PATH_DELIMITER, objectName.length() - 2) :
+ objectName.lastIndexOf(PATH_DELIMITER);
+ return index < 0 ?
+ fromStringPathComponents(resourceId.getBucketName(), /* objectName= */
+ null, /* allowEmptyObjectName= */ true) :
+ fromStringPathComponents(resourceId.getBucketName(), objectName.substring(0, index + 1),
+ /* allowEmptyObjectName= */ false);
+ }
+
+ /**
+ * Constructs and returns full path for the given bucket and object names.
+ */
+ public static URI fromResourceId(StorageResourceId resourceId, boolean allowEmptyObjectName) {
+ return fromStringPathComponents(resourceId.getBucketName(), resourceId.getObjectName(),
+ allowEmptyObjectName);
+ }
+
+ /**
+ * Constructs and returns full path for the given bucket and object names.
+ */
+ public static URI fromStringPathComponents(String bucketName, String objectName,
+ boolean allowEmptyObjectName) {
+ if (allowEmptyObjectName && bucketName == null && objectName == null) {
+ return GoogleCloudStorageFileSystem.GCSROOT;
+ }
+
+ String authority = StringPaths.validateBucketName(bucketName);
+ String path = PATH_DELIMITER + StringPaths.validateObjectName(objectName, allowEmptyObjectName);
+
+ try {
+ return new URI(SCHEME, authority, path,
+ /* query= */ null,
+ /* fragment= */ null);
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException(
+ String.format("Invalid bucket name (%s) or object name (%s)", bucketName, objectName), e);
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/VerificationAttributes.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/VerificationAttributes.java
new file mode 100644
index 00000000000..4155482fc7d
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/VerificationAttributes.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2016 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import java.util.Arrays;
+import javax.annotation.Nullable;
+
+/**
+ * GCS provided validation attributes for a single object.
+ */
+class VerificationAttributes {
+ private final byte[] md5hash;
+ private final byte[] crc32c;
+
+ VerificationAttributes(@Nullable byte[] md5hash, @Nullable byte[] crc32c) {
+ this.md5hash = md5hash;
+ this.crc32c = crc32c;
+ }
+
+ /**
+ * MD5 hash of an object, if available.
+ */
+ @Nullable
+ byte[] getMd5hash() {
+ return md5hash;
+ }
+
+ /**
+ * CRC32c checksum of an object, if available.
+ */
+ @Nullable
+ byte[] getCrc32c() {
+ return crc32c;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof VerificationAttributes)) {
+ return false;
+ }
+ VerificationAttributes that = (VerificationAttributes) o;
+ return Arrays.equals(md5hash, that.md5hash) && Arrays.equals(crc32c, that.crc32c);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = Arrays.hashCode(md5hash);
+ result = 31 * result + Arrays.hashCode(crc32c);
+ return result;
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/package-info.java b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/package-info.java
new file mode 100644
index 00000000000..eedfb7822ac
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/main/java/org/apache/hadoop/fs/gs/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Google Cloud Storage Filesystem.
+ */
+
+package org.apache.hadoop.fs.gs;
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/TestStorageResourceId.java b/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/TestStorageResourceId.java
new file mode 100644
index 00000000000..e0a39b2d7e4
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/TestStorageResourceId.java
@@ -0,0 +1,285 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import java.net.URI;
+
+import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestStorageResourceId {
+ @Test
+ public void testConstructorInvalid() {
+ assertThrows(IllegalArgumentException.class, () -> {
+ new StorageResourceId(null);
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ new StorageResourceId("");
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ new StorageResourceId(null, null);
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ new StorageResourceId("foo", null);
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ new StorageResourceId("", null);
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ new StorageResourceId(null, null, 0L);
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ new StorageResourceId("foo", null, 0L);
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ new StorageResourceId("", null, 0L);
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ new StorageResourceId(null, 0L);
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ new StorageResourceId("", 0L);
+ });
+ }
+
+ @Test
+ public void testFromStringPathInvalid() {
+ assertThrows(IllegalArgumentException.class, () -> {
+ StorageResourceId.fromStringPath(null);
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ StorageResourceId.fromStringPath("");
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ StorageResourceId.fromStringPath("foo");
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ StorageResourceId.fromStringPath("/foo/bar");
+ });
+
+ assertThrows(IllegalArgumentException.class, () -> {
+ StorageResourceId.fromStringPath("gs:///foo/bar");
+ });
+ }
+
+ @Test
+ public void testConstructor() {
+ String bucketName = "testbucketname";
+ String objectName = "a/b/c.txt";
+
+ verify(new StorageResourceId(bucketName), bucketName,
+ StorageResourceId.UNKNOWN_GENERATION_ID, null, false,
+ true, true, false, false);
+
+ verify(new StorageResourceId(bucketName, objectName), bucketName,
+ StorageResourceId.UNKNOWN_GENERATION_ID, objectName, false,
+ false, false, true, false);
+
+ long genId = System.currentTimeMillis();
+ verify(new StorageResourceId(bucketName, objectName, genId), bucketName,
+ genId, objectName, true,
+ false, false, true, false);
+
+ verify(new StorageResourceId(bucketName, genId), bucketName,
+ genId, null, true,
+ true, true, false, false);
+ }
+
+ @Test
+ public void testEqualsBucket() {
+ StorageResourceId bucket1 = new StorageResourceId("test-bucket");
+ StorageResourceId bucket2 = new StorageResourceId("test-bucket");
+ assertTrue(bucket1.equals(bucket2));
+ assertEquals(bucket1.hashCode(), bucket2.hashCode());
+ }
+
+ @Test
+ public void testEqualsObject() {
+ StorageResourceId obj1 = new StorageResourceId("test-bucket", "test-object");
+ StorageResourceId obj2 = new StorageResourceId("test-bucket", "test-object");
+ assertTrue(obj1.equals(obj2));
+ assertEquals(obj1.hashCode(), obj2.hashCode());
+ }
+
+ @Test
+ public void testEqualsDifferentBucket() {
+ StorageResourceId bucket1 = new StorageResourceId("test-bucket");
+ StorageResourceId bucket2 = new StorageResourceId("other-bucket");
+ assertFalse(bucket1.equals(bucket2));
+ }
+
+ @Test
+ public void testEqualsDifferentObject() {
+ StorageResourceId obj1 = new StorageResourceId("test-bucket", "test-object");
+ StorageResourceId obj2 = new StorageResourceId("test-bucket", "other-object");
+ assertFalse(obj1.equals(obj2));
+ }
+
+ @Test
+ public void testToDirectoryIdFromFile() {
+ StorageResourceId fileId = new StorageResourceId("my-bucket", "path/to/file.txt");
+ StorageResourceId dirId = fileId.toDirectoryId();
+
+ assertNotSame(fileId, dirId); // Should return a new instance
+ assertTrue(dirId.isDirectory());
+ assertEquals("my-bucket", dirId.getBucketName());
+ assertEquals("path/to/file.txt/", dirId.getObjectName());
+ assertEquals(fileId.getGenerationId(), dirId.getGenerationId());
+ }
+
+ @Test
+ public void testToDirectoryIdFromDirectoryObject() {
+ StorageResourceId dirIdOriginal = new StorageResourceId("my-bucket", "path/to/dir/");
+ StorageResourceId dirIdConverted = dirIdOriginal.toDirectoryId();
+
+ assertSame(dirIdOriginal, dirIdConverted); // Should return the same instance
+ assertTrue(dirIdConverted.isDirectory());
+ assertEquals("path/to/dir/", dirIdConverted.getObjectName());
+ }
+
+ @Test
+ public void testToDirectoryIdFromBucket() {
+ StorageResourceId bucketId = new StorageResourceId("my-bucket");
+ StorageResourceId convertedId = bucketId.toDirectoryId();
+ assertSame(bucketId, convertedId);
+ assertTrue(convertedId.isBucket());
+ }
+
+ @Test
+ public void testFromStringPathRoot() {
+ StorageResourceId id = StorageResourceId.fromStringPath("gs://");
+ assertTrue(id.isRoot());
+ }
+
+ @Test
+ public void testFromStringPathBucket() {
+ StorageResourceId id = StorageResourceId.fromStringPath("gs://my-bucket");
+ assertTrue(id.isBucket());
+ assertEquals("my-bucket", id.getBucketName());
+ assertNull(id.getObjectName());
+ assertEquals(StorageResourceId.UNKNOWN_GENERATION_ID, id.getGenerationId());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "gs://my-bucket/object",
+ "gs://my-bucket/folder/file.txt",
+ "gs://my-bucket/folder/"
+ })
+ public void testFromStringPathObject(String path) {
+ String expectedBucket = path.split("/")[2];
+ String expectedObject =
+ path.substring(path.indexOf(expectedBucket) + expectedBucket.length() + 1);
+
+ StorageResourceId id = StorageResourceId.fromStringPath(path);
+ assertTrue(id.isStorageObject());
+ assertEquals(expectedBucket, id.getBucketName());
+ assertEquals(expectedObject, id.getObjectName());
+ assertEquals(StorageResourceId.UNKNOWN_GENERATION_ID, id.getGenerationId());
+ }
+
+ @Test
+ public void testFromStringPathObjectWithGenerationId() {
+ long genId = 12345L;
+ StorageResourceId id = StorageResourceId.fromStringPath("gs://my-bucket/object.txt", genId);
+ assertTrue(id.isStorageObject());
+ assertEquals("my-bucket", id.getBucketName());
+ assertEquals("object.txt", id.getObjectName());
+ assertEquals(genId, id.getGenerationId());
+ assertTrue(id.hasGenerationId());
+ }
+
+ @Test
+ public void testFromUriPathBucket() throws Exception {
+ URI uri = new URI("gs://my-bucket");
+ StorageResourceId id = StorageResourceId.fromUriPath(uri, true);
+ assertTrue(id.isBucket());
+ assertEquals("my-bucket", id.getBucketName());
+ assertNull(id.getObjectName());
+ }
+
+ @Test
+ public void testFromUriPathObject() throws Exception {
+ URI uri = new URI("gs://my-bucket/path/to/file.txt");
+ StorageResourceId id = StorageResourceId.fromUriPath(uri, false);
+ assertTrue(id.isStorageObject());
+ assertEquals("my-bucket", id.getBucketName());
+ assertEquals("path/to/file.txt", id.getObjectName());
+ }
+
+ @Test
+ public void testFromUriPathObjectWithGenerationId() throws Exception {
+ URI uri = new URI("gs://my-bucket/object.txt");
+ long genId = 54321L;
+ StorageResourceId id = StorageResourceId.fromUriPath(uri, false, genId);
+ assertTrue(id.isStorageObject());
+ assertEquals("my-bucket", id.getBucketName());
+ assertEquals("object.txt", id.getObjectName());
+ assertEquals(genId, id.getGenerationId());
+ assertTrue(id.hasGenerationId());
+ }
+
+ @Test
+ public void testFromUriPathBucketWithGenerationId() throws Exception {
+ assertThrows(IllegalArgumentException.class, () -> {
+ URI uri = new URI("gs://my-bucket");
+ long genId = 54321L;
+ StorageResourceId.fromUriPath(uri, false, genId);
+ });
+ }
+
+ private static void verify(
+ StorageResourceId id,
+ String bucketName,
+ long generationId,
+ String objectName,
+ boolean hasGenerationId,
+ boolean isBucket,
+ boolean isDirectory,
+ boolean isStorageObject,
+ boolean isRoot) {
+ assertEquals(bucketName, id.getBucketName());
+ assertEquals(generationId, id.getGenerationId());
+ assertEquals(objectName, id.getObjectName());
+ assertEquals(hasGenerationId, id.hasGenerationId());
+ assertEquals(isBucket, id.isBucket());
+ assertEquals(isDirectory, id.isDirectory());
+ assertEquals(isStorageObject, id.isStorageObject());
+ assertEquals(isRoot, id.isRoot());
+ }
+}
diff --git a/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/TestStringPaths.java b/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/TestStringPaths.java
new file mode 100644
index 00000000000..16234e0ce1d
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/TestStringPaths.java
@@ -0,0 +1,164 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import org.junit.Test;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+public class TestStringPaths {
+ @Test
+ public void testValidateBucketNameValid() {
+ assertEquals("my-bucket", StringPaths.validateBucketName("my-bucket"));
+ assertEquals("my.bucket", StringPaths.validateBucketName("my.bucket"));
+ assertEquals("my_bucket", StringPaths.validateBucketName("my_bucket"));
+ assertEquals("bucket123", StringPaths.validateBucketName("bucket123"));
+ assertEquals("a", StringPaths.validateBucketName("a"));
+ assertEquals("long-bucket-name-with-numbers-123",
+ StringPaths.validateBucketName("long-bucket-name-with-numbers-123"));
+ }
+
+ @Test
+ public void testValidateBucketNameEndsWithSlash() {
+ assertEquals("my-bucket", StringPaths.validateBucketName("my-bucket/"));
+ assertEquals("another-bucket", StringPaths.validateBucketName("another-bucket/"));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testValidateBucketNameEmpty() {
+ StringPaths.validateBucketName("");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testValidateBucketNameNull() {
+ StringPaths.validateBucketName(null);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testValidateBucketNameInvalidChars() {
+ StringPaths.validateBucketName("my bucket"); // Space
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testValidateBucketNameInvalidChars2() {
+ StringPaths.validateBucketName("my@bucket"); // @ symbol
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testValidateBucketNameUpperCase() {
+ StringPaths.validateBucketName("MyBucket"); // Uppercase
+ }
+
+ @Test
+ public void testValidateObjectNameValid() {
+ assertEquals("path/to/object",
+ StringPaths.validateObjectName("path/to/object", false));
+ assertEquals("object", StringPaths.validateObjectName("object", false));
+ assertEquals("dir/",
+ StringPaths.validateObjectName("dir/", false)); // Still valid after validation
+ assertEquals("", StringPaths.validateObjectName("/", true)); // Slash becomes empty if allowed
+ assertEquals("", StringPaths.validateObjectName("", true));
+ }
+
+ @Test
+ public void testValidateObjectNameLeadingSlash() {
+ assertEquals("path/to/object", StringPaths.validateObjectName("/path/to/object", false));
+ assertEquals("object", StringPaths.validateObjectName("/object", false));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testValidateObjectNameEmptyNotAllowed() {
+ StringPaths.validateObjectName("", false);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testValidateObjectNameNullNotAllowed() {
+ StringPaths.validateObjectName(null, false);
+ }
+
+ @Test
+ public void testValidateObjectNameEmptyAllowed() {
+ assertEquals("", StringPaths.validateObjectName("", true));
+ assertEquals("", StringPaths.validateObjectName(null, true));
+ assertEquals("", StringPaths.validateObjectName("/", true)); // Single slash becomes empty
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testValidateObjectNameConsecutiveSlashes() {
+ StringPaths.validateObjectName("path//to/object", false);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testValidateObjectNameConsecutiveSlashesAtStart() {
+ StringPaths.validateObjectName("//path/to/object", false);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testValidateObjectNameConsecutiveSlashesAtEnd() {
+ StringPaths.validateObjectName("path/to/object//", false);
+ }
+
+ @Test
+ public void testFromComponentsValid() {
+ assertEquals("gs://my-bucket/path/to/object",
+ StringPaths.fromComponents("my-bucket", "path/to/object"));
+ assertEquals("gs://my-bucket/dir/", StringPaths.fromComponents("my-bucket", "dir/"));
+ assertEquals("gs://my-bucket/", StringPaths.fromComponents("my-bucket", ""));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testFromComponentsNullBucketNonNullObject() {
+ StringPaths.fromComponents(null, "path/to/object");
+ }
+
+ @Test
+ public void testFromComponentsNullBucketAndObject() {
+ assertEquals("gs://", StringPaths.fromComponents(null, null));
+ }
+
+ @Test
+ public void testIsDirectoryPath() {
+ assertTrue(StringPaths.isDirectoryPath("dir/"));
+ assertTrue(StringPaths.isDirectoryPath("path/to/dir/"));
+ assertFalse(StringPaths.isDirectoryPath("file.txt"));
+ assertFalse(StringPaths.isDirectoryPath("path/to/file.txt"));
+ assertFalse(StringPaths.isDirectoryPath(""));
+ assertFalse(StringPaths.isDirectoryPath(null));
+ }
+
+ @Test
+ public void testToFilePath() {
+ assertEquals("path/to/file", StringPaths.toFilePath("path/to/file/"));
+ assertEquals("file.txt", StringPaths.toFilePath("file.txt"));
+ assertEquals("dir", StringPaths.toFilePath("dir/"));
+ assertEquals("", StringPaths.toFilePath(""));
+ assertNull(StringPaths.toFilePath(null));
+ }
+
+ // --- Tests for toDirectoryPath ---
+
+ @Test
+ public void testToDirectoryPath() {
+ assertEquals("path/to/dir/", StringPaths.toDirectoryPath("path/to/dir"));
+ assertEquals("dir/", StringPaths.toDirectoryPath("dir/"));
+ assertEquals("file/", StringPaths.toDirectoryPath("file"));
+ assertEquals("", StringPaths.toDirectoryPath(""));
+ assertNull(StringPaths.toDirectoryPath(null));
+ }
+}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/TestUriPaths.java b/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/TestUriPaths.java
new file mode 100644
index 00000000000..fe93a28dc43
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/TestUriPaths.java
@@ -0,0 +1,150 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.gs;
+
+import java.net.URI;
+
+import org.junit.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class TestUriPaths {
+ @Test
+ public void testToDirectoryFile() throws Exception {
+ URI fileUri = new URI("gs://my-bucket/path/to/file.txt");
+ URI expectedDirUri = new URI("gs://my-bucket/path/to/file.txt/");
+ // Temporarily override the behavior for testing purposes
+ // This is not a clean mocking strategy for static methods, but demonstrates the test intent.
+ // In a real environment, you'd use PowerMock or refactor.
+ URI result = UriPaths.toDirectory(fileUri);
+ assertEquals(expectedDirUri, result);
+ }
+
+ @Test
+ public void testToDirectoryAlreadyDirectory() throws Exception {
+ URI dirUri = new URI("gs://my-bucket/path/to/dir/");
+ URI result = UriPaths.toDirectory(dirUri);
+ assertEquals(dirUri, result);
+ }
+
+ @Test
+ public void testToDirectoryRootBucket() throws Exception {
+ URI bucketUri = new URI("gs://my-bucket");
+ URI result = UriPaths.toDirectory(bucketUri);
+ assertEquals(bucketUri, result); // Buckets are implicitly directories
+ }
+
+ @Test
+ public void testGetParentPathFile() throws Exception {
+ URI uri = new URI("gs://my-bucket/path/to/file.txt");
+ URI expectedParent = new URI("gs://my-bucket/path/to/");
+ assertEquals(expectedParent, UriPaths.getParentPath(uri));
+ }
+
+ @Test
+ public void testGetParentPathDirectory() throws Exception {
+ URI uri = new URI("gs://my-bucket/path/to/dir/");
+ URI expectedParent = new URI("gs://my-bucket/path/to/");
+ assertEquals(expectedParent, UriPaths.getParentPath(uri));
+ }
+
+ @Test
+ public void testGetParentPathObjectAtBucketRoot() throws Exception {
+ URI uri = new URI("gs://my-bucket/file.txt");
+ URI expectedParent = new URI("gs://my-bucket/");
+ assertEquals(expectedParent, UriPaths.getParentPath(uri));
+ }
+
+ @Test
+ public void testGetParentPathDirectoryAtBucketRoot() throws Exception {
+ URI uri = new URI("gs://my-bucket/dir/");
+ URI expectedParent = new URI("gs://my-bucket/");
+ assertEquals(expectedParent, UriPaths.getParentPath(uri));
+ }
+
+ @Test
+ public void testGetParentPathBucket() throws Exception {
+ URI uri = new URI("gs://my-bucket");
+ assertEquals(GoogleCloudStorageFileSystem.GCSROOT, UriPaths.getParentPath(uri));
+ }
+
+ @Test
+ public void testFromResourceIdObject() throws Exception {
+ StorageResourceId resourceId = new StorageResourceId("my-bucket", "path/to/object");
+ URI expectedUri = new URI("gs://my-bucket/path/to/object");
+ assertEquals(expectedUri, UriPaths.fromResourceId(resourceId, false));
+ }
+
+ @Test
+ public void testFromResourceIdDirectory() throws Exception {
+ StorageResourceId resourceId = new StorageResourceId("my-bucket", "path/to/dir/");
+ URI expectedUri = new URI("gs://my-bucket/path/to/dir/");
+ assertEquals(expectedUri, UriPaths.fromResourceId(resourceId, false));
+ }
+
+ @Test
+ public void testFromResourceIdBucket() throws Exception {
+ StorageResourceId resourceId = new StorageResourceId("my-bucket");
+ URI expectedUri = new URI("gs://my-bucket/");
+ assertEquals(expectedUri, UriPaths.fromResourceId(resourceId, true));
+ }
+
+ @Test
+ public void testFromResourceIdEmptyObjectAllowed() throws Exception {
+ StorageResourceId resourceId = new StorageResourceId("my-bucket");
+ URI expectedUri = new URI("gs://my-bucket/");
+ assertEquals(expectedUri, UriPaths.fromResourceId(resourceId, true));
+ }
+
+ @Test
+ public void testFromResourceIdNullObjectAllowed() throws Exception {
+ StorageResourceId resourceId = new StorageResourceId("my-bucket");
+ URI expectedUri = new URI("gs://my-bucket/");
+ assertEquals(expectedUri, UriPaths.fromResourceId(resourceId, true));
+ }
+
+ @Test
+ public void testFromStringPathComponentsValid() throws Exception {
+ assertEquals(new URI("gs://my-bucket/path/to/object"),
+ UriPaths.fromStringPathComponents("my-bucket", "path/to/object", false));
+ assertEquals(new URI("gs://my-bucket/path/to/dir/"),
+ UriPaths.fromStringPathComponents("my-bucket", "path/to/dir/", false));
+ assertEquals(new URI("gs://my-bucket/"),
+ UriPaths.fromStringPathComponents("my-bucket", null, true));
+ assertEquals(new URI("gs://my-bucket/"),
+ UriPaths.fromStringPathComponents("my-bucket", "", true));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testFromStringPathComponentsNullBucketNameNotAllowed() {
+ UriPaths.fromStringPathComponents(null, "object", false);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testFromStringPathComponentsEmptyObjectNameNotAllowed() {
+ UriPaths.fromStringPathComponents("my-bucket", "", false);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testFromStringPathComponentsConsecutiveSlashes() {
+ UriPaths.fromStringPathComponents("my-bucket", "path//to/object", false);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testFromStringPathComponentsInvalidBucketName() {
+ UriPaths.fromStringPathComponents("MyBucket", "object", false); // Uppercase
+ }
+}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/package-info.java b/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/package-info.java
new file mode 100644
index 00000000000..fe289cc6d3d
--- /dev/null
+++ b/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Google Cloud Storage Filesystem tests.
+ */
+package org.apache.hadoop.fs.gs;
\ No newline at end of file
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index 8c1256a177c..70df9b02d0f 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -53,6 +53,7 @@
<module>hadoop-fs2img</module>
<module>hadoop-benchmark</module>
<module>hadoop-compat-bench</module>
+ <module>hadoop-gcp</module>
</modules>
<build>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org