You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by pv...@apache.org on 2018/05/09 16:19:28 UTC
nifi git commit: NIFI-5156: Updated GCP SDK to latest version
Repository: nifi
Updated Branches:
refs/heads/master 7a4990e7f -> f742a3a6a
NIFI-5156: Updated GCP SDK to latest version
Signed-off-by: Pierre Villard <pi...@gmail.com>
This closes #2680.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/f742a3a6
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/f742a3a6
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/f742a3a6
Branch: refs/heads/master
Commit: f742a3a6accd7695a32b1e4c42c2ec93619a7b9a
Parents: 7a4990e
Author: zenfenan <si...@gmail.com>
Authored: Sat May 5 14:45:40 2018 +0530
Committer: Pierre Villard <pi...@gmail.com>
Committed: Wed May 9 18:19:17 2018 +0200
----------------------------------------------------------------------
.../nifi-gcp-bundle/nifi-gcp-processors/pom.xml | 16 +------------
.../processors/gcp/AbstractGCPProcessor.java | 5 ++--
.../gcp/storage/AbstractGCSProcessor.java | 24 ++++++++------------
.../processors/gcp/storage/ListGCSBucket.java | 2 +-
.../MockCredentialsFactoryProcessor.java | 6 ++---
.../MockCredentialsServiceProcessor.java | 6 ++---
.../processors/gcp/storage/AbstractGCSIT.java | 6 ++---
.../processors/gcp/storage/AbstractGCSTest.java | 5 +---
.../gcp/storage/FetchGCSObjectTest.java | 5 ----
.../gcp/storage/ListGCSBucketTest.java | 3 ++-
nifi-nar-bundles/nifi-gcp-bundle/pom.xml | 16 +++++++++++++
11 files changed, 40 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/f742a3a6/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml
index c9119b8..eeb757b 100644
--- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml
@@ -63,21 +63,12 @@
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
- <artifactId>google-cloud</artifactId>
- <version>0.8.0</version>
+ <artifactId>google-cloud-storage</artifactId>
<exclusions>
- <exclusion> <!-- NIFI-3089 -->
- <groupId>org.json</groupId>
- <artifactId>json</artifactId>
- </exclusion>
<exclusion>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</exclusion>
- <exclusion> <!-- NIFI-5124 -->
- <groupId>commons-fileupload</groupId>
- <artifactId>commons-fileupload</artifactId>
- </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -85,11 +76,6 @@
<artifactId>json</artifactId>
<version>1.8</version>
</dependency>
- <dependency>
- <groupId>commons-fileupload</groupId>
- <artifactId>commons-fileupload</artifactId>
- <version>1.3.3</version>
- </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/nifi/blob/f742a3a6/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/AbstractGCPProcessor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/AbstractGCPProcessor.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/AbstractGCPProcessor.java
index 5f51194..0505d63 100644
--- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/AbstractGCPProcessor.java
+++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/AbstractGCPProcessor.java
@@ -17,8 +17,8 @@
package org.apache.nifi.processors.gcp;
import com.google.auth.oauth2.GoogleCredentials;
-import com.google.cloud.HttpServiceOptions;
import com.google.cloud.Service;
+import com.google.cloud.ServiceOptions;
import com.google.common.collect.ImmutableList;
import org.apache.nifi.annotation.lifecycle.OnScheduled;
import org.apache.nifi.components.PropertyDescriptor;
@@ -36,8 +36,7 @@ import java.util.List;
*/
public abstract class AbstractGCPProcessor<
CloudService extends Service<CloudServiceOptions>,
- CloudServiceRpc,
- CloudServiceOptions extends HttpServiceOptions<CloudService, CloudServiceRpc, CloudServiceOptions>> extends AbstractProcessor {
+ CloudServiceOptions extends ServiceOptions<CloudService, CloudServiceOptions>> extends AbstractProcessor {
public static final PropertyDescriptor PROJECT_ID = new PropertyDescriptor
.Builder().name("gcp-project-id")
http://git-wip-us.apache.org/repos/asf/nifi/blob/f742a3a6/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/AbstractGCSProcessor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/AbstractGCSProcessor.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/AbstractGCSProcessor.java
index 7cc854d..bed596b 100644
--- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/AbstractGCSProcessor.java
+++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/AbstractGCSProcessor.java
@@ -18,12 +18,12 @@ package org.apache.nifi.processors.gcp.storage;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.api.gax.retrying.RetrySettings;
import com.google.auth.http.HttpTransportFactory;
import com.google.auth.oauth2.GoogleCredentials;
-import com.google.cloud.RetryParams;
+import com.google.cloud.http.HttpTransportOptions;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageOptions;
-import com.google.cloud.storage.spi.StorageRpc;
import com.google.common.collect.ImmutableList;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.processor.ProcessContext;
@@ -44,7 +44,7 @@ import java.util.Set;
*
* Every GCS processor operation requires a bucket, whether it's reading or writing from said bucket.
*/
-public abstract class AbstractGCSProcessor extends AbstractGCPProcessor<Storage, StorageRpc, StorageOptions> {
+public abstract class AbstractGCSProcessor extends AbstractGCPProcessor<Storage, StorageOptions> {
public static final Relationship REL_SUCCESS =
new Relationship.Builder().name("success")
.description("FlowFiles are routed to this relationship after a successful Google Cloud Storage operation.")
@@ -80,25 +80,19 @@ public abstract class AbstractGCSProcessor extends AbstractGCPProcessor<Storage,
StorageOptions.Builder storageOptionsBuilder = StorageOptions.newBuilder()
.setCredentials(credentials)
.setProjectId(projectId)
- .setRetryParams(RetryParams.newBuilder()
- .setRetryMaxAttempts(retryCount)
- .setRetryMinAttempts(retryCount)
+ .setRetrySettings(RetrySettings.newBuilder()
+ .setMaxAttempts(retryCount)
.build());
if (!StringUtils.isBlank(proxyHost) && proxyPort > 0) {
- storageOptionsBuilder.setHttpTransportFactory(new HttpTransportFactory() {
+ storageOptionsBuilder.setTransportOptions(HttpTransportOptions.newBuilder().setHttpTransportFactory(new HttpTransportFactory() {
@Override
public HttpTransport create() {
- final HttpTransport transport = new NetHttpTransport.Builder()
- .setProxy(
- new Proxy(
- Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort)
- )
- )
+ return new NetHttpTransport.Builder()
+ .setProxy(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort)))
.build();
- return transport;
}
- });
+ }).build());
}
return storageOptionsBuilder.build();
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/f742a3a6/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/ListGCSBucket.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/ListGCSBucket.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/ListGCSBucket.java
index 6bb3812..46b48fb 100644
--- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/ListGCSBucket.java
+++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/ListGCSBucket.java
@@ -16,7 +16,7 @@
*/
package org.apache.nifi.processors.gcp.storage;
-import com.google.cloud.Page;
+import com.google.api.gax.paging.Page;
import com.google.cloud.storage.Acl;
import com.google.cloud.storage.Blob;
import com.google.cloud.storage.BlobInfo;
http://git-wip-us.apache.org/repos/asf/nifi/blob/f742a3a6/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/factory/MockCredentialsFactoryProcessor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/factory/MockCredentialsFactoryProcessor.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/factory/MockCredentialsFactoryProcessor.java
index 0734c40..84614e9 100644
--- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/factory/MockCredentialsFactoryProcessor.java
+++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/factory/MockCredentialsFactoryProcessor.java
@@ -17,8 +17,8 @@
package org.apache.nifi.processors.gcp.credentials.factory;
import com.google.auth.oauth2.GoogleCredentials;
-import com.google.cloud.HttpServiceOptions;
import com.google.cloud.Service;
+import com.google.cloud.ServiceOptions;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
@@ -53,8 +53,8 @@ public class MockCredentialsFactoryProcessor extends AbstractGCPProcessor {
}
@Override
- protected HttpServiceOptions getServiceOptions(ProcessContext context, GoogleCredentials credentials) {
- HttpServiceOptions mockOptions = mock(HttpServiceOptions.class);
+ protected ServiceOptions getServiceOptions(ProcessContext context, GoogleCredentials credentials) {
+ ServiceOptions mockOptions = mock(ServiceOptions.class);
Service mockService = mock(Service.class);
when(mockOptions.getService()).thenReturn(mockService);
http://git-wip-us.apache.org/repos/asf/nifi/blob/f742a3a6/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/service/MockCredentialsServiceProcessor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/service/MockCredentialsServiceProcessor.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/service/MockCredentialsServiceProcessor.java
index 36c71d6..e351e4e 100644
--- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/service/MockCredentialsServiceProcessor.java
+++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/service/MockCredentialsServiceProcessor.java
@@ -17,8 +17,8 @@
package org.apache.nifi.processors.gcp.credentials.service;
import com.google.auth.oauth2.GoogleCredentials;
-import com.google.cloud.HttpServiceOptions;
import com.google.cloud.Service;
+import com.google.cloud.ServiceOptions;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
@@ -43,8 +43,8 @@ public class MockCredentialsServiceProcessor extends AbstractGCPProcessor {
}
@Override
- protected HttpServiceOptions getServiceOptions(ProcessContext context, GoogleCredentials credentials) {
- HttpServiceOptions mockOptions = mock(HttpServiceOptions.class);
+ protected ServiceOptions getServiceOptions(ProcessContext context, GoogleCredentials credentials) {
+ ServiceOptions mockOptions = mock(ServiceOptions.class);
Service mockService = mock(Service.class);
when(mockOptions.getService()).thenReturn(mockService);
http://git-wip-us.apache.org/repos/asf/nifi/blob/f742a3a6/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSIT.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSIT.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSIT.java
index 725f824..d66df78 100644
--- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSIT.java
+++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSIT.java
@@ -33,7 +33,6 @@ import org.junit.BeforeClass;
import org.junit.experimental.categories.Category;
import java.util.Arrays;
-import java.util.Iterator;
import static org.junit.Assert.fail;
@@ -78,10 +77,9 @@ public abstract class AbstractGCSIT {
public static void tearDown() {
try {
// Empty the bucket before deleting it.
- Iterator<Blob> blobIterator = storage.list(BUCKET, Storage.BlobListOption.versions(true)).iterateAll();
+ Iterable<Blob> blobIterable = storage.list(BUCKET, Storage.BlobListOption.versions(true)).iterateAll();
- while(blobIterator.hasNext()) {
- Blob blob = blobIterator.next();
+ for (final Blob blob : blobIterable) {
storage.delete(blob.getBlobId());
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/f742a3a6/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSTest.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSTest.java
index 6b35cde..8b94c47 100644
--- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSTest.java
+++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSTest.java
@@ -88,10 +88,7 @@ public abstract class AbstractGCSTest {
PROJECT_ID, options.getProjectId());
assertEquals("Retry counts should match",
- RETRIES.intValue(), options.getRetryParams().getRetryMinAttempts());
-
- assertEquals("Retry counts should match",
- RETRIES.intValue(), options.getRetryParams().getRetryMaxAttempts());
+ RETRIES.intValue(), options.getRetrySettings().getMaxAttempts());
assertSame("Credentials should be configured correctly",
mockCredentials, options.getCredentials());
http://git-wip-us.apache.org/repos/asf/nifi/blob/f742a3a6/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/FetchGCSObjectTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/FetchGCSObjectTest.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/FetchGCSObjectTest.java
index 0d8b209..5e51270 100644
--- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/FetchGCSObjectTest.java
+++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/FetchGCSObjectTest.java
@@ -142,11 +142,6 @@ public class FetchGCSObjectTest extends AbstractGCSTest {
}
@Override
- public void chunkSize(int i) {
-
- }
-
- @Override
public void setChunkSize(int i) {
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/f742a3a6/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/ListGCSBucketTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/ListGCSBucketTest.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/ListGCSBucketTest.java
index 6c21d91..eca81bb 100644
--- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/ListGCSBucketTest.java
+++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/ListGCSBucketTest.java
@@ -16,7 +16,8 @@
*/
package org.apache.nifi.processors.gcp.storage;
-import com.google.cloud.Page;
+
+import com.google.api.gax.paging.Page;
import com.google.cloud.storage.Acl;
import com.google.cloud.storage.Blob;
import com.google.cloud.storage.BlobInfo;
http://git-wip-us.apache.org/repos/asf/nifi/blob/f742a3a6/nifi-nar-bundles/nifi-gcp-bundle/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-gcp-bundle/pom.xml b/nifi-nar-bundles/nifi-gcp-bundle/pom.xml
index 36a33be..341633c 100644
--- a/nifi-nar-bundles/nifi-gcp-bundle/pom.xml
+++ b/nifi-nar-bundles/nifi-gcp-bundle/pom.xml
@@ -26,6 +26,22 @@
<version>1.7.0-SNAPSHOT</version>
<packaging>pom</packaging>
+ <properties>
+ <google.cloud.sdk.version>0.47.0-alpha</google.cloud.sdk.version>
+ </properties>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>google-cloud</artifactId>
+ <version>${google.cloud.sdk.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
<modules>
<module>nifi-gcp-services-api</module>
<module>nifi-gcp-services-api-nar</module>