You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2016/10/13 00:13:07 UTC
[1/2] incubator-beam git commit: Converts Datastore to use AutoValue
Repository: incubator-beam
Updated Branches:
refs/heads/master e2868d687 -> a5d129361
Converts Datastore to use AutoValue
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/251e3e1e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/251e3e1e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/251e3e1e
Branch: refs/heads/master
Commit: 251e3e1ef2868b9901be3dbd27793b613855704a
Parents: e2868d6
Author: Eugene Kirpichov <ki...@google.com>
Authored: Wed Sep 28 17:29:11 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Wed Oct 12 17:12:32 2016 -0700
----------------------------------------------------------------------
sdks/java/io/google-cloud-platform/pom.xml | 6 ++
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 95 +++++++-------------
2 files changed, 38 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/251e3e1e/sdks/java/io/google-cloud-platform/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/pom.xml b/sdks/java/io/google-cloud-platform/pom.xml
index 15c0447..1a598bb 100644
--- a/sdks/java/io/google-cloud-platform/pom.xml
+++ b/sdks/java/io/google-cloud-platform/pom.xml
@@ -213,6 +213,12 @@
<artifactId>avro</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.google.auto.value</groupId>
+ <artifactId>auto-value</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
<!-- test -->
<dependency>
<groupId>org.apache.beam</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/251e3e1e/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
index 45b2d6f..bfbff32 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
@@ -35,6 +35,7 @@ import com.google.api.client.auth.oauth2.Credential;
import com.google.api.client.util.BackOff;
import com.google.api.client.util.BackOffUtils;
import com.google.api.client.util.Sleeper;
+import com.google.auto.value.AutoValue;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableList;
@@ -194,7 +195,7 @@ public class DatastoreV1 {
* {@link DatastoreV1.Read#withNamespace}, {@link DatastoreV1.Read#withNumQuerySplits}.
*/
public DatastoreV1.Read read() {
- return new DatastoreV1.Read(null, null, null, 0);
+ return new AutoValue_DatastoreV1_Read.Builder().setNumQuerySplits(0).build();
}
/**
@@ -203,7 +204,8 @@ public class DatastoreV1 {
*
* @see DatastoreIO
*/
- public static class Read extends PTransform<PBegin, PCollection<Entity>> {
+ @AutoValue
+ public abstract static class Read extends PTransform<PBegin, PCollection<Entity>> {
private static final Logger LOG = LoggerFactory.getLogger(Read.class);
/** An upper bound on the number of splits for a query. */
@@ -222,16 +224,23 @@ public class DatastoreV1 {
*/
static final int QUERY_BATCH_LIMIT = 500;
- @Nullable
- private final String projectId;
+ @Nullable public abstract String getProjectId();
+ @Nullable public abstract Query getQuery();
+ @Nullable public abstract String getNamespace();
+ public abstract int getNumQuerySplits();
- @Nullable
- private final Query query;
+ public abstract String toString();
- @Nullable
- private final String namespace;
+ abstract Builder toBuilder();
- private final int numQuerySplits;
+ @AutoValue.Builder
+ abstract static class Builder {
+ abstract Builder setProjectId(String projectId);
+ abstract Builder setQuery(Query query);
+ abstract Builder setNamespace(String namespace);
+ abstract Builder setNumQuerySplits(int numQuerySplits);
+ abstract Read build();
+ }
/**
* Computes the number of splits to be performed on the given query by querying the estimated
@@ -345,25 +354,12 @@ public class DatastoreV1 {
}
/**
- * Note that only {@code namespace} is really {@code @Nullable}. The other parameters may be
- * {@code null} as a matter of build order, but if they are {@code null} at instantiation time,
- * an error will be thrown.
- */
- private Read(@Nullable String projectId, @Nullable Query query, @Nullable String namespace,
- int numQuerySplits) {
- this.projectId = projectId;
- this.query = query;
- this.namespace = namespace;
- this.numQuerySplits = numQuerySplits;
- }
-
- /**
* Returns a new {@link DatastoreV1.Read} that reads from the Cloud Datastore for the specified
* project.
*/
public DatastoreV1.Read withProjectId(String projectId) {
checkNotNull(projectId, "projectId");
- return new DatastoreV1.Read(projectId, query, namespace, numQuerySplits);
+ return toBuilder().setProjectId(projectId).build();
}
/**
@@ -378,14 +374,14 @@ public class DatastoreV1 {
checkNotNull(query, "query");
checkArgument(!query.hasLimit() || query.getLimit().getValue() > 0,
"Invalid query limit %s: must be positive", query.getLimit().getValue());
- return new DatastoreV1.Read(projectId, query, namespace, numQuerySplits);
+ return toBuilder().setQuery(query).build();
}
/**
* Returns a new {@link DatastoreV1.Read} that reads from the given namespace.
*/
public DatastoreV1.Read withNamespace(String namespace) {
- return new DatastoreV1.Read(projectId, query, namespace, numQuerySplits);
+ return toBuilder().setNamespace(namespace).build();
}
/**
@@ -405,29 +401,11 @@ public class DatastoreV1 {
* </ul>
*/
public DatastoreV1.Read withNumQuerySplits(int numQuerySplits) {
- return new DatastoreV1.Read(projectId, query, namespace,
- Math.min(Math.max(numQuerySplits, 0), NUM_QUERY_SPLITS_MAX));
- }
-
- @Nullable
- public Query getQuery() {
- return query;
- }
-
- @Nullable
- public String getProjectId() {
- return projectId;
+ return toBuilder()
+ .setNumQuerySplits(Math.min(Math.max(numQuerySplits, 0), NUM_QUERY_SPLITS_MAX))
+ .build();
}
- @Nullable
- public String getNamespace() {
- return namespace;
- }
-
-
- /**
- * {@inheritDoc}
- */
@Override
public PCollection<Entity> apply(PBegin input) {
V1Options v1Options = V1Options.from(getProjectId(), getQuery(),
@@ -451,8 +429,8 @@ public class DatastoreV1 {
* a {@code PCollection<Entity>}.
*/
PCollection<KV<Integer, Query>> queries = input
- .apply(Create.of(query))
- .apply(ParDo.of(new SplitQueryFn(v1Options, numQuerySplits)));
+ .apply(Create.of(getQuery()))
+ .apply(ParDo.of(new SplitQueryFn(v1Options, getNumQuerySplits())));
PCollection<Query> shardedQueries = queries
.apply(GroupByKey.<Integer, Query>create())
@@ -467,31 +445,22 @@ public class DatastoreV1 {
@Override
public void validate(PBegin input) {
- checkNotNull(projectId, "projectId");
- checkNotNull(query, "query");
+ checkNotNull(getProjectId(), "projectId");
+ checkNotNull(getQuery(), "query");
}
@Override
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
builder
- .addIfNotNull(DisplayData.item("projectId", projectId)
+ .addIfNotNull(DisplayData.item("projectId", getProjectId())
.withLabel("ProjectId"))
- .addIfNotNull(DisplayData.item("namespace", namespace)
+ .addIfNotNull(DisplayData.item("namespace", getNamespace())
.withLabel("Namespace"))
- .addIfNotNull(DisplayData.item("query", query.toString())
+ .addIfNotNull(DisplayData.item("query", getQuery().toString())
.withLabel("Query"));
}
- @Override
- public String toString() {
- return MoreObjects.toStringHelper(getClass())
- .add("projectId", projectId)
- .add("query", query)
- .add("namespace", namespace)
- .toString();
- }
-
/**
* A class for v1 Cloud Datastore related options.
*/
@@ -596,7 +565,7 @@ public class DatastoreV1 {
}
@Override
- public void populateDisplayData(Builder builder) {
+ public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
builder
.addIfNotNull(DisplayData.item("projectId", options.getProjectId())
[2/2] incubator-beam git commit: Closes #1031
Posted by dh...@apache.org.
Closes #1031
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/a5d12936
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/a5d12936
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/a5d12936
Branch: refs/heads/master
Commit: a5d12936158f5725a12a7f1f23eb90f86c4e68b1
Parents: e2868d6 251e3e1
Author: Dan Halperin <dh...@google.com>
Authored: Wed Oct 12 17:12:33 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Wed Oct 12 17:12:33 2016 -0700
----------------------------------------------------------------------
sdks/java/io/google-cloud-platform/pom.xml | 6 ++
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 95 +++++++-------------
2 files changed, 38 insertions(+), 63 deletions(-)
----------------------------------------------------------------------