You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2016/08/10 17:39:22 UTC
[2/2] incubator-beam git commit: Use Datastore AncestorKey for
AutoComplete example
Use Datastore AncestorKey for AutoComplete example
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/a41dab86
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/a41dab86
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/a41dab86
Branch: refs/heads/master
Commit: a41dab8685c086f60d0cd72b61ce7affd3559a34
Parents: 95e7f01
Author: Vikas Kedigehalli <vi...@google.com>
Authored: Thu Aug 4 11:29:25 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Wed Aug 10 10:39:15 2016 -0700
----------------------------------------------------------------------
.../beam/examples/complete/AutoComplete.java | 21 +++++++++++++++-----
1 file changed, 16 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a41dab86/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java
----------------------------------------------------------------------
diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java b/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java
index 1ab39c9..baae100 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java
@@ -18,6 +18,7 @@
package org.apache.beam.examples.complete;
import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.datastore.v1beta3.client.DatastoreHelper.makeKey;
import static com.google.datastore.v1beta3.client.DatastoreHelper.makeValue;
import org.apache.beam.examples.common.ExampleBigQueryTableOptions;
@@ -61,7 +62,6 @@ import com.google.common.base.MoreObjects;
import com.google.datastore.v1beta3.Entity;
import com.google.datastore.v1beta3.Key;
import com.google.datastore.v1beta3.Value;
-import com.google.datastore.v1beta3.client.DatastoreHelper;
import org.joda.time.Duration;
@@ -384,18 +384,23 @@ public class AutoComplete {
/**
* Takes as input a the top candidates per prefix, and emits an entity
* suitable for writing to Datastore.
+ *
+ * <p>Note: We use ancestor keys for strong consistency. See the Cloud Datastore documentation on
+ * <a href="https://cloud.google.com/datastore/docs/concepts/structuring_for_strong_consistency">
+ * Structuring Data for Strong Consistency</a>
*/
static class FormatForDatastore extends DoFn<KV<String, List<CompletionCandidate>>, Entity> {
private String kind;
-
- public FormatForDatastore(String kind) {
+ private String ancestorKey;
+ public FormatForDatastore(String kind, String ancestorKey) {
this.kind = kind;
+ this.ancestorKey = ancestorKey;
}
@ProcessElement
public void processElement(ProcessContext c) {
Entity.Builder entityBuilder = Entity.newBuilder();
- Key key = DatastoreHelper.makeKey(kind, c.element().getKey()).build();
+ Key key = makeKey(makeKey(kind, ancestorKey).build(), kind, c.element().getKey()).build();
entityBuilder.setKey(key);
List<Value> candidates = new ArrayList<>();
@@ -444,6 +449,11 @@ public class AutoComplete {
Boolean getOutputToDatastore();
void setOutputToDatastore(Boolean value);
+ @Description("Datastore ancestor key")
+ @Default.String("root")
+ String getDatastoreAncestorKey();
+ void setDatastoreAncestorKey(String value);
+
@Description("Datastore output project ID, defaults to project ID")
String getOutputProject();
void setOutputProject(String value);
@@ -476,7 +486,8 @@ public class AutoComplete {
if (options.getOutputToDatastore()) {
toWrite
- .apply("FormatForDatastore", ParDo.of(new FormatForDatastore(options.getKind())))
+ .apply("FormatForDatastore", ParDo.of(new FormatForDatastore(options.getKind(),
+ options.getDatastoreAncestorKey())))
.apply(DatastoreIO.v1beta3().write().withProjectId(MoreObjects.firstNonNull(
options.getOutputProject(), options.getProject())));
}