You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by cc...@apache.org on 2018/09/21 19:02:16 UTC
[beam] branch revert-6372-autocomplete-it created (now e3ddecd)
This is an automated email from the ASF dual-hosted git repository.
ccy pushed a change to branch revert-6372-autocomplete-it
in repository https://gitbox.apache.org/repos/asf/beam.git.
at e3ddecd Revert "Adding Autocomplete IT for Java"
This branch includes the following new commits:
new e3ddecd Revert "Adding Autocomplete IT for Java"
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[beam] 01/01: Revert "Adding Autocomplete IT for Java"
Posted by cc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ccy pushed a commit to branch revert-6372-autocomplete-it
in repository https://gitbox.apache.org/repos/asf/beam.git
commit e3ddecdda6587233b333da4aa8a0e0dba66c7337
Author: Charles Chen <ch...@users.noreply.github.com>
AuthorDate: Fri Sep 21 12:02:06 2018 -0700
Revert "Adding Autocomplete IT for Java"
---
.../beam/examples/complete/AutoComplete.java | 41 +--------------
.../beam/examples/complete/AutoCompleteIT.java | 59 ----------------------
2 files changed, 2 insertions(+), 98 deletions(-)
diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java b/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java
index 34e4247..0f9b8fd 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java
@@ -51,7 +51,6 @@ import org.apache.beam.sdk.options.Description;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.options.StreamingOptions;
import org.apache.beam.sdk.options.Validation;
-import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.transforms.Count;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.Filter;
@@ -61,7 +60,6 @@ import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.Partition;
import org.apache.beam.sdk.transforms.Partition.PartitionFn;
import org.apache.beam.sdk.transforms.SerializableFunction;
-import org.apache.beam.sdk.transforms.Sum;
import org.apache.beam.sdk.transforms.Top;
import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
import org.apache.beam.sdk.transforms.windowing.SlidingWindows;
@@ -433,17 +431,6 @@ public class AutoComplete {
void setOutputToBigQuery(Boolean value);
- @Description("Whether to send output to checksum Transform.")
- @Default.Boolean(true)
- Boolean getOutputToChecksum();
-
- void setOutputToChecksum(Boolean value);
-
- @Description("Expected result of the checksum transform.")
- Long getExpectedChecksum();
-
- void setExpectedChecksum(Long value);
-
@Description("Whether output to Cloud Datastore")
@Default.Boolean(false)
Boolean getOutputToDatastore();
@@ -462,7 +449,8 @@ public class AutoComplete {
void setOutputProject(String value);
}
- public static void runAutocompletePipeline(Options options) throws IOException {
+ public static void main(String[] args) throws IOException {
+ Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);
options.setBigQuerySchema(FormatForBigquery.getSchema());
ExampleUtils exampleUtils = new ExampleUtils(options);
@@ -518,35 +506,10 @@ public class AutoComplete {
: BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE));
}
- if (options.getOutputToChecksum()) {
- PCollection<Long> checksum =
- toWrite
- .apply(
- ParDo.of(
- new DoFn<KV<String, List<CompletionCandidate>>, Long>() {
- @ProcessElement
- public void process(ProcessContext c) {
- KV<String, List<CompletionCandidate>> elm = c.element();
- Long listHash =
- c.element().getValue().stream().mapToLong(cc -> cc.hashCode()).sum();
- c.output(Long.valueOf(elm.getKey().hashCode()) + listHash);
- }
- }))
- .apply(Sum.longsGlobally());
-
- PAssert.that(checksum).containsInAnyOrder(options.getExpectedChecksum());
- }
-
// Run the pipeline.
PipelineResult result = p.run();
// ExampleUtils will try to cancel the pipeline and the injector before the program exists.
exampleUtils.waitToFinish(result);
}
-
- public static void main(String[] args) throws IOException {
- Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);
-
- runAutocompletePipeline(options);
- }
}
diff --git a/examples/java/src/test/java/org/apache/beam/examples/complete/AutoCompleteIT.java b/examples/java/src/test/java/org/apache/beam/examples/complete/AutoCompleteIT.java
deleted file mode 100644
index 6b01d1a..0000000
--- a/examples/java/src/test/java/org/apache/beam/examples/complete/AutoCompleteIT.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.examples.complete;
-
-import org.apache.beam.examples.complete.AutoComplete.Options;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.testing.TestPipelineOptions;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Run integration tests for a basic AutoComplete pipeline. */
-@RunWith(JUnit4.class)
-public class AutoCompleteIT {
-
- public static final String DEFAULT_INPUT =
- "gs://apache-beam-samples/shakespeare/kinglear-hashtag.txt";
-
- public static final Long DEFAULT_INPUT_CHECKSUM = -25447108232L;
-
- /** Options for the Autocomplete Integration test. */
- public interface AutoCompleteITOptions extends TestPipelineOptions, Options {}
-
- @BeforeClass
- public static void setUp() {
- PipelineOptionsFactory.register(TestPipelineOptions.class);
- }
-
- @Test
- public void testE2EAutoComplete() throws Exception {
- AutoCompleteITOptions options =
- TestPipeline.testingPipelineOptions().as(AutoCompleteITOptions.class);
-
- options.setInputFile(DEFAULT_INPUT);
- options.setOutputToBigQuery(false);
- options.setOutputToDatastore(false);
- options.setOutputToChecksum(true);
- options.setExpectedChecksum(DEFAULT_INPUT_CHECKSUM);
-
- AutoComplete.runAutocompletePipeline(options);
- }
-}