You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by cc...@apache.org on 2018/09/21 19:02:17 UTC
[beam] 01/01: Revert "Adding Autocomplete IT for Java"
This is an automated email from the ASF dual-hosted git repository.
ccy pushed a commit to branch revert-6372-autocomplete-it
in repository https://gitbox.apache.org/repos/asf/beam.git
commit e3ddecdda6587233b333da4aa8a0e0dba66c7337
Author: Charles Chen <ch...@users.noreply.github.com>
AuthorDate: Fri Sep 21 12:02:06 2018 -0700
Revert "Adding Autocomplete IT for Java"
---
.../beam/examples/complete/AutoComplete.java | 41 +--------------
.../beam/examples/complete/AutoCompleteIT.java | 59 ----------------------
2 files changed, 2 insertions(+), 98 deletions(-)
diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java b/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java
index 34e4247..0f9b8fd 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java
@@ -51,7 +51,6 @@ import org.apache.beam.sdk.options.Description;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.options.StreamingOptions;
import org.apache.beam.sdk.options.Validation;
-import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.transforms.Count;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.Filter;
@@ -61,7 +60,6 @@ import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.Partition;
import org.apache.beam.sdk.transforms.Partition.PartitionFn;
import org.apache.beam.sdk.transforms.SerializableFunction;
-import org.apache.beam.sdk.transforms.Sum;
import org.apache.beam.sdk.transforms.Top;
import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
import org.apache.beam.sdk.transforms.windowing.SlidingWindows;
@@ -433,17 +431,6 @@ public class AutoComplete {
void setOutputToBigQuery(Boolean value);
- @Description("Whether to send output to checksum Transform.")
- @Default.Boolean(true)
- Boolean getOutputToChecksum();
-
- void setOutputToChecksum(Boolean value);
-
- @Description("Expected result of the checksum transform.")
- Long getExpectedChecksum();
-
- void setExpectedChecksum(Long value);
-
@Description("Whether output to Cloud Datastore")
@Default.Boolean(false)
Boolean getOutputToDatastore();
@@ -462,7 +449,8 @@ public class AutoComplete {
void setOutputProject(String value);
}
- public static void runAutocompletePipeline(Options options) throws IOException {
+ public static void main(String[] args) throws IOException {
+ Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);
options.setBigQuerySchema(FormatForBigquery.getSchema());
ExampleUtils exampleUtils = new ExampleUtils(options);
@@ -518,35 +506,10 @@ public class AutoComplete {
: BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE));
}
- if (options.getOutputToChecksum()) {
- PCollection<Long> checksum =
- toWrite
- .apply(
- ParDo.of(
- new DoFn<KV<String, List<CompletionCandidate>>, Long>() {
- @ProcessElement
- public void process(ProcessContext c) {
- KV<String, List<CompletionCandidate>> elm = c.element();
- Long listHash =
- c.element().getValue().stream().mapToLong(cc -> cc.hashCode()).sum();
- c.output(Long.valueOf(elm.getKey().hashCode()) + listHash);
- }
- }))
- .apply(Sum.longsGlobally());
-
- PAssert.that(checksum).containsInAnyOrder(options.getExpectedChecksum());
- }
-
// Run the pipeline.
PipelineResult result = p.run();
// ExampleUtils will try to cancel the pipeline and the injector before the program exists.
exampleUtils.waitToFinish(result);
}
-
- public static void main(String[] args) throws IOException {
- Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);
-
- runAutocompletePipeline(options);
- }
}
diff --git a/examples/java/src/test/java/org/apache/beam/examples/complete/AutoCompleteIT.java b/examples/java/src/test/java/org/apache/beam/examples/complete/AutoCompleteIT.java
deleted file mode 100644
index 6b01d1a..0000000
--- a/examples/java/src/test/java/org/apache/beam/examples/complete/AutoCompleteIT.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.examples.complete;
-
-import org.apache.beam.examples.complete.AutoComplete.Options;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.testing.TestPipelineOptions;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Run integration tests for a basic AutoComplete pipeline. */
-@RunWith(JUnit4.class)
-public class AutoCompleteIT {
-
- public static final String DEFAULT_INPUT =
- "gs://apache-beam-samples/shakespeare/kinglear-hashtag.txt";
-
- public static final Long DEFAULT_INPUT_CHECKSUM = -25447108232L;
-
- /** Options for the Autocomplete Integration test. */
- public interface AutoCompleteITOptions extends TestPipelineOptions, Options {}
-
- @BeforeClass
- public static void setUp() {
- PipelineOptionsFactory.register(TestPipelineOptions.class);
- }
-
- @Test
- public void testE2EAutoComplete() throws Exception {
- AutoCompleteITOptions options =
- TestPipeline.testingPipelineOptions().as(AutoCompleteITOptions.class);
-
- options.setInputFile(DEFAULT_INPUT);
- options.setOutputToBigQuery(false);
- options.setOutputToDatastore(false);
- options.setOutputToChecksum(true);
- options.setExpectedChecksum(DEFAULT_INPUT_CHECKSUM);
-
- AutoComplete.runAutocompletePipeline(options);
- }
-}