You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ie...@apache.org on 2017/08/23 17:09:48 UTC
[40/55] [abbrv] beam git commit: Update to Beam 2.2.0-SNAPSHOT and
final touches before merge into master
Update to Beam 2.2.0-SNAPSHOT and final touches before merge into master
Clean pom, exclude nexmark packages from aggregated javadoc, put spark logs in WARN
Update execution matrix in README: Flink termination of streaming pipelines is now ok as far as Nexmark is concerned
Remove how to run Nexmark on Flink on Google Cloud Platform from README
Update command lines in README after moving nexmark from integration to sdks module
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/0acd7201
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/0acd7201
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/0acd7201
Branch: refs/heads/master
Commit: 0acd72016b046640dd753d60a569403488f595a5
Parents: f4333df
Author: Etienne Chauchot <ec...@gmail.com>
Authored: Fri Aug 11 16:41:00 2017 +0200
Committer: Ismaël Mejía <ie...@gmail.com>
Committed: Wed Aug 23 19:07:29 2017 +0200
----------------------------------------------------------------------
.../java/nexmark/BEAM_ON_FLINK_ON_GCP.md | 282 -------------------
sdks/java/javadoc/ant.xml | 1 +
sdks/java/nexmark/README.md | 56 ++--
sdks/java/nexmark/pom.xml | 35 +--
.../apache/beam/sdk/nexmark/package-info.java | 2 +-
.../nexmark/src/main/resources/log4j.properties | 4 +-
6 files changed, 34 insertions(+), 346 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/0acd7201/integration/java/nexmark/BEAM_ON_FLINK_ON_GCP.md
----------------------------------------------------------------------
diff --git a/integration/java/nexmark/BEAM_ON_FLINK_ON_GCP.md b/integration/java/nexmark/BEAM_ON_FLINK_ON_GCP.md
deleted file mode 100644
index 6a7fd34..0000000
--- a/integration/java/nexmark/BEAM_ON_FLINK_ON_GCP.md
+++ /dev/null
@@ -1,282 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-# Running NexMark on Beam on Flink on Google Compute Platform
-
-Here's how to create a cluster of VMs on Google Compute Platform, deploy
-Flink to them, and invoke a NexMark pipeline using the Beam-on-Flink
-runner.
-
-These instructions are somewhat baroque and I hope they can be
-simplified over time.
-
-## Prerequisites
-
-You'll need:
-
-* the Google Cloud SDK
-* a clone of the Beam repository
-* a Flink binary distribution
-* a project on Google Compute Platform.
-
-## Establish the shell environment
-
-```
-# Beam root
-BEAM=<path to Beam source directory>
-# Flink root
-FLINK_VER=flink-1.0.3
-FLINK=<path to Flink distribution directory>
-# Google Cloud project
-PROJECT=<your project id>
-# Google Cloud zone
-ZONE=<your project zone>
-# Cloud commands
-GCLOUD=<path to gcloud command>
-GSUTIL=<path to gsutil command>
-```
-
-## Establish VM names for Flink master and workers
-
-```
-MASTER=flink-m
-NUM_WORKERS=5
-WORKERS=""
-for (( i = 0; i < $NUM_WORKERS; i++ )); do
- WORKERS="$WORKERS flink-w-$i"
-done
-ALL="$MASTER $WORKERS"
-```
-
-## Build Beam
-
-```
-( cd $BEAM && mvn clean install )
-```
-
-## Bring up the cluster
-
-Establish project defaults and authenticate:
-```
-$GCLOUD init
-$GCLOUD auth login
-```
-
-Build Google Cloud Dataproc cluster:
-```
-$GCLOUD beta dataproc clusters create \
- --project=$PROJECT \
- --zone=$ZONE \
- --bucket=nexmark \
- --scopes=cloud-platform \
- --num-workers=$NUM_WORKERS \
- --image-version=preview \
- flink
-```
-
-Force google_compute_engine ssh keys to be generated locally:
-```
-$GCLOUD compute ssh \
- --project=$PROJECT \
- --zone=$ZONE \
- $MASTER \
- --command 'exit'
-```
-
-Open ports on the VMs:
-```
-$GCLOUD compute firewall-rules create allow-monitoring --allow tcp:8080-8081
-$GCLOUD compute firewall-rules create allow-debug --allow tcp:5555
-```
-
-Establish keys on master and workers
-**CAUTION:** This will leave your private key on your master VM.
-Better would be to create a key just for inter-worker ssh.
-```
-for m in $ALL; do
- echo "*** $m ***"
- $GCLOUD beta compute scp \
- --project=$PROJECT \
- --zone=$ZONE \
- ~/.ssh/google_compute_engine.pub $m:~/.ssh/
-done
-$GCLOUD beta compute scp \
- --project=$PROJECT \
- --zone=$ZONE \
- ~/.ssh/google_compute_engine $MASTER:~/.ssh/
-```
-
-Collect IP addresses for workers:
-```
-MASTER_EXT_IP=$($GCLOUD compute instances describe \
- --project=$PROJECT \
- --zone=$ZONE \
- $MASTER | grep natIP: | sed 's/[ ]*natIP:[ ]*//')
-MASTER_IP=$($GCLOUD compute instances describe \
- --project=$PROJECT \
- --zone=$ZONE \
- $MASTER | grep networkIP: | sed 's/[ ]*networkIP:[ ]*//')
-WORKER_IPS=""
-for m in $WORKERS; do
- echo "*** $m ***"
- WORKER_IP=$($GCLOUD compute instances describe \
- --project=$PROJECT \
- --zone=$ZONE \
- $m | grep networkIP: | sed 's/[ ]*networkIP:[ ]*//')
- WORKER_IPS="$WORKER_IPS $WORKER_IP"
-done
-```
-
-Configure Flink:
-```
-cat $FLINK/conf/flink-conf.yaml \
- | sed "s|.*\(jobmanager.rpc.address\):.*|\1: $MASTER_IP|g" \
- | sed "s|.*\(jobmanager.heap.mb\):.*|\1: 4096|g" \
- | sed "s|.*\(taskmanager.heap.mb\):.*|\1: 8192|g" \
- | sed "s|.*\(parallelism.default\):.*|\1: $(($NUM_WORKERS * 4))|g" \
- | sed "s|.*\(fs.hdfs.hadoopconf\):.*|\1: /etc/hadoop/conf|g" \
- | sed "s|.*\(taskmanager.numberOfTaskSlots\):.*|\1: 4|g" \
- | sed "s|.*\(jobmanager.web.submit.enable\):.*|\1: false|g" \
- | sed "s|.*\(env.ssh.opts\):.*||g" \
- > ~/flink-conf.yaml
-cat $FLINK/conf/log4j.properties \
- | sed "s|.*\(log4j.rootLogger\)=.*|\1=ERROR, file|g" \
- > ~/log4j.properties
-echo "env.ssh.opts: -i /home/$USER/.ssh/google_compute_engine -o StrictHostKeyChecking=no" >> ~/flink-conf.yaml
-echo "$MASTER_IP:8081" > ~/masters
-echo -n > ~/slaves
-for ip in $WORKER_IPS; do
- echo $ip >> ~/slaves
-done
-cp -f \
- ~/flink-conf.yaml \
- ~/masters ~/slaves \
- ~/log4j.properties \
- $FLINK/conf/
-```
-
-Package configured Flink for distribution to workers:
-```
-( cd ~/ && tar -cvzf ~/flink.tgz $FLINK/* )
-```
-
-Distribute:
-```
-$GSUTIL cp ~/flink.tgz gs://nexmark
-for m in $ALL; do
- echo "*** $m ***"
- $GCLOUD compute ssh \
- --project=$PROJECT \
- --zone=$ZONE \
- $m \
- --command 'gsutil cp gs://nexmark/flink.tgz ~/ && tar -xvzf ~/flink.tgz'
-done
-```
-
-Start the Flink cluster:
-```
-$GCLOUD compute ssh \
- --project=$PROJECT \
- --zone=$ZONE \
- $MASTER \
- --command "~/$FLINK_VER/bin/start-cluster.sh"
-```
-
-Bring up the Flink monitoring UI:
-```
-/usr/bin/google-chrome $MASTER_EXT_IP:8081 &
-```
-
-## Run NexMark
-
-Distribute the Beam + NexMark jar to all workers:
-```
-$GSUTIL cp $BEAM/integration/java/target/java-integration-all-bundled-0.2.0-incubating-SNAPSHOT.jar gs://nexmark
-for m in $ALL; do
- echo "*** $m ***"
- $GCLOUD compute ssh \
- --project=$PROJECT \
- --zone=$ZONE \
- $m \
- --command "gsutil cp gs://nexmark/java-integration-all-bundled-0.2.0-incubating-SNAPSHOT.jar ~/$FLINK_VER/lib/"
-done
-```
-
-Create a Pubsub topic and subscription for testing:
-```
-$GCLOUD alpha pubsub \
- --project=$PROJECT \
- topics create flink_test
-
-$GCLOUD alpha pubsub \
- --project=$PROJECT \
- subscriptions create flink_test \
- --topic flink_test \
- --ack-deadline=60 \
- --topic-project=$PROJECT
-```
-
-Launch!
-**NOTE:** As of flink-1.0.3 this will throw a `NullPointerException`
-in `org.apache.beam.sdk.io.PubsubUnboundedSink$WriterFn.startBundle`.
-See Jira issue [BEAM-196](https://issues.apache.org/jira/browse/BEAM-196).
-
-```
-$GCLOUD compute ssh \
- --project=$PROJECT \
- --zone=$ZONE \
- $MASTER \
- --command "~/$FLINK_VER/bin/flink run \
- -c org.apache.beam.integration.nexmark.drivers.NexmarkFlinkDriver \
- ~/$FLINK_VER/lib/java-integration-all-bundled-0.2.0-incubating-SNAPSHOT.jar \
- --project=$PROJECT \
- --streaming=true \
- --query=0 \
- --sourceType=PUBSUB \
- --pubSubMode=COMBINED \
- --pubsubTopic=flink_test \
- --resourceNameMode=VERBATIM \
- --manageResources=false \
- --monitorJobs=false \
- --numEventGenerators=5 \
- --firstEventRate=1000 \
- --nextEventRate=1000 \
- --isRateLimited=true \
- --numEvents=0 \
- --useWallclockEventTime=true \
- --usePubsubPublishTime=true"
-```
-
-## Teardown the cluster
-
-Stop the Flink cluster:
-```
-$GCLOUD compute ssh \
- --project=$PROJECT \
- --zone=$ZONE \
- $MASTER \
- --command "~/$FLINK_VER/bin/stop-cluster.sh"
-```
-
-Teardown the Dataproc cluster:
-```
-$GCLOUD beta dataproc clusters delete \
- --project=$PROJECT \
- flink
-```
http://git-wip-us.apache.org/repos/asf/beam/blob/0acd7201/sdks/java/javadoc/ant.xml
----------------------------------------------------------------------
diff --git a/sdks/java/javadoc/ant.xml b/sdks/java/javadoc/ant.xml
index 9a1880a..152b9dd 100644
--- a/sdks/java/javadoc/ant.xml
+++ b/sdks/java/javadoc/ant.xml
@@ -30,6 +30,7 @@
<fileset dir="..">
<include name="**/src/main/java/**/*.java"/>
<exclude name="**/maven-archetypes/**"/>
+ <exclude name="**/nexmark/**"/>
</fileset>
<!-- For each pathname, turn X/src/main/java/Y to Y. This
results in one Java source tree. -->
http://git-wip-us.apache.org/repos/asf/beam/blob/0acd7201/sdks/java/nexmark/README.md
----------------------------------------------------------------------
diff --git a/sdks/java/nexmark/README.md b/sdks/java/nexmark/README.md
index a9acd63..f252943 100644
--- a/sdks/java/nexmark/README.md
+++ b/sdks/java/nexmark/README.md
@@ -17,7 +17,7 @@
under the License.
-->
-# NEXMark integration suite
+# NEXMark test suite
This is a suite of pipelines inspired by the 'continuous data stream'
queries in [http://datalab.cs.pdx.edu/niagaraST/NEXMark/]
@@ -139,7 +139,7 @@ Available suites are:
--runner=DataflowRunner \
--tempLocation=gs://talend-imejia/nexmark/temp/ \
--stagingLocation=gs://talend-imejia/nexmark/temp/staging/ \
- --filesToStage=target/beam-integration-java-0.7.0-SNAPSHOT.jar
+ --filesToStage=target/beam-sdks-java-nexmark-2.1.0-SNAPSHOT.jar
## Direct specific configuration
@@ -183,19 +183,19 @@ Open issues are tracked [here](https://github.com../../../../../issues):
| Query | Direct | Spark | Flink | Apex |
| ----: | ------ | ------------------------------------------------------------ | ---------------------------------------------------------- | ------------------------------------------------------------ |
-| 0 | ok | ok | [BEAM-593](https://issues.apache.org/jira/browse/BEAM-593) | ok |
-| 1 | ok | ok | [BEAM-593](https://issues.apache.org/jira/browse/BEAM-593) | ok |
-| 2 | ok | ok | [BEAM-593](https://issues.apache.org/jira/browse/BEAM-593) | ok |
-| 3 | ok | [BEAM-1035](https://issues.apache.org/jira/browse/BEAM-1035) | [BEAM-593](https://issues.apache.org/jira/browse/BEAM-593) | [BEAM-1037](https://issues.apache.org/jira/browse/BEAM-1037) |
-| 4 | ok | ok | [BEAM-593](https://issues.apache.org/jira/browse/BEAM-593) | ok |
-| 5 | ok | ok | [BEAM-593](https://issues.apache.org/jira/browse/BEAM-593) | ok |
-| 6 | ok | ok | [BEAM-593](https://issues.apache.org/jira/browse/BEAM-593) | ok |
-| 7 | ok | [BEAM-2112](https://issues.apache.org/jira/browse/BEAM-2112) | [BEAM-593](https://issues.apache.org/jira/browse/BEAM-593) | ok |
-| 8 | ok | ok | [BEAM-593](https://issues.apache.org/jira/browse/BEAM-593) | ok |
-| 9 | ok | ok | [BEAM-593](https://issues.apache.org/jira/browse/BEAM-593) | ok |
-| 10 | ok | ok | [BEAM-593](https://issues.apache.org/jira/browse/BEAM-593) | ok |
-| 11 | ok | ok | [BEAM-593](https://issues.apache.org/jira/browse/BEAM-593) | ok |
-| 12 | ok | ok | [BEAM-593](https://issues.apache.org/jira/browse/BEAM-593) | ok |
+| 0 | ok | ok | ok | ok |
+| 1 | ok | ok | ok | ok |
+| 2 | ok | ok | ok | ok |
+| 3 | ok | [BEAM-1035](https://issues.apache.org/jira/browse/BEAM-1035) | ok | [BEAM-1037](https://issues.apache.org/jira/browse/BEAM-1037) |
+| 4 | ok | ok | ok | ok |
+| 5 | ok | ok | ok | ok |
+| 6 | ok | ok | ok | ok |
+| 7 | ok | [BEAM-2112](https://issues.apache.org/jira/browse/BEAM-2112) | ok | ok |
+| 8 | ok | ok | ok | ok |
+| 9 | ok | ok | ok | ok |
+| 10 | ok | ok | ok | ok |
+| 11 | ok | ok | ok | ok |
+| 12 | ok | ok | ok | ok |
## Batch / Synthetic / Cluster
@@ -219,44 +219,44 @@ TODO
Batch Mode
- mvn exec:java -Dexec.mainClass=org.apache.beam.integration.nexmark.Main -Pdirect-runner -Dexec.args="--runner=DirectRunner --suite=SMOKE --streaming=false --manageResources=false --monitorJobs=true --enforceEncodability=true --enforceImmutability=true"
+ mvn exec:java -Dexec.mainClass=org.apache.beam.sdk.nexmark.Main -Pdirect-runner -Dexec.args="--runner=DirectRunner --suite=SMOKE --streaming=false --manageResources=false --monitorJobs=true --enforceEncodability=true --enforceImmutability=true"
Streaming Mode
- mvn exec:java -Dexec.mainClass=org.apache.beam.integration.nexmark.Main -Pdirect-runner -Dexec.args="--runner=DirectRunner --suite=SMOKE --streaming=true --manageResources=false --monitorJobs=true --enforceEncodability=true --enforceImmutability=true"
+ mvn exec:java -Dexec.mainClass=org.apache.beam.sdk.nexmark.Main -Pdirect-runner -Dexec.args="--runner=DirectRunner --suite=SMOKE --streaming=true --manageResources=false --monitorJobs=true --enforceEncodability=true --enforceImmutability=true"
## Running SMOKE suite on the SparkRunner (local)
Batch Mode
- mvn exec:java -Dexec.mainClass=org.apache.beam.integration.nexmark.Main -Pspark-runner "-Dexec.args=--runner=SparkRunner --suite=SMOKE --streamTimeout=60 --streaming=false --manageResources=false --monitorJobs=true"
+ mvn exec:java -Dexec.mainClass=org.apache.beam.sdk.nexmark.Main -Pspark-runner "-Dexec.args=--runner=SparkRunner --suite=SMOKE --streamTimeout=60 --streaming=false --manageResources=false --monitorJobs=true"
Streaming Mode
- mvn exec:java -Dexec.mainClass=org.apache.beam.integration.nexmark.Main -Pspark-runner "-Dexec.args=--runner=SparkRunner --suite=SMOKE --streamTimeout=60 --streaming=true --manageResources=false --monitorJobs=true"
+ mvn exec:java -Dexec.mainClass=org.apache.beam.sdk.nexmark.Main -Pspark-runner "-Dexec.args=--runner=SparkRunner --suite=SMOKE --streamTimeout=60 --streaming=true --manageResources=false --monitorJobs=true"
## Running SMOKE suite on the FlinkRunner (local)
Batch Mode
- mvn exec:java -Dexec.mainClass=org.apache.beam.integration.nexmark.Main -Pflink-runner "-Dexec.args=--runner=FlinkRunner --suite=SMOKE --streamTimeout=60 --streaming=false --manageResources=false --monitorJobs=true --flinkMaster=local"
+ mvn exec:java -Dexec.mainClass=org.apache.beam.sdk.nexmark.Main -Pflink-runner "-Dexec.args=--runner=FlinkRunner --suite=SMOKE --streamTimeout=60 --streaming=false --manageResources=false --monitorJobs=true --flinkMaster=local"
Streaming Mode
- mvn exec:java -Dexec.mainClass=org.apache.beam.integration.nexmark.Main -Pflink-runner "-Dexec.args=--runner=FlinkRunner --suite=SMOKE --streamTimeout=60 --streaming=true --manageResources=false --monitorJobs=true --flinkMaster=local"
+ mvn exec:java -Dexec.mainClass=org.apache.beam.sdk.nexmark.Main -Pflink-runner "-Dexec.args=--runner=FlinkRunner --suite=SMOKE --streamTimeout=60 --streaming=true --manageResources=false --monitorJobs=true --flinkMaster=local"
## Running SMOKE suite on the ApexRunner (local)
Batch Mode
- mvn exec:java -Dexec.mainClass=org.apache.beam.integration.nexmark.Main -Papex-runner "-Dexec.args=--runner=ApexRunner --suite=SMOKE --streamTimeout=60 --streaming=false --manageResources=false --monitorJobs=false"
+ mvn exec:java -Dexec.mainClass=org.apache.beam.sdk.nexmark.Main -Papex-runner "-Dexec.args=--runner=ApexRunner --suite=SMOKE --streamTimeout=60 --streaming=false --manageResources=false --monitorJobs=false"
Streaming Mode
- mvn exec:java -Dexec.mainClass=org.apache.beam.integration.nexmark.Main -Papex-runner "-Dexec.args=--runner=ApexRunner --suite=SMOKE --streamTimeout=60 --streaming=true --manageResources=false --monitorJobs=false"
+ mvn exec:java -Dexec.mainClass=org.apache.beam.sdk.nexmark.Main -Papex-runner "-Dexec.args=--runner=ApexRunner --suite=SMOKE --streamTimeout=60 --streaming=true --manageResources=false --monitorJobs=false"
## Running SMOKE suite on Google Cloud Dataflow
@@ -269,8 +269,8 @@ Submit to Google Dataflow service
```
-java -cp integration/java/target/beam-integration-java-nexmark-bundled-2.1.0-SNAPSHOT.jar \
- org.apache.beam.integration.nexmark.Main \
+java -cp sdks/java/nexmark/target/beam-sdks-java-nexmark-bundled-2.1.0-SNAPSHOT.jar \
+ org.apache.beam.sdk.nexmark.Main \
--runner=DataflowRunner
--project=<your project> \
--zone=<your zone> \
@@ -303,8 +303,8 @@ java -cp integration/java/target/beam-integration-java-nexmark-bundled-2.1.0-SNA
```
```
-java -cp integration/java/target/beam-integration-java-nexmark-bundled-2.1.0-SNAPSHOT.jar \
- org.apache.beam.integration.nexmark.Main \
+java -cp sdks/java/nexmark/target/beam-sdks-java-nexmark-bundled-2.1.0-SNAPSHOT.jar \
+ org.apache.beam.sdk.nexmark.Main \
--runner=DataflowRunner
--project=<your project> \
--zone=<your zone> \
@@ -336,5 +336,5 @@ Building package
Submit to the cluster
- spark-submit --master yarn-client --class org.apache.beam.integration.nexmark.Main --driver-memory 512m --executor-memory 512m --executor-cores 1 beam-integration-java-nexmark-bundled-2.1.0-SNAPSHOT.jar --runner=SparkRunner --query=0 --streamTimeout=60 --streaming=false --manageResources=false --monitorJobs=true
+ spark-submit --master yarn-client --class org.apache.beam.sdk.nexmark.Main --driver-memory 512m --executor-memory 512m --executor-cores 1 beam-sdks-java-nexmark-bundled-2.1.0-SNAPSHOT.jar --runner=SparkRunner --query=0 --streamTimeout=60 --streaming=false --manageResources=false --monitorJobs=true
http://git-wip-us.apache.org/repos/asf/beam/blob/0acd7201/sdks/java/nexmark/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/nexmark/pom.xml b/sdks/java/nexmark/pom.xml
index c1b6025..67628a1 100644
--- a/sdks/java/nexmark/pom.xml
+++ b/sdks/java/nexmark/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.2.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -35,7 +35,7 @@
<!--
The direct runner is available by default.
- You can also include it on the classpath explicitly with -P direct-runner
+ You can also include it on the classpath explicitly using the profile below
-->
<profile>
<id>direct-runner</id>
@@ -51,7 +51,6 @@
</dependencies>
</profile>
- <!-- Include the Apache Apex runner with -P apex-runner -->
<profile>
<id>apex-runner</id>
<dependencies>
@@ -60,27 +59,9 @@
<artifactId>beam-runners-apex</artifactId>
<scope>runtime</scope>
</dependency>
- <!--
- Apex depends on httpclient version 4.3.5, project has a transitive dependency to httpclient 4.0.1 from
- google-http-client. Apex dependency version being specified explicitly so that it gets picked up. This
- can be removed when the project no longer has a dependency on a different httpclient version.
- -->
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>4.3.5</version>
- <scope>runtime</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
</dependencies>
</profile>
- <!-- Include the Apache Flink runner with -P flink-runner -->
<profile>
<id>flink-runner</id>
<dependencies>
@@ -92,7 +73,6 @@
</dependencies>
</profile>
- <!-- Include the Apache Spark runner -P spark-runner -->
<profile>
<id>spark-runner</id>
<dependencies>
@@ -122,7 +102,6 @@
</dependencies>
</profile>
- <!-- Include the Google Cloud Dataflow runner -P dataflow-runner -->
<profile>
<id>dataflow-runner</id>
<dependencies>
@@ -186,23 +165,15 @@
</execution>
</executions>
</plugin>
-
- <!-- Coverage analysis for unit tests. -->
- <plugin>
- <groupId>org.jacoco</groupId>
- <artifactId>jacoco-maven-plugin</artifactId>
- </plugin>
</plugins>
</build>
<dependencies>
- <!-- Java SDK -->
<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-core</artifactId>
</dependency>
- <!-- IOs -->
<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-io-google-cloud-platform</artifactId>
@@ -218,7 +189,6 @@
<artifactId>google-api-services-bigquery</artifactId>
</dependency>
- <!-- Extra libraries -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
@@ -282,7 +252,6 @@
<artifactId>hamcrest-all</artifactId>
</dependency>
- <!-- Test -->
<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>beam-runners-direct-java</artifactId>
http://git-wip-us.apache.org/repos/asf/beam/blob/0acd7201/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/package-info.java
----------------------------------------------------------------------
diff --git a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/package-info.java b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/package-info.java
index 7500a24..62218a4 100644
--- a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/package-info.java
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/package-info.java
@@ -16,6 +16,6 @@
* limitations under the License.
*/
/**
- * Nexmark.
+ * Nexmark test suite.
*/
package org.apache.beam.sdk.nexmark;
http://git-wip-us.apache.org/repos/asf/beam/blob/0acd7201/sdks/java/nexmark/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/sdks/java/nexmark/src/main/resources/log4j.properties b/sdks/java/nexmark/src/main/resources/log4j.properties
index 7dd57b5..14f8acd 100644
--- a/sdks/java/nexmark/src/main/resources/log4j.properties
+++ b/sdks/java/nexmark/src/main/resources/log4j.properties
@@ -27,14 +27,14 @@ log4j.logger.org.apache.beam.runners.direct=WARN
log4j.logger.org.apache.beam.sdk=WARN
# Nexmark specific
-log4j.logger.org.apache.beam.integration.nexmark=WARN
+log4j.logger.org.apache.beam.sdk.nexmark=WARN
# Settings to quiet third party logs that are too verbose
log4j.logger.org.spark_project.jetty=WARN
log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR
# Setting to quiet spark logs, Beam logs should standout
-log4j.logger.org.apache.beam.runners.spark=INFO
+log4j.logger.org.apache.beam.runners.spark=WARN
log4j.logger.org.apache.spark=WARN
log4j.logger.org.spark-project=WARN
log4j.logger.io.netty=INFO