You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by fh...@apache.org on 2019/08/19 16:10:02 UTC
[flink-playgrounds] 01/02: [FLINK-12749] [playgrounds] initial
version of flink-cluster-playground
This is an automated email from the ASF dual-hosted git repository.
fhueske pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-playgrounds.git
commit eedb8b8bcb6e896251795b87f1e772b143750407
Author: Konstantin Knauf <kn...@gmail.com>
AuthorDate: Mon Jul 22 08:07:27 2019 +0200
[FLINK-12749] [playgrounds] initial version of flink-cluster-playground
---
README.md | 20 ++++++
flink-cluster-playground/conf/flink-conf.yaml | 30 +++++++++
flink-cluster-playground/conf/log4j-cli.properties | 43 +++++++++++++
.../conf/log4j-console.properties | 39 ++++++++++++
flink-cluster-playground/docker-compose.yaml | 72 ++++++++++++++++++++++
5 files changed, 204 insertions(+)
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..8fe2903
--- /dev/null
+++ b/README.md
@@ -0,0 +1,20 @@
+# Apache Flink Playgrounds
+
+Apache Flink is an open source stream processing framework with powerful stream- and batch-
+processing capabilities.
+
+Learn more about Flink at [http://flink.apache.org/](http://flink.apache.org/)
+
+## Playgrounds
+
+This repository contains the configuration files for two Apache Flink playgrounds.
+
+* The [Flink Cluster Playground](../master/flink-cluster-playground) consists of a Flink Session Cluster, a Kafka Cluster and a simple
+Flink Job. It is explained in detail as part of
+[Apache Flink's "Getting Started" guide](https://ci.apache.org/projects/flink/flink-docs-stable/getting-started/docker-playgrounds/flink-cluster-playground.html).
+
+* The interactive SQL playground is still under development and will be added shortly.
+
+## About
+
+Apache Flink is an open source project of The Apache Software Foundation (ASF).
\ No newline at end of file
diff --git a/flink-cluster-playground/conf/flink-conf.yaml b/flink-cluster-playground/conf/flink-conf.yaml
new file mode 100644
index 0000000..5c8d0e6
--- /dev/null
+++ b/flink-cluster-playground/conf/flink-conf.yaml
@@ -0,0 +1,30 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+jobmanager.rpc.address: jobmanager
+blob.server.port: 6124
+query.server.port: 6125
+
+taskmanager.numberOfTaskSlots: 2
+
+state.backend: filesystem
+state.checkpoints.dir: file:///tmp/flink-checkpoints-directory
+state.savepoints.dir: file:///tmp/flink-savepoints-directory
+
+heartbeat.interval: 1000
+heartbeat.timeout: 5000
diff --git a/flink-cluster-playground/conf/log4j-cli.properties b/flink-cluster-playground/conf/log4j-cli.properties
new file mode 100644
index 0000000..a6c8d36
--- /dev/null
+++ b/flink-cluster-playground/conf/log4j-cli.properties
@@ -0,0 +1,43 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+log4j.rootLogger=INFO, file
+
+# Log all infos in the given file
+log4j.appender.file=org.apache.log4j.FileAppender
+log4j.appender.file.file=${log.file}
+log4j.appender.file.append=false
+log4j.appender.file.layout=org.apache.log4j.PatternLayout
+log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+
+
+# Log output from org.apache.flink.yarn to the console. This is used by the
+# CliFrontend class when using a per-job YARN cluster.
+log4j.logger.org.apache.flink.yarn=INFO, console
+log4j.logger.org.apache.flink.yarn.cli.FlinkYarnSessionCli=INFO, console
+log4j.logger.org.apache.hadoop=INFO, console
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+
+# suppress the warning that hadoop native libraries are not loaded (irrelevant for the client)
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=OFF
+
+# suppress the irrelevant (wrong) warnings from the netty channel handler
+log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file
diff --git a/flink-cluster-playground/conf/log4j-console.properties b/flink-cluster-playground/conf/log4j-console.properties
new file mode 100644
index 0000000..b458d58
--- /dev/null
+++ b/flink-cluster-playground/conf/log4j-console.properties
@@ -0,0 +1,39 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# This affects logging for both user code and Flink
+log4j.rootLogger=INFO, console
+
+# Uncomment this if you want to _only_ change Flink's logging
+#log4j.logger.org.apache.flink=INFO
+
+# The following lines keep the log level of common libraries/connectors on
+# log level INFO. The root logger does not override this. You have to manually
+# change the log levels here.
+log4j.logger.akka=INFO
+log4j.logger.org.apache.kafka=INFO
+log4j.logger.org.apache.hadoop=INFO
+log4j.logger.org.apache.zookeeper=INFO
+
+# Log all infos to the console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+
+# Suppress the irrelevant (wrong) warnings from the Netty channel handler
+log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, console
diff --git a/flink-cluster-playground/docker-compose.yaml b/flink-cluster-playground/docker-compose.yaml
new file mode 100644
index 0000000..7842762
--- /dev/null
+++ b/flink-cluster-playground/docker-compose.yaml
@@ -0,0 +1,72 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+version: "2.1"
+services:
+ client:
+ image: flink:1.9-scala_2.11
+ command: "flink run -d -p 2 /opt/flink/examples/streaming/ClickEventCount.jar --bootstrap.servers kafka:9092 --checkpointing --event-time"
+ depends_on:
+ - jobmanager
+ - kafka
+ volumes:
+ - ./conf:/opt/flink/conf
+ environment:
+ - JOB_MANAGER_RPC_ADDRESS=jobmanager
+ clickevent-generator:
+ image: flink:1.9-scala_2.11
+ command: "java -classpath /opt/flink/examples/streaming/ClickEventCount.jar:/opt/flink/lib/* org.apache.flink.streaming.examples.windowing.clickeventcount.ClickEventGenerator --bootstrap.servers kafka:9092 --topic input"
+ depends_on:
+ - kafka
+ jobmanager:
+ image: flink:1.9-scala_2.11
+ command: "jobmanager.sh start-foreground"
+ ports:
+ - 8081:8081
+ volumes:
+ - ./conf:/opt/flink/conf
+ - flink-checkpoint-directory:/tmp/flink-checkpoint-directory
+ - /tmp/flink-savepoints-directory:/tmp/flink-savepoints-directory
+ environment:
+ - JOB_MANAGER_RPC_ADDRESS=jobmanager
+ taskmanager:
+ image: flink:1.9-scala_2.11
+ depends_on:
+ - jobmanager
+ command: "taskmanager.sh start-foreground"
+ volumes:
+ - ./conf:/opt/flink/conf
+ - flink-checkpoint-directory:/tmp/flink-checkpoint-directory
+ - /tmp/flink-savepoints-directory:/tmp/flink-savepoints-directory
+ environment:
+ - JOB_MANAGER_RPC_ADDRESS=jobmanager
+ zookeeper:
+ image: wurstmeister/zookeeper:3.4.6
+ kafka:
+ image: wurstmeister/kafka:2.12-2.2.1
+ environment:
+ KAFKA_ADVERTISED_LISTENERS: INSIDE://:9092,OUTSIDE://:9094
+ KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9094
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
+ KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
+ KAFKA_CREATE_TOPICS: "input:2:1, output:2:1"
+ KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+ ports:
+ - 9094:9094
+volumes:
+ flink-checkpoint-directory:
\ No newline at end of file