You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bahir.apache.org by ck...@apache.org on 2017/03/06 09:21:24 UTC
bahir-website git commit: [BAHIR-92] Add Flink extensions in
update-doc.sh
Repository: bahir-website
Updated Branches:
refs/heads/master 0d53ccdd4 -> 4142e1e34
[BAHIR-92] Add Flink extensions in update-doc.sh
Refactoring update-doc.sh script to include Bahir-Flink extensions
and make it easier to add future extensions.
Closes #8
Project: http://git-wip-us.apache.org/repos/asf/bahir-website/repo
Commit: http://git-wip-us.apache.org/repos/asf/bahir-website/commit/4142e1e3
Tree: http://git-wip-us.apache.org/repos/asf/bahir-website/tree/4142e1e3
Diff: http://git-wip-us.apache.org/repos/asf/bahir-website/diff/4142e1e3
Branch: refs/heads/master
Commit: 4142e1e34462c3037e0db5366bee28f27b00ff37
Parents: 0d53ccd
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Mon Mar 6 01:20:35 2017 -0800
Committer: Christian Kadner <ck...@us.ibm.com>
Committed: Mon Mar 6 01:20:35 2017 -0800
----------------------------------------------------------------------
.../templates/flink-streaming-activemq.template | 27 +++++
.../templates/flink-streaming-akka.template | 27 +++++
.../templates/flink-streaming-flume.template | 27 +++++
.../templates/flink-streaming-netty.template | 27 +++++
.../templates/flink-streaming-redis.template | 27 +++++
update-doc.sh | 117 ++++++++++++-------
6 files changed, 212 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bahir-website/blob/4142e1e3/site/docs/flink/templates/flink-streaming-activemq.template
----------------------------------------------------------------------
diff --git a/site/docs/flink/templates/flink-streaming-activemq.template b/site/docs/flink/templates/flink-streaming-activemq.template
new file mode 100644
index 0000000..0116e96
--- /dev/null
+++ b/site/docs/flink/templates/flink-streaming-activemq.template
@@ -0,0 +1,27 @@
+---
+layout: page
+title: Apache Flink Streaming Connector for ActiveMQ
+description: Apache Flink Streaming Connector for ActiveMQ
+group: nav-right
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+{% include JB/setup %}
+
http://git-wip-us.apache.org/repos/asf/bahir-website/blob/4142e1e3/site/docs/flink/templates/flink-streaming-akka.template
----------------------------------------------------------------------
diff --git a/site/docs/flink/templates/flink-streaming-akka.template b/site/docs/flink/templates/flink-streaming-akka.template
new file mode 100644
index 0000000..c0d6790
--- /dev/null
+++ b/site/docs/flink/templates/flink-streaming-akka.template
@@ -0,0 +1,27 @@
+---
+layout: page
+title: Apache Flink Streaming Connector for Akka
+description: Apache Flink Streaming Connector for Akka
+group: nav-right
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+{% include JB/setup %}
+
http://git-wip-us.apache.org/repos/asf/bahir-website/blob/4142e1e3/site/docs/flink/templates/flink-streaming-flume.template
----------------------------------------------------------------------
diff --git a/site/docs/flink/templates/flink-streaming-flume.template b/site/docs/flink/templates/flink-streaming-flume.template
new file mode 100644
index 0000000..699ee05
--- /dev/null
+++ b/site/docs/flink/templates/flink-streaming-flume.template
@@ -0,0 +1,27 @@
+---
+layout: page
+title: Apache Flink Streaming Connector for Apache Flume
+description: Apache Flink Streaming Connector for Apache Flume
+group: nav-right
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+{% include JB/setup %}
+
http://git-wip-us.apache.org/repos/asf/bahir-website/blob/4142e1e3/site/docs/flink/templates/flink-streaming-netty.template
----------------------------------------------------------------------
diff --git a/site/docs/flink/templates/flink-streaming-netty.template b/site/docs/flink/templates/flink-streaming-netty.template
new file mode 100644
index 0000000..fc57b6d
--- /dev/null
+++ b/site/docs/flink/templates/flink-streaming-netty.template
@@ -0,0 +1,27 @@
+---
+layout: page
+title: Apache Flink Streaming Connector for Netty
+description: Apache Flink Streaming Connector for Netty
+group: nav-right
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+{% include JB/setup %}
+
http://git-wip-us.apache.org/repos/asf/bahir-website/blob/4142e1e3/site/docs/flink/templates/flink-streaming-redis.template
----------------------------------------------------------------------
diff --git a/site/docs/flink/templates/flink-streaming-redis.template b/site/docs/flink/templates/flink-streaming-redis.template
new file mode 100644
index 0000000..8d3c7dc
--- /dev/null
+++ b/site/docs/flink/templates/flink-streaming-redis.template
@@ -0,0 +1,27 @@
+---
+layout: page
+title: Apache Flink Streaming Connector for Redis
+description: Apache Flink Streaming Connector for Redis
+group: nav-right
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+{% include JB/setup %}
+
http://git-wip-us.apache.org/repos/asf/bahir-website/blob/4142e1e3/update-doc.sh
----------------------------------------------------------------------
diff --git a/update-doc.sh b/update-doc.sh
index 81e4a84..d5615a5 100755
--- a/update-doc.sh
+++ b/update-doc.sh
@@ -43,62 +43,99 @@
set -e
BASE_DIR=$(pwd)
-WEBSITE_TEMPLATES_DIR=$BASE_DIR/site/docs/spark/templates
-WEBSITE_DOC_DIR=$BASE_DIR/site/docs/spark/current
-BAHIR_SOURCE_DIR=$BASE_DIR/target/bahir
+
+SPARK_WEBSITE_TEMPLATES_DIR=$BASE_DIR/site/docs/spark/templates
+SPARK_WEBSITE_DOC_DIR=$BASE_DIR/site/docs/spark/current
+SPARK_REPO_NAME=bahir
+SPARK_BAHIR_SOURCE_DIR=$BASE_DIR/target/$SPARK_REPO_NAME
+
+FLINK_WEBSITE_TEMPLATES_DIR=$BASE_DIR/site/docs/flink/templates
+FLINK_WEBSITE_DOC_DIR=$BASE_DIR/site/docs/flink/current
+FLINK_REPO_NAME=bahir-flink
+FLINK_BAHIR_SOURCE_DIR=$BASE_DIR/target/$FLINK_REPO_NAME
function checkout_code {
# Checkout code
+ cd "$BASE_DIR" # make sure we're in the base dir
+
rm -rf target
mkdir target
cd target
- git clone https://git-wip-us.apache.org/repos/asf/bahir.git --quiet
- cd bahir
- git checkout $GIT_REF
- git_hash=`git rev-parse --short HEAD`
- echo "Checked out Bahir git hash $git_hash"
+ git clone "https://git-wip-us.apache.org/repos/asf/$1.git" --quiet
+ cd "$1"
+ git checkout "$GIT_REF"
+ git_hash=$(git rev-parse --short HEAD)
+ echo "Checked out $1 git hash $git_hash"
git clean -d -f -x
- cd "$BASE_DIR" #return to base dir
+ cd "$BASE_DIR" # return to base dir
}
-checkout_code
+function update_docs {
+ WEBSITE_TEMPLATES_DIR=$1
+ WEBSITE_DOC_DIR=$2
+ SOURCE_DIR=$3
+ shift 3
+
+ [ ! -d "$WEBSITE_DOC_DIR" ] && mkdir "$WEBSITE_DOC_DIR"
+
+ while [ $# -ne 0 ]; do
+ echo "Syncing document $WEBSITE_DOC_DIR/$1.md with source $SOURCE_DIR/$2/README.md"
+ rm -f "$WEBSITE_DOC_DIR/$1.md"
+ cp "$WEBSITE_TEMPLATES_DIR/$1.template" "$WEBSITE_DOC_DIR/$1.md"
+ cat "$SOURCE_DIR/$2/README.md" >> "$WEBSITE_DOC_DIR/$1.md"
+ shift 2
+ done
+}
-rm -rf $WEBSITE_DOC_DIR/spark*.md
+function check_version_strings {
+ if grep -q -r "[0-9]-SNAPSHOT" "$1"/*.md ; then
+ echo
+ echo "TODO: Replace '...-SNAPSHOT' version strings:"
+ echo
+ grep -r -n "[0-9]-SNAPSHOT" "$1"/*.md | sed -e 's|'"$(pwd)"/'||g' | grep --color "[0-9.]*-SNAPSHOT"
+ echo
+ echo "i.e. to replace '2.1.0-SNAPSHOT' with '2.0.2' run the following command:"
+ echo
+ echo " perl -i -pe 's/2.1.0-SNAPSHOT/2.0.2/g' $1/*.md"
+ echo
+ else
+ echo
+ echo "Generated files:"
+ echo
+ ls "$1"/*.md | xargs -n1 | sed -e 's|'"$(pwd -P)"/'||g'
+ fi
+}
-cp $WEBSITE_TEMPLATES_DIR/spark-sql-streaming-mqtt.template $WEBSITE_DOC_DIR/spark-sql-streaming-mqtt.md
-cat $BAHIR_SOURCE_DIR/sql-streaming-mqtt/README.md >> $WEBSITE_DOC_DIR/spark-sql-streaming-mqtt.md
+echo
+echo "================= Updating Apache Spark Extension documents ================="
+echo
-cp $WEBSITE_TEMPLATES_DIR/spark-streaming-akka.template $WEBSITE_DOC_DIR/spark-streaming-akka.md
-cat $BAHIR_SOURCE_DIR/streaming-akka/README.md >> $WEBSITE_DOC_DIR/spark-streaming-akka.md
+checkout_code $SPARK_REPO_NAME
-cp $WEBSITE_TEMPLATES_DIR/spark-streaming-mqtt.template $WEBSITE_DOC_DIR/spark-streaming-mqtt.md
-cat $BAHIR_SOURCE_DIR/streaming-mqtt/README.md >> $WEBSITE_DOC_DIR/spark-streaming-mqtt.md
+update_docs "$SPARK_WEBSITE_TEMPLATES_DIR" "$SPARK_WEBSITE_DOC_DIR" "$SPARK_BAHIR_SOURCE_DIR" \
+ spark-sql-streaming-mqtt sql-streaming-mqtt \
+ spark-streaming-akka streaming-akka \
+ spark-streaming-mqtt streaming-mqtt \
+ spark-streaming-twitter streaming-twitter \
+ spark-streaming-zeromq streaming-zeromq
-cp $WEBSITE_TEMPLATES_DIR/spark-streaming-twitter.template $WEBSITE_DOC_DIR/spark-streaming-twitter.md
-cat $BAHIR_SOURCE_DIR/streaming-twitter/README.md >> $WEBSITE_DOC_DIR/spark-streaming-twitter.md
+check_version_strings "$SPARK_WEBSITE_DOC_DIR"
-cp $WEBSITE_TEMPLATES_DIR/spark-streaming-zeromq.template $WEBSITE_DOC_DIR/spark-streaming-zeromq.md
-cat $BAHIR_SOURCE_DIR/streaming-zeromq/README.md >> $WEBSITE_DOC_DIR/spark-streaming-zeromq.md
+echo
+echo "================= Updating Apache Flink Extension documents ================="
+echo
-set +e
+checkout_code $FLINK_REPO_NAME
+
+update_docs "$FLINK_WEBSITE_TEMPLATES_DIR" "$FLINK_WEBSITE_DOC_DIR" "$FLINK_BAHIR_SOURCE_DIR" \
+ flink-streaming-activemq flink-connector-activemq \
+ flink-streaming-akka flink-connector-akka \
+ flink-streaming-flume flink-connector-flume \
+ flink-streaming-netty flink-connector-netty \
+ flink-streaming-redis flink-connector-redis
-echo "Done."
-
-if grep -q -r "[0-9]-SNAPSHOT" $WEBSITE_DOC_DIR/spark*.md ; then
- echo
- echo "TODO: Replace '...-SNAPSHOT' version strings:"
- echo
- grep -r -n "[0-9]-SNAPSHOT" $WEBSITE_DOC_DIR/spark*.md | sed -e 's|'$(pwd)/'||g' | grep --color "[0-9.]*-SNAPSHOT"
- echo
- echo "i.e. to replace '2.1.0-SNAPSHOT' with '2.0.2' run the following command:"
- echo
- echo " perl -i -pe 's/2.1.0-SNAPSHOT/2.0.2/g' $WEBSITE_DOC_DIR/spark*.md"
- echo
-else
- echo
- echo "Generated files:"
- echo
- ls $WEBSITE_DOC_DIR/spark*.md | xargs -n1 | sed -e 's|'$(pwd -P)/'||g'
-fi
+check_version_strings "$FLINK_WEBSITE_DOC_DIR"
+
+set +e