You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by aj...@apache.org on 2023/01/23 20:34:53 UTC
[samza] branch master updated: [SAMZA-2772] Manage Jekyll dependencies with Gradle (#1647)
This is an automated email from the ASF dual-hosted git repository.
ajothomas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/samza.git
The following commit(s) were added to refs/heads/master by this push:
new d102d208e [SAMZA-2772] Manage Jekyll dependencies with Gradle (#1647)
d102d208e is described below
commit d102d208e3bc356fbdd6027643661e7768ab2ed3
Author: Adam Faris <af...@linkedin.com>
AuthorDate: Mon Jan 23 12:34:46 2023 -0800
[SAMZA-2772] Manage Jekyll dependencies with Gradle (#1647)
* Update Jekyll & swap redcarpet for kramdown
* Add jruby & jekyll dependencies to build.gradle
* buildscript.gradle update
* buildscript update
* downgrade jruby-gradle plugin so it is compatible with JDK8
* update documentation and finalize changes
* Update Apache RAT excludes for clean build
* This is a empty commit to trigger remote build tests
* trigger build tests by adding new line to build.gradle
* pin jekyll dependency
* remove jekyll incremental build flag
* Downgrade Jruby & Rouge so it works on RHEL/CentOS 7
* Replace 'jekyll build' with gradle task in publish-site.sh
---
.gitignore | 2 ++
bin/publish-site.sh | 7 +++---
build.gradle | 47 ++++++++++++++++++++++++++++++++++++-
docs/Gemfile | 23 ------------------
docs/Gemfile.lock | 59 -----------------------------------------------
docs/README.md | 22 ++++++++----------
docs/_blog/index.md | 2 +-
docs/_config.yml | 6 ++---
docs/index.md.bak | 39 -------------------------------
docs/meetups/index.html | 2 +-
docs/talks/index.html | 2 +-
gradle/buildscript.gradle | 5 ++--
settings.gradle | 1 +
13 files changed, 72 insertions(+), 145 deletions(-)
diff --git a/.gitignore b/.gitignore
index 05d33d058..4a73f40db 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,6 +18,8 @@ dashboard-deployable.tgz
deployable.tar
dist-dashboard
docs/_site
+docs/.jekyll-cache
+docs/.jekyll-metadata
.gradle
build
**/bin
diff --git a/bin/publish-site.sh b/bin/publish-site.sh
index 41e78e1a7..b06a3bb03 100755
--- a/bin/publish-site.sh
+++ b/bin/publish-site.sh
@@ -32,15 +32,16 @@ if test -z "$COMMENT" || test -z "$USER"; then
exit 0
fi
-echo "Using uer: $USER"
+echo "Using user: $USER"
echo "Using comment: $COMMENT"
echo "Generating javadocs."
$BASE_DIR/bin/generate-javadocs.sh
echo "Building site."
-cd $DOCS_DIR
-bundle exec jekyll build
+cd "$BASE_DIR"
+./gradlew docs:jekyllbuild
+cd "$DOCS_DIR"
echo "Replacing version"
./_docs/replace-versioned.sh
diff --git a/build.gradle b/build.gradle
index 1b6fcaddd..7b3dc4b88 100644
--- a/build.gradle
+++ b/build.gradle
@@ -84,7 +84,7 @@ rat {
'docs/sitemap.xml',
'docs/learn/documentation/*/api/javadocs/**',
'docs/learn/documentation/*/rest/javadocs/**',
- 'docs/Gemfile.lock',
+ 'docs/.jekyll-cache/**',
'gradle/wrapper/**',
'gradlew',
'gradlew.bat',
@@ -93,6 +93,8 @@ rat {
'RELEASE.md',
'samza-core/src/test/resources/classloader/samza-framework-api-classes.txt',
'samza-core/src/test/resources/*.txt',
+ 'samza-core/bin/test/*.txt',
+ 'samza-core/bin/test/classloader/samza-framework-api-classes.txt',
'samza-test/src/main/resources/**',
'samza-hdfs/src/main/resources/**',
'samza-hdfs/src/test/resources/**',
@@ -141,6 +143,48 @@ subprojects {
}
}
+project(":docs") {
+ apply plugin: "com.github.jruby-gradle.base"
+ apply plugin: 'java'
+
+ repositories {
+ ruby.gems()
+ }
+
+ jruby {
+ jrubyVersion = '9.3.9.0'
+ }
+
+ dependencies {
+ gems "rubygems:jekyll-sass-converter:2.2.0"
+ gems "rubygems:jekyll:4.3.1"
+ gems "rubygems:http_parser.rb:0.6.0"
+ gems "rubygems:webrick:1.7.0"
+ gems "rubygems:json:2.6.3"
+ gems "rubygems:rouge:3.30.0"
+ gems "rubygems:kramdown:2.4.0"
+ gems "rubygems:kramdown-parser-gfm:1.1.0"
+ }
+
+ task jekyllBuild(type: com.github.jrubygradle.JRubyExec) {
+ script "jekyll"
+ scriptArgs "build", "$projectDir", "-d", "$projectDir/_site"
+ }
+
+ task jekyllServeLocal(type: com.github.jrubygradle.JRubyExec) {
+ def baseurl = ""
+ script "jekyll"
+ scriptArgs "serve", "--watch", "--source", "$projectDir" , "--destination", "$projectDir/_site", "--trace", "--baseurl", "$baseurl"
+ }
+
+ task jekyllServePublic(type: com.github.jrubygradle.JRubyExec) {
+ def baseurl = ""
+ script "jekyll"
+ scriptArgs "serve", "--watch", "--source", "$projectDir" , "--destination", "$projectDir/_site", "--trace", "--baseurl", "$baseurl", "--host", "0.0.0.0"
+ }
+
+}
+
project(':samza-api') {
apply plugin: 'java'
@@ -976,3 +1020,4 @@ project(":samza-test_$scalaSuffix") {
wrapper {
gradleVersion = project.gradleVersion
}
+
diff --git a/docs/Gemfile b/docs/Gemfile
deleted file mode 100644
index 84b00ab60..000000000
--- a/docs/Gemfile
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-source 'https://rubygems.org'
-
-gem 'jekyll', '3.4.5'
-gem 'json', '2.1.0'
-gem 'redcarpet', '3.4.0'
-gem 'pygments.rb', '1.2.1'
diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock
deleted file mode 100644
index 0ab92cf1e..000000000
--- a/docs/Gemfile.lock
+++ /dev/null
@@ -1,59 +0,0 @@
-GEM
- remote: https://rubygems.org/
- specs:
- addressable (2.5.2)
- public_suffix (>= 2.0.2, < 4.0)
- colorator (1.1.0)
- ffi (1.9.23)
- forwardable-extended (2.6.0)
- jekyll (3.4.5)
- addressable (~> 2.4)
- colorator (~> 1.0)
- jekyll-sass-converter (~> 1.0)
- jekyll-watch (~> 1.1)
- kramdown (~> 1.3)
- liquid (~> 3.0)
- mercenary (~> 0.3.3)
- pathutil (~> 0.9)
- rouge (~> 1.7)
- safe_yaml (~> 1.0)
- jekyll-sass-converter (1.5.2)
- sass (~> 3.4)
- jekyll-watch (1.5.0)
- listen (~> 3.0, < 3.1)
- json (2.1.0)
- kramdown (1.16.2)
- liquid (3.0.6)
- listen (3.0.8)
- rb-fsevent (~> 0.9, >= 0.9.4)
- rb-inotify (~> 0.9, >= 0.9.7)
- mercenary (0.3.6)
- multi_json (1.13.1)
- pathutil (0.16.1)
- forwardable-extended (~> 2.6)
- public_suffix (2.0.5)
- pygments.rb (1.2.1)
- multi_json (>= 1.0.0)
- rb-fsevent (0.10.3)
- rb-inotify (0.9.10)
- ffi (>= 0.5.0, < 2)
- redcarpet (3.4.0)
- rouge (1.11.1)
- safe_yaml (1.0.4)
- sass (3.5.6)
- sass-listen (~> 4.0.0)
- sass-listen (4.0.0)
- rb-fsevent (~> 0.9, >= 0.9.4)
- rb-inotify (~> 0.9, >= 0.9.7)
-
-PLATFORMS
- ruby
-
-DEPENDENCIES
- jekyll (= 3.4.5)
- json (= 2.1.0)
- pygments.rb (= 1.2.1)
- redcarpet (= 3.4.0)
-
-BUNDLED WITH
- 1.16.1
diff --git a/docs/README.md b/docs/README.md
index a3f7ab335..ae070b751 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -16,30 +16,28 @@
-->
## Setup
-Samza's documentation uses Jekyll to build a website out of markdown pages. Prerequisites:
+Samza's documentation uses Jekyll to build a website out of markdown pages. Jekyll dependencies are now managed with Gradle.
-1. You need [Ruby](https://www.ruby-lang.org/) installed on your machine (run `ruby --version` to check)
-2. Install [Bundler](http://bundler.io/) by running `sudo gem install bundler`
-3. To install Jekyll and its dependencies, change to the `docs` directory and run `bundle install`
+To compile the website in the _site directory, execute:
-To serve the website on [localhost:4000](http://localhost:4000/) (run from the `docs` directory):
+ ./gradlew docs:jekyllBuild
- bundle exec jekyll serve --watch --baseurl ""
+To serve the website on [localhost:4000](http://localhost:4000/), run the following:
-To compile the website in the \_site directory, execute:
+ ./gradlew docs:jekyllServeLocal
- bundle exec jekyll build
+To serve the website on so its visible to other hosts on port 4000, run the following:
-To test the site, run:
+ ./gradlew docs:jekyllServePublic
+
+The jekyllServe* tasks do not end as they will wait for Jekyll to exit. You will need to cntl-c jekyllServeLocal or JekyllServePublic to stop the task.
- bundle exec jekyll serve --watch --baseurl ""
-(add --host 0.0.0.0 if you want it to be visible from other hosts)
## Versioning
If you're working with versioned content (anything in the learn or img directories), you'll also need to run a script that generates the appropriate directories:
- _docs/local-site-test.sh
+ _docs/local-site-test.sh
Run this script after starting the site server with the `serve` command above. Otherwise, you may see some errors when trying to view versioned content.
In addition, the local-site-test.sh script must be run every time a change is made to versioned content locally in order to trigger a refresh with Jekyll.
diff --git a/docs/_blog/index.md b/docs/_blog/index.md
index 3a422f856..695d864db 100644
--- a/docs/_blog/index.md
+++ b/docs/_blog/index.md
@@ -22,7 +22,7 @@ exclude_from_loop: true
<div class="news_cards">
- {% assign sorted = (site.blog | sort: 'date') | reverse %}
+ {% assign sorted = site.blog | sort: 'date' | reverse %}
{% for post in sorted %}
diff --git a/docs/_config.yml b/docs/_config.yml
index 77dc3548c..e05d1e81b 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -16,10 +16,10 @@
# under the License.
permalink: /:categories/:title
name: Samza
-highlighter: pygments
-markdown: redcarpet
+highlighter: rouge
+markdown: kramdown
exclude: ['_notes']
-redcarpet:
+kramdown:
extensions: ['with_toc_data', 'smart', 'strikethrough', 'tables']
exclude: [_docs]
baseurl: http://samza.apache.org
diff --git a/docs/index.md.bak b/docs/index.md.bak
deleted file mode 100644
index 4e6c19c0d..000000000
--- a/docs/index.md.bak
+++ /dev/null
@@ -1,39 +0,0 @@
----
-layout: default
----
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-## What is Samza?
-
-Apache Samza is a distributed stream processing framework. It uses <a target="_blank" href="http://kafka.apache.org">Apache Kafka</a> for messaging, and <a target="_blank" href="http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/YARN.html">Apache Hadoop YARN</a> to provide fault tolerance, processor isolation, security, and resource management.
-
-* **Simple API:** Unlike most low-level messaging system APIs, Samza provides a very simple callback-based "process message" API comparable to MapReduce.
-* **Managed state:** Samza manages snapshotting and restoration of a stream processor's state. When the processor is restarted, Samza restores its state to a consistent snapshot. Samza is built to handle large amounts of state (many gigabytes per partition).
-* **Fault tolerance:** Whenever a machine in the cluster fails, Samza works with YARN to transparently migrate your tasks to another machine.
-* **Durability:** Samza uses Kafka to guarantee that messages are processed in the order they were written to a partition, and that no messages are ever lost.
-* **Scalability:** Samza is partitioned and distributed at every level. Kafka provides ordered, partitioned, replayable, fault-tolerant streams. YARN provides a distributed environment for Samza containers to run in.
-* **Pluggable:** Though Samza works out of the box with Kafka and YARN, Samza provides a pluggable API that lets you run Samza with other messaging systems and execution environments.
-* **Processor isolation:** Samza works with Apache YARN, which supports Hadoop's security model, and resource isolation through Linux CGroups.
-
-Check out [Hello Samza](/startup/hello-samza/{{site.version}}) to try Samza. Read the [Background](/learn/documentation/{{site.version}}/introduction/background.html) page to learn more about Samza.
-
-### Apache Software Foundation
-
-Apache Samza is a top level project of the [Apache Software Foundation](http://www.apache.org/).
-
-
diff --git a/docs/meetups/index.html b/docs/meetups/index.html
index af3bbfe39..f36ca2d81 100644
--- a/docs/meetups/index.html
+++ b/docs/meetups/index.html
@@ -21,7 +21,7 @@ title: Samza Meetups
-->
<div class="meetups" data-plugin="paginate" data-max-posts="2" data-previous=".paginate-previous--meetup" data-next=".paginate-next--meetup" data-pagination=".pagination--meetup">
- {% assign sorted = (site.meetups | sort: 'date') | reverse %}
+ {% assign sorted = site.meetups | sort: 'date' | reverse %}
{% for meetup in sorted %}
{% if meetup.exclude_from_loop or meetup.date == site.time %}
{% continue %}
diff --git a/docs/talks/index.html b/docs/talks/index.html
index 0ee8fa610..5242375b8 100644
--- a/docs/talks/index.html
+++ b/docs/talks/index.html
@@ -21,7 +21,7 @@ title: Samza Talks
-->
<div class="talks" data-plugin="paginate" data-max-posts="2" data-previous=".paginate-previous--talk" data-next=".paginate-next--talk" data-pagination=".pagination--talk">
- {% assign sorted = (site.talks | sort: 'date') | reverse %}
+ {% assign sorted = site.talks | sort: 'date' | reverse %}
{% for talk in sorted %}
{% if talk.exclude_from_loop %}
{% continue %}
diff --git a/gradle/buildscript.gradle b/gradle/buildscript.gradle
index 6b52d2f2f..2abedd63c 100644
--- a/gradle/buildscript.gradle
+++ b/gradle/buildscript.gradle
@@ -18,9 +18,9 @@
*/
repositories {
repositories {
- // For gradle-nexus-plugin
+ // For plugins listed below
maven {
- url 'http://jcenter.bintray.com/'
+ url "https://plugins.gradle.org/m2/"
}
}
}
@@ -28,4 +28,5 @@ repositories {
dependencies {
classpath 'de.obqo.gradle:gradle-lesscss-plugin:1.0-1.3.3'
classpath 'com.bmuschko:gradle-nexus-plugin:2.3.1'
+ classpath 'com.github.jruby-gradle:jruby-gradle-plugin:2.0.2'
}
diff --git a/settings.gradle b/settings.gradle
index a29de6aec..792d9b67c 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -18,6 +18,7 @@
*/
include \
+ 'docs',
'samza-api',
'samza-sql',
'samza-shell',