You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@wayang.apache.org by gi...@apache.org on 2024/03/06 10:04:10 UTC
(incubator-wayang-website) branch deploy updated: deploy: 344485e80777af255530c747aa008fca8c7c5241
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch deploy
in repository https://gitbox.apache.org/repos/asf/incubator-wayang-website.git
The following commit(s) were added to refs/heads/deploy by this push:
new d732e225 deploy: 344485e80777af255530c747aa008fca8c7c5241
d732e225 is described below
commit d732e22551ab874f2fedf78a6252b938e6d1bbb7
Author: 2pk03 <2p...@users.noreply.github.com>
AuthorDate: Wed Mar 6 10:04:05 2024 +0000
deploy: 344485e80777af255530c747aa008fca8c7c5241
---
404.html | 4 +-
assets/js/29e20dbd.80bb1200.js | 1 +
assets/js/29e20dbd.cfb2394c.js | 1 -
assets/js/34f284a7.747c7b73.js | 1 +
assets/js/3df8af31.d8e13fa1.js | 1 -
assets/js/3df8af31.e6d0197b.js | 1 +
assets/js/3e5cd75b.1fb44ffe.js | 1 +
assets/js/3e5cd75b.4b329537.js | 1 -
.../{814f3328.478ed4d8.js => 814f3328.36233fa8.js} | 2 +-
assets/js/b2f554cd.23ab19eb.js | 1 -
assets/js/b2f554cd.6a9dc8d5.js | 1 +
.../{e82464af.d16505e9.js => e82464af.295b901b.js} | 2 +-
assets/js/effa2a56.5c0a5b22.js | 1 -
assets/js/effa2a56.865c2d23.js | 1 +
assets/js/main.3d1f1383.js | 2 -
assets/js/main.f140bb99.js | 2 +
...js.LICENSE.txt => main.f140bb99.js.LICENSE.txt} | 0
assets/js/runtime~main.06eee15f.js | 1 +
assets/js/runtime~main.d19f8cc4.js | 1 -
blog/Apache Kafka meets Apache Wayang/index.html | 8 +-
blog/archive/index.html | 6 +-
blog/atom.xml | 106 +++++++++------------
blog/index.html | 106 ++++++++++++---------
.../kafka => kafka-meets-wayang-1}/index.html | 55 ++---------
blog/rss.xml | 106 +++++++++------------
.../index.html | 106 ++++++++++++---------
blog/tags/index.html | 6 +-
blog/tags/kafka/index.html | 106 ++++++++++++---------
blog/tags/wayang/index.html | 106 ++++++++++++---------
blog/website_update/index.html | 8 +-
docs/community/committer/index.html | 4 +-
docs/community/contribute/index.html | 4 +-
docs/community/mailinglist/index.html | 4 +-
docs/community/repositories/index.html | 4 +-
docs/community/team/index.html | 4 +-
docs/guide/adding-operators/index.html | 4 +-
docs/guide/api-documentation/index.html | 4 +-
docs/guide/developing-in-wayang/index.html | 4 +-
docs/guide/examples/index.html | 4 +-
docs/guide/getting-started/index.html | 4 +-
docs/guide/installation/index.html | 4 +-
docs/guide/ml4all/index.html | 4 +-
docs/guide/optimizer/index.html | 4 +-
docs/introduction/about/index.html | 4 +-
docs/introduction/benchmark/index.html | 4 +-
docs/introduction/features/index.html | 4 +-
docs/start/download/index.html | 4 +-
index.html | 4 +-
lunr-index-1709719224803.json | 1 -
lunr-index-1709719417673.json | 1 +
lunr-index.json | 2 +-
search-doc-1709719224803.json | 1 -
search-doc-1709719417673.json | 1 +
search-doc.json | 2 +-
sitemap.xml | 2 +-
55 files changed, 406 insertions(+), 420 deletions(-)
diff --git a/404.html b/404.html
index c535403d..6138d66a 100644
--- a/404.html
+++ b/404.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Page Not Found | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/404.html"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docusaurus_tag" content="default"><meta d [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/assets/js/29e20dbd.80bb1200.js b/assets/js/29e20dbd.80bb1200.js
new file mode 100644
index 00000000..6b934fd4
--- /dev/null
+++ b/assets/js/29e20dbd.80bb1200.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[578],{5944:(e,a,t)=>{t.r(a),t.d(a,{assets:()=>c,contentTitle:()=>s,default:()=>p,frontMatter:()=>o,metadata:()=>r,toc:()=>l});var n=t(5893),i=t(1151);const o={slug:"Apache Kafka meets Apache Wayang",title:"Apache Kafka meets Apache Wayang - Part 2",authors:"kamir",tags:["wayang","kafka","cross organization data collaboration"]},s="Apache Wayang meets Apache Kafka - Part 2",r={permalink:"/blog/Apache [...]
\ No newline at end of file
diff --git a/assets/js/29e20dbd.cfb2394c.js b/assets/js/29e20dbd.cfb2394c.js
deleted file mode 100644
index a2a8ef14..00000000
--- a/assets/js/29e20dbd.cfb2394c.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[578],{5944:(e,a,t)=>{t.r(a),t.d(a,{assets:()=>c,contentTitle:()=>s,default:()=>p,frontMatter:()=>o,metadata:()=>r,toc:()=>h});var n=t(5893),i=t(1151);const o={slug:"Apache Kafka meets Apache Wayang",title:"Apache Kafka meets Apache Wayang - Part 2",authors:"kamir",tags:["wayang","kafka","cross organization data collaboration"]},s="Apache Wayang meets Apache Kafka - Part 2",r={permalink:"/blog/Apache [...]
\ No newline at end of file
diff --git a/assets/js/34f284a7.747c7b73.js b/assets/js/34f284a7.747c7b73.js
new file mode 100644
index 00000000..c4a45ddc
--- /dev/null
+++ b/assets/js/34f284a7.747c7b73.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[8815],{9702:(e,a,t)=>{t.r(a),t.d(a,{assets:()=>c,contentTitle:()=>s,default:()=>p,frontMatter:()=>o,metadata:()=>r,toc:()=>l});var n=t(5893),i=t(1151);const o={slug:"Apache Kafka meets Apache Wayang",title:"Apache Kafka meets Apache Wayang - Part 2",authors:"kamir",tags:["wayang","kafka","cross organization data collaboration"]},s="Apache Wayang meets Apache Kafka - Part 2",r={permalink:"/blog/Apach [...]
\ No newline at end of file
diff --git a/assets/js/3df8af31.d8e13fa1.js b/assets/js/3df8af31.d8e13fa1.js
deleted file mode 100644
index 9e7be84f..00000000
--- a/assets/js/3df8af31.d8e13fa1.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[6603],{5149:(a,e,t)=>{t.r(e),t.d(e,{assets:()=>l,contentTitle:()=>r,default:()=>p,frontMatter:()=>i,metadata:()=>s,toc:()=>c});var n=t(5893),o=t(1151);const i={slug:"Apache Kafka meets Apache Wayang",title:"Apache Kafka meets Apache Wayang - Part 1",authors:"kamir",tags:["wayang","kafka","cross organization data collaboration"]},r="Apache Wayang meets Apache Kafka - Part 1",s={permalink:"/blog/Apach [...]
\ No newline at end of file
diff --git a/assets/js/3df8af31.e6d0197b.js b/assets/js/3df8af31.e6d0197b.js
new file mode 100644
index 00000000..2e705075
--- /dev/null
+++ b/assets/js/3df8af31.e6d0197b.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[6603],{5149:(a,e,t)=>{t.r(e),t.d(e,{assets:()=>l,contentTitle:()=>r,default:()=>p,frontMatter:()=>i,metadata:()=>s,toc:()=>c});var n=t(5893),o=t(1151);const i={slug:"kafka-meets-wayang-1",title:"Apache Kafka meets Apache Wayang - Part 1",authors:"kamir",tags:["wayang","kafka","cross organization data collaboration"]},r="Apache Wayang meets Apache Kafka - Part 1",s={permalink:"/blog/kafka-meets-wayan [...]
\ No newline at end of file
diff --git a/assets/js/3e5cd75b.1fb44ffe.js b/assets/js/3e5cd75b.1fb44ffe.js
new file mode 100644
index 00000000..120a8075
--- /dev/null
+++ b/assets/js/3e5cd75b.1fb44ffe.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[8045],{6722:(a,e,t)=>{t.r(e),t.d(e,{assets:()=>l,contentTitle:()=>r,default:()=>p,frontMatter:()=>i,metadata:()=>s,toc:()=>c});var n=t(5893),o=t(1151);const i={slug:"kafka-meets-wayang-1",title:"Apache Kafka meets Apache Wayang - Part 1",authors:"kamir",tags:["wayang","kafka","cross organization data collaboration"]},r="Apache Wayang meets Apache Kafka - Part 1",s={permalink:"/blog/kafka-meets-wayan [...]
\ No newline at end of file
diff --git a/assets/js/3e5cd75b.4b329537.js b/assets/js/3e5cd75b.4b329537.js
deleted file mode 100644
index ebce70b2..00000000
--- a/assets/js/3e5cd75b.4b329537.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[8045],{6722:(a,e,t)=>{t.r(e),t.d(e,{assets:()=>l,contentTitle:()=>r,default:()=>p,frontMatter:()=>i,metadata:()=>s,toc:()=>c});var n=t(5893),o=t(1151);const i={slug:"Apache Kafka meets Apache Wayang",title:"Apache Kafka meets Apache Wayang - Part 1",authors:"kamir",tags:["wayang","kafka","cross organization data collaboration"]},r="Apache Wayang meets Apache Kafka - Part 1",s={permalink:"/blog/Apach [...]
\ No newline at end of file
diff --git a/assets/js/814f3328.478ed4d8.js b/assets/js/814f3328.36233fa8.js
similarity index 64%
rename from assets/js/814f3328.478ed4d8.js
rename to assets/js/814f3328.36233fa8.js
index 762c1380..856ddf7b 100644
--- a/assets/js/814f3328.478ed4d8.js
+++ b/assets/js/814f3328.36233fa8.js
@@ -1 +1 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[2535],{5641:e=>{e.exports=JSON.parse('{"title":"All our posts","items":[{"title":"Apache Kafka meets Apache Wayang - Part 2","permalink":"/blog/Apache Kafka meets Apache Wayang","unlisted":false},{"title":"Apache Kafka meets Apache Wayang - Part 1","permalink":"/blog/Apache Kafka meets Apache Wayang","unlisted":false},{"title":"Website updated","permalink":"/blog/website_update","unlisted":false}]}')}}]);
\ No newline at end of file
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[2535],{5641:e=>{e.exports=JSON.parse('{"title":"All our posts","items":[{"title":"Apache Kafka meets Apache Wayang - Part 2","permalink":"/blog/Apache Kafka meets Apache Wayang","unlisted":false},{"title":"Apache Kafka meets Apache Wayang - Part 1","permalink":"/blog/kafka-meets-wayang-1","unlisted":false},{"title":"Website updated","permalink":"/blog/website_update","unlisted":false}]}')}}]);
\ No newline at end of file
diff --git a/assets/js/b2f554cd.23ab19eb.js b/assets/js/b2f554cd.23ab19eb.js
deleted file mode 100644
index 6af91bc2..00000000
--- a/assets/js/b2f554cd.23ab19eb.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[1477],{10:a=>{a.exports=JSON.parse('{"blogPosts":[{"id":"Apache Kafka meets Apache Wayang","metadata":{"permalink":"/blog/Apache Kafka meets Apache Wayang","source":"@site/blog/2024-03-06-kafka-meets-wayang-2.md","title":"Apache Kafka meets Apache Wayang - Part 2","description":"In the second part of the article series we describe the implementation of the Kafka Source and Kafka Sink component for A [...]
\ No newline at end of file
diff --git a/assets/js/b2f554cd.6a9dc8d5.js b/assets/js/b2f554cd.6a9dc8d5.js
new file mode 100644
index 00000000..37d0e883
--- /dev/null
+++ b/assets/js/b2f554cd.6a9dc8d5.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[1477],{10:a=>{a.exports=JSON.parse('{"blogPosts":[{"id":"Apache Kafka meets Apache Wayang","metadata":{"permalink":"/blog/Apache Kafka meets Apache Wayang","source":"@site/blog/2024-03-06-kafka-meets-wayang-2.md","title":"Apache Kafka meets Apache Wayang - Part 2","description":"In the second part of the article series we describe the implementation of the Kafka Source and Kafka Sink component for A [...]
\ No newline at end of file
diff --git a/assets/js/e82464af.d16505e9.js b/assets/js/e82464af.295b901b.js
similarity index 58%
rename from assets/js/e82464af.d16505e9.js
rename to assets/js/e82464af.295b901b.js
index a18062d9..b63be810 100644
--- a/assets/js/e82464af.d16505e9.js
+++ b/assets/js/e82464af.295b901b.js
@@ -1 +1 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[7555],{1757:(e,t,a)=>{a.r(t),a.d(t,{assets:()=>i,contentTitle:()=>r,default:()=>p,frontMatter:()=>o,metadata:()=>u,toc:()=>c});var n=a(5893),s=a(1151);const o={slug:"website_update",title:"Website updated",authors:["alo.alt"],tags:["wayang"]},r=void 0,u={permalink:"/blog/website_update",source:"@site/blog/2024-01-25-website_update.md",title:"Website updated",description:"We're updated our website an [...]
\ No newline at end of file
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[7555],{1757:(e,t,a)=>{a.r(t),a.d(t,{assets:()=>i,contentTitle:()=>r,default:()=>p,frontMatter:()=>o,metadata:()=>u,toc:()=>c});var n=a(5893),s=a(1151);const o={slug:"website_update",title:"Website updated",authors:["alo.alt"],tags:["wayang"]},r=void 0,u={permalink:"/blog/website_update",source:"@site/blog/2024-01-25-website_update.md",title:"Website updated",description:"We're updated our website an [...]
\ No newline at end of file
diff --git a/assets/js/effa2a56.5c0a5b22.js b/assets/js/effa2a56.5c0a5b22.js
deleted file mode 100644
index fd76a80c..00000000
--- a/assets/js/effa2a56.5c0a5b22.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[3399],{897:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>o,contentTitle:()=>l,default:()=>h,frontMatter:()=>i,metadata:()=>r,toc:()=>c});var a=t(5893),s=t(1151);const i={slug:"website_update",title:"Website updated",authors:["alo.alt"],tags:["wayang"]},l="Website updated",r={permalink:"/blog/website_update",source:"@site/blog/2024-01-25-website_update.md",title:"Website updated",description:"We're updated our [...]
\ No newline at end of file
diff --git a/assets/js/effa2a56.865c2d23.js b/assets/js/effa2a56.865c2d23.js
new file mode 100644
index 00000000..8cb77162
--- /dev/null
+++ b/assets/js/effa2a56.865c2d23.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[3399],{897:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>o,contentTitle:()=>l,default:()=>d,frontMatter:()=>i,metadata:()=>r,toc:()=>c});var s=t(5893),a=t(1151);const i={slug:"website_update",title:"Website updated",authors:["alo.alt"],tags:["wayang"]},l="Website updated",r={permalink:"/blog/website_update",source:"@site/blog/2024-01-25-website_update.md",title:"Website updated",description:"We're updated our [...]
\ No newline at end of file
diff --git a/assets/js/main.3d1f1383.js b/assets/js/main.3d1f1383.js
deleted file mode 100644
index 6054f427..00000000
--- a/assets/js/main.3d1f1383.js
+++ /dev/null
@@ -1,2 +0,0 @@
-/*! For license information please see main.3d1f1383.js.LICENSE.txt */
-(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[179],{723:(e,t,n)=>{"use strict";n.d(t,{Z:()=>p});n(7294);var r=n(8356),a=n.n(r),o=n(6887);const i={"01a85c17":[()=>Promise.all([n.e(532),n.e(4013)]).then(n.bind(n,1223)),"@theme/BlogTagsListPage",1223],"05a94acb":[()=>n.e(3617).then(n.bind(n,7856)),"@site/docs/community/committer.md",7856],"092604a4":[()=>n.e(5088).then(n.bind(n,45)),"@site/docs/introduction/benchmark.md",45],17896441:[()=>Promise.all([n.e(532) [...]
\ No newline at end of file
diff --git a/assets/js/main.f140bb99.js b/assets/js/main.f140bb99.js
new file mode 100644
index 00000000..26e82d5e
--- /dev/null
+++ b/assets/js/main.f140bb99.js
@@ -0,0 +1,2 @@
+/*! For license information please see main.f140bb99.js.LICENSE.txt */
+(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[179],{723:(e,t,n)=>{"use strict";n.d(t,{Z:()=>p});n(7294);var r=n(8356),a=n.n(r),o=n(6887);const i={"01a85c17":[()=>Promise.all([n.e(532),n.e(4013)]).then(n.bind(n,1223)),"@theme/BlogTagsListPage",1223],"05a94acb":[()=>n.e(3617).then(n.bind(n,7856)),"@site/docs/community/committer.md",7856],"092604a4":[()=>n.e(5088).then(n.bind(n,45)),"@site/docs/introduction/benchmark.md",45],17896441:[()=>Promise.all([n.e(532) [...]
\ No newline at end of file
diff --git a/assets/js/main.3d1f1383.js.LICENSE.txt b/assets/js/main.f140bb99.js.LICENSE.txt
similarity index 100%
rename from assets/js/main.3d1f1383.js.LICENSE.txt
rename to assets/js/main.f140bb99.js.LICENSE.txt
diff --git a/assets/js/runtime~main.06eee15f.js b/assets/js/runtime~main.06eee15f.js
new file mode 100644
index 00000000..cd674ea8
--- /dev/null
+++ b/assets/js/runtime~main.06eee15f.js
@@ -0,0 +1 @@
+(()=>{"use strict";var e,a,c,t,d,r={},f={};function o(e){var a=f[e];if(void 0!==a)return a.exports;var c=f[e]={exports:{}};return r[e].call(c.exports,c,c.exports,o),c.exports}o.m=r,e=[],o.O=(a,c,t,d)=>{if(!c){var r=1/0;for(i=0;i<e.length;i++){c=e[i][0],t=e[i][1],d=e[i][2];for(var f=!0,n=0;n<c.length;n++)(!1&d||r>=d)&&Object.keys(o.O).every((e=>o.O[e](c[n])))?c.splice(n--,1):(f=!1,d<r&&(r=d));if(f){e.splice(i--,1);var b=t();void 0!==b&&(a=b)}}return a}d=d||0;for(var i=e.length;i>0&&e[i-1] [...]
\ No newline at end of file
diff --git a/assets/js/runtime~main.d19f8cc4.js b/assets/js/runtime~main.d19f8cc4.js
deleted file mode 100644
index 5bafed69..00000000
--- a/assets/js/runtime~main.d19f8cc4.js
+++ /dev/null
@@ -1 +0,0 @@
-(()=>{"use strict";var e,a,t,c,r,d={},f={};function o(e){var a=f[e];if(void 0!==a)return a.exports;var t=f[e]={exports:{}};return d[e].call(t.exports,t,t.exports,o),t.exports}o.m=d,e=[],o.O=(a,t,c,r)=>{if(!t){var d=1/0;for(i=0;i<e.length;i++){t=e[i][0],c=e[i][1],r=e[i][2];for(var f=!0,n=0;n<t.length;n++)(!1&r||d>=r)&&Object.keys(o.O).every((e=>o.O[e](t[n])))?t.splice(n--,1):(f=!1,r<d&&(d=r));if(f){e.splice(i--,1);var b=c();void 0!==b&&(a=b)}}return a}r=r||0;for(var i=e.length;i>0&&e[i-1] [...]
\ No newline at end of file
diff --git a/blog/Apache Kafka meets Apache Wayang/index.html b/blog/Apache Kafka meets Apache Wayang/index.html
index ca5680f0..0e516041 100644
--- a/blog/Apache Kafka meets Apache Wayang/index.html
+++ b/blog/Apache Kafka meets Apache Wayang/index.html
@@ -5,11 +5,11 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Apache Kafka meets Apache Wayang - Part 2 | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/Apache Kafka meets Apache Wayang"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-r [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
We look into the “Read- and Write-Path” for our data items, called <em>DataQuanta</em>.</p>
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="apache-wayangs-read--write-path-for-kafka-topics">Apache Wayang’s Read & Write Path for Kafka topics<a href="#apache-wayangs-read--write-path-for-kafka-topics" class="hash-link" aria-label="Direct link to Apache Wayang’s Read & Write Path for Kafka topics" title="Direct link to Apache Wayang’s Read & Write Path for Kafka topics"></a></h2>
<p>To describe the read and write paths for data in the context of the created Apache Wayang code snippet, the primary classes and interfaces we need to understand are as follows:</p>
@@ -66,7 +66,7 @@ The layer above handles the mapping of the components at startup time.
All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.
-Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg blogPostFooterDetailsFull_mRVl"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li [...]
+Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg blogPostFooterDetailsFull_mRVl"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li [...]
<p> Apache Wayang is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that th [...]
<p>
Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. <br>
diff --git a/blog/archive/index.html b/blog/archive/index.html
index 7171b70f..51ae4b16 100644
--- a/blog/archive/index.html
+++ b/blog/archive/index.html
@@ -5,11 +5,11 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Archive | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/archive"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docusaurus_tag" content="default"><meta data [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
<p> Apache Wayang is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that th [...]
<p>
Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. <br>
diff --git a/blog/atom.xml b/blog/atom.xml
index d0aed44c..79ecf5f2 100644
--- a/blog/atom.xml
+++ b/blog/atom.xml
@@ -81,68 +81,54 @@ Finally, in part four we bring all puzzles together, and show the full implement
</entry>
<entry>
<title type="html"><![CDATA[Apache Kafka meets Apache Wayang - Part 1]]></title>
- <id>https://wayang.apache.org/blog/Apache Kafka meets Apache Wayang</id>
- <link href="https://wayang.apache.org/blog/Apache Kafka meets Apache Wayang"/>
+ <id>https://wayang.apache.org/blog/kafka-meets-wayang-1</id>
+ <link href="https://wayang.apache.org/blog/kafka-meets-wayang-1"/>
<updated>2024-03-05T00:00:00.000Z</updated>
<summary type="html"><![CDATA[Intro]]></summary>
- <content type="html"><![CDATA[<p>In the second part of the article series we describe the implementation of the Kafka Source and Kafka Sink component for Apache Wayang.
-We look into the “Read- and Write-Path” for our data items, called <em>DataQuanta</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="apache-wayangs-read--write-path-for-kafka-topics">Apache Wayang’s Read & Write Path for Kafka topics<a href="https://wayang.apache.org/blog/Apache%20Kafka%20meets%20Apache%20Wayang#apache-wayangs-read--write-path-for-kafka-topics" class="hash-link" aria-label="Direct link to Apache Wayang’s Read & Write Path for Kafka topics" title="Direct link to Apache Wayang’s Read & Write Path for Kafka topics"></a></h2>
-<p>To describe the read and write paths for data in the context of the created Apache Wayang code snippet, the primary classes and interfaces we need to understand are as follows:</p>
-<p><strong>WayangContext:</strong> This class is essential for initializing the Wayang processing environment.
-It allows you to configure the execution environment and register plugins that define which platforms Wayang can use for data processing tasks, such as <em>Java.basicPlugin()</em> for local Java execution.</p>
-<p><strong>JavaPlanBuilder:</strong> This class is used to build and define the data processing pipeline (or plan) in Wayang.
-It provides a fluent API to specify the operations to be performed on the data, from reading the input to processing it and writing the output.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-path">Read Path<a href="https://wayang.apache.org/blog/Apache%20Kafka%20meets%20Apache%20Wayang#read-path" class="hash-link" aria-label="Direct link to Read Path" title="Direct link to Read Path"></a></h3>
-<p>The read path describes how data is ingested from a source into the Wayang processing pipeline:</p>
-<p><em>Reading from Kafka Topic:</em> The method <em>readKafkaTopic(topicName)</em> is used to ingest data from a specified Kafka topic.
-This is the starting point of the data processing pipeline, where topicName represents the name of the Kafka topic from which data is read.</p>
-<p><em>Data Tokenization and Preparation:</em> Once the data is read from Kafka, it undergoes several transformations such as Splitting, Filtering, and Mapping.
-What follows are the procedures known as Reducing, Grouping, Co-Grouping, and Counting.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="write-path">Write Path<a href="https://wayang.apache.org/blog/Apache%20Kafka%20meets%20Apache%20Wayang#write-path" class="hash-link" aria-label="Direct link to Write Path" title="Direct link to Write Path"></a></h3>
-<p><em>Writing to Kafka Topic:</em> The final step in the pipeline involves writing the processed data back to a Kafka topic using <em>.writeKafkaTopic(...)</em>.
-This method takes parameters that specify the target Kafka topic, a serialization function to format the data as strings, and additional configuration for load profile estimation, which optimizes the writing process.</p>
-<p>This read-write path provides a comprehensive flow of data from ingestion from Kafka, through various processing steps, and finally back to Kafka, showcasing a full cycle of data processing within Apache Wayang's abstracted environment and is implemented in our example program shown in <em>listing 1</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="implementation-of-input--and-output-operators">Implementation of Input- and Output Operators<a href="https://wayang.apache.org/blog/Apache%20Kafka%20meets%20Apache%20Wayang#implementation-of-input--and-output-operators" class="hash-link" aria-label="Direct link to Implementation of Input- and Output Operators" title="Direct link to Implementation of Input- and Output Operators"></a></h2>
-<p>The next section shows how a new pair of operators can be implemented to extend Apache Wayang’s capabilities on the input and output side.
-We created the Kafka Source and Kafka Sink components so that our cross organizational data collaboration scenario can be implemented using data streaming infrastructure.</p>
-<p><strong>Level 1 – Wayang execution plan with abstract operators</strong></p>
-<p>The implementation of our Kafka Source and Kafka Sink components for Apache Wayang requires new methods and classes on three layers.
-First of all in the API package.
-Here we use the JavaPlanBuilder to expose the function for selecting a Kafka topic as the source to be used by client.<br>
-<!-- -->The class <em>JavaPlanBuilder</em> in package <em>org.apache.wayang.api</em> in the project <em>wayang-api/wayang-api-scala-java</em> exposes our new functionality to our external client.
-An instance of the JavaPlanBuilder is used to define the data processing pipeline.
-We use its <em>readKafkaTopic()</em> which specifies the source Kafka topic to read from, and for the write path we use the <em>writeKafkaTopic()</em> method.
-Both Methods do only trigger activities in the background.</p>
-<p>For the output side, we use the <em>DataQuantaBuilder</em> class, which offers an implementation of the writeKafkaTopic function.
-This function is designed to send processed data, referred to as DataQuanta, to a specified Kafka topic.
-Essentially, it marks the final step in a data processing sequence constructed using the Apache Wayang framework.</p>
-<p>In the DataQuanta class we implemented the methods writeKafkaTopic and writeKafkaTopicJava which use the KafkaTopicSink class.
-In this API layer we use the Scala programming language, but we utilize the Java classes, implemented in the layer below.</p>
-<p><strong>Level 2 – Wiring between Platform Abstraction and Implementation</strong></p>
-<p>The second layer builds the bridge between the WayangContext and PlanBuilders which work together with DataQuanta and the DataQuantaBuilder.</p>
-<p>Also, the mapping between the abstract components and the specific implementations are defined in this layer.</p>
-<p>Therefore, the mappings package has a class <em>Mappings</em> in which all relevant input and output operators are listed.
-We use it to register the KafkaSourceMapping and a KafkaSinkMapping for the particular platform, Java in our case.
-These classes allow the Apache Wayang framework to use the Java implementation of the KafkaTopicSource component (and KafkaTopicSink respectively).
-While the Wayang execution plan uses the higher abstractions, here on the “platform level” we have to link the specific implementation for the target platform.
-In our case this leads to a Java program running on a JVM which is set up by the Apache Wayang framework using the logical components of the execution plan.</p>
-<p>Those mappings link the real implementation of our operators the ones used in an execution plan.
-The JavaKafkaTopicSource and the JavaKafkaTopicSink extend the KafkaTopicSource and KafkaTopicSink so that the lower level implementation of those classes become available within Wayang’s Java Platform context.</p>
-<p>In this layer, the KafkaConsumer class and the KafkaProducer class are used, but both are configured and instantiated in the next layer underneath.
-All this is done in the project <em>wayang-plarforms/wayang-java</em>.</p>
-<p><strong>Layer 3 – Input/Output Connector Layer</strong></p>
-<p>The <em>KafkaTopicSource</em> and <em>KafkaTopicSink</em> classes build the third layer of our implementation.
-Both are implemented in Java programming language.
-In this layer, the real Kafka-Client logic is defined.
-Details about consumer and producers, client configuration, and schema handling have to be handled here.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="https://wayang.apache.org/blog/Apache%20Kafka%20meets%20Apache%20Wayang#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary"></a></h2>
-<p>Both classes in the third layer implement the Kafka client logic which is needed by the Wayang-execution plan when external data flows should be established.
-The layer above handles the mapping of the components at startup time.
-All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="https://wayang.apache.org/blog/Apache%20Kafka%20meets%20Apache%20Wayang#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
-<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.
-Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p>]]></content>
+ <content type="html"><![CDATA[<h2 class="anchor anchorWithStickyNavbar_LWe7" id="intro">Intro<a href="https://wayang.apache.org/blog/kafka-meets-wayang-1#intro" class="hash-link" aria-label="Direct link to Intro" title="Direct link to Intro"></a></h2>
+<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
+The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
+<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
+We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
+<p>The use case behind this work is an imaginary data collaboration scenario.
+We see this example and the demand for a solution already in many places.<br>
+<!-- -->For us this is motivation enough to propose a solution.
+This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
+This reduces complexity of data management and cost dramatically.</p>
+<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
+This analysis pattern will also be relevant in the context of data analysis along supply chains, another typical example where data from many stakeholder together is needed but never managed in one place, for good reasons.</p>
+<p>Data federation can help us to unlock the hidden value of all those isolated data lakes.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="a-cross-organizational-data-sharing-scenario">A cross organizational data sharing scenario<a href="https://wayang.apache.org/blog/kafka-meets-wayang-1#a-cross-organizational-data-sharing-scenario" class="hash-link" aria-label="Direct link to A cross organizational data sharing scenario" title="Direct link to A cross organizational data sharing scenario"></a></h2>
+<p>Our goal is the implementation of a cross organization decentralized data processing scenario, in which protected local data should be processed in combination with public data from public sources in a collaborative manner.
+Instead of copying all data into a central data lake or a central data platform we decided to use federated analytics.
+Apache Wayang is the tool we work with.
+In our case, the public data is hosted on publicly available websites or data pods.
+A client can use the HTTP(S) protocol to read the data which is given in a well defined format.
+For simplicity we decided to use CSV format.
+When we look into the data of each participant we have a different perspective.</p>
+<p>Our processing procedure should calculate a particular metric on the <em>local data</em> of each participant.
+An example of such a metric is the average spending of all users on a particular product category per month.
+This can vary from partner to partner, hence, we want to be able to calculate a peer-group comparison so that each partner can see its own metric compared with a global average calculated from contributions by all partners.
+Such a process requires global averaging and local averaging.
+And due to governance constraints, we can’t bring all raw data together in one place.</p>
+<p>Instead, we want to use Apache Wayang for this purpose.
+We simplify the procedure and split it into two phases.
+Phase one is the process, which allows each participant to calculate the local metrics.
+This requires only local data. The second phase requires data from all collaborating partners.
+The monthly sum and counter values per partner and category are needed in one place by all other parties.
+Hence, the algorithm of the first phase stores the local results locally, and the contributions to the global results in an externally accessible Kafka topic.
+We assume this is done by each of the partners.</p>
+<p>Now we have a scenario, in which an Apache Wayang process must be able to read data from multiple Apache Kafka topics from multiple Apache Kafka clusters but finally writes into a single Kafka topic, which then can be accessed by all the participating clients.</p>
+<p><img decoding="async" loading="lazy" alt="images/image-1.png" src="https://wayang.apache.org/assets/images/image-1-9cc35d5aea2b867d7e5759a96bd02334.png" width="904" height="550" class="img_ev3q"></p>
+<p>The illustration shows the data flows in such a scenario.
+Jobs with red border are executed by the participants in isolation within their own data processing environments.
+But they share some of the data, using publicly accessible Kafka topics, marked by A. Job 4 is the Apache Wayang job in our focus: here we intent to read data from 3 different source systems, and write results into a fourth system (marked as B), which can be accesses by all participants again.</p>
+<p>With this in mind we want to implement an Apache Wayang application which implements the illustrated <em>Job 4</em>.
+Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
+Our assumption is, that in the beginning, there won’t be much data.</p>
+<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
+Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.
+Later, we want to switch to Apache Spark.</p>]]></content>
<author>
<name>Mirko Kämpf</name>
<uri>https://github.com/kamir</uri>
diff --git a/blog/index.html b/blog/index.html
index 14c123cf..d0f15344 100644
--- a/blog/index.html
+++ b/blog/index.html
@@ -5,54 +5,68 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Blog | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" p [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
-<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
-The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
-<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
-We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
-<p>The use case behind this work is an imaginary data collaboration scenario.
-We see this example and the demand for a solution already in many places.<br>
-<!-- -->For us this is motivation enough to propose a solution.
-This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
-This reduces complexity of data management and cost dramatically.</p>
-<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
-This analysis pattern will also be relevant in the context of data analysis along supply chains, another typical example where data from many stakeholder together is needed but never managed in one place, for good reasons.</p>
-<p>Data federation can help us to unlock the hidden value of all those isolated data lakes.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="a-cross-organizational-data-sharing-scenario">A cross organizational data sharing scenario<a href="#a-cross-organizational-data-sharing-scenario" class="hash-link" aria-label="Direct link to A cross organizational data sharing scenario" title="Direct link to A cross organizational data sharing scenario"></a></h2>
-<p>Our goal is the implementation of a cross organization decentralized data processing scenario, in which protected local data should be processed in combination with public data from public sources in a collaborative manner.
-Instead of copying all data into a central data lake or a central data platform we decided to use federated analytics.
-Apache Wayang is the tool we work with.
-In our case, the public data is hosted on publicly available websites or data pods.
-A client can use the HTTP(S) protocol to read the data which is given in a well defined format.
-For simplicity we decided to use CSV format.
-When we look into the data of each participant we have a different perspective.</p>
-<p>Our processing procedure should calculate a particular metric on the <em>local data</em> of each participant.
-An example of such a metric is the average spending of all users on a particular product category per month.
-This can vary from partner to partner, hence, we want to be able to calculate a peer-group comparison so that each partner can see its own metric compared with a global average calculated from contributions by all partners.
-Such a process requires global averaging and local averaging.
-And due to governance constraints, we can’t bring all raw data together in one place.</p>
-<p>Instead, we want to use Apache Wayang for this purpose.
-We simplify the procedure and split it into two phases.
-Phase one is the process, which allows each participant to calculate the local metrics.
-This requires only local data. The second phase requires data from all collaborating partners.
-The monthly sum and counter values per partner and category are needed in one place by all other parties.
-Hence, the algorithm of the first phase stores the local results locally, and the contributions to the global results in an externally accessible Kafka topic.
-We assume this is done by each of the partners.</p>
-<p>Now we have a scenario, in which an Apache Wayang process must be able to read data from multiple Apache Kafka topics from multiple Apache Kafka clusters but finally writes into a single Kafka topic, which then can be accessed by all the participating clients.</p>
-<p><img decoding="async" loading="lazy" alt="images/image-1.png" src="/assets/images/image-1-9cc35d5aea2b867d7e5759a96bd02334.png" width="904" height="550" class="img_ev3q"></p>
-<p>The illustration shows the data flows in such a scenario.
-Jobs with red border are executed by the participants in isolation within their own data processing environments.
-But they share some of the data, using publicly accessible Kafka topics, marked by A. Job 4 is the Apache Wayang job in our focus: here we intent to read data from 3 different source systems, and write results into a fourth system (marked as B), which can be accesses by all participants again.</p>
-<p>With this in mind we want to implement an Apache Wayang application which implements the illustrated <em>Job 4</em>.
-Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
-Our assumption is, that in the beginning, there won’t be much data.</p>
-<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
-Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.
-Later, we want to switch to Apache Spark.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/cross-organization-data-collaboration">cross organization [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+We look into the “Read- and Write-Path” for our data items, called <em>DataQuanta</em>.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="apache-wayangs-read--write-path-for-kafka-topics">Apache Wayang’s Read & Write Path for Kafka topics<a href="#apache-wayangs-read--write-path-for-kafka-topics" class="hash-link" aria-label="Direct link to Apache Wayang’s Read & Write Path for Kafka topics" title="Direct link to Apache Wayang’s Read & Write Path for Kafka topics"></a></h2>
+<p>To describe the read and write paths for data in the context of the created Apache Wayang code snippet, the primary classes and interfaces we need to understand are as follows:</p>
+<p><strong>WayangContext:</strong> This class is essential for initializing the Wayang processing environment.
+It allows you to configure the execution environment and register plugins that define which platforms Wayang can use for data processing tasks, such as <em>Java.basicPlugin()</em> for local Java execution.</p>
+<p><strong>JavaPlanBuilder:</strong> This class is used to build and define the data processing pipeline (or plan) in Wayang.
+It provides a fluent API to specify the operations to be performed on the data, from reading the input to processing it and writing the output.</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-path">Read Path<a href="#read-path" class="hash-link" aria-label="Direct link to Read Path" title="Direct link to Read Path"></a></h3>
+<p>The read path describes how data is ingested from a source into the Wayang processing pipeline:</p>
+<p><em>Reading from Kafka Topic:</em> The method <em>readKafkaTopic(topicName)</em> is used to ingest data from a specified Kafka topic.
+This is the starting point of the data processing pipeline, where topicName represents the name of the Kafka topic from which data is read.</p>
+<p><em>Data Tokenization and Preparation:</em> Once the data is read from Kafka, it undergoes several transformations such as Splitting, Filtering, and Mapping.
+What follows are the procedures known as Reducing, Grouping, Co-Grouping, and Counting.</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="write-path">Write Path<a href="#write-path" class="hash-link" aria-label="Direct link to Write Path" title="Direct link to Write Path"></a></h3>
+<p><em>Writing to Kafka Topic:</em> The final step in the pipeline involves writing the processed data back to a Kafka topic using <em>.writeKafkaTopic(...)</em>.
+This method takes parameters that specify the target Kafka topic, a serialization function to format the data as strings, and additional configuration for load profile estimation, which optimizes the writing process.</p>
+<p>This read-write path provides a comprehensive flow of data from ingestion from Kafka, through various processing steps, and finally back to Kafka, showcasing a full cycle of data processing within Apache Wayang's abstracted environment and is implemented in our example program shown in <em>listing 1</em>.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="implementation-of-input--and-output-operators">Implementation of Input- and Output Operators<a href="#implementation-of-input--and-output-operators" class="hash-link" aria-label="Direct link to Implementation of Input- and Output Operators" title="Direct link to Implementation of Input- and Output Operators"></a></h2>
+<p>The next section shows how a new pair of operators can be implemented to extend Apache Wayang’s capabilities on the input and output side.
+We created the Kafka Source and Kafka Sink components so that our cross organizational data collaboration scenario can be implemented using data streaming infrastructure.</p>
+<p><strong>Level 1 – Wayang execution plan with abstract operators</strong></p>
+<p>The implementation of our Kafka Source and Kafka Sink components for Apache Wayang requires new methods and classes on three layers.
+First of all in the API package.
+Here we use the JavaPlanBuilder to expose the function for selecting a Kafka topic as the source to be used by client.<br>
+<!-- -->The class <em>JavaPlanBuilder</em> in package <em>org.apache.wayang.api</em> in the project <em>wayang-api/wayang-api-scala-java</em> exposes our new functionality to our external client.
+An instance of the JavaPlanBuilder is used to define the data processing pipeline.
+We use its <em>readKafkaTopic()</em> which specifies the source Kafka topic to read from, and for the write path we use the <em>writeKafkaTopic()</em> method.
+Both Methods do only trigger activities in the background.</p>
+<p>For the output side, we use the <em>DataQuantaBuilder</em> class, which offers an implementation of the writeKafkaTopic function.
+This function is designed to send processed data, referred to as DataQuanta, to a specified Kafka topic.
+Essentially, it marks the final step in a data processing sequence constructed using the Apache Wayang framework.</p>
+<p>In the DataQuanta class we implemented the methods writeKafkaTopic and writeKafkaTopicJava which use the KafkaTopicSink class.
+In this API layer we use the Scala programming language, but we utilize the Java classes, implemented in the layer below.</p>
+<p><strong>Level 2 – Wiring between Platform Abstraction and Implementation</strong></p>
+<p>The second layer builds the bridge between the WayangContext and PlanBuilders which work together with DataQuanta and the DataQuantaBuilder.</p>
+<p>Also, the mapping between the abstract components and the specific implementations are defined in this layer.</p>
+<p>Therefore, the mappings package has a class <em>Mappings</em> in which all relevant input and output operators are listed.
+We use it to register the KafkaSourceMapping and a KafkaSinkMapping for the particular platform, Java in our case.
+These classes allow the Apache Wayang framework to use the Java implementation of the KafkaTopicSource component (and KafkaTopicSink respectively).
+While the Wayang execution plan uses the higher abstractions, here on the “platform level” we have to link the specific implementation for the target platform.
+In our case this leads to a Java program running on a JVM which is set up by the Apache Wayang framework using the logical components of the execution plan.</p>
+<p>Those mappings link the real implementation of our operators the ones used in an execution plan.
+The JavaKafkaTopicSource and the JavaKafkaTopicSink extend the KafkaTopicSource and KafkaTopicSink so that the lower level implementation of those classes become available within Wayang’s Java Platform context.</p>
+<p>In this layer, the KafkaConsumer class and the KafkaProducer class are used, but both are configured and instantiated in the next layer underneath.
+All this is done in the project <em>wayang-plarforms/wayang-java</em>.</p>
+<p><strong>Layer 3 – Input/Output Connector Layer</strong></p>
+<p>The <em>KafkaTopicSource</em> and <em>KafkaTopicSink</em> classes build the third layer of our implementation.
+Both are implemented in Java programming language.
+In this layer, the real Kafka-Client logic is defined.
+Details about consumer and producers, client configuration, and schema handling have to be handled here.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary"></a></h2>
+<p>Both classes in the third layer implement the Kafka client logic which is needed by the Wayang-execution plan when external data flows should be established.
+The layer above handles the mapping of the components at startup time.
+All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
+<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.
+Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class= [...]
<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
diff --git a/blog/tags/kafka/index.html b/blog/kafka-meets-wayang-1/index.html
similarity index 51%
copy from blog/tags/kafka/index.html
copy to blog/kafka-meets-wayang-1/index.html
index ef93e905..60b48ec6 100644
--- a/blog/tags/kafka/index.html
+++ b/blog/kafka-meets-wayang-1/index.html
@@ -1,15 +1,15 @@
<!doctype html>
-<html lang="en" dir="ltr" class="blog-wrapper blog-tags-post-list-page plugin-blog plugin-id-default" data-has-hydrated="false">
+<html lang="en" dir="ltr" class="blog-wrapper blog-post-page plugin-blog plugin-id-default" data-has-hydrated="false">
<head>
<meta charset="UTF-8">
<meta name="generator" content="Docusaurus v3.1.1">
-<title data-rh="true">2 posts tagged with "kafka" | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/tags/kafka"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:l [...]
+<title data-rh="true">Apache Kafka meets Apache Wayang - Part 1 | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/kafka-meets-wayang-1"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" nam [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
@@ -52,50 +52,7 @@ Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> avai
Our assumption is, that in the beginning, there won’t be much data.</p>
<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.
-Later, we want to switch to Apache Spark.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/cross-organization-data-collaboration">cross organization [...]
-<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
-The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
-<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
-We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
-<p>The use case behind this work is an imaginary data collaboration scenario.
-We see this example and the demand for a solution already in many places.<br>
-<!-- -->For us this is motivation enough to propose a solution.
-This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
-This reduces complexity of data management and cost dramatically.</p>
-<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
-This analysis pattern will also be relevant in the context of data analysis along supply chains, another typical example where data from many stakeholder together is needed but never managed in one place, for good reasons.</p>
-<p>Data federation can help us to unlock the hidden value of all those isolated data lakes.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="a-cross-organizational-data-sharing-scenario">A cross organizational data sharing scenario<a href="#a-cross-organizational-data-sharing-scenario" class="hash-link" aria-label="Direct link to A cross organizational data sharing scenario" title="Direct link to A cross organizational data sharing scenario"></a></h2>
-<p>Our goal is the implementation of a cross organization decentralized data processing scenario, in which protected local data should be processed in combination with public data from public sources in a collaborative manner.
-Instead of copying all data into a central data lake or a central data platform we decided to use federated analytics.
-Apache Wayang is the tool we work with.
-In our case, the public data is hosted on publicly available websites or data pods.
-A client can use the HTTP(S) protocol to read the data which is given in a well defined format.
-For simplicity we decided to use CSV format.
-When we look into the data of each participant we have a different perspective.</p>
-<p>Our processing procedure should calculate a particular metric on the <em>local data</em> of each participant.
-An example of such a metric is the average spending of all users on a particular product category per month.
-This can vary from partner to partner, hence, we want to be able to calculate a peer-group comparison so that each partner can see its own metric compared with a global average calculated from contributions by all partners.
-Such a process requires global averaging and local averaging.
-And due to governance constraints, we can’t bring all raw data together in one place.</p>
-<p>Instead, we want to use Apache Wayang for this purpose.
-We simplify the procedure and split it into two phases.
-Phase one is the process, which allows each participant to calculate the local metrics.
-This requires only local data. The second phase requires data from all collaborating partners.
-The monthly sum and counter values per partner and category are needed in one place by all other parties.
-Hence, the algorithm of the first phase stores the local results locally, and the contributions to the global results in an externally accessible Kafka topic.
-We assume this is done by each of the partners.</p>
-<p>Now we have a scenario, in which an Apache Wayang process must be able to read data from multiple Apache Kafka topics from multiple Apache Kafka clusters but finally writes into a single Kafka topic, which then can be accessed by all the participating clients.</p>
-<p><img decoding="async" loading="lazy" alt="images/image-1.png" src="/assets/images/image-1-9cc35d5aea2b867d7e5759a96bd02334.png" width="904" height="550" class="img_ev3q"></p>
-<p>The illustration shows the data flows in such a scenario.
-Jobs with red border are executed by the participants in isolation within their own data processing environments.
-But they share some of the data, using publicly accessible Kafka topics, marked by A. Job 4 is the Apache Wayang job in our focus: here we intent to read data from 3 different source systems, and write results into a fourth system (marked as B), which can be accesses by all participants again.</p>
-<p>With this in mind we want to implement an Apache Wayang application which implements the illustrated <em>Job 4</em>.
-Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
-Our assumption is, that in the beginning, there won’t be much data.</p>
-<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
-Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.
-Later, we want to switch to Apache Spark.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/cross-organization-data-collaboration">cross organization [...]
+Later, we want to switch to Apache Spark.</p></div><footer class="row docusaurus-mt-lg blogPostFooterDetailsFull_mRVl"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/cross-organization-data-co [...]
<p> Apache Wayang is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that th [...]
<p>
Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. <br>
diff --git a/blog/rss.xml b/blog/rss.xml
index 3a7869e0..36213d9c 100644
--- a/blog/rss.xml
+++ b/blog/rss.xml
@@ -78,68 +78,54 @@ Finally, in part four we bring all puzzles together, and show the full implement
</item>
<item>
<title><![CDATA[Apache Kafka meets Apache Wayang - Part 1]]></title>
- <link>https://wayang.apache.org/blog/Apache Kafka meets Apache Wayang</link>
- <guid>https://wayang.apache.org/blog/Apache Kafka meets Apache Wayang</guid>
+ <link>https://wayang.apache.org/blog/kafka-meets-wayang-1</link>
+ <guid>https://wayang.apache.org/blog/kafka-meets-wayang-1</guid>
<pubDate>Tue, 05 Mar 2024 00:00:00 GMT</pubDate>
<description><![CDATA[Intro]]></description>
- <content:encoded><![CDATA[<p>In the second part of the article series we describe the implementation of the Kafka Source and Kafka Sink component for Apache Wayang.
-We look into the “Read- and Write-Path” for our data items, called <em>DataQuanta</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="apache-wayangs-read--write-path-for-kafka-topics">Apache Wayang’s Read & Write Path for Kafka topics<a href="https://wayang.apache.org/blog/Apache%20Kafka%20meets%20Apache%20Wayang#apache-wayangs-read--write-path-for-kafka-topics" class="hash-link" aria-label="Direct link to Apache Wayang’s Read & Write Path for Kafka topics" title="Direct link to Apache Wayang’s Read & Write Path for Kafka topics"></a></h2>
-<p>To describe the read and write paths for data in the context of the created Apache Wayang code snippet, the primary classes and interfaces we need to understand are as follows:</p>
-<p><strong>WayangContext:</strong> This class is essential for initializing the Wayang processing environment.
-It allows you to configure the execution environment and register plugins that define which platforms Wayang can use for data processing tasks, such as <em>Java.basicPlugin()</em> for local Java execution.</p>
-<p><strong>JavaPlanBuilder:</strong> This class is used to build and define the data processing pipeline (or plan) in Wayang.
-It provides a fluent API to specify the operations to be performed on the data, from reading the input to processing it and writing the output.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-path">Read Path<a href="https://wayang.apache.org/blog/Apache%20Kafka%20meets%20Apache%20Wayang#read-path" class="hash-link" aria-label="Direct link to Read Path" title="Direct link to Read Path"></a></h3>
-<p>The read path describes how data is ingested from a source into the Wayang processing pipeline:</p>
-<p><em>Reading from Kafka Topic:</em> The method <em>readKafkaTopic(topicName)</em> is used to ingest data from a specified Kafka topic.
-This is the starting point of the data processing pipeline, where topicName represents the name of the Kafka topic from which data is read.</p>
-<p><em>Data Tokenization and Preparation:</em> Once the data is read from Kafka, it undergoes several transformations such as Splitting, Filtering, and Mapping.
-What follows are the procedures known as Reducing, Grouping, Co-Grouping, and Counting.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="write-path">Write Path<a href="https://wayang.apache.org/blog/Apache%20Kafka%20meets%20Apache%20Wayang#write-path" class="hash-link" aria-label="Direct link to Write Path" title="Direct link to Write Path"></a></h3>
-<p><em>Writing to Kafka Topic:</em> The final step in the pipeline involves writing the processed data back to a Kafka topic using <em>.writeKafkaTopic(...)</em>.
-This method takes parameters that specify the target Kafka topic, a serialization function to format the data as strings, and additional configuration for load profile estimation, which optimizes the writing process.</p>
-<p>This read-write path provides a comprehensive flow of data from ingestion from Kafka, through various processing steps, and finally back to Kafka, showcasing a full cycle of data processing within Apache Wayang's abstracted environment and is implemented in our example program shown in <em>listing 1</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="implementation-of-input--and-output-operators">Implementation of Input- and Output Operators<a href="https://wayang.apache.org/blog/Apache%20Kafka%20meets%20Apache%20Wayang#implementation-of-input--and-output-operators" class="hash-link" aria-label="Direct link to Implementation of Input- and Output Operators" title="Direct link to Implementation of Input- and Output Operators"></a></h2>
-<p>The next section shows how a new pair of operators can be implemented to extend Apache Wayang’s capabilities on the input and output side.
-We created the Kafka Source and Kafka Sink components so that our cross organizational data collaboration scenario can be implemented using data streaming infrastructure.</p>
-<p><strong>Level 1 – Wayang execution plan with abstract operators</strong></p>
-<p>The implementation of our Kafka Source and Kafka Sink components for Apache Wayang requires new methods and classes on three layers.
-First of all in the API package.
-Here we use the JavaPlanBuilder to expose the function for selecting a Kafka topic as the source to be used by client.<br>
-<!-- -->The class <em>JavaPlanBuilder</em> in package <em>org.apache.wayang.api</em> in the project <em>wayang-api/wayang-api-scala-java</em> exposes our new functionality to our external client.
-An instance of the JavaPlanBuilder is used to define the data processing pipeline.
-We use its <em>readKafkaTopic()</em> which specifies the source Kafka topic to read from, and for the write path we use the <em>writeKafkaTopic()</em> method.
-Both Methods do only trigger activities in the background.</p>
-<p>For the output side, we use the <em>DataQuantaBuilder</em> class, which offers an implementation of the writeKafkaTopic function.
-This function is designed to send processed data, referred to as DataQuanta, to a specified Kafka topic.
-Essentially, it marks the final step in a data processing sequence constructed using the Apache Wayang framework.</p>
-<p>In the DataQuanta class we implemented the methods writeKafkaTopic and writeKafkaTopicJava which use the KafkaTopicSink class.
-In this API layer we use the Scala programming language, but we utilize the Java classes, implemented in the layer below.</p>
-<p><strong>Level 2 – Wiring between Platform Abstraction and Implementation</strong></p>
-<p>The second layer builds the bridge between the WayangContext and PlanBuilders which work together with DataQuanta and the DataQuantaBuilder.</p>
-<p>Also, the mapping between the abstract components and the specific implementations are defined in this layer.</p>
-<p>Therefore, the mappings package has a class <em>Mappings</em> in which all relevant input and output operators are listed.
-We use it to register the KafkaSourceMapping and a KafkaSinkMapping for the particular platform, Java in our case.
-These classes allow the Apache Wayang framework to use the Java implementation of the KafkaTopicSource component (and KafkaTopicSink respectively).
-While the Wayang execution plan uses the higher abstractions, here on the “platform level” we have to link the specific implementation for the target platform.
-In our case this leads to a Java program running on a JVM which is set up by the Apache Wayang framework using the logical components of the execution plan.</p>
-<p>Those mappings link the real implementation of our operators the ones used in an execution plan.
-The JavaKafkaTopicSource and the JavaKafkaTopicSink extend the KafkaTopicSource and KafkaTopicSink so that the lower level implementation of those classes become available within Wayang’s Java Platform context.</p>
-<p>In this layer, the KafkaConsumer class and the KafkaProducer class are used, but both are configured and instantiated in the next layer underneath.
-All this is done in the project <em>wayang-plarforms/wayang-java</em>.</p>
-<p><strong>Layer 3 – Input/Output Connector Layer</strong></p>
-<p>The <em>KafkaTopicSource</em> and <em>KafkaTopicSink</em> classes build the third layer of our implementation.
-Both are implemented in Java programming language.
-In this layer, the real Kafka-Client logic is defined.
-Details about consumer and producers, client configuration, and schema handling have to be handled here.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="https://wayang.apache.org/blog/Apache%20Kafka%20meets%20Apache%20Wayang#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary"></a></h2>
-<p>Both classes in the third layer implement the Kafka client logic which is needed by the Wayang-execution plan when external data flows should be established.
-The layer above handles the mapping of the components at startup time.
-All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="https://wayang.apache.org/blog/Apache%20Kafka%20meets%20Apache%20Wayang#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
-<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.
-Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p>]]></content:encoded>
+ <content:encoded><![CDATA[<h2 class="anchor anchorWithStickyNavbar_LWe7" id="intro">Intro<a href="https://wayang.apache.org/blog/kafka-meets-wayang-1#intro" class="hash-link" aria-label="Direct link to Intro" title="Direct link to Intro"></a></h2>
+<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
+The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
+<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
+We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
+<p>The use case behind this work is an imaginary data collaboration scenario.
+We see this example and the demand for a solution already in many places.<br>
+<!-- -->For us this is motivation enough to propose a solution.
+This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
+This reduces complexity of data management and cost dramatically.</p>
+<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
+This analysis pattern will also be relevant in the context of data analysis along supply chains, another typical example where data from many stakeholder together is needed but never managed in one place, for good reasons.</p>
+<p>Data federation can help us to unlock the hidden value of all those isolated data lakes.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="a-cross-organizational-data-sharing-scenario">A cross organizational data sharing scenario<a href="https://wayang.apache.org/blog/kafka-meets-wayang-1#a-cross-organizational-data-sharing-scenario" class="hash-link" aria-label="Direct link to A cross organizational data sharing scenario" title="Direct link to A cross organizational data sharing scenario"></a></h2>
+<p>Our goal is the implementation of a cross organization decentralized data processing scenario, in which protected local data should be processed in combination with public data from public sources in a collaborative manner.
+Instead of copying all data into a central data lake or a central data platform we decided to use federated analytics.
+Apache Wayang is the tool we work with.
+In our case, the public data is hosted on publicly available websites or data pods.
+A client can use the HTTP(S) protocol to read the data which is given in a well defined format.
+For simplicity we decided to use CSV format.
+When we look into the data of each participant we have a different perspective.</p>
+<p>Our processing procedure should calculate a particular metric on the <em>local data</em> of each participant.
+An example of such a metric is the average spending of all users on a particular product category per month.
+This can vary from partner to partner, hence, we want to be able to calculate a peer-group comparison so that each partner can see its own metric compared with a global average calculated from contributions by all partners.
+Such a process requires global averaging and local averaging.
+And due to governance constraints, we can’t bring all raw data together in one place.</p>
+<p>Instead, we want to use Apache Wayang for this purpose.
+We simplify the procedure and split it into two phases.
+Phase one is the process, which allows each participant to calculate the local metrics.
+This requires only local data. The second phase requires data from all collaborating partners.
+The monthly sum and counter values per partner and category are needed in one place by all other parties.
+Hence, the algorithm of the first phase stores the local results locally, and the contributions to the global results in an externally accessible Kafka topic.
+We assume this is done by each of the partners.</p>
+<p>Now we have a scenario, in which an Apache Wayang process must be able to read data from multiple Apache Kafka topics from multiple Apache Kafka clusters but finally writes into a single Kafka topic, which then can be accessed by all the participating clients.</p>
+<p><img decoding="async" loading="lazy" alt="images/image-1.png" src="https://wayang.apache.org/assets/images/image-1-9cc35d5aea2b867d7e5759a96bd02334.png" width="904" height="550" class="img_ev3q"></p>
+<p>The illustration shows the data flows in such a scenario.
+Jobs with red border are executed by the participants in isolation within their own data processing environments.
+But they share some of the data, using publicly accessible Kafka topics, marked by A. Job 4 is the Apache Wayang job in our focus: here we intent to read data from 3 different source systems, and write results into a fourth system (marked as B), which can be accesses by all participants again.</p>
+<p>With this in mind we want to implement an Apache Wayang application which implements the illustrated <em>Job 4</em>.
+Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
+Our assumption is, that in the beginning, there won’t be much data.</p>
+<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
+Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.
+Later, we want to switch to Apache Spark.</p>]]></content:encoded>
<category>wayang</category>
<category>kafka</category>
<category>cross organization data collaboration</category>
diff --git a/blog/tags/cross-organization-data-collaboration/index.html b/blog/tags/cross-organization-data-collaboration/index.html
index e9782856..b04342d4 100644
--- a/blog/tags/cross-organization-data-collaboration/index.html
+++ b/blog/tags/cross-organization-data-collaboration/index.html
@@ -5,54 +5,68 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">2 posts tagged with "cross organization data collaboration" | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/tags/cross-organization-data-collaboration"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusa [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
-<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
-The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
-<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
-We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
-<p>The use case behind this work is an imaginary data collaboration scenario.
-We see this example and the demand for a solution already in many places.<br>
-<!-- -->For us this is motivation enough to propose a solution.
-This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
-This reduces complexity of data management and cost dramatically.</p>
-<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
-This analysis pattern will also be relevant in the context of data analysis along supply chains, another typical example where data from many stakeholder together is needed but never managed in one place, for good reasons.</p>
-<p>Data federation can help us to unlock the hidden value of all those isolated data lakes.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="a-cross-organizational-data-sharing-scenario">A cross organizational data sharing scenario<a href="#a-cross-organizational-data-sharing-scenario" class="hash-link" aria-label="Direct link to A cross organizational data sharing scenario" title="Direct link to A cross organizational data sharing scenario"></a></h2>
-<p>Our goal is the implementation of a cross organization decentralized data processing scenario, in which protected local data should be processed in combination with public data from public sources in a collaborative manner.
-Instead of copying all data into a central data lake or a central data platform we decided to use federated analytics.
-Apache Wayang is the tool we work with.
-In our case, the public data is hosted on publicly available websites or data pods.
-A client can use the HTTP(S) protocol to read the data which is given in a well defined format.
-For simplicity we decided to use CSV format.
-When we look into the data of each participant we have a different perspective.</p>
-<p>Our processing procedure should calculate a particular metric on the <em>local data</em> of each participant.
-An example of such a metric is the average spending of all users on a particular product category per month.
-This can vary from partner to partner, hence, we want to be able to calculate a peer-group comparison so that each partner can see its own metric compared with a global average calculated from contributions by all partners.
-Such a process requires global averaging and local averaging.
-And due to governance constraints, we can’t bring all raw data together in one place.</p>
-<p>Instead, we want to use Apache Wayang for this purpose.
-We simplify the procedure and split it into two phases.
-Phase one is the process, which allows each participant to calculate the local metrics.
-This requires only local data. The second phase requires data from all collaborating partners.
-The monthly sum and counter values per partner and category are needed in one place by all other parties.
-Hence, the algorithm of the first phase stores the local results locally, and the contributions to the global results in an externally accessible Kafka topic.
-We assume this is done by each of the partners.</p>
-<p>Now we have a scenario, in which an Apache Wayang process must be able to read data from multiple Apache Kafka topics from multiple Apache Kafka clusters but finally writes into a single Kafka topic, which then can be accessed by all the participating clients.</p>
-<p><img decoding="async" loading="lazy" alt="images/image-1.png" src="/assets/images/image-1-9cc35d5aea2b867d7e5759a96bd02334.png" width="904" height="550" class="img_ev3q"></p>
-<p>The illustration shows the data flows in such a scenario.
-Jobs with red border are executed by the participants in isolation within their own data processing environments.
-But they share some of the data, using publicly accessible Kafka topics, marked by A. Job 4 is the Apache Wayang job in our focus: here we intent to read data from 3 different source systems, and write results into a fourth system (marked as B), which can be accesses by all participants again.</p>
-<p>With this in mind we want to implement an Apache Wayang application which implements the illustrated <em>Job 4</em>.
-Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
-Our assumption is, that in the beginning, there won’t be much data.</p>
-<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
-Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.
-Later, we want to switch to Apache Spark.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/cross-organization-data-collaboration">cross organization [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+We look into the “Read- and Write-Path” for our data items, called <em>DataQuanta</em>.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="apache-wayangs-read--write-path-for-kafka-topics">Apache Wayang’s Read & Write Path for Kafka topics<a href="#apache-wayangs-read--write-path-for-kafka-topics" class="hash-link" aria-label="Direct link to Apache Wayang’s Read & Write Path for Kafka topics" title="Direct link to Apache Wayang’s Read & Write Path for Kafka topics"></a></h2>
+<p>To describe the read and write paths for data in the context of the created Apache Wayang code snippet, the primary classes and interfaces we need to understand are as follows:</p>
+<p><strong>WayangContext:</strong> This class is essential for initializing the Wayang processing environment.
+It allows you to configure the execution environment and register plugins that define which platforms Wayang can use for data processing tasks, such as <em>Java.basicPlugin()</em> for local Java execution.</p>
+<p><strong>JavaPlanBuilder:</strong> This class is used to build and define the data processing pipeline (or plan) in Wayang.
+It provides a fluent API to specify the operations to be performed on the data, from reading the input to processing it and writing the output.</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-path">Read Path<a href="#read-path" class="hash-link" aria-label="Direct link to Read Path" title="Direct link to Read Path"></a></h3>
+<p>The read path describes how data is ingested from a source into the Wayang processing pipeline:</p>
+<p><em>Reading from Kafka Topic:</em> The method <em>readKafkaTopic(topicName)</em> is used to ingest data from a specified Kafka topic.
+This is the starting point of the data processing pipeline, where topicName represents the name of the Kafka topic from which data is read.</p>
+<p><em>Data Tokenization and Preparation:</em> Once the data is read from Kafka, it undergoes several transformations such as Splitting, Filtering, and Mapping.
+What follows are the procedures known as Reducing, Grouping, Co-Grouping, and Counting.</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="write-path">Write Path<a href="#write-path" class="hash-link" aria-label="Direct link to Write Path" title="Direct link to Write Path"></a></h3>
+<p><em>Writing to Kafka Topic:</em> The final step in the pipeline involves writing the processed data back to a Kafka topic using <em>.writeKafkaTopic(...)</em>.
+This method takes parameters that specify the target Kafka topic, a serialization function to format the data as strings, and additional configuration for load profile estimation, which optimizes the writing process.</p>
+<p>This read-write path provides a comprehensive flow of data from ingestion from Kafka, through various processing steps, and finally back to Kafka, showcasing a full cycle of data processing within Apache Wayang's abstracted environment and is implemented in our example program shown in <em>listing 1</em>.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="implementation-of-input--and-output-operators">Implementation of Input- and Output Operators<a href="#implementation-of-input--and-output-operators" class="hash-link" aria-label="Direct link to Implementation of Input- and Output Operators" title="Direct link to Implementation of Input- and Output Operators"></a></h2>
+<p>The next section shows how a new pair of operators can be implemented to extend Apache Wayang’s capabilities on the input and output side.
+We created the Kafka Source and Kafka Sink components so that our cross organizational data collaboration scenario can be implemented using data streaming infrastructure.</p>
+<p><strong>Level 1 – Wayang execution plan with abstract operators</strong></p>
+<p>The implementation of our Kafka Source and Kafka Sink components for Apache Wayang requires new methods and classes on three layers.
+First of all in the API package.
+Here we use the JavaPlanBuilder to expose the function for selecting a Kafka topic as the source to be used by client.<br>
+<!-- -->The class <em>JavaPlanBuilder</em> in package <em>org.apache.wayang.api</em> in the project <em>wayang-api/wayang-api-scala-java</em> exposes our new functionality to our external client.
+An instance of the JavaPlanBuilder is used to define the data processing pipeline.
+We use its <em>readKafkaTopic()</em> which specifies the source Kafka topic to read from, and for the write path we use the <em>writeKafkaTopic()</em> method.
+Both Methods do only trigger activities in the background.</p>
+<p>For the output side, we use the <em>DataQuantaBuilder</em> class, which offers an implementation of the writeKafkaTopic function.
+This function is designed to send processed data, referred to as DataQuanta, to a specified Kafka topic.
+Essentially, it marks the final step in a data processing sequence constructed using the Apache Wayang framework.</p>
+<p>In the DataQuanta class we implemented the methods writeKafkaTopic and writeKafkaTopicJava which use the KafkaTopicSink class.
+In this API layer we use the Scala programming language, but we utilize the Java classes, implemented in the layer below.</p>
+<p><strong>Level 2 – Wiring between Platform Abstraction and Implementation</strong></p>
+<p>The second layer builds the bridge between the WayangContext and PlanBuilders which work together with DataQuanta and the DataQuantaBuilder.</p>
+<p>Also, the mapping between the abstract components and the specific implementations are defined in this layer.</p>
+<p>Therefore, the mappings package has a class <em>Mappings</em> in which all relevant input and output operators are listed.
+We use it to register the KafkaSourceMapping and a KafkaSinkMapping for the particular platform, Java in our case.
+These classes allow the Apache Wayang framework to use the Java implementation of the KafkaTopicSource component (and KafkaTopicSink respectively).
+While the Wayang execution plan uses the higher abstractions, here on the “platform level” we have to link the specific implementation for the target platform.
+In our case this leads to a Java program running on a JVM which is set up by the Apache Wayang framework using the logical components of the execution plan.</p>
+<p>Those mappings link the real implementation of our operators the ones used in an execution plan.
+The JavaKafkaTopicSource and the JavaKafkaTopicSink extend the KafkaTopicSource and KafkaTopicSink so that the lower level implementation of those classes become available within Wayang’s Java Platform context.</p>
+<p>In this layer, the KafkaConsumer class and the KafkaProducer class are used, but both are configured and instantiated in the next layer underneath.
+All this is done in the project <em>wayang-plarforms/wayang-java</em>.</p>
+<p><strong>Layer 3 – Input/Output Connector Layer</strong></p>
+<p>The <em>KafkaTopicSource</em> and <em>KafkaTopicSink</em> classes build the third layer of our implementation.
+Both are implemented in Java programming language.
+In this layer, the real Kafka-Client logic is defined.
+Details about consumer and producers, client configuration, and schema handling have to be handled here.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary"></a></h2>
+<p>Both classes in the third layer implement the Kafka client logic which is needed by the Wayang-execution plan when external data flows should be established.
+The layer above handles the mapping of the components at startup time.
+All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
+<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.
+Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class= [...]
<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
diff --git a/blog/tags/index.html b/blog/tags/index.html
index 5c5928e4..d25912a5 100644
--- a/blog/tags/index.html
+++ b/blog/tags/index.html
@@ -5,11 +5,11 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Tags | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/tags"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="tr [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
<p> Apache Wayang is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that th [...]
<p>
Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. <br>
diff --git a/blog/tags/kafka/index.html b/blog/tags/kafka/index.html
index ef93e905..64433b9c 100644
--- a/blog/tags/kafka/index.html
+++ b/blog/tags/kafka/index.html
@@ -5,54 +5,68 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">2 posts tagged with "kafka" | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/tags/kafka"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:l [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
-<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
-The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
-<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
-We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
-<p>The use case behind this work is an imaginary data collaboration scenario.
-We see this example and the demand for a solution already in many places.<br>
-<!-- -->For us this is motivation enough to propose a solution.
-This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
-This reduces complexity of data management and cost dramatically.</p>
-<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
-This analysis pattern will also be relevant in the context of data analysis along supply chains, another typical example where data from many stakeholder together is needed but never managed in one place, for good reasons.</p>
-<p>Data federation can help us to unlock the hidden value of all those isolated data lakes.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="a-cross-organizational-data-sharing-scenario">A cross organizational data sharing scenario<a href="#a-cross-organizational-data-sharing-scenario" class="hash-link" aria-label="Direct link to A cross organizational data sharing scenario" title="Direct link to A cross organizational data sharing scenario"></a></h2>
-<p>Our goal is the implementation of a cross organization decentralized data processing scenario, in which protected local data should be processed in combination with public data from public sources in a collaborative manner.
-Instead of copying all data into a central data lake or a central data platform we decided to use federated analytics.
-Apache Wayang is the tool we work with.
-In our case, the public data is hosted on publicly available websites or data pods.
-A client can use the HTTP(S) protocol to read the data which is given in a well defined format.
-For simplicity we decided to use CSV format.
-When we look into the data of each participant we have a different perspective.</p>
-<p>Our processing procedure should calculate a particular metric on the <em>local data</em> of each participant.
-An example of such a metric is the average spending of all users on a particular product category per month.
-This can vary from partner to partner, hence, we want to be able to calculate a peer-group comparison so that each partner can see its own metric compared with a global average calculated from contributions by all partners.
-Such a process requires global averaging and local averaging.
-And due to governance constraints, we can’t bring all raw data together in one place.</p>
-<p>Instead, we want to use Apache Wayang for this purpose.
-We simplify the procedure and split it into two phases.
-Phase one is the process, which allows each participant to calculate the local metrics.
-This requires only local data. The second phase requires data from all collaborating partners.
-The monthly sum and counter values per partner and category are needed in one place by all other parties.
-Hence, the algorithm of the first phase stores the local results locally, and the contributions to the global results in an externally accessible Kafka topic.
-We assume this is done by each of the partners.</p>
-<p>Now we have a scenario, in which an Apache Wayang process must be able to read data from multiple Apache Kafka topics from multiple Apache Kafka clusters but finally writes into a single Kafka topic, which then can be accessed by all the participating clients.</p>
-<p><img decoding="async" loading="lazy" alt="images/image-1.png" src="/assets/images/image-1-9cc35d5aea2b867d7e5759a96bd02334.png" width="904" height="550" class="img_ev3q"></p>
-<p>The illustration shows the data flows in such a scenario.
-Jobs with red border are executed by the participants in isolation within their own data processing environments.
-But they share some of the data, using publicly accessible Kafka topics, marked by A. Job 4 is the Apache Wayang job in our focus: here we intent to read data from 3 different source systems, and write results into a fourth system (marked as B), which can be accesses by all participants again.</p>
-<p>With this in mind we want to implement an Apache Wayang application which implements the illustrated <em>Job 4</em>.
-Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
-Our assumption is, that in the beginning, there won’t be much data.</p>
-<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
-Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.
-Later, we want to switch to Apache Spark.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/cross-organization-data-collaboration">cross organization [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+We look into the “Read- and Write-Path” for our data items, called <em>DataQuanta</em>.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="apache-wayangs-read--write-path-for-kafka-topics">Apache Wayang’s Read & Write Path for Kafka topics<a href="#apache-wayangs-read--write-path-for-kafka-topics" class="hash-link" aria-label="Direct link to Apache Wayang’s Read & Write Path for Kafka topics" title="Direct link to Apache Wayang’s Read & Write Path for Kafka topics"></a></h2>
+<p>To describe the read and write paths for data in the context of the created Apache Wayang code snippet, the primary classes and interfaces we need to understand are as follows:</p>
+<p><strong>WayangContext:</strong> This class is essential for initializing the Wayang processing environment.
+It allows you to configure the execution environment and register plugins that define which platforms Wayang can use for data processing tasks, such as <em>Java.basicPlugin()</em> for local Java execution.</p>
+<p><strong>JavaPlanBuilder:</strong> This class is used to build and define the data processing pipeline (or plan) in Wayang.
+It provides a fluent API to specify the operations to be performed on the data, from reading the input to processing it and writing the output.</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-path">Read Path<a href="#read-path" class="hash-link" aria-label="Direct link to Read Path" title="Direct link to Read Path"></a></h3>
+<p>The read path describes how data is ingested from a source into the Wayang processing pipeline:</p>
+<p><em>Reading from Kafka Topic:</em> The method <em>readKafkaTopic(topicName)</em> is used to ingest data from a specified Kafka topic.
+This is the starting point of the data processing pipeline, where topicName represents the name of the Kafka topic from which data is read.</p>
+<p><em>Data Tokenization and Preparation:</em> Once the data is read from Kafka, it undergoes several transformations such as Splitting, Filtering, and Mapping.
+What follows are the procedures known as Reducing, Grouping, Co-Grouping, and Counting.</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="write-path">Write Path<a href="#write-path" class="hash-link" aria-label="Direct link to Write Path" title="Direct link to Write Path"></a></h3>
+<p><em>Writing to Kafka Topic:</em> The final step in the pipeline involves writing the processed data back to a Kafka topic using <em>.writeKafkaTopic(...)</em>.
+This method takes parameters that specify the target Kafka topic, a serialization function to format the data as strings, and additional configuration for load profile estimation, which optimizes the writing process.</p>
+<p>This read-write path provides a comprehensive flow of data from ingestion from Kafka, through various processing steps, and finally back to Kafka, showcasing a full cycle of data processing within Apache Wayang's abstracted environment and is implemented in our example program shown in <em>listing 1</em>.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="implementation-of-input--and-output-operators">Implementation of Input- and Output Operators<a href="#implementation-of-input--and-output-operators" class="hash-link" aria-label="Direct link to Implementation of Input- and Output Operators" title="Direct link to Implementation of Input- and Output Operators"></a></h2>
+<p>The next section shows how a new pair of operators can be implemented to extend Apache Wayang’s capabilities on the input and output side.
+We created the Kafka Source and Kafka Sink components so that our cross organizational data collaboration scenario can be implemented using data streaming infrastructure.</p>
+<p><strong>Level 1 – Wayang execution plan with abstract operators</strong></p>
+<p>The implementation of our Kafka Source and Kafka Sink components for Apache Wayang requires new methods and classes on three layers.
+First of all in the API package.
+Here we use the JavaPlanBuilder to expose the function for selecting a Kafka topic as the source to be used by client.<br>
+<!-- -->The class <em>JavaPlanBuilder</em> in package <em>org.apache.wayang.api</em> in the project <em>wayang-api/wayang-api-scala-java</em> exposes our new functionality to our external client.
+An instance of the JavaPlanBuilder is used to define the data processing pipeline.
+We use its <em>readKafkaTopic()</em> which specifies the source Kafka topic to read from, and for the write path we use the <em>writeKafkaTopic()</em> method.
+Both Methods do only trigger activities in the background.</p>
+<p>For the output side, we use the <em>DataQuantaBuilder</em> class, which offers an implementation of the writeKafkaTopic function.
+This function is designed to send processed data, referred to as DataQuanta, to a specified Kafka topic.
+Essentially, it marks the final step in a data processing sequence constructed using the Apache Wayang framework.</p>
+<p>In the DataQuanta class we implemented the methods writeKafkaTopic and writeKafkaTopicJava which use the KafkaTopicSink class.
+In this API layer we use the Scala programming language, but we utilize the Java classes, implemented in the layer below.</p>
+<p><strong>Level 2 – Wiring between Platform Abstraction and Implementation</strong></p>
+<p>The second layer builds the bridge between the WayangContext and PlanBuilders which work together with DataQuanta and the DataQuantaBuilder.</p>
+<p>Also, the mapping between the abstract components and the specific implementations are defined in this layer.</p>
+<p>Therefore, the mappings package has a class <em>Mappings</em> in which all relevant input and output operators are listed.
+We use it to register the KafkaSourceMapping and a KafkaSinkMapping for the particular platform, Java in our case.
+These classes allow the Apache Wayang framework to use the Java implementation of the KafkaTopicSource component (and KafkaTopicSink respectively).
+While the Wayang execution plan uses the higher abstractions, here on the “platform level” we have to link the specific implementation for the target platform.
+In our case this leads to a Java program running on a JVM which is set up by the Apache Wayang framework using the logical components of the execution plan.</p>
+<p>Those mappings link the real implementation of our operators the ones used in an execution plan.
+The JavaKafkaTopicSource and the JavaKafkaTopicSink extend the KafkaTopicSource and KafkaTopicSink so that the lower level implementation of those classes become available within Wayang’s Java Platform context.</p>
+<p>In this layer, the KafkaConsumer class and the KafkaProducer class are used, but both are configured and instantiated in the next layer underneath.
+All this is done in the project <em>wayang-plarforms/wayang-java</em>.</p>
+<p><strong>Layer 3 – Input/Output Connector Layer</strong></p>
+<p>The <em>KafkaTopicSource</em> and <em>KafkaTopicSink</em> classes build the third layer of our implementation.
+Both are implemented in Java programming language.
+In this layer, the real Kafka-Client logic is defined.
+Details about consumer and producers, client configuration, and schema handling have to be handled here.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary"></a></h2>
+<p>Both classes in the third layer implement the Kafka client logic which is needed by the Wayang-execution plan when external data flows should be established.
+The layer above handles the mapping of the components at startup time.
+All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
+<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.
+Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class= [...]
<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
diff --git a/blog/tags/wayang/index.html b/blog/tags/wayang/index.html
index 2e097ad6..88e85925 100644
--- a/blog/tags/wayang/index.html
+++ b/blog/tags/wayang/index.html
@@ -5,54 +5,68 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">3 posts tagged with "wayang" | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/tags/wayang"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
-<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
-The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
-<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
-We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
-<p>The use case behind this work is an imaginary data collaboration scenario.
-We see this example and the demand for a solution already in many places.<br>
-<!-- -->For us this is motivation enough to propose a solution.
-This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
-This reduces complexity of data management and cost dramatically.</p>
-<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
-This analysis pattern will also be relevant in the context of data analysis along supply chains, another typical example where data from many stakeholder together is needed but never managed in one place, for good reasons.</p>
-<p>Data federation can help us to unlock the hidden value of all those isolated data lakes.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="a-cross-organizational-data-sharing-scenario">A cross organizational data sharing scenario<a href="#a-cross-organizational-data-sharing-scenario" class="hash-link" aria-label="Direct link to A cross organizational data sharing scenario" title="Direct link to A cross organizational data sharing scenario"></a></h2>
-<p>Our goal is the implementation of a cross organization decentralized data processing scenario, in which protected local data should be processed in combination with public data from public sources in a collaborative manner.
-Instead of copying all data into a central data lake or a central data platform we decided to use federated analytics.
-Apache Wayang is the tool we work with.
-In our case, the public data is hosted on publicly available websites or data pods.
-A client can use the HTTP(S) protocol to read the data which is given in a well defined format.
-For simplicity we decided to use CSV format.
-When we look into the data of each participant we have a different perspective.</p>
-<p>Our processing procedure should calculate a particular metric on the <em>local data</em> of each participant.
-An example of such a metric is the average spending of all users on a particular product category per month.
-This can vary from partner to partner, hence, we want to be able to calculate a peer-group comparison so that each partner can see its own metric compared with a global average calculated from contributions by all partners.
-Such a process requires global averaging and local averaging.
-And due to governance constraints, we can’t bring all raw data together in one place.</p>
-<p>Instead, we want to use Apache Wayang for this purpose.
-We simplify the procedure and split it into two phases.
-Phase one is the process, which allows each participant to calculate the local metrics.
-This requires only local data. The second phase requires data from all collaborating partners.
-The monthly sum and counter values per partner and category are needed in one place by all other parties.
-Hence, the algorithm of the first phase stores the local results locally, and the contributions to the global results in an externally accessible Kafka topic.
-We assume this is done by each of the partners.</p>
-<p>Now we have a scenario, in which an Apache Wayang process must be able to read data from multiple Apache Kafka topics from multiple Apache Kafka clusters but finally writes into a single Kafka topic, which then can be accessed by all the participating clients.</p>
-<p><img decoding="async" loading="lazy" alt="images/image-1.png" src="/assets/images/image-1-9cc35d5aea2b867d7e5759a96bd02334.png" width="904" height="550" class="img_ev3q"></p>
-<p>The illustration shows the data flows in such a scenario.
-Jobs with red border are executed by the participants in isolation within their own data processing environments.
-But they share some of the data, using publicly accessible Kafka topics, marked by A. Job 4 is the Apache Wayang job in our focus: here we intent to read data from 3 different source systems, and write results into a fourth system (marked as B), which can be accesses by all participants again.</p>
-<p>With this in mind we want to implement an Apache Wayang application which implements the illustrated <em>Job 4</em>.
-Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
-Our assumption is, that in the beginning, there won’t be much data.</p>
-<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
-Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.
-Later, we want to switch to Apache Spark.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/cross-organization-data-collaboration">cross organization [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+We look into the “Read- and Write-Path” for our data items, called <em>DataQuanta</em>.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="apache-wayangs-read--write-path-for-kafka-topics">Apache Wayang’s Read & Write Path for Kafka topics<a href="#apache-wayangs-read--write-path-for-kafka-topics" class="hash-link" aria-label="Direct link to Apache Wayang’s Read & Write Path for Kafka topics" title="Direct link to Apache Wayang’s Read & Write Path for Kafka topics"></a></h2>
+<p>To describe the read and write paths for data in the context of the created Apache Wayang code snippet, the primary classes and interfaces we need to understand are as follows:</p>
+<p><strong>WayangContext:</strong> This class is essential for initializing the Wayang processing environment.
+It allows you to configure the execution environment and register plugins that define which platforms Wayang can use for data processing tasks, such as <em>Java.basicPlugin()</em> for local Java execution.</p>
+<p><strong>JavaPlanBuilder:</strong> This class is used to build and define the data processing pipeline (or plan) in Wayang.
+It provides a fluent API to specify the operations to be performed on the data, from reading the input to processing it and writing the output.</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-path">Read Path<a href="#read-path" class="hash-link" aria-label="Direct link to Read Path" title="Direct link to Read Path"></a></h3>
+<p>The read path describes how data is ingested from a source into the Wayang processing pipeline:</p>
+<p><em>Reading from Kafka Topic:</em> The method <em>readKafkaTopic(topicName)</em> is used to ingest data from a specified Kafka topic.
+This is the starting point of the data processing pipeline, where topicName represents the name of the Kafka topic from which data is read.</p>
+<p><em>Data Tokenization and Preparation:</em> Once the data is read from Kafka, it undergoes several transformations such as Splitting, Filtering, and Mapping.
+What follows are the procedures known as Reducing, Grouping, Co-Grouping, and Counting.</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="write-path">Write Path<a href="#write-path" class="hash-link" aria-label="Direct link to Write Path" title="Direct link to Write Path"></a></h3>
+<p><em>Writing to Kafka Topic:</em> The final step in the pipeline involves writing the processed data back to a Kafka topic using <em>.writeKafkaTopic(...)</em>.
+This method takes parameters that specify the target Kafka topic, a serialization function to format the data as strings, and additional configuration for load profile estimation, which optimizes the writing process.</p>
+<p>This read-write path provides a comprehensive flow of data from ingestion from Kafka, through various processing steps, and finally back to Kafka, showcasing a full cycle of data processing within Apache Wayang's abstracted environment and is implemented in our example program shown in <em>listing 1</em>.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="implementation-of-input--and-output-operators">Implementation of Input- and Output Operators<a href="#implementation-of-input--and-output-operators" class="hash-link" aria-label="Direct link to Implementation of Input- and Output Operators" title="Direct link to Implementation of Input- and Output Operators"></a></h2>
+<p>The next section shows how a new pair of operators can be implemented to extend Apache Wayang’s capabilities on the input and output side.
+We created the Kafka Source and Kafka Sink components so that our cross organizational data collaboration scenario can be implemented using data streaming infrastructure.</p>
+<p><strong>Level 1 – Wayang execution plan with abstract operators</strong></p>
+<p>The implementation of our Kafka Source and Kafka Sink components for Apache Wayang requires new methods and classes on three layers.
+First of all in the API package.
+Here we use the JavaPlanBuilder to expose the function for selecting a Kafka topic as the source to be used by client.<br>
+<!-- -->The class <em>JavaPlanBuilder</em> in package <em>org.apache.wayang.api</em> in the project <em>wayang-api/wayang-api-scala-java</em> exposes our new functionality to our external client.
+An instance of the JavaPlanBuilder is used to define the data processing pipeline.
+We use its <em>readKafkaTopic()</em> which specifies the source Kafka topic to read from, and for the write path we use the <em>writeKafkaTopic()</em> method.
+Both Methods do only trigger activities in the background.</p>
+<p>For the output side, we use the <em>DataQuantaBuilder</em> class, which offers an implementation of the writeKafkaTopic function.
+This function is designed to send processed data, referred to as DataQuanta, to a specified Kafka topic.
+Essentially, it marks the final step in a data processing sequence constructed using the Apache Wayang framework.</p>
+<p>In the DataQuanta class we implemented the methods writeKafkaTopic and writeKafkaTopicJava which use the KafkaTopicSink class.
+In this API layer we use the Scala programming language, but we utilize the Java classes, implemented in the layer below.</p>
+<p><strong>Level 2 – Wiring between Platform Abstraction and Implementation</strong></p>
+<p>The second layer builds the bridge between the WayangContext and PlanBuilders which work together with DataQuanta and the DataQuantaBuilder.</p>
+<p>Also, the mapping between the abstract components and the specific implementations are defined in this layer.</p>
+<p>Therefore, the mappings package has a class <em>Mappings</em> in which all relevant input and output operators are listed.
+We use it to register the KafkaSourceMapping and a KafkaSinkMapping for the particular platform, Java in our case.
+These classes allow the Apache Wayang framework to use the Java implementation of the KafkaTopicSource component (and KafkaTopicSink respectively).
+While the Wayang execution plan uses the higher abstractions, here on the “platform level” we have to link the specific implementation for the target platform.
+In our case this leads to a Java program running on a JVM which is set up by the Apache Wayang framework using the logical components of the execution plan.</p>
+<p>Those mappings link the real implementation of our operators the ones used in an execution plan.
+The JavaKafkaTopicSource and the JavaKafkaTopicSink extend the KafkaTopicSource and KafkaTopicSink so that the lower level implementation of those classes become available within Wayang’s Java Platform context.</p>
+<p>In this layer, the KafkaConsumer class and the KafkaProducer class are used, but both are configured and instantiated in the next layer underneath.
+All this is done in the project <em>wayang-plarforms/wayang-java</em>.</p>
+<p><strong>Layer 3 – Input/Output Connector Layer</strong></p>
+<p>The <em>KafkaTopicSource</em> and <em>KafkaTopicSink</em> classes build the third layer of our implementation.
+Both are implemented in Java programming language.
+In this layer, the real Kafka-Client logic is defined.
+Details about consumer and producers, client configuration, and schema handling have to be handled here.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary"></a></h2>
+<p>Both classes in the third layer implement the Kafka client logic which is needed by the Wayang-execution plan when external data flows should be established.
+The layer above handles the mapping of the components at startup time.
+All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
+<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.
+Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class= [...]
<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
diff --git a/blog/website_update/index.html b/blog/website_update/index.html
index 0605fe9d..a9b269f0 100644
--- a/blog/website_update/index.html
+++ b/blog/website_update/index.html
@@ -5,11 +5,11 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Website updated | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/website_update"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docusaurus_tag" content="defa [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
<p>Author: <a href="https://github.com/2pk03" target="_blank" rel="noopener noreferrer">2pk03</a></p>
<p>We switched to a new CMS. That's all.</p>
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="cheatsheet">Cheatsheet<a href="#cheatsheet" class="hash-link" aria-label="Direct link to Cheatsheet" title="Direct link to Cheatsheet"></a></h2>
@@ -39,7 +39,7 @@
<li>Line one</li>
<li>Line two</li>
<li>Line three</li>
-</ul></div><footer class="row docusaurus-mt-lg blogPostFooterDetailsFull_mRVl"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li></ul></div></footer></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Blog post page navigation"><a class="pagination-nav__link pagination-nav__link--prev" href="/blog/Apache Kafka meets Apache Wayang"><div class="pagi [...]
+</ul></div><footer class="row docusaurus-mt-lg blogPostFooterDetailsFull_mRVl"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li></ul></div></footer></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Blog post page navigation"><a class="pagination-nav__link pagination-nav__link--prev" href="/blog/kafka-meets-wayang-1"><div class="pagination-nav__ [...]
<p> Apache Wayang is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that th [...]
<p>
Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. <br>
diff --git a/docs/community/committer/index.html b/docs/community/committer/index.html
index da256986..9085879d 100644
--- a/docs/community/committer/index.html
+++ b/docs/community/committer/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Becoming a committer | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/community/committer"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/community/contribute/index.html b/docs/community/contribute/index.html
index 75a2ff27..979bf9a5 100644
--- a/docs/community/contribute/index.html
+++ b/docs/community/contribute/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">How to contribute | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/community/contribute"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" c [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/community/mailinglist/index.html b/docs/community/mailinglist/index.html
index 4633fe94..959081b2 100644
--- a/docs/community/mailinglist/index.html
+++ b/docs/community/mailinglist/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Mailinglists | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/community/mailinglist"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" conte [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/community/repositories/index.html b/docs/community/repositories/index.html
index 446de43d..94f2a85a 100644
--- a/docs/community/repositories/index.html
+++ b/docs/community/repositories/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Repositories | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/community/repositories"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" cont [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/community/team/index.html b/docs/community/team/index.html
index f1071e09..42c2c099 100644
--- a/docs/community/team/index.html
+++ b/docs/community/team/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Team | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/community/team"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta d [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/guide/adding-operators/index.html b/docs/guide/adding-operators/index.html
index 09a8a66e..2e2aab05 100644
--- a/docs/guide/adding-operators/index.html
+++ b/docs/guide/adding-operators/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Adding an operator in Wayang | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/guide/adding-operators"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearc [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/guide/api-documentation/index.html b/docs/guide/api-documentation/index.html
index 0a9c1960..b4c7f537 100644
--- a/docs/guide/api-documentation/index.html
+++ b/docs/guide/api-documentation/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">API documentation | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/guide/api-documentation"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/guide/developing-in-wayang/index.html b/docs/guide/developing-in-wayang/index.html
index 00b2c0e1..f5821402 100644
--- a/docs/guide/developing-in-wayang/index.html
+++ b/docs/guide/developing-in-wayang/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Developing in Wayang | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/guide/developing-in-wayang"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:la [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/guide/examples/index.html b/docs/guide/examples/index.html
index aa5d3306..b9c94296 100644
--- a/docs/guide/examples/index.html
+++ b/docs/guide/examples/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Usage Examples | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/guide/examples"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="e [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/guide/getting-started/index.html b/docs/guide/getting-started/index.html
index 3dbe492e..3dcbe4e2 100644
--- a/docs/guide/getting-started/index.html
+++ b/docs/guide/getting-started/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Getting started | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/guide/getting-started"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" co [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/guide/installation/index.html b/docs/guide/installation/index.html
index 83e95233..f76c24fa 100644
--- a/docs/guide/installation/index.html
+++ b/docs/guide/installation/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">How to build Wayang | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/guide/installation"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" c [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/guide/ml4all/index.html b/docs/guide/ml4all/index.html
index 4326fb15..3867b089 100644
--- a/docs/guide/ml4all/index.html
+++ b/docs/guide/ml4all/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Scalable Machine Learning | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/guide/ml4all"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" c [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/guide/optimizer/index.html b/docs/guide/optimizer/index.html
index 0909fb0c..188c727f 100644
--- a/docs/guide/optimizer/index.html
+++ b/docs/guide/optimizer/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Data processing optimizer | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/guide/optimizer"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/introduction/about/index.html b/docs/introduction/about/index.html
index edf36a3e..2cf273d0 100644
--- a/docs/introduction/about/index.html
+++ b/docs/introduction/about/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">What is Wayang? | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/introduction/about"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" conte [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/introduction/benchmark/index.html b/docs/introduction/benchmark/index.html
index 993d431f..662e9773 100644
--- a/docs/introduction/benchmark/index.html
+++ b/docs/introduction/benchmark/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Benchmarking Wayang | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/introduction/benchmark"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:languag [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/introduction/features/index.html b/docs/introduction/features/index.html
index 1fe581aa..5757e869 100644
--- a/docs/introduction/features/index.html
+++ b/docs/introduction/features/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Features | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/introduction/features"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content=" [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/start/download/index.html b/docs/start/download/index.html
index e284df47..ecfcf92e 100644
--- a/docs/start/download/index.html
+++ b/docs/start/download/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Download Apache Wayang | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/start/download"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" co [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/index.html b/index.html
index 05193e6e..f05736c2 100644
--- a/index.html
+++ b/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Apache Wayang (incubating) | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docusaurus_tag" content="default"><me [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.d19f8cc4.js" defer="defer"></script>
-<script src="/assets/js/main.3d1f1383.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.06eee15f.js" defer="defer"></script>
+<script src="/assets/js/main.f140bb99.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/lunr-index-1709719224803.json b/lunr-index-1709719224803.json
deleted file mode 100644
index 2dd50cc5..00000000
--- a/lunr-index-1709719224803.json
+++ /dev/null
@@ -1 +0,0 @@
-{"version":"2.3.9","fields":["title","content","keywords"],"fieldVectors":[["title/0",[0,754.742,1,697.992]],["content/0",[]],["keywords/0",[]],["title/1",[2,1134.943]],["content/1",[3,6.397,4,9.894,5,5.653,6,11.887,7,11.887,8,7.321,9,11.887,10,11.887,11,9.971,12,4.08,13,9.483,14,7.321,15,11.887,16,11.887]],["keywords/1",[]],["title/2",[17,167.422,18,302.596,19,516.393,20,49.045,21,302.596,22,318.023]],["content/2",[]],["keywords/2",[]],["title/3",[12,162.494,17,98.237,18,277.399,23,326. [...]
\ No newline at end of file
diff --git a/lunr-index-1709719417673.json b/lunr-index-1709719417673.json
new file mode 100644
index 00000000..eefc201c
--- /dev/null
+++ b/lunr-index-1709719417673.json
@@ -0,0 +1 @@
+{"version":"2.3.9","fields":["title","content","keywords"],"fieldVectors":[["title/0",[0,161.042,1,271.853,2,461.928,3,47.784,4,283.183,5,309.503]],["content/0",[]],["keywords/0",[]],["title/1",[0,94.759,1,249.341,6,333.603,7,259.733,8,167.699,9,283.873,10,298.171,11,314.519]],["content/1",[0,1.786,3,1.339,7,6.574,9,7.185,10,5.62,12,5.351,13,4.574,14,6.72,15,4.52,16,2.53,17,7.986,18,7.986,19,7.749,20,6.72,21,3.255,22,5.111,23,3.563,24,6.288,25,7.261,26,5.928,27,4.45,28,7.547,29,5.111,30, [...]
\ No newline at end of file
diff --git a/lunr-index.json b/lunr-index.json
index 2dd50cc5..eefc201c 100644
--- a/lunr-index.json
+++ b/lunr-index.json
@@ -1 +1 @@
-{"version":"2.3.9","fields":["title","content","keywords"],"fieldVectors":[["title/0",[0,754.742,1,697.992]],["content/0",[]],["keywords/0",[]],["title/1",[2,1134.943]],["content/1",[3,6.397,4,9.894,5,5.653,6,11.887,7,11.887,8,7.321,9,11.887,10,11.887,11,9.971,12,4.08,13,9.483,14,7.321,15,11.887,16,11.887]],["keywords/1",[]],["title/2",[17,167.422,18,302.596,19,516.393,20,49.045,21,302.596,22,318.023]],["content/2",[]],["keywords/2",[]],["title/3",[12,162.494,17,98.237,18,277.399,23,326. [...]
\ No newline at end of file
+{"version":"2.3.9","fields":["title","content","keywords"],"fieldVectors":[["title/0",[0,161.042,1,271.853,2,461.928,3,47.784,4,283.183,5,309.503]],["content/0",[]],["keywords/0",[]],["title/1",[0,94.759,1,249.341,6,333.603,7,259.733,8,167.699,9,283.873,10,298.171,11,314.519]],["content/1",[0,1.786,3,1.339,7,6.574,9,7.185,10,5.62,12,5.351,13,4.574,14,6.72,15,4.52,16,2.53,17,7.986,18,7.986,19,7.749,20,6.72,21,3.255,22,5.111,23,3.563,24,6.288,25,7.261,26,5.928,27,4.45,28,7.547,29,5.111,30, [...]
\ No newline at end of file
diff --git a/search-doc-1709719224803.json b/search-doc-1709719224803.json
deleted file mode 100644
index 7e3c47b1..00000000
--- a/search-doc-1709719224803.json
+++ /dev/null
@@ -1 +0,0 @@
-{"searchDocs":[{"title":"Website updated","type":0,"sectionRef":"#","url":"/blog/website_update","content":"","keywords":"","version":null},{"title":"Cheatsheet","type":1,"pageTitle":"Website updated","url":"/blog/website_update#cheatsheet","content":" List: Line one Line one.oneLine one.two Line two Line two.oneLine two.two Line three ...... Another style for a list: Line oneLine twoLine three ","version":null,"tagName":"h2"},{"title":"Apache Kafka meets Apache Wayang - Part 2","typ [...]
\ No newline at end of file
diff --git a/search-doc-1709719417673.json b/search-doc-1709719417673.json
new file mode 100644
index 00000000..1a8752cf
--- /dev/null
+++ b/search-doc-1709719417673.json
@@ -0,0 +1 @@
+{"searchDocs":[{"title":"Apache Kafka meets Apache Wayang - Part 2","type":0,"sectionRef":"#","url":"/blog/Apache Kafka meets Apache Wayang","content":"","keywords":"","version":null},{"title":"Apache Wayang’s Read & Write Path for Kafka topics","type":1,"pageTitle":"Apache Kafka meets Apache Wayang - Part 2","url":"/blog/Apache Kafka meets Apache Wayang#apache-wayangs-read--write-path-for-kafka-topics","content":" To describe the read and write paths for data in the context of the crea [...]
\ No newline at end of file
diff --git a/search-doc.json b/search-doc.json
index 7e3c47b1..1a8752cf 100644
--- a/search-doc.json
+++ b/search-doc.json
@@ -1 +1 @@
-{"searchDocs":[{"title":"Website updated","type":0,"sectionRef":"#","url":"/blog/website_update","content":"","keywords":"","version":null},{"title":"Cheatsheet","type":1,"pageTitle":"Website updated","url":"/blog/website_update#cheatsheet","content":" List: Line one Line one.oneLine one.two Line two Line two.oneLine two.two Line three ...... Another style for a list: Line oneLine twoLine three ","version":null,"tagName":"h2"},{"title":"Apache Kafka meets Apache Wayang - Part 2","typ [...]
\ No newline at end of file
+{"searchDocs":[{"title":"Apache Kafka meets Apache Wayang - Part 2","type":0,"sectionRef":"#","url":"/blog/Apache Kafka meets Apache Wayang","content":"","keywords":"","version":null},{"title":"Apache Wayang’s Read & Write Path for Kafka topics","type":1,"pageTitle":"Apache Kafka meets Apache Wayang - Part 2","url":"/blog/Apache Kafka meets Apache Wayang#apache-wayangs-read--write-path-for-kafka-topics","content":" To describe the read and write paths for data in the context of the crea [...]
\ No newline at end of file
diff --git a/sitemap.xml b/sitemap.xml
index 3d24067d..08b126e4 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -1 +1 @@
-<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:news="http://www.google.com/schemas/sitemap-news/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1" xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"><url><loc>https://wayang.apache.org/blog</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://wayang.apache.org/blog/Apache%20Kafka%20me [...]
\ No newline at end of file
+<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:news="http://www.google.com/schemas/sitemap-news/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1" xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"><url><loc>https://wayang.apache.org/blog</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://wayang.apache.org/blog/Apache%20Kafka%20me [...]
\ No newline at end of file